From d425cbe2f0df08aacd9f5093a758455309ab0cf5 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Thu, 9 Jul 2026 13:38:52 +0000 Subject: [PATCH] 8286300: Port JEP 425 to S390X 8377034: Enable full JSR166TestCase.java test for s390x 8380035: compiler/intrinsics/TestReturnOopSetForJFRWriteCheckpoint.java crashes on s390x 8335163: [s390x] test failure - PrintClasses.java Co-authored-by: Andrew Haley Co-authored-by: Richard Reingruber Reviewed-by: rrich, aph, pchilanomate --- .../cpu/s390/abstractInterpreter_s390.cpp | 8 +- src/hotspot/cpu/s390/assembler_s390.hpp | 5 +- src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp | 4 +- src/hotspot/cpu/s390/c1_Runtime1_s390.cpp | 49 +- .../cpu/s390/continuationEntry_s390.hpp | 6 +- .../s390/continuationEntry_s390.inline.hpp | 23 +- .../continuationFreezeThaw_s390.inline.hpp | 277 ++++++++- .../s390/continuationHelper_s390.inline.hpp | 116 ++-- src/hotspot/cpu/s390/frame_s390.cpp | 85 ++- src/hotspot/cpu/s390/frame_s390.hpp | 49 +- src/hotspot/cpu/s390/frame_s390.inline.hpp | 106 +++- src/hotspot/cpu/s390/globals_s390.hpp | 2 +- src/hotspot/cpu/s390/interp_masm_s390.cpp | 126 +++- src/hotspot/cpu/s390/interp_masm_s390.hpp | 8 +- src/hotspot/cpu/s390/macroAssembler_s390.cpp | 114 +++- src/hotspot/cpu/s390/macroAssembler_s390.hpp | 26 +- .../cpu/s390/macroAssembler_s390.inline.hpp | 14 +- src/hotspot/cpu/s390/nativeInst_s390.cpp | 31 +- src/hotspot/cpu/s390/nativeInst_s390.hpp | 44 +- src/hotspot/cpu/s390/s390.ad | 35 ++ src/hotspot/cpu/s390/sharedRuntime_s390.cpp | 582 +++++++++++++++++- .../cpu/s390/smallRegisterMap_s390.inline.hpp | 17 +- .../stackChunkFrameStream_s390.inline.hpp | 90 ++- .../cpu/s390/stackChunkOop_s390.inline.hpp | 11 +- .../cpu/s390/stubDeclarations_s390.hpp | 4 +- src/hotspot/cpu/s390/stubGenerator_s390.cpp | 184 +++++- .../templateInterpreterGenerator_s390.cpp | 71 ++- src/hotspot/cpu/s390/templateTable_s390.cpp | 8 +- src/hotspot/cpu/s390/upcallLinker_s390.cpp | 4 + .../share/oops/stackChunkOop.inline.hpp | 4 +- src/hotspot/share/runtime/continuation.cpp | 2 +- .../share/runtime/continuationFreezeThaw.cpp | 29 +- src/hotspot/share/runtime/frame.cpp | 6 +- src/hotspot/share/runtime/sharedRuntime.cpp | 10 +- test/hotspot/jtreg/ProblemList.txt | 2 - test/jdk/ProblemList.txt | 18 - .../util/concurrent/tck/JSR166TestCase.java | 14 - 37 files changed, 1840 insertions(+), 344 deletions(-) diff --git a/src/hotspot/cpu/s390/abstractInterpreter_s390.cpp b/src/hotspot/cpu/s390/abstractInterpreter_s390.cpp index 96990f0ce94..c54f1a4b010 100644 --- a/src/hotspot/cpu/s390/abstractInterpreter_s390.cpp +++ b/src/hotspot/cpu/s390/abstractInterpreter_s390.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -197,8 +197,10 @@ void AbstractInterpreter::layout_activation(Method* method, assert(is_bottom_frame && (sender_sp == caller->unextended_sp()), "must initialize sender_sp of bottom skeleton frame when pushing it"); } else { - assert(caller->is_entry_frame() || caller->is_upcall_stub_frame(), "is there a new frame type??"); - sender_sp = caller->sp(); // Call_stub only uses it's fp. + // For entry, upcall_stub, and native frames, sender_sp is simply the caller's sp. + // These frames use the standard C ABI and don't require adjustment. + assert(caller->is_entry_frame() || caller->is_upcall_stub_frame() || caller->is_native_frame(), "is there a new frame type??"); + sender_sp = caller->sp(); } interpreter_frame->interpreter_frame_set_method(method); diff --git a/src/hotspot/cpu/s390/assembler_s390.hpp b/src/hotspot/cpu/s390/assembler_s390.hpp index c0cee5bd555..95ae442bb49 100644 --- a/src/hotspot/cpu/s390/assembler_s390.hpp +++ b/src/hotspot/cpu/s390/assembler_s390.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016, 2024 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -3279,6 +3279,9 @@ class Assembler : public AbstractAssembler { static bool is_z_nop(address x) { return is_z_nop(* (short *) x); } + static bool is_z_illtrap(address x) { + return *(uint16_t*)x == 0u; + } static bool is_z_br(long x) { return is_z_bcr(x) && ((x & 0x00f0) == 0x00f0); } diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp index 08f922a0b9a..db3f2f6218f 100644 --- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp @@ -524,6 +524,7 @@ void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { __ z_nop(); __ z_brasl(Z_R14, op->addr()); add_call_info(code_offset(), op->info()); + __ post_call_nop(); } void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { @@ -539,7 +540,7 @@ void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { // CALL to fixup routine. Fixup routine uses ScopeDesc info // to determine who we intended to call. __ relocate(virtual_call_Relocation::spec(virtual_call_oop_addr)); - call(op, relocInfo::none); + call(op, relocInfo::none); // call will emit a post call nop, see above method. } void LIR_Assembler::move_regs(Register from_reg, Register to_reg) { @@ -2792,6 +2793,7 @@ void LIR_Assembler::rt_call(LIR_Opr result, address dest, if (info != nullptr) { add_call_info_here(info); } + __ post_call_nop(); } void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) { diff --git a/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp b/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp index e78b04fe911..d26db67d078 100644 --- a/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp +++ b/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016, 2023 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -52,13 +52,8 @@ int StubAssembler::call_RT(Register oop_result1, Register metadata_result, addre set_num_rt_args(0); // Nothing on stack. assert(!(oop_result1->is_valid() || metadata_result->is_valid()) || oop_result1 != metadata_result, "registers must be different"); - // We cannot trust that code generated by the C++ compiler saves R14 - // to z_abi_160.return_pc, because sometimes it spills R14 using stmg at - // z_abi_160.gpr14 (e.g. InterpreterRuntime::_new()). - // Therefore we load the PC into Z_R1_scratch and let set_last_Java_frame() save - // it into the frame anchor. - address pc = get_PC(Z_R1_scratch); - int call_offset = (int)(pc - addr_at(0)); + Label resume; + z_larl(Z_R1_scratch, resume); set_last_Java_frame(Z_SP, Z_R1_scratch); // ARG1 must hold thread address. @@ -67,9 +62,12 @@ int StubAssembler::call_RT(Register oop_result1, Register metadata_result, addre address return_pc = nullptr; align_call_far_patchable(this->pc()); return_pc = call_c_opt(entry_point); + + bind(resume); + int call_offset = offset(); assert(return_pc != nullptr, "const section overflow"); - reset_last_Java_frame(); + reset_last_Java_frame(/* check_last_java_sp= */ false); // Check for pending exceptions. { @@ -208,8 +206,37 @@ void Runtime1::initialize_pd() { } uint Runtime1::runtime_blob_current_thread_offset(frame f) { - Unimplemented(); - return 0; + CodeBlob* cb = f.cb(); + assert(cb == Runtime1::blob_for(StubId::c1_monitorenter_id) || + cb == Runtime1::blob_for(StubId::c1_monitorenter_nofpu_id), "must be"); + assert(cb != nullptr && cb->is_runtime_stub(), "invalid frame"); + + // Calculate the offset of Z_thread (Z_R8) in the saved register area. + // Both c1_monitorenter_id and c1_monitorenter_nofpu_id have the same frame layout: + // - c1_monitorenter_id uses RegisterSaver::all_registers (saves FPU regs) + // - c1_monitorenter_nofpu_id uses RegisterSaver::all_integer_registers (excludes FPU regs but reserves space) + // + // From RegisterSaver_LiveRegs and RegisterSaver_LiveIntRegs: + // Both have 15 float register slots (F0, F2-F15, F1 is excluded as scratch) + // Then integer registers: R2, R3, R4, R5, R6, R7, R8, R9, R10, R11, R12, R13 + // Z_thread is Z_R8, which is the 7th integer register (index 6 from R2) + // + // Stack layout from SP: + // [0..159] : z_abi_160 + // [160..279] : 15 float register slots (15 * 8 = 120 bytes) + // [280..327] : R2-R7 (6 * 8 = 48 bytes) + // [328..335] : R8 (Z_thread) <- this is what we need + // + // Offset = 160 + 120 + 48 = 328 bytes from SP + // Return value is in 64-bit words: 328 / 8 = 41 + + const int float_reg_slots = 15; // F0, F2-F15 (F1 is scratch, excluded) + const int int_regs_before_r8 = 6; // R2, R3, R4, R5, R6, R7 + const int z_thread_offset = frame::z_abi_160_size + + (float_reg_slots * 8) + + (int_regs_before_r8 * 8); + + return z_thread_offset / wordSize; } OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) { diff --git a/src/hotspot/cpu/s390/continuationEntry_s390.hpp b/src/hotspot/cpu/s390/continuationEntry_s390.hpp index e4e611d2b15..15b1347ce0a 100644 --- a/src/hotspot/cpu/s390/continuationEntry_s390.hpp +++ b/src/hotspot/cpu/s390/continuationEntry_s390.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2026 IBM Corporation. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,8 +26,11 @@ #ifndef CPU_S390_CONTINUATIONENTRY_S390_HPP #define CPU_S390_CONTINUATIONENTRY_S390_HPP +#include "runtime/frame.hpp" + class ContinuationEntryPD { - // empty + // This is needed to position the ContinuationEntry at the unextended sp of the entry frame + frame::z_abi_160_base _abi; }; #endif // CPU_S390_CONTINUATIONENTRY_S390_HPP diff --git a/src/hotspot/cpu/s390/continuationEntry_s390.inline.hpp b/src/hotspot/cpu/s390/continuationEntry_s390.inline.hpp index 1d4e3c2439d..58ff8f0d194 100644 --- a/src/hotspot/cpu/s390/continuationEntry_s390.inline.hpp +++ b/src/hotspot/cpu/s390/continuationEntry_s390.inline.hpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2026 IBM Corporation. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,22 +26,28 @@ #ifndef CPU_S390_CONTINUATIONENTRY_S390_INLINE_HPP #define CPU_S390_CONTINUATIONENTRY_S390_INLINE_HPP +#include "oops/method.inline.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" +#include "utilities/macros.hpp" #include "runtime/continuationEntry.hpp" -// TODO: Implement - inline frame ContinuationEntry::to_frame() const { - Unimplemented(); - return frame(); + static CodeBlob* cb = CodeCache::find_blob_fast(entry_pc()); + assert(cb != nullptr, ""); + assert(cb->as_nmethod()->method()->is_continuation_enter_intrinsic(), ""); + return frame(entry_sp(), entry_pc(), entry_sp(), entry_fp(), cb); } inline intptr_t* ContinuationEntry::entry_fp() const { - Unimplemented(); - return nullptr; + return (intptr_t*)((address)this + size()); } inline void ContinuationEntry::update_register_map(RegisterMap* map) const { - Unimplemented(); + // No register map update needed for s390. + // In the Java calling convention on s390, all registers are volatile (caller-saved), + // so there are no non-volatile (callee-saved) registers that need to be tracked + // in the register map for continuation entry frames. } #endif // CPU_S390_CONTINUATIONENTRY_S390_INLINE_HPP diff --git a/src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp b/src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp index 1102a745ac0..2f7660052c0 100644 --- a/src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp +++ b/src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2026 IBM Corporation. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,98 +30,316 @@ #include "runtime/frame.hpp" #include "runtime/frame.inline.hpp" +inline void patch_callee_link(const frame& f, intptr_t* fp) { + *ContinuationHelper::Frame::callee_link_address(f) = fp; +} + +inline void patch_callee_link_relative(const frame& f, intptr_t* fp) { + intptr_t* la = (intptr_t*)ContinuationHelper::Frame::callee_link_address(f); + intptr_t new_value = fp - la; + *la = new_value; +} + inline void FreezeBase::set_top_frame_metadata_pd(const frame& hf) { - Unimplemented(); + stackChunkOop chunk = _cont.tail(); + assert(chunk->is_in_chunk(hf.sp()), "hf.sp()=" PTR_FORMAT, p2i(hf.sp())); + + hf.own_abi()->return_pc = (uint64_t)hf.pc(); + if (hf.is_interpreted_frame()) { + patch_callee_link_relative(hf, hf.fp()); + } else { +#ifdef ASSERT + // See also FreezeBase::patch_pd() + patch_callee_link(hf, (intptr_t*)badAddress); +#endif + } } template inline frame FreezeBase::sender(const frame& f) { - Unimplemented(); - return frame(); + assert(FKind::is_instance(f), ""); + + if (FKind::interpreted) { + return frame(f.sender_sp(), f.sender_pc(), f.interpreter_frame_sender_sp()); + } + + intptr_t* sender_sp = f.sender_sp(); + address sender_pc = f.sender_pc(); + assert(sender_sp != f.sp(), "must have changed"); + int slot = 0; + CodeBlob* sender_cb = CodeCache::find_blob_and_oopmap(sender_pc, slot); + return sender_cb != nullptr + ? frame(sender_sp, sender_sp, nullptr, sender_pc, sender_cb, slot == -1 ? nullptr : sender_cb->oop_map_for_slot(slot, sender_pc)) + : frame(sender_sp, sender_pc, sender_sp); } template frame FreezeBase::new_heap_frame(frame& f, frame& caller) { - Unimplemented(); - return frame(); + assert(FKind::is_instance(f), ""); + intptr_t *sp, *fp; + if (FKind::interpreted) { + intptr_t locals_offset = *f.addr_at(_z_ijava_idx(locals)); + + // If the caller.is_empty(), i.e. we're freezing into an empty chunk, then we set + // the chunk's argsize in finalize_freeze and make room for it above the unextended_sp + // See also comment on StackChunkFrameStream::interpreter_frame_size() + + int overlap = + (caller.is_interpreted_frame() || caller.is_empty()) + ? ContinuationHelper::InterpretedFrame::stack_argsize(f) + frame::metadata_words_at_top + : 0; + + // Calculate the new frame's FP in the heap chunk. + // Starting from caller's unextended_sp, we: + // - subtract 1 for the z_parent_ijava_frame_abi (which sits just below the locals) + // - subtract locals_offset (distance from FP to locals in the original frame) + // - add overlap (to account for shared stack args when caller is interpreted or empty) + // This positions FP such that locals are correctly placed relative to the caller's frame. + fp = caller.unextended_sp() - 1 - locals_offset + overlap; + + // esp points one slot below the last argument + intptr_t* x86_64_like_unextended_sp = f.interpreter_frame_esp() + 1 - frame::metadata_words_at_top; + + sp = fp - (f.fp() - x86_64_like_unextended_sp); + assert (sp <= fp && (fp <= caller.unextended_sp() || caller.is_interpreted_frame()), + "sp=" PTR_FORMAT " fp=" PTR_FORMAT " caller.unextended_sp()=" PTR_FORMAT " caller.is_interpreted_frame()=%d", + p2i(sp), p2i(fp), p2i(caller.unextended_sp()), caller.is_interpreted_frame()); + caller.set_sp(fp); + + assert(_cont.tail()->is_in_chunk(sp), ""); + + frame hf(sp, sp, fp, f.pc(), nullptr, nullptr, true /* on_heap */); + // frame_top() and frame_bottom() read these before relativize_interpreted_frame_metadata() is called + *hf.addr_at(_z_ijava_idx(locals)) = locals_offset; + *hf.addr_at(_z_ijava_idx(esp)) = f.interpreter_frame_esp() - f.fp(); + return hf; + } else { + int fsize = FKind::size(f); + sp = caller.unextended_sp() - fsize; + if (caller.is_interpreted_frame()) { + // If the caller is interpreted, our stackargs are not supposed to overlap with it + // so we make more room by moving sp down by argsize + int argsize = FKind::stack_argsize(f); + sp -= argsize + frame::metadata_words_at_top; + } + fp = sp + fsize; + caller.set_sp(fp); + + assert(_cont.tail()->is_in_chunk(sp), ""); + + return frame(sp, sp, fp, f.pc(), nullptr, nullptr, true /* on_heap */); + } } void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { - Unimplemented(); + // Nothing to do on s390 and ppc. On x86/aarch64/riscv, the unextended_sp is stored + // in interpreter_frame_last_sp and needs to be restored from there. On s390/ppc, + // the frame structure doesn't have interpreter_frame_last_sp; instead, the unextended_sp + // is directly maintained in the frame and doesn't need adjustment. } inline void FreezeBase::prepare_freeze_interpreted_top_frame(frame& f) { - Unimplemented(); + // Nothing to do. We don't save a last sp because we cannot use sp as esp. + // Instead the top frame is trimmed when making an i2i call. The original + // top_frame_sp is set when the frame is pushed (see generate_fixed_frame()). + // An interpreter top frame that was just thawed is resized to top_frame_sp by the + // resume adapter (see generate_cont_resume_interpreter_adapter()). So the assertion is + // false, if we freeze again right after thawing as we do when redoing a vm call wasn't + // successful. + assert(_thread->interp_redoing_vm_call() || + ((intptr_t*)f.at_relative(_z_ijava_idx(top_frame_sp)) == f.unextended_sp()), + "top_frame_sp:" PTR_FORMAT " usp:" PTR_FORMAT, f.at_relative(_z_ijava_idx(top_frame_sp)), p2i(f.unextended_sp())); } inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { - Unimplemented(); + intptr_t* vfp = f.fp(); + intptr_t* hfp = hf.fp(); + assert(f.fp() > (intptr_t*)f.interpreter_frame_esp(), ""); + + // There is alignment padding between vfp and f's locals array in the original + // frame, because we freeze the padding (see recurse_freeze_interpreted_frame) + // in order to keep the same relativized locals pointer, we don't need to change it here. + + // Make sure that monitors is already relativized. + assert(hf.at_absolute(_z_ijava_idx(monitors)) <= -(frame::z_ijava_state_size / wordSize), ""); + // Make sure that esp is already relativized. + assert(hf.at_absolute(_z_ijava_idx(esp)) <= hf.at_absolute(_z_ijava_idx(monitors)), ""); + // top_frame_sp is already relativized + + // hfp == hf.sp() + (f.fp() - f.sp()) is not true on ppc because the stack frame has room for + // the maximal expression stack and the expression stack in the heap frame is trimmed. + assert(hf.fp() == hf.interpreter_frame_esp() + (f.fp() - f.interpreter_frame_esp()), ""); + assert(hf.fp() <= (intptr_t*)hf.at(_z_ijava_idx(locals)), ""); } inline void FreezeBase::patch_pd(frame& hf, const frame& caller) { - Unimplemented(); + if (caller.is_interpreted_frame()) { + assert(!caller.is_empty(), ""); + patch_callee_link_relative(caller, caller.fp()); + } +#ifdef ASSERT + else { + // For compiled frames the back link is actually redundant. It gets computed + // as unextended_sp + frame_size. + + // Note a difference from x86_64: the link is not made relative if the caller + // is a compiled frame because there rbp is used as a non-volatile register by + // c1/c2 so it could be a computed value local to the caller. + + // See also: + // - FreezeBase::set_top_frame_metadata_pd + // - StackChunkFrameStream::fp() + // - UseContinuationFastPath: compiled frames are copied in a batch w/o patching the back link. + // The backlinks are restored when thawing (see Thaw::patch_caller_links()) + patch_callee_link(hf, (intptr_t*)badAddress); + } +#endif } inline void FreezeBase::patch_pd_unused(intptr_t* sp) { - Unimplemented(); } inline void FreezeBase::patch_stack_pd(intptr_t* frame_sp, intptr_t* heap_sp) { - Unimplemented(); + // Nothing to do. The backchain is reconstructed when thawing (see Thaw::patch_caller_links()) } inline intptr_t* AnchorMark::anchor_mark_set_pd() { - Unimplemented(); - return nullptr; + // Nothing to do on s390 because the interpreter does not use SP as expression stack pointer. + // Instead there is a dedicated register Z_esp which is not affected by VM calls. + return _top_frame.sp(); } inline void AnchorMark::anchor_mark_clear_pd() { - Unimplemented(); + // Nothing to do. See anchor_mark_set_pd(). } inline frame ThawBase::new_entry_frame() { - Unimplemented(); - return frame(); + intptr_t* sp = _cont.entrySP(); + return frame(sp, _cont.entryPC(), sp, _cont.entryFP()); } template frame ThawBase::new_stack_frame(const frame& hf, frame& caller, bool bottom) { - Unimplemented(); - return frame(); + assert(FKind::is_instance(hf), ""); + + assert(is_aligned(caller.fp(), frame::frame_alignment), PTR_FORMAT, p2i(caller.fp())); + // caller.sp() can be unaligned. This is fixed below. + if (FKind::interpreted) { + // Note: we have to overlap with the caller, at least if it is interpreted, to match the + // max_thawing_size calculation during freeze. See also comment above. + intptr_t* heap_sp = hf.unextended_sp(); + const int fsize = ContinuationHelper::InterpretedFrame::frame_bottom(hf) - hf.unextended_sp(); + const int overlap = !caller.is_interpreted_frame() ? 0 + : ContinuationHelper::InterpretedFrame::stack_argsize(hf) + frame::metadata_words_at_top; + intptr_t* frame_sp = caller.unextended_sp() + overlap - fsize; + intptr_t* fp = frame_sp + (hf.fp() - heap_sp); + // align fp + int padding = fp - align_down(fp, frame::frame_alignment); + fp -= padding; + // alignment of sp is done by callee or in finish_thaw() + frame_sp -= padding; + + // On s390 esp points to the first free slot on the expression stack (see frame_s390.hpp). + // The assertion verifies that frame_sp + metadata_words_at_top points to the slot above esp, + // which corresponds to the last parameter position. + DEBUG_ONLY(intptr_t* esp = fp + *hf.addr_at(_z_ijava_idx(esp));) + assert(frame_sp + frame::metadata_words_at_top == esp+1, " frame_sp=" PTR_FORMAT " esp=" PTR_FORMAT, p2i(frame_sp), p2i(esp)); + caller.set_sp(fp); + frame f(frame_sp, hf.pc(), frame_sp, fp); + // we need to set the locals so that the caller of new_stack_frame() can call + // ContinuationHelper::InterpretedFrame::frame_bottom + // copy relativized locals from the heap frame + *f.addr_at(_z_ijava_idx(locals)) = *hf.addr_at(_z_ijava_idx(locals)); + + return f; + } else { + int fsize = FKind::size(hf); + int argsize = FKind::stack_argsize(hf); + intptr_t* frame_sp = caller.sp() - fsize; + + if ((bottom && argsize > 0) || caller.is_interpreted_frame()) { + assert(!_should_patch_caller_pc, "what??"); + _should_patch_caller_pc = caller.is_interpreted_frame(); + frame_sp -= argsize + frame::metadata_words_at_top; + frame_sp = align_down(frame_sp, frame::alignment_in_bytes); + caller.set_sp(frame_sp + fsize); + } + + assert(hf.cb() != nullptr, ""); + assert(hf.oop_map() != nullptr, ""); + intptr_t* fp = frame_sp + fsize; + return frame(frame_sp, frame_sp, fp, hf.pc(), hf.cb(), hf.oop_map(), false); + } } inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, const frame& f) { - Unimplemented(); + // Make sure that monitors is still relativized. + assert(f.at_absolute(_z_ijava_idx(monitors)) <= -(frame::z_ijava_state_size / wordSize), ""); + // Make sure that esp is still relativized. + assert(f.at_absolute(_z_ijava_idx(esp)) <= f.at_absolute(_z_ijava_idx(monitors)), ""); + // Keep top_frame_sp relativized. } inline intptr_t* ThawBase::align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom) { - Unimplemented(); + // Unused. Alignment is done directly in new_stack_frame() / finish_thaw(). return nullptr; } inline void ThawBase::patch_pd(frame& f, const frame& caller) { - Unimplemented(); + patch_callee_link(caller, caller.fp()); + // Prevent assertion if f gets deoptimized right away before it's fully initialized + f.mark_not_fully_initialized(); } inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) { - Unimplemented(); + assert(f.own_abi()->callers_sp == (uint64_t)caller_sp, "should have been fixed by patch_caller_links"); } inline intptr_t* ThawBase::push_cleanup_continuation() { - Unimplemented(); - return nullptr; + frame enterSpecial = new_entry_frame(); + frame::z_common_abi* enterSpecial_abi = (frame::z_common_abi*)enterSpecial.sp(); + + enterSpecial_abi->return_pc = (intptr_t)ContinuationEntry::cleanup_pc(); + + log_develop_trace(continuations, preempt)("push_cleanup_continuation enterSpecial sp: " INTPTR_FORMAT " cleanup pc: " INTPTR_FORMAT, + p2i(enterSpecial_abi), + p2i(ContinuationEntry::cleanup_pc())); + + return enterSpecial.sp(); } inline intptr_t* ThawBase::push_preempt_adapter() { - Unimplemented(); - return nullptr; + frame enterSpecial = new_entry_frame(); + frame::z_common_abi* enterSpecial_abi = (frame::z_common_abi*)enterSpecial.sp(); + + enterSpecial_abi->return_pc = (intptr_t)StubRoutines::cont_preempt_stub(); + + log_develop_trace(continuations, preempt)("push_preempt_adapter enterSpecial sp: " INTPTR_FORMAT " adapter pc: " INTPTR_FORMAT, + p2i(enterSpecial_abi), + p2i(StubRoutines::cont_preempt_stub())); + + return enterSpecial.sp(); } template inline void Thaw::patch_caller_links(intptr_t* sp, intptr_t* bottom) { - Unimplemented(); + for (intptr_t* callers_sp; sp < bottom; sp = callers_sp) { + address pc = (address)((frame::z_java_abi*) sp)->return_pc; + assert(pc != nullptr, ""); + // see ThawBase::patch_return() which gets called just before + bool is_entry_frame = pc == StubRoutines::cont_returnBarrier() || pc == _cont.entryPC(); + if (is_entry_frame) { + callers_sp = _cont.entryFP(); + } else { + assert(!Interpreter::contains(pc), "sp:" PTR_FORMAT " pc:" PTR_FORMAT, p2i(sp), p2i(pc)); + CodeBlob* cb = CodeCache::find_blob(pc); + callers_sp = sp + cb->frame_size(); + } + // set the back link + ((frame::z_java_abi*) sp)->callers_sp = (intptr_t) callers_sp; + } } inline void ThawBase::prefetch_chunk_pd(void* start, int size) { - Unimplemented(); + // TODO: implement in future; } #endif // CPU_S390_CONTINUATION_S390_INLINE_HPP diff --git a/src/hotspot/cpu/s390/continuationHelper_s390.inline.hpp b/src/hotspot/cpu/s390/continuationHelper_s390.inline.hpp index fb7d998c458..11944a8f040 100644 --- a/src/hotspot/cpu/s390/continuationHelper_s390.inline.hpp +++ b/src/hotspot/cpu/s390/continuationHelper_s390.inline.hpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2026 IBM Corporation. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,108 +28,133 @@ #include "runtime/continuationHelper.hpp" -// TODO: Implement - -template -static inline intptr_t** link_address(const frame& f) { - Unimplemented(); - return nullptr; -} - static inline void patch_return_pc_with_preempt_stub(frame& f) { - Unimplemented(); + if (f.is_runtime_frame()) { + // Patch the pc of the now old last Java frame (we already set the anchor to enterSpecial) + // so that when target returns to Java it will actually return to the preempt cleanup stub. + // We step over the runtime stub frame and patch the return PC in the caller's frame. + intptr_t* caller_sp = f.sp() + f.cb()->frame_size(); + frame::z_common_abi* abi = (frame::z_common_abi*)caller_sp; + abi->return_pc = (uint64_t)StubRoutines::cont_preempt_stub(); + } else { + // The target will check for preemption once it returns to the interpreter + // or the native wrapper code and will manually jump to the preempt stub. + JavaThread *thread = JavaThread::current(); + DEBUG_ONLY(Method* m = f.is_interpreted_frame() ? f.interpreter_frame_method() : f.cb()->as_nmethod()->method();) + assert(m->is_object_wait0() || thread->interp_at_preemptable_vmcall_cnt() > 0, + "preemptable VM call not using call_VM_preemptable"); + thread->set_preempt_alternate_return(StubRoutines::cont_preempt_stub()); + } } inline int ContinuationHelper::frame_align_words(int size) { - Unimplemented(); + // S390 requires 8-byte (1-word) frame alignment, not 16-byte like other platforms. + // Because frames are already 8-byte aligned, no additional padding words are needed. + // Other platforms (x86, aarch64, ppc) return size & 1 to ensure 16-byte alignment, + // but s390's 8-byte alignment requirement is already satisfied. return 0; } -inline intptr_t* ContinuationHelper::frame_align_pointer(intptr_t* sp) { - Unimplemented(); - return nullptr; +inline intptr_t* ContinuationHelper::frame_align_pointer(intptr_t* p) { + return align_down(p, frame::frame_alignment); } template inline void ContinuationHelper::update_register_map(const frame& f, RegisterMap* map) { - Unimplemented(); + // All registers are considered volatile and saved in the caller (Java) frame if needed. + // No register map update required for s390. } inline void ContinuationHelper::update_register_map_with_callee(const frame& f, RegisterMap* map) { - Unimplemented(); + // All registers are considered volatile and saved in the caller (Java) frame if needed. + // No register map update required for s390. } inline void ContinuationHelper::push_pd(const frame& f) { - Unimplemented(); + f.own_abi()->callers_sp = (uint64_t)f.fp(); } inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, ContinuationEntry* cont) { - Unimplemented(); + // No frame pointer update needed for s390. + // Unlike x86/aarch64, s390 doesn't require setting last_Java_fp in the anchor. } inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) { - Unimplemented(); + // No frame pointer update needed for s390. + // Unlike x86/aarch64, s390 doesn't require setting last_Java_fp in the anchor. } #ifdef ASSERT inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) { - Unimplemented(); - return false; + intptr_t* sp = f.sp(); + address pc = *(address*)(sp - frame::sender_sp_ret_address_offset()); + intptr_t* fp = (intptr_t*)f.own_abi()->callers_sp; + assert(f.raw_pc() == pc, "f.ra_pc: " INTPTR_FORMAT " actual: " INTPTR_FORMAT, p2i(f.raw_pc()), p2i(pc)); + assert(f.fp() == fp, "f.fp: " INTPTR_FORMAT " actual: " INTPTR_FORMAT, p2i(f.fp()), p2i(fp)); + return f.raw_pc() == pc && f.fp() == fp; } #endif inline intptr_t** ContinuationHelper::Frame::callee_link_address(const frame& f) { - Unimplemented(); - return nullptr; -} - -template -static inline intptr_t* real_fp(const frame& f) { - Unimplemented(); - return nullptr; + return (intptr_t**)&f.own_abi()->callers_sp; } inline address* ContinuationHelper::InterpretedFrame::return_pc_address(const frame& f) { - Unimplemented(); - return nullptr; + return (address*)&f.callers_abi()->return_pc; } inline void ContinuationHelper::InterpretedFrame::patch_sender_sp(frame& f, const frame& caller) { - Unimplemented(); + intptr_t* sp = caller.unextended_sp(); + if (!f.is_heap_frame() && caller.is_interpreted_frame()) { + // When the caller is an interpreted frame, we need to use the caller's top_frame_sp + // instead of unextended_sp. This is because the interpreter resizes the caller's + // frame before making a call + sp = (intptr_t*)caller.at_relative(_z_ijava_idx(top_frame_sp)); + } + assert(f.is_interpreted_frame(), ""); + assert(f.is_heap_frame() || is_aligned(sp, frame::alignment_in_bytes), ""); + intptr_t* la = f.addr_at(_z_ijava_idx(sender_sp)); + *la = f.is_heap_frame() ? (intptr_t)(sp - f.fp()) : (intptr_t)sp; } inline address* ContinuationHelper::Frame::return_pc_address(const frame& f) { - Unimplemented(); - return nullptr; + return (address*)&f.callers_abi()->return_pc; } inline address ContinuationHelper::Frame::real_pc(const frame& f) { - Unimplemented(); - return nullptr; + return (address)f.own_abi()->return_pc; } inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc) { - Unimplemented(); + f.own_abi()->return_pc = (uint64_t)pc; } inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, InterpreterOopMap* mask) { // inclusive; this will be copied with the frame - Unimplemented(); - return nullptr; + int expression_stack_sz = expression_stack_size(f, mask); + intptr_t* res = (intptr_t*)f.interpreter_frame_monitor_end() - expression_stack_sz; + assert(res <= (intptr_t*)f.ijava_state() - expression_stack_sz, + "res=" PTR_FORMAT " f.ijava_state()=" PTR_FORMAT " expression_stack_sz=%d", + p2i(res), p2i(f.ijava_state()), expression_stack_sz); + assert(res >= f.unextended_sp(), + "res: " INTPTR_FORMAT " ijava_state: " INTPTR_FORMAT " esp: " INTPTR_FORMAT " unextended_sp: " INTPTR_FORMAT " expression_stack_size: %d", + p2i(res), p2i(f.ijava_state()), f.ijava_state()->esp, p2i(f.unextended_sp()), expression_stack_sz); + return res; } inline intptr_t* ContinuationHelper::InterpretedFrame::frame_bottom(const frame& f) { // exclusive; this will not be copied with the frame - Unimplemented(); - return nullptr; + return (intptr_t*)f.at_relative(_z_ijava_idx(locals)) + 1; // exclusive; this will not be copied with the frame } inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, int callee_argsize, bool callee_interpreted) { - Unimplemented(); - return nullptr; + intptr_t* pseudo_unextended_sp = f.interpreter_frame_esp() + 1 - frame::metadata_words_at_top; + // callee_argsize includes metadata (frame::metadata_words_at_top). + // When the callee is interpreted, we add callee_argsize to account for the arguments + // that are part of the caller's frame but logically belong to the callee. + return pseudo_unextended_sp + (callee_interpreted ? callee_argsize : 0); } inline intptr_t* ContinuationHelper::InterpretedFrame::callers_sp(const frame& f) { - Unimplemented(); - return nullptr; + return f.fp(); } #endif // CPU_S390_CONTINUATIONHELPER_S390_INLINE_HPP diff --git a/src/hotspot/cpu/s390/frame_s390.cpp b/src/hotspot/cpu/s390/frame_s390.cpp index b602d0adce5..af4c670132a 100644 --- a/src/hotspot/cpu/s390/frame_s390.cpp +++ b/src/hotspot/cpu/s390/frame_s390.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016, 2023 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -54,6 +54,10 @@ void RegisterMap::check_location_valid() { // Profiling/safepoint support bool frame::safe_for_sender(JavaThread *thread) { + if (is_heap_frame()) { + return true; + } + address sp = (address)_sp; address fp = (address)_fp; address unextended_sp = (address)_unextended_sp; @@ -120,6 +124,13 @@ bool frame::safe_for_sender(JavaThread *thread) { intptr_t* sender_sp = (intptr_t*) fp; address sender_pc = (address) sender_abi->return_pc; + if (Continuation::is_return_barrier_entry(sender_pc)) { + // If our sender_pc is the return barrier, then our "real" sender is the continuation entry + frame s = Continuation::continuation_bottom_sender(thread, *this, sender_sp); + sender_sp = s.sp(); + sender_pc = s.pc(); + } + // We must always be able to find a recognizable pc. CodeBlob* sender_blob = CodeCache::find_blob(sender_pc); if (sender_blob == nullptr) { @@ -192,7 +203,8 @@ void frame::interpreter_frame_set_locals(intptr_t* locs) { // sender_sp intptr_t* frame::interpreter_frame_sender_sp() const { - return sender_sp(); + assert(is_interpreted_frame(), "interpreted frame expected"); + return (intptr_t*)at(_z_ijava_idx(sender_sp)); } frame frame::sender_for_entry_frame(RegisterMap *map) const { @@ -244,16 +256,52 @@ frame frame::sender_for_upcall_stub_frame(RegisterMap* map) const { frame fr(jfa->last_Java_sp(), jfa->last_Java_pc()); return fr; + } +#if defined(ASSERT) +static address get_register_address_in_stub(const frame& stub_fr, VMReg reg) { + RegisterMap map(nullptr, + RegisterMap::UpdateMap::include, + RegisterMap::ProcessFrames::skip, + RegisterMap::WalkContinuation::skip); + stub_fr.oop_map()->update_register_map(&stub_fr, &map); + return map.location(reg, stub_fr.sp()); +} +#endif + JavaThread** frame::saved_thread_address(const frame& f) { - Unimplemented(); - return nullptr; + CodeBlob* cb = f.cb(); + assert(cb != nullptr && cb->is_runtime_stub(), "invalid frame"); + + JavaThread** thread_addr; +#ifdef COMPILER1 + if (cb == Runtime1::blob_for(StubId::c1_monitorenter_id) || + cb == Runtime1::blob_for(StubId::c1_monitorenter_nofpu_id)) { + thread_addr = (JavaThread**)(f.sp() + Runtime1::runtime_blob_current_thread_offset(f)); + } else +#endif + { + // c2 only saves Z_fp in the stub frame so nothing to do. + thread_addr = nullptr; + } + assert(get_register_address_in_stub(f, SharedRuntime::thread_register()) == (address)thread_addr, "wrong thread address"); + return thread_addr; } frame frame::sender_for_interpreter_frame(RegisterMap *map) const { - // Pass callers sender_sp as unextended_sp. - return frame(sender_sp(), sender_pc(), (intptr_t*)(ijava_state()->sender_sp)); + // This is the sp before any possible extension (adapter/locals). + intptr_t* unextended_sp = interpreter_frame_sender_sp(); + address sender_pc = this->sender_pc(); + if (Continuation::is_return_barrier_entry(sender_pc)) { + if (map->walk_cont()) { // about to walk into an h-stack + return Continuation::top_frame(*this, map); + } else { + return Continuation::continuation_bottom_sender(map->thread(), *this, sender_sp()); + } + } + + return frame(sender_sp(), sender_pc, unextended_sp); } void frame::patch_pc(Thread* thread, address pc) { @@ -284,7 +332,7 @@ void frame::patch_pc(Thread* thread, address pc) { #ifdef ASSERT { - frame f(this->sp(), pc, this->unextended_sp()); + frame f(sp(), unextended_sp(), fp(), pc, cb(), oop_map(), is_heap_frame()); assert(f.is_deoptimized_frame() == this->is_deoptimized_frame() && f.pc() == this->pc() && f.raw_pc() == this->raw_pc(), "must be (f.is_deoptimized_frame(): %d this->is_deoptimized_frame(): %d " "f.pc(): " INTPTR_FORMAT " this->pc(): " INTPTR_FORMAT " f.raw_pc(): " INTPTR_FORMAT " this->raw_pc(): " INTPTR_FORMAT ")", @@ -648,6 +696,8 @@ extern "C" void bt_max(intptr_t *start_sp, intptr_t *top_pc, int max_frames) { } #if !defined(PRODUCT) +#define DESCRIBE_ADDRESS_MAGIC(name) \ + values.describe(frame_no, (intptr_t*)&ijava_state()->name, #name "_number_debug"); #define DESCRIBE_ADDRESS(name) \ values.describe(frame_no, (intptr_t*)&ijava_state()->name, #name); @@ -656,25 +706,38 @@ void frame::describe_pd(FrameValues& values, int frame_no) { if (is_interpreted_frame()) { // Describe z_ijava_state elements. DESCRIBE_ADDRESS(method); + DESCRIBE_ADDRESS(mirror); DESCRIBE_ADDRESS(locals); DESCRIBE_ADDRESS(monitors); DESCRIBE_ADDRESS(cpoolCache); DESCRIBE_ADDRESS(bcp); - DESCRIBE_ADDRESS(mdx); DESCRIBE_ADDRESS(esp); - DESCRIBE_ADDRESS(sender_sp); + DESCRIBE_ADDRESS(mdx); DESCRIBE_ADDRESS(top_frame_sp); + DESCRIBE_ADDRESS(sender_sp); DESCRIBE_ADDRESS(oop_tmp); DESCRIBE_ADDRESS(lresult); DESCRIBE_ADDRESS(fresult); + DESCRIBE_ADDRESS_MAGIC(magic); + } + + if (is_java_frame() || Continuation::is_continuation_enterSpecial(*this)) { + intptr_t* ret_pc_loc = (intptr_t*)&own_abi()->return_pc; + address ret_pc = *(address*)ret_pc_loc; + values.describe(frame_no, ret_pc_loc, + Continuation::is_return_barrier_entry(ret_pc) ? "return address (return barrier)" : "return address"); } } #endif // !PRODUCT intptr_t *frame::initial_deoptimization_info() { - // Used to reset the saved FP. - return fp(); + // `this` is the caller of the deoptee. We want to trim it, if compiled, to + // unextended_sp. This is necessary if the deoptee frame is the bottom frame + // of a continuation on stack (more frames could be in a StackChunk) as it + // will pop its stack args. Otherwise the recursion in + // FreezeBase::recurse_freeze_java_frame() would not stop at the bottom frame. + return is_compiled_frame() ? unextended_sp() : sp(); } BasicObjectLock* frame::interpreter_frame_monitor_end() const { diff --git a/src/hotspot/cpu/s390/frame_s390.hpp b/src/hotspot/cpu/s390/frame_s390.hpp index 664a49fdd21..36fc5970cd8 100644 --- a/src/hotspot/cpu/s390/frame_s390.hpp +++ b/src/hotspot/cpu/s390/frame_s390.hpp @@ -130,6 +130,7 @@ enum { z_native_abi_size = sizeof(z_native_abi), + z_abi_160_base_size = sizeof(z_abi_160_base), z_abi_160_size = sizeof(z_abi_160_base) }; @@ -442,6 +443,14 @@ private: + + #ifdef ASSERT + enum special_backlink_values : uint64_t { + NOT_FULLY_INITIALIZED = 0xDEADBEEF8 + }; + bool is_fully_initialized() const { return (uint64_t)_fp != NOT_FULLY_INITIALIZED; } +#endif // ASSERT + // STACK: // ... // [THIS_FRAME] <-- this._sp (stack pointer for this frame) @@ -452,10 +461,16 @@ // NOTE: Stack pointer is now held in the base class, so remove it from here. // Needed by deoptimization. - intptr_t* _unextended_sp; + union { + intptr_t* _unextended_sp; + int _offset_unextended_sp; // for use in stack-chunk frames + }; // Frame pointer for this frame. - intptr_t* _fp; + union { + intptr_t* _fp; // frame pointer + int _offset_fp; // relative frame pointer for use in stack-chunk frames + }; public: @@ -464,17 +479,25 @@ // Accessors inline intptr_t* fp() const { assert_absolute(); return _fp; } + void set_fp(intptr_t* newfp) { _fp = newfp; } + int offset_fp() const { assert_offset(); return _offset_fp; } + void set_offset_fp(int value) { assert_on_heap(); _offset_fp = value; } + + // Mark a frame as not fully initialized. Must not be used for frames in the valid back chain. + void mark_not_fully_initialized() const { DEBUG_ONLY(own_abi()->callers_sp = NOT_FULLY_INITIALIZED;) } private: // Initialize frame members (_pc and _sp must be given) inline void setup(); - // Constructors - public: + + // Constructors + inline frame(intptr_t* sp, intptr_t* fp, address pc); // To be used, if sp was not extended to match callee's calling convention. inline frame(intptr_t* sp, address pc, intptr_t* unextended_sp = nullptr, intptr_t* fp = nullptr, CodeBlob* cb = nullptr); + inline frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map, bool on_heap); inline frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map = nullptr); // Access frame via stack pointer. @@ -495,11 +518,8 @@ // template interpreter state inline z_ijava_state* ijava_state_unchecked() const; - private: - - inline z_ijava_state* ijava_state() const; - public: + inline z_ijava_state* ijava_state() const; inline intptr_t* interpreter_frame_esp() const; // Where z_ijava_state.esp is saved. @@ -542,14 +562,19 @@ unsigned long flags, int max_frames = 0); enum { - metadata_words = 0, + // size, in words, of frame metadata (e.g. pc and link) + metadata_words = sizeof(z_java_abi) >> LogBytesPerWord, metadata_words_at_bottom = 0, - metadata_words_at_top = 0, - frame_alignment = 16, + metadata_words_at_top = sizeof(z_java_abi) >> LogBytesPerWord, + // in bytes + frame_alignment = 8, // size, in words, of maximum shift in frame position due to alignment - align_wiggle = 1 + align_wiggle = 0 }; static jint interpreter_frame_expression_stack_direction() { return -1; } + // returns the sending frame, without applying any barriers + inline frame sender_raw(RegisterMap* map) const; + #endif // CPU_S390_FRAME_S390_HPP diff --git a/src/hotspot/cpu/s390/frame_s390.inline.hpp b/src/hotspot/cpu/s390/frame_s390.inline.hpp index 6fcd36c57d1..e31b0d5a426 100644 --- a/src/hotspot/cpu/s390/frame_s390.inline.hpp +++ b/src/hotspot/cpu/s390/frame_s390.inline.hpp @@ -26,7 +26,8 @@ #ifndef CPU_S390_FRAME_S390_INLINE_HPP #define CPU_S390_FRAME_S390_INLINE_HPP -#include "code/codeCache.hpp" +#include "code/codeBlob.inline.hpp" +#include "code/codeCache.inline.hpp" #include "code/vmreg.inline.hpp" #include "runtime/sharedRuntime.hpp" #include "utilities/align.hpp" @@ -44,14 +45,25 @@ inline void frame::setup() { _cb = CodeCache::find_blob(_pc); } - if (_fp == nullptr) { - _fp = (intptr_t*)own_abi()->callers_sp; - } - if (_unextended_sp == nullptr) { _unextended_sp = _sp; } + if (_fp == nullptr) { + // The back link for compiled frames on the heap is not valid + if (is_heap_frame()) { + // fp for interpreted frames should have been derelativized and passed to the constructor + assert(is_compiled_frame() + || is_native_frame() // native wrapper (nmethod) for j.l.Object::wait0 + || is_runtime_frame(), // e.g. Runtime1::monitorenter, SharedRuntime::complete_monitor_locking_C + "sp:" PTR_FORMAT " fp:" PTR_FORMAT " name:%s", p2i(_sp), p2i(_unextended_sp + _cb->frame_size()), _cb->name()); + // The back link for compiled frames on the heap is invalid. + _fp = _unextended_sp + _cb->frame_size(); + } else { + _fp = (intptr_t *) own_abi()->callers_sp; + } + } + // When thawing continuation frames the _unextended_sp passed to the constructor is not aligend assert(_on_heap || (is_aligned(_sp, alignment_in_bytes) && is_aligned(_fp, alignment_in_bytes)), "invalid alignment sp:" PTR_FORMAT " unextended_sp:" PTR_FORMAT " fp:" PTR_FORMAT, p2i(_sp), p2i(_unextended_sp), p2i(_fp)); @@ -70,7 +82,12 @@ inline void frame::setup() { } } - // assert(_on_heap || is_aligned(_sp, frame::frame_alignment), "SP must be 8-byte aligned"); + // Continuation frames on the java heap are not aligned. + // When thawing interpreted frames the sp can be unaligned (see new_stack_frame()). + assert(_on_heap || + ((is_aligned(_sp, alignment_in_bytes) || is_interpreted_frame()) && + (is_aligned(_fp, alignment_in_bytes) || !is_fully_initialized())), + "invalid alignment sp:" PTR_FORMAT " unextended_sp:" PTR_FORMAT " fp:" PTR_FORMAT, p2i(_sp), p2i(_unextended_sp), p2i(_fp)); } // Constructors @@ -87,11 +104,26 @@ inline frame::frame(intptr_t* sp, address pc, intptr_t* unextended_sp, intptr_t* inline frame::frame(intptr_t* sp) : frame(sp, nullptr) {} +inline frame::frame(intptr_t* sp, intptr_t* fp, address pc) + : _sp(sp), _pc(pc), _cb(nullptr), _oop_map(nullptr), _deopt_state(unknown), + _on_heap(false), DEBUG_ONLY(_frame_index(-1) COMMA) _unextended_sp(nullptr), _fp(fp) { + setup(); +} + inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map) :_sp(sp), _pc(pc), _cb(cb), _oop_map(oop_map), _on_heap(false), DEBUG_ONLY(_frame_index(-1) COMMA) _unextended_sp(unextended_sp), _fp(fp) { setup(); } +inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map, bool on_heap) + :_sp(sp), _pc(pc), _cb(cb), _oop_map(oop_map), _on_heap(on_heap), DEBUG_ONLY(_frame_index(-1) COMMA) _unextended_sp(unextended_sp), _fp(fp) { + // In thaw, non-heap frames use this constructor to pass oop_map. I don't know why. + assert(_on_heap || _cb != nullptr, "these frames are always heap frames"); + if (cb != nullptr) { + setup(); + } +} + // Generic constructor. Used by pns() in debug.cpp only #ifndef PRODUCT inline frame::frame(void* sp, void* pc, void* unextended_sp) @@ -295,11 +327,11 @@ inline JavaCallWrapper** frame::entry_frame_call_wrapper_addr() const { } inline oop frame::saved_oop_result(RegisterMap* map) const { - return *((oop*) map->location(Z_R2->as_VMReg(), nullptr)); // R2 is return register. + return *((oop*) map->location(Z_R2->as_VMReg(), sp())); // R2 is return register. } inline void frame::set_saved_oop_result(RegisterMap* map, oop obj) { - *((oop*) map->location(Z_R2->as_VMReg(), nullptr)) = obj; // R2 is return register. + *((oop*) map->location(Z_R2->as_VMReg(), sp())) = obj; // R2 is return register. } inline intptr_t* frame::real_fp() const { @@ -307,40 +339,55 @@ inline intptr_t* frame::real_fp() const { } inline int frame::compiled_frame_stack_argsize() const { - Unimplemented(); - return 0; + assert(cb()->is_nmethod(), "what ?"); + return (cb()->as_nmethod()->num_stack_arg_slots() * VMRegImpl::stack_slot_size) >> LogBytesPerWord; } inline void frame::interpreted_frame_oop_map(InterpreterOopMap* mask) const { - Unimplemented(); + assert(mask != nullptr, ""); + Method* m = interpreter_frame_method(); + int bci = interpreter_frame_bci(); + m->mask_for(bci, mask); // OopMapCache::compute_one_oop_map(m, bci, mask); } inline int frame::sender_sp_ret_address_offset() { - Unimplemented(); - return 0; + return -(int)(_z_common_abi(return_pc) >> LogBytesPerWord); } inline void frame::set_unextended_sp(intptr_t* value) { - Unimplemented(); + _unextended_sp = value; } inline int frame::offset_unextended_sp() const { - Unimplemented(); - return 0; + assert_offset(); return _offset_unextended_sp; } inline void frame::set_offset_unextended_sp(int value) { - Unimplemented(); + assert_on_heap(); _offset_unextended_sp = value; } //------------------------------------------------------------------------------ // frame::sender inline frame frame::sender(RegisterMap* map) const { + frame result = sender_raw(map); + + if (map->process_frames() && !map->in_cont()) { + StackWatermarkSet::on_iteration(map->thread(), result); + } + + return result; +} + +inline frame frame::sender_raw(RegisterMap* map) const { // Default is we don't have to follow them. The sender_for_xxx will // update it accordingly. map->set_include_argument_oops(false); + if (map->in_cont()) { // already in an h-stack + return map->stack_chunk()->sender(*this, map); + } + if (is_entry_frame()) return sender_for_entry_frame(map); if (is_upcall_stub_frame()) return sender_for_upcall_stub_frame(map); if (is_interpreted_frame()) return sender_for_interpreter_frame(map); @@ -362,12 +409,31 @@ inline frame frame::sender_for_compiled_frame(RegisterMap *map) const { // Now adjust the map. if (map->update_map()) { // Tell GC to use argument oopmaps for some runtime stubs that need it. - map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread())); - if (_cb->oop_maps() != nullptr) { - OopMapSet::update_register_map(this, map); + + // For C1, some runtime stubs don't have oop maps (e.g., slow_subtype_check, + // unwind_exception), so set this flag outside of update_register_map to ensure + // the GC can handle arguments correctly even when oop_map() is null. + if (!_cb->is_nmethod()) { // compiled frames do not use callee-saved registers + map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread())); + if (oop_map() != nullptr) { + _oop_map->update_register_map(this, map); + } + } else { + assert(!_cb->caller_must_gc_arguments(map->thread()), ""); + assert(!map->include_argument_oops(), ""); + assert(oop_map() == nullptr || !oop_map()->has_any(OopMapValue::callee_saved_value), "callee-saved value in compiled frame"); } } + assert(sender_sp != sp(), "must have changed"); + + if (Continuation::is_return_barrier_entry(sender_pc)) { + if (map->walk_cont()) { // about to walk into an h-stack + return Continuation::top_frame(*this, map); + } else { + return Continuation::continuation_bottom_sender(map->thread(), *this, sender_sp); + } + } return frame(sender_sp, sender_pc); } diff --git a/src/hotspot/cpu/s390/globals_s390.hpp b/src/hotspot/cpu/s390/globals_s390.hpp index 80ed6d1acc8..745a6171ca2 100644 --- a/src/hotspot/cpu/s390/globals_s390.hpp +++ b/src/hotspot/cpu/s390/globals_s390.hpp @@ -64,7 +64,7 @@ define_pd_global(intx, StackRedPages, DEFAULT_STACK_RED_PAGES); define_pd_global(intx, StackShadowPages, DEFAULT_STACK_SHADOW_PAGES); define_pd_global(intx, StackReservedPages, DEFAULT_STACK_RESERVED_PAGES); -define_pd_global(bool, VMContinuations, false); +define_pd_global(bool, VMContinuations, true); define_pd_global(bool, RewriteBytecodes, true); define_pd_global(bool, RewriteFrequentPairs, true); diff --git a/src/hotspot/cpu/s390/interp_masm_s390.cpp b/src/hotspot/cpu/s390/interp_masm_s390.cpp index d50cb833e68..5d86a0c3182 100644 --- a/src/hotspot/cpu/s390/interp_masm_s390.cpp +++ b/src/hotspot/cpu/s390/interp_masm_s390.cpp @@ -165,6 +165,109 @@ void InterpreterMacroAssembler::dispatch_via(TosState state, address *table) { // to perform additional, template interpreter specific tasks before actually // calling their MacroAssembler counterparts. +void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, address entry_point, + Register arg_1, bool check_exceptions) { + if (!Continuations::enabled()) { + call_VM(oop_result, entry_point, arg_1, check_exceptions); + return; + } + call_VM_preemptable(oop_result, entry_point, arg_1, noreg /* arg_2 */, check_exceptions); +} + +void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, address entry_point, + Register arg_1, Register arg_2, bool check_exceptions) { + if (!Continuations::enabled()) { + call_VM(oop_result, entry_point, arg_1, arg_2, check_exceptions); + return; + } + + Label resume_pc, not_preempted; + Register tmp = Z_R1_scratch; + assert(InterpreterRuntime::is_preemptable_call(entry_point), "VM call not preemptable, should use call_VM()"); + assert_different_registers(arg_1, tmp); + assert_different_registers(arg_2, tmp); + +#ifdef ASSERT + { + NearLabel L1; + asm_assert_mem8_is_zero(in_bytes(JavaThread::preempt_alternate_return_offset()), Z_thread, + "Should not have alternate return address set", 100); + // We check this counter in patch_return_pc_with_preempt_stub() during freeze. + z_asi(Address(Z_thread, JavaThread::interp_at_preemptable_vmcall_cnt_offset()), 1); + z_lt(tmp, Address(Z_thread, JavaThread::interp_at_preemptable_vmcall_cnt_offset())); + z_brh(L1); + stop("call_VM_preemptable_helper: should be > 0"); + bind(L1); + } +#endif // ASSERT + + lgr_if_needed(Z_ARG2, arg_1); + assert(arg_2 != Z_ARG2, "smashed argument"); + + if (arg_2 != noreg) { + lgr_if_needed(Z_ARG3, arg_2); + } + + // Force freeze slow path. + push_cont_fastpath(); + // Make VM call. In case of preemption set last_pc to the one we want to resume to. + // Note: call_VM_base will use resume_pc label to set last_Java_pc. + call_VM(noreg, entry_point, false /*check_exceptions*/, &resume_pc /* last_java_pc */); + pop_cont_fastpath(); + + +#ifdef ASSERT + { + NearLabel L; + z_asi(Address(Z_thread, JavaThread::interp_at_preemptable_vmcall_cnt_offset()), -1); + z_lt(tmp, Address(Z_thread, JavaThread::interp_at_preemptable_vmcall_cnt_offset())); + z_brnl(L); + stop("call_VM_preemptable_helper: should be >= 0"); + bind(L); + } +#endif // ASSERT + + // Check if preempted. + z_ltg(tmp, Address(Z_thread, JavaThread::preempt_alternate_return_offset())); + z_brz(not_preempted); + + // Preempted. Frames are already frozen on heap. + z_mvghi(Address(Z_thread, JavaThread::preempt_alternate_return_offset()), 0); + z_br(tmp); // branch to handler in Z_R1_scratch + + bind(resume_pc); // Location to resume execution + restore_after_resume(); + + bind(not_preempted); + + if (check_exceptions) { + NearLabel ok; + load_and_test_long(tmp, Address(Z_thread, Thread::pending_exception_offset())); + z_bre(ok); + load_const_optimized(tmp, StubRoutines::forward_exception_entry()); + z_br(tmp); + bind(ok); + } + + // get oop result if there is one and reset the value in the thread + if (oop_result->is_valid()) { + get_vm_result_oop(oop_result); + } +} + +void InterpreterMacroAssembler::restore_after_resume() { + if (!Continuations::enabled()) return; + load_const_optimized(Z_R1, Interpreter::cont_resume_interpreter_adapter()); + call(Z_R1); +#ifdef ASSERT + NearLabel ok; + z_cg(Z_fp, _z_common_abi(callers_sp), Z_SP); + z_bre(ok); + stop(FILE_AND_LINE ": FP is expected in Z_fp"); + bind(ok); +#endif // ASSERT +} + void InterpreterMacroAssembler::call_VM_leaf_base(address entry_point) { bool allow_relocation = true; // Fenerally valid variant. Assume code is relocated. // interpreter specific @@ -193,20 +296,20 @@ void InterpreterMacroAssembler::call_VM_base(Register oop_result, Register last_ save_esp(); // super call MacroAssembler::call_VM_base(oop_result, last_java_sp, - entry_point, allow_relocation, check_exceptions); + entry_point, allow_relocation, check_exceptions, nullptr); restore_bcp(); } void InterpreterMacroAssembler::call_VM_base(Register oop_result, Register last_java_sp, address entry_point, bool allow_relocation, - bool check_exceptions) { + bool check_exceptions, Label* last_java_pc) { // interpreter specific save_bcp(); save_esp(); // super call MacroAssembler::call_VM_base(oop_result, last_java_sp, - entry_point, allow_relocation, check_exceptions); + entry_point, allow_relocation, check_exceptions, last_java_pc); restore_bcp(); } @@ -697,7 +800,7 @@ void InterpreterMacroAssembler::get_monitors(Register reg) { bind(ok); #endif // ASSERT mem2reg_opt(reg, Address(Z_fp, _z_ijava_state_neg(monitors))); - z_slag(reg, reg, Interpreter::logStackElementSize); + z_slag(reg, reg, Interpreter::logStackElementSize); // sign preserved z_agr(reg, Z_fp); } @@ -968,6 +1071,14 @@ void InterpreterMacroAssembler::remove_activation(TosState state, bool install_monitor_exception, bool notify_jvmti) { BLOCK_COMMENT("remove_activation {"); + +#ifdef ASSERT + { + asm_assert_mem8_is_zero(in_bytes(JavaThread::preempt_alternate_return_offset()), Z_thread, + "remove_activation: should not have alternate return address set", 101); + } +#endif // ASSERT + unlock_if_synchronized_method(state, throw_monitor_exception, install_monitor_exception); // Save result (push state before jvmti call and pop it afterwards) and notify jvmti. @@ -1003,6 +1114,7 @@ void InterpreterMacroAssembler::remove_activation(TosState state, verify_oop(Z_tos, state); pop_interpreter_frame(return_pc, Z_ARG2, Z_ARG3); + pop_cont_fastpath(); BLOCK_COMMENT("} remove_activation"); } @@ -1023,9 +1135,9 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) { z_bru(done); bind(slow_case); - call_VM(noreg, - CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), - monitor); + call_VM_preemptable(noreg, + CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), + monitor); bind(done); } diff --git a/src/hotspot/cpu/s390/interp_masm_s390.hpp b/src/hotspot/cpu/s390/interp_masm_s390.hpp index a210588d062..6921fd05ff0 100644 --- a/src/hotspot/cpu/s390/interp_masm_s390.hpp +++ b/src/hotspot/cpu/s390/interp_masm_s390.hpp @@ -46,7 +46,8 @@ class InterpreterMacroAssembler: public MacroAssembler { Register last_java_sp, address entry_point, bool allow_relocation, - bool check_exceptions); + bool check_exceptions, + Label *last_java_pc); // Base routine for all dispatches. void dispatch_base(TosState state, address* table, bool generate_poll = false); @@ -55,9 +56,14 @@ class InterpreterMacroAssembler: public MacroAssembler { InterpreterMacroAssembler(CodeBuffer* c) : MacroAssembler(c) {} + void restore_after_resume(); virtual void check_and_handle_popframe(Register java_thread); virtual void check_and_handle_earlyret(Register java_thread); + // Use for vthread preemption + void call_VM_preemptable(Register oop_result, address entry_point, Register arg_1, bool check_exceptions = true); + void call_VM_preemptable(Register oop_result, address entry_point, Register arg_1, Register arg_2, bool check_exceptions = true); + void jump_to_entry(address entry, Register Rscratch); virtual void load_earlyret_value(TosState state); diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp index ea75d483e5f..5d5c7570e27 100644 --- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp @@ -32,6 +32,7 @@ #include "gc/shared/barrierSetAssembler.hpp" #include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/interpreter.hpp" +#include "interpreter/interpreterRuntime.hpp" #include "gc/shared/cardTableBarrierSet.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" @@ -1932,6 +1933,12 @@ unsigned long MacroAssembler::patched_branch(address dest_pos, unsigned long ins // Only called when binding labels (share/vm/asm/assembler.cpp) // Pass arguments as intended. Do not pre-calculate distance. void MacroAssembler::pd_patch_instruction(address branch, address target, const char* file, int line) { + + if (is_load_const(branch)) { + patch_const(branch, (long)target); + return; + } + unsigned long stub_inst; int inst_len = get_instruction(branch, &stub_inst); @@ -2249,7 +2256,8 @@ void MacroAssembler::call_VM_base(Register oop_result, Register last_java_sp, address entry_point, bool allow_relocation, - bool check_exceptions) { // Defaults to true. + bool check_exceptions, // Defaults to true. + Label *last_java_pc) { // Allow_relocation indicates, if true, that the generated code shall // be fit for code relocation or referenced data relocation. In other // words: all addresses must be considered variable. PC-relative addressing @@ -2263,7 +2271,7 @@ void MacroAssembler::call_VM_base(Register oop_result, last_java_sp = Z_SP; // Load Z_SP as SP. } - set_top_ijava_frame_at_SP_as_last_Java_frame(last_java_sp, Z_R1, allow_relocation); + set_top_ijava_frame_at_SP_as_last_Java_frame(last_java_sp, Z_R1, allow_relocation, last_java_pc); // ARG1 must hold thread address. z_lgr(Z_ARG1, Z_thread); @@ -2309,14 +2317,14 @@ void MacroAssembler::call_VM_base(Register oop_result, address entry_point, bool check_exceptions) { // Defaults to true. bool allow_relocation = true; - call_VM_base(oop_result, last_java_sp, entry_point, allow_relocation, check_exceptions); + call_VM_base(oop_result, last_java_sp, entry_point, allow_relocation, check_exceptions, nullptr); } // VM calls without explicit last_java_sp. -void MacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions) { +void MacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions, Label* last_java_pc) { // Call takes possible detour via InterpreterMacroAssembler. - call_VM_base(oop_result, noreg, entry_point, true, check_exceptions); + call_VM_base(oop_result, noreg, entry_point, true, check_exceptions, last_java_pc); } void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, bool check_exceptions) { @@ -2348,7 +2356,7 @@ void MacroAssembler::call_VM(Register oop_result, address entry_point, Register void MacroAssembler::call_VM_static(Register oop_result, address entry_point, bool check_exceptions) { // Call takes possible detour via InterpreterMacroAssembler. - call_VM_base(oop_result, noreg, entry_point, false, check_exceptions); + call_VM_base(oop_result, noreg, entry_point, false, check_exceptions, nullptr); } void MacroAssembler::call_VM_static(Register oop_result, address entry_point, Register arg_1, Register arg_2, @@ -2366,7 +2374,7 @@ void MacroAssembler::call_VM_static(Register oop_result, address entry_point, Re void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, bool check_exceptions) { // Call takes possible detour via InterpreterMacroAssembler. - call_VM_base(oop_result, last_java_sp, entry_point, true, check_exceptions); + call_VM_base(oop_result, last_java_sp, entry_point, true, check_exceptions, nullptr); } void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, bool check_exceptions) { @@ -3810,19 +3818,21 @@ void MacroAssembler::set_last_Java_frame(Register last_Java_sp, Register last_Ja BLOCK_COMMENT("} set_last_Java_frame"); } -void MacroAssembler::reset_last_Java_frame(bool allow_relocation) { +void MacroAssembler::reset_last_Java_frame(bool check_last_java_sp, bool allow_relocation) { BLOCK_COMMENT("reset_last_Java_frame {"); - if (allow_relocation) { - asm_assert_mem8_isnot_zero(in_bytes(JavaThread::last_Java_sp_offset()), - Z_thread, - "SP was not set, still zero", - 0x202); - } else { - asm_assert_mem8_isnot_zero_static(in_bytes(JavaThread::last_Java_sp_offset()), - Z_thread, - "SP was not set, still zero", - 0x202); + if (check_last_java_sp) { + if (allow_relocation) { + asm_assert_mem8_isnot_zero(in_bytes(JavaThread::last_Java_sp_offset()), + Z_thread, + "SP was not set, still zero", + 0x202); + } else { + asm_assert_mem8_isnot_zero_static(in_bytes(JavaThread::last_Java_sp_offset()), + Z_thread, + "SP was not set, still zero", + 0x202); + } } // _last_Java_sp = 0 @@ -3836,15 +3846,14 @@ void MacroAssembler::reset_last_Java_frame(bool allow_relocation) { return; } -void MacroAssembler::set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1, bool allow_relocation) { +void MacroAssembler::set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1, bool allow_relocation, Label* jpc) { assert_different_registers(sp, tmp1); - // We cannot trust that code generated by the C++ compiler saves R14 - // to z_abi_160.return_pc, because sometimes it spills R14 using stmg at - // z_abi_160.gpr14 (e.g. InterpreterRuntime::_new()). - // Therefore we load the PC into tmp1 and let set_last_Java_frame() save - // it into the frame anchor. - get_PC(tmp1); + if (jpc == nullptr || jpc->is_bound()) { + load_const_optimized(tmp1, jpc == nullptr ? pc() : target(*jpc)); + } else { + load_const(tmp1, *jpc); + } set_last_Java_frame(/*sp=*/sp, /*pc=*/tmp1, allow_relocation); } @@ -5890,7 +5899,7 @@ bool is_excluded(Register excluded_register[], Register reg, int n) { } void MacroAssembler::clobber_volatile_registers(Register excluded_register[], int n) { - const int magic_number = 0x82; + const int magic_number = 0xbadbad; for (int i = 0; i < 6 /* R0 to R5 */; i++) { Register reg = as_Register(i); @@ -5899,6 +5908,26 @@ void MacroAssembler::clobber_volatile_registers(Register excluded_register[], in } } } + +void MacroAssembler::clobber_nonvolatile_registers() { + BLOCK_COMMENT("clobber_nonvolatile_registers {"); + static const Register regs[] = { + Z_R6, + Z_R7, + // don't zap Z_thread (Z_R8) + Z_R9, + Z_R10, + Z_R11, + Z_R12, + Z_R13 + }; + Register bad = regs[0]; + load_const_optimized(bad, 0xbad0101babe11111); + for (uint32_t i = 1; i < (sizeof(regs) / sizeof(Register)); i++) { + z_lgr(regs[i], bad); + } + BLOCK_COMMENT("} clobber_nonvolatile_registers"); +} #endif // ASSERT // Save and restore functions: Exclude Z_R0. @@ -6742,6 +6771,39 @@ void MacroAssembler::pop_count_int_with_ext3(Register r_dst, Register r_src) { BLOCK_COMMENT("} pop_count_int_with_ext3"); } +void MacroAssembler::post_call_nop() { + // Make inline again when loom is always enabled. + if (!Continuations::enabled()) { + return; + } + nop(); + // TODO: + // 1. https://bugs.openjdk.org/browse/JDK-8300002 + // 2. https://bugs.openjdk.org/browse/JDK-8290965 +} + +void MacroAssembler::push_cont_fastpath() { + BLOCK_COMMENT("push_cont_fastpath {"); + if (!Continuations::enabled()) return; + NearLabel done; + z_clg(Z_SP, Address(Z_thread, JavaThread::cont_fastpath_offset())); + z_brnh(done); // bcondNotHigh -> less than equal + z_stg(Z_SP, Address(Z_thread, JavaThread::cont_fastpath_offset())); + bind(done); + BLOCK_COMMENT("} push_cont_fastpath"); +} + +void MacroAssembler::pop_cont_fastpath() { + BLOCK_COMMENT("pop_cont_fastpath {"); + if (!Continuations::enabled()) return; + NearLabel done; + z_clg(Z_SP, Address(Z_thread, JavaThread::cont_fastpath_offset())); + z_brl(done); + z_mvghi(Address(Z_thread, JavaThread::cont_fastpath_offset()), 0); + bind(done); + BLOCK_COMMENT("} pop_cont_fastpath"); +} + // LOAD HALFWORD IMMEDIATE ON CONDITION (32 <- 16) void MacroAssembler::load_on_condition_imm_32(Register dst, int64_t i2, branch_condition cc) { if (VM_Version::has_LoadStoreConditional2()) { // z_lochi works on z13 or above diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.hpp b/src/hotspot/cpu/s390/macroAssembler_s390.hpp index 8e2834ba9b7..9dd4054a36c 100644 --- a/src/hotspot/cpu/s390/macroAssembler_s390.hpp +++ b/src/hotspot/cpu/s390/macroAssembler_s390.hpp @@ -1,7 +1,7 @@ /* * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016, 2024 SAP SE. All rights reserved. - * Copyright (c) 2024 IBM Corporation. All rights reserved. + * Copyright (c) 2024, 2026, IBM Corporation. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -525,12 +525,13 @@ class MacroAssembler: public Assembler { Register last_java_sp, // To set up last_Java_frame in stubs; use noreg otherwise. address entry_point, // The entry point. bool allow_relocation, // Flag to request generation of relocatable code. - bool check_exception); // Flag which indicates if exception should be checked. + bool check_exception, // Flag which indicates if exception should be checked. + Label *last_java_pc); // Call into the VM. // Passes the thread pointer (in Z_ARG1) as a prepended argument. // Makes sure oop return values are visible to the GC. - void call_VM(Register oop_result, address entry_point, bool check_exceptions = true); + void call_VM(Register oop_result, address entry_point, bool check_exceptions = true, Label* last_java_pc = nullptr); void call_VM(Register oop_result, address entry_point, Register arg_1, bool check_exceptions = true); void call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, bool check_exceptions = true); void call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, @@ -575,6 +576,8 @@ class MacroAssembler: public Assembler { // Get the pc where the last call will return to. Returns _last_calls_return_pc. inline address last_calls_return_pc(); + void post_call_nop(); + static int ic_check_size(); int ic_check(int end_alignment); @@ -805,14 +808,14 @@ class MacroAssembler: public Assembler { // Support for last Java frame (but use call_VM instead where possible). private: void set_last_Java_frame(Register last_Java_sp, Register last_Java_pc, bool allow_relocation); - void reset_last_Java_frame(bool allow_relocation); - void set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1, bool allow_relocation); + void reset_last_Java_frame(bool check_last_java_sp, bool allow_relocation); + void set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1, bool allow_relocation, Label* last_java_pc = nullptr); public: inline void set_last_Java_frame(Register last_java_sp, Register last_Java_pc); inline void set_last_Java_frame_static(Register last_java_sp, Register last_Java_pc); - inline void reset_last_Java_frame(void); - inline void reset_last_Java_frame_static(void); - inline void set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1); + inline void reset_last_Java_frame(bool check_last_java_sp = true); + inline void reset_last_Java_frame_static(bool check_last_java_sp = true); + inline void set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1, Label* jpc = nullptr); inline void set_top_ijava_frame_at_SP_as_last_Java_frame_static(Register sp, Register tmp1); void set_thread_state(JavaThreadState new_state); @@ -979,6 +982,10 @@ class MacroAssembler: public Assembler { } void asm_assert_frame_size(Register expected_size, Register tmp, const char* msg, int id); + // Load bad values into registers that are nonvolatile according to the ABI except Z_thread. + // This is done after vthread preemption and before vthread resume. + void clobber_nonvolatile_registers() NOT_DEBUG_RETURN; + // Save and restore functions: Exclude Z_R0. void save_volatile_regs( Register dst, int offset, bool include_fp, bool include_flags); void restore_volatile_regs(Register src, int offset, bool include_fp, bool include_flags); @@ -1109,6 +1116,9 @@ class MacroAssembler: public Assembler { void pop_count_int_with_ext3(Register dst, Register src); void pop_count_long_with_ext3(Register dst, Register src); + void push_cont_fastpath(); + void pop_cont_fastpath(); + void load_on_condition_imm_32(Register dst, int64_t i2, branch_condition cc); void load_on_condition_imm_64(Register dst, int64_t i2, branch_condition cc); diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.inline.hpp b/src/hotspot/cpu/s390/macroAssembler_s390.inline.hpp index 72724fb66d1..24bec32f8b4 100644 --- a/src/hotspot/cpu/s390/macroAssembler_s390.inline.hpp +++ b/src/hotspot/cpu/s390/macroAssembler_s390.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016, 2023 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -295,16 +295,16 @@ inline void MacroAssembler::set_last_Java_frame_static(Register last_Java_sp, Re set_last_Java_frame(last_Java_sp, last_Java_pc, false); } -inline void MacroAssembler::reset_last_Java_frame(void) { - reset_last_Java_frame(true); +inline void MacroAssembler::reset_last_Java_frame(bool check_last_java_sp) { + reset_last_Java_frame(check_last_java_sp, true); } -inline void MacroAssembler::reset_last_Java_frame_static(void) { - reset_last_Java_frame(false); +inline void MacroAssembler::reset_last_Java_frame_static(bool check_last_java_sp) { + reset_last_Java_frame(check_last_java_sp, false); } -inline void MacroAssembler::set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1) { - set_top_ijava_frame_at_SP_as_last_Java_frame(sp, tmp1, true); +inline void MacroAssembler::set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1, Label *jpc) { + set_top_ijava_frame_at_SP_as_last_Java_frame(sp, tmp1, true, jpc); } inline void MacroAssembler::set_top_ijava_frame_at_SP_as_last_Java_frame_static(Register sp, Register tmp1) { diff --git a/src/hotspot/cpu/s390/nativeInst_s390.cpp b/src/hotspot/cpu/s390/nativeInst_s390.cpp index 546f8b13397..3520e9a3493 100644 --- a/src/hotspot/cpu/s390/nativeInst_s390.cpp +++ b/src/hotspot/cpu/s390/nativeInst_s390.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -630,3 +630,32 @@ void NativeGeneralJump::replace_mt_safe(address instr_addr, address code_buffer) *(intptr_t*)instr_addr = load_const_bytes | bytes_after_jump; ICache::invalidate_range(instr_addr, 6); } + +void NativeDeoptInstruction::verify() { +} + +void NativePostCallNop::make_deopt() { + NativeDeoptInstruction::insert(addr_at(0)); +} + +void NativeDeoptInstruction::insert(address code_pos) { + ResourceMark rm; + int code_size = 2; // z_illtrap is of 2 bytes + CodeBuffer cb(code_pos, code_size + 1); + MacroAssembler* a = new MacroAssembler(&cb); + a->z_illtrap(); + // forcing CPU to reload these 2 bytes of instruction by setting current range invalid + ICache::invalidate_range(code_pos, code_size); +} + +bool NativeDeoptInstruction::is_deopt_at(address instr){ + // Check if the instruction is an illtrap (illegal instruction used for deoptimization) + if (!Assembler::is_z_illtrap(instr)) return false; + + // Verify the instruction belongs to an nmethod + CodeBlob* cb = CodeCache::find_blob(instr); + if (cb == nullptr || !cb->is_nmethod()) { + return false; + } + return true; +} diff --git a/src/hotspot/cpu/s390/nativeInst_s390.hpp b/src/hotspot/cpu/s390/nativeInst_s390.hpp index 9852bc410b1..0ba97830bb7 100644 --- a/src/hotspot/cpu/s390/nativeInst_s390.hpp +++ b/src/hotspot/cpu/s390/nativeInst_s390.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016, 2024 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -82,6 +82,11 @@ class NativeInstruction { bool is_illegal(); + bool is_nop() const { + // TODO update: https://bugs.openjdk.org/browse/JDK-8290965 + return Assembler::is_z_nop(addr_at(0)); + } + // Bcrl is currently the only accepted instruction here. bool is_jump(); @@ -650,39 +655,40 @@ class NativeGeneralJump: public NativeInstruction { class NativePostCallNop: public NativeInstruction { public: enum z_specific_constants { - // Once the check is implemented, this has to specify number of bytes checked on the first - // read. If the check would read beyond size of the instruction at the deopt handler stub - // code entry point, then it has to happen in two stages - to prevent out of bounds access - // in case the return address points to the entry point which could be at the end of page. - first_check_size = 0 // check is unimplemented + // The check reads a 2-byte nop instruction. Since s390 nop is 2 bytes (BCR instruction), + // we can safely read it in a single stage without risk of out-of-bounds access. + // The nop instruction is checked by is_nop() which reads a short (2 bytes). + first_check_size = 2 }; - bool check() const { Unimplemented(); return false; } + bool check() const { return is_nop(); } bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const { return false; } bool patch(int32_t oopmap_slot, int32_t cb_offset) { Unimplemented(); return false; } - void make_deopt() { Unimplemented(); } + void make_deopt(); }; inline NativePostCallNop* nativePostCallNop_at(address address) { - // Unimplemented(); + NativePostCallNop* nop = (NativePostCallNop*) address; + if (nop->check()) { + return nop; + } return nullptr; } class NativeDeoptInstruction: public NativeInstruction { public: - address instruction_address() const { Unimplemented(); return nullptr; } - address next_instruction_address() const { Unimplemented(); return nullptr; } + enum { + instruction_offset = 0 + }; - void verify() { Unimplemented(); } + address instruction_address() const { return addr_at(instruction_offset); } + address next_instruction_address() const { return instruction_address() + Assembler::instr_len(addr_at(0)); } - static bool is_deopt_at(address instr) { - // Unimplemented(); - return false; - } + void verify(); + + static bool is_deopt_at(address instr); // MT-safe patching - static void insert(address code_pos) { - Unimplemented(); - } + static void insert(address code_pos); }; #endif // CPU_S390_NATIVEINST_S390_HPP diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad index c0e51bd2bfd..6cdf40cda9c 100644 --- a/src/hotspot/cpu/s390/s390.ad +++ b/src/hotspot/cpu/s390/s390.ad @@ -2361,6 +2361,7 @@ encode %{ unsigned int actual_ret_off = __ offset(); assert(start_off + size_of_code == actual_ret_off, "wrong return_pc"); #endif + __ post_call_nop(); %} enc_class z_enc_java_static_call(method meth) %{ @@ -2393,6 +2394,7 @@ encode %{ } __ clear_inst_mark(); + __ post_call_nop(); %} // Java dynamic call @@ -2449,6 +2451,7 @@ encode %{ __ z_basr(Z_R14, Z_R1_scratch); unsigned int ret_off = __ offset(); } + __ post_call_nop(); %} enc_class z_enc_cmov_reg(cmpOp cmp, iRegI dst, iRegI src) %{ @@ -5557,6 +5560,38 @@ instruct compareAndSwapN_bool(iRegP mem_ptr, rarg5RegN oldval, iRegN_P2N newval, ins_pipe(pipe_class_dummy); %} +instruct compareAndExchangeN(iRegN res, iRegP mem_ptr, rarg5RegN oldval, iRegN_P2N newval, flagsReg cr) %{ + match(Set res (CompareAndExchangeN mem_ptr (Binary oldval newval))); + predicate(n->as_LoadStore()->barrier_data() == 0); + effect(TEMP_DEF res, USE mem_ptr, USE_KILL oldval, KILL cr); + format %{ "$res = CompareAndExchangeN $oldval,$newval,$mem_ptr" %} + ins_encode %{ + Register Rcomp = reg_to_register_object($oldval$$reg); + Register Rnew = reg_to_register_object($newval$$reg); + Register Raddr = reg_to_register_object($mem_ptr$$reg); + Register Rres = reg_to_register_object($res$$reg); + __ z_lr(Rres, Rcomp); + __ z_cs(Rres, Rnew, 0, Raddr); + %} + ins_pipe(pipe_class_dummy); +%} + +instruct compareAndExchangeP(iRegP res, iRegP mem_ptr, rarg5RegP oldval, iRegP_N2P newval, flagsReg cr) %{ + match(Set res (CompareAndExchangeP mem_ptr (Binary oldval newval))); + predicate(n->as_LoadStore()->barrier_data() == 0); + effect(TEMP_DEF res, USE mem_ptr, USE_KILL oldval, KILL cr); + format %{ "$res = CompareAndExchangeP $oldval,$newval,$mem_ptr" %} + ins_encode %{ + Register Rcomp = reg_to_register_object($oldval$$reg); + Register Rnew = reg_to_register_object($newval$$reg); + Register Raddr = reg_to_register_object($mem_ptr$$reg); + Register Rres = reg_to_register_object($res$$reg); + __ z_lgr(Rres, Rcomp); + __ z_csg(Rres, Rnew, 0, Raddr); + %} + ins_pipe(pipe_class_dummy); +%} + //----------Atomic operations on memory (GetAndSet*, GetAndAdd*)--------------- // Exploit: direct memory arithmetic diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp index e5a27e66968..1a13e76e930 100644 --- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp +++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016, 2024 SAP SE. All rights reserved. + * Copyright (c) 2026 IBM Corporation. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -37,6 +38,8 @@ #include "oops/klass.inline.hpp" #include "prims/methodHandles.hpp" #include "registerSaver_s390.hpp" +#include "runtime/continuation.hpp" +#include "runtime/continuationEntry.inline.hpp" #include "runtime/jniHandles.hpp" #include "runtime/safepointMechanism.hpp" #include "runtime/sharedRuntime.hpp" @@ -1339,6 +1342,395 @@ static void move32_64(MacroAssembler *masm, // Wrap a JNI call. //---------------------------------------------------------------------- #undef USE_RESIZE_FRAME + +static void check_continuation_enter_argument(VMReg actual_vmreg, + Register expected_reg, + const char* name) { + assert(!actual_vmreg->is_stack(), "%s cannot be on stack", name); + assert(actual_vmreg->as_Register() == expected_reg, + "%s is in unexpected register: %s instead of %s", + name, actual_vmreg->as_Register()->name(), expected_reg->name()); +} + +//---------------------------- continuation_enter_setup --------------------------- +// +// Frame setup. +// +// Arguments: +// None. +// +// Results: +// Z_SP: pointer to blank ContinuationEntry in the pushed frame. +// +// Kills: +// Nothing +// +static OopMap* continuation_enter_setup(MacroAssembler* masm, int& framesize_words) { + + assert(ContinuationEntry::size() % VMRegImpl::stack_slot_size == 0, ""); + assert(in_bytes(ContinuationEntry::cont_offset()) % VMRegImpl::stack_slot_size == 0, ""); + assert(in_bytes(ContinuationEntry::chunk_offset()) % VMRegImpl::stack_slot_size == 0, ""); + + const int frame_size_in_bytes = (int)ContinuationEntry::size(); + assert(is_aligned(frame_size_in_bytes, frame::alignment_in_bytes), "alignment error"); + + framesize_words = frame_size_in_bytes / wordSize; + + DEBUG_ONLY(__ block_comment("continuation_enter_setup {")); + __ save_return_pc(); // preserve current Z_R14 + __ push_frame(frame_size_in_bytes); + + OopMap* map = new OopMap((int)frame_size_in_bytes / VMRegImpl::stack_slot_size, 0 /* arg_slots*/); + __ z_mvc(Address(Z_SP, ContinuationEntry::parent_offset()), /* move to */ + Address(Z_thread, JavaThread::cont_entry_offset()), /* move from */ + sizeof(ContinuationEntry*) /* size of data to be moved */ + ); + __ z_stg(Z_SP, Address(Z_thread, JavaThread::cont_entry_offset())); + DEBUG_ONLY(__ block_comment("} continuation_enter_setup")); + return map; +} + +//---------------------------- fill_continuation_entry --------------------------- +// +// Initialize the new ContinuationEntry. +// +// Arguments: +// Z_SP : pointer to blank Continuation entry +// reg_cont_obj : pointer to the continuation +// reg_flags : flags / isVirtualThread +// +// Results: +// Z_SP : pointer to filled out ContinuationEntry +// +// Kills: +// This is peace driven method, doesn't kill anyone. +// +static void fill_continuation_entry(MacroAssembler* masm, Register reg_cont_obj, Register reg_flags) { + assert_different_registers(reg_cont_obj, reg_flags); + DEBUG_ONLY(__ block_comment("fill_continuation_entry {")); +#ifdef ASSERT + assert(Immediate::is_simm16(ContinuationEntry::cookie_value()), "update below instruction"); + __ z_mvhi(Address(Z_SP, ContinuationEntry::cookie_offset()), ContinuationEntry::cookie_value()); +#endif //ASSERT + __ z_stg(reg_cont_obj, Address(Z_SP, ContinuationEntry::cont_offset())); + __ z_st(reg_flags, Address(Z_SP, ContinuationEntry::flags_offset())); + __ z_mvghi(Address(Z_SP, ContinuationEntry::chunk_offset()), 0); + __ z_mvhi( Address(Z_SP, ContinuationEntry::argsize_offset()), 0); + __ z_mvhi( Address(Z_SP, ContinuationEntry::pin_count_offset()), 0); + + __ z_mvc(Address(Z_SP, ContinuationEntry::parent_cont_fastpath_offset()), /* move to */ + Address(Z_thread, JavaThread::cont_fastpath_offset()), /* move from */ + sizeof(ContinuationEntry*) /* size of data to be moved */ + ); + + __ z_mvghi(Address(Z_thread, JavaThread::cont_fastpath_offset()), 0); + + DEBUG_ONLY(__ block_comment("} fill_continuation_entry")); +} + +//---------------------------- continuation_enter_cleanup --------------------------- +// +// Copy corresponding attributes from the top ContinuationEntry to the JavaThread +// before deleting it. +// +// Arguments: +// Z_SP: pointer to the ContinuationEntry +// +// Results: +// None. +// +// Kills: +// Z_R0_scratch (in debug builds) +// Z_R10 (when CheckJNICalls is enabled) +// +static void continuation_enter_cleanup(MacroAssembler* masm) { + __ block_comment("continuation_enter_cleanup {"); + +#ifdef ASSERT + __ z_cg(Z_SP, Address(Z_thread, JavaThread::cont_entry_offset())); + __ asm_assert(Assembler::bcondEqual, FILE_AND_LINE ": incorrect Z_SP", 0x1bb); + + __ z_lgf(Z_R0, Address(Z_SP, ContinuationEntry::cookie_offset())); + __ z_cfi(Z_R0, ContinuationEntry::cookie_value()); + __ asm_assert(Assembler::bcondEqual, FILE_AND_LINE ": incorrect cookie value", 0x1cc); +#endif // ASSERT + + __ z_mvc(Address(Z_thread, JavaThread::cont_fastpath_offset()), /* move to */ + Address(Z_SP, ContinuationEntry::parent_cont_fastpath_offset()), /* move from */ + sizeof(ContinuationEntry*) /* size of data to be moved */ + ); + + __ z_mvc(Address(Z_thread, JavaThread::cont_entry_offset()), /* move to */ + Address(Z_SP, ContinuationEntry::parent_offset()), /* move from */ + sizeof(ContinuationEntry*) /* size of data to be moved */ + ); + + __ block_comment("} continuation_enter_cleanup"); +} +static void gen_continuation_enter(MacroAssembler* masm, + const VMRegPair* regs, + int& exception_offset, + OopMapSet* oop_maps, + int& frame_complete, + int& framesize_words, + int& interpreted_entry_offset, + int& compiled_entry_offset) { + // enterSpecial(Continuation c, boolean isContinue, boolean isVirtualThread) + int pos_cont_obj = 0; + int pos_is_cont = 1; + int pos_is_virtual = 2; + + // The platform-specific calling convention may present the arguments in various registers. + // To simplify the rest of the code, we expect the arguments to reside at these known + // registers, and we additionally check the placement here in case calling convention ever + // changes. + Register reg_cont_obj = Z_ARG1; + Register reg_is_cont = Z_ARG2; + Register reg_is_virtual = Z_ARG3; + + check_continuation_enter_argument(regs[pos_cont_obj].first(), reg_cont_obj, "Continuation object"); + check_continuation_enter_argument(regs[pos_is_cont].first(), reg_is_cont, "isContinue"); + check_continuation_enter_argument(regs[pos_is_virtual].first(), reg_is_virtual, "isVirtualThread"); + + address resolve_static_call = SharedRuntime::get_resolve_static_call_stub(); + + address start = __ pc(); + + Label L_thaw, L_exit; + + // i2i entry used at interp_only_mode only + interpreted_entry_offset = __ pc() - start; + { +#ifdef ASSERT + NearLabel is_interp_only; + __ load_and_test_int(Z_R0_scratch, Address(Z_thread, JavaThread::interp_only_mode_offset())); + __ z_brnz(is_interp_only); + __ stop("enterSpecial interpreter entry called when not in interp_only_mode"); + __ bind(is_interp_only); +#endif + + // Read interpreter arguments into registers (this is an ad-hoc i2c adapter) + // s390x stores frame pointer in the slot 0, so argument will be loaded from slot 1 + __ z_lg(reg_cont_obj, Address(Z_esp, Interpreter::stackElementSize*3)); + __ z_llgf(reg_is_cont, Address(Z_esp, Interpreter::stackElementSize*2)); + __ z_llgf(reg_is_virtual, Address(Z_esp, Interpreter::stackElementSize*1)); + + __ push_cont_fastpath(); + + OopMap* map = continuation_enter_setup(masm, framesize_words); + + // The frame is complete here, but we only record it for the compiled entry, so the frame would appear unsafe, + // but that's okay because at the very worst we'll miss an async sample, but we're in interp_only_mode anyway. + + __ verify_oop(reg_cont_obj); + + fill_continuation_entry(masm, reg_cont_obj, reg_is_virtual); + + // If isContinue, call to thaw. Otherwise, call Continuation.enter(Continuation c, boolean isContinue) + __ compare32_and_branch(reg_is_cont, 0, Assembler::bcondNotZero, L_thaw); + + // --- call Continuation.enter(Continuation c, boolean isContinue) + + // Emit compiled static call. The call will be always resolved to the c2i + // entry of Continuation.enter(Continuation c, boolean isContinue). + // There are special cases in SharedRuntime::resolve_static_call_C() and + // SharedRuntime::resolve_sub_helper_internal() to achieve this + // See also corresponding call below. + // Make sure the call is patchable + + __ align(NativeCall::call_far_pcrelative_displacement_alignment, + __ offset() + NativeCall::call_far_pcrelative_displacement_offset); + + // Emit stub for static call + address stub = CompiledDirectCall::emit_to_interp_stub(masm, __ pc()); + if (stub == nullptr) { + fatal("CodeCache is full at gen_continuation_enter"); + } + __ relocate(relocInfo::static_call_type); + __ z_nop(); + __ z_brasl(Z_R14, resolve_static_call); + oop_maps->add_gc_map(__ pc() - start, map); + __ post_call_nop(); + __ branch_optimized(Assembler::bcondAlways, L_exit); + } + + // compiled entry + __ align(CodeEntryAlignment); + compiled_entry_offset = __ pc() - start; + + OopMap* map = continuation_enter_setup(masm, framesize_words); + + // Frame is now completed as far as size and linkage. + + frame_complete =__ pc() - start; + + __ verify_oop(reg_cont_obj); + + fill_continuation_entry(masm, reg_cont_obj, reg_is_virtual); + + // If isContinue, call to thaw. Otherwise, call Continuation.enter(Continuation c, boolean isContinue) + __ z_ltr(reg_is_cont, reg_is_cont); + __ branch_optimized(Assembler::bcondNotEqual, L_thaw); // was reg_is_cont equal to 0 ? + + // --- call Continuation.enter(Continuation c, boolean isContinue) + + // Make sure the call is patchable + __ align(NativeCall::call_far_pcrelative_displacement_alignment, + __ offset() + NativeCall::call_far_pcrelative_displacement_offset); + + // Emit stub for static call + address stub = CompiledDirectCall::emit_to_interp_stub(masm, __ pc()); + guarantee(stub != nullptr, "CodeCache is full at gen_continuation_enter"); + + assert((__ offset() + NativeCall::call_far_pcrelative_displacement_offset) % NativeCall::call_far_pcrelative_displacement_alignment == 0, + "must be aligned (offset=%d)", __ offset()); + + // The call needs to be resolved. There's a special case for this in + // SharedRuntime::find_callee_info_helper() which calls + // LinkResolver::resolve_continuation_enter() which resolves the call to + // Continuation.enter(Continuation c, boolean isContinue). + __ relocate(relocInfo::static_call_type); + __ z_nop(); + __ z_brasl(Z_R14, resolve_static_call); + oop_maps->add_gc_map(__ pc() - start, map); + __ post_call_nop(); + + __ branch_optimized(Assembler::bcondAlways, L_exit); + + // --- Thawing path + + __ bind(L_thaw); + ContinuationEntry::_thaw_call_pc_offset = __ pc() - start; + __ load_const_optimized(Z_R1_scratch, StubRoutines::cont_thaw()); + __ call(Z_R1_scratch); + oop_maps->add_gc_map(__ pc() - start, map->deep_copy()); + ContinuationEntry::_return_pc_offset = __ pc() - start; + __ post_call_nop(); + + // --- Normal exit (resolve/thawing) + __ bind(L_exit); + ContinuationEntry::_cleanup_offset = __ pc() - start; + continuation_enter_cleanup(masm); + + // Pop frame and return + DEBUG_ONLY(__ z_lg(Z_R0, Address(Z_SP, 0))); + __ add2reg(Z_SP, framesize_words * wordSize); + +#ifdef ASSERT + NearLabel ok; + __ z_cgr(Z_R0, Z_SP); + __ z_bre(ok); + __ stop("inconsistent frame size"); + __ bind(ok); +#endif // ASSERT + + __ restore_return_pc(); + __ z_br(Z_R14); + + // --- Exception handling path + exception_offset = __ pc() - start; + + continuation_enter_cleanup(masm); + + // Load caller's return pc + __ z_lg(Z_ARG2, _z_common_abi(callers_sp), Z_SP); + __ z_lg(Z_ARG2, _z_common_abi(return_pc), Z_ARG2); + + __ save_return_pc(); + __ push_frame_abi160(0 + 2 * BytesPerWord); + + __ z_stg(Z_ARG1, 0 * BytesPerWord + frame::z_abi_160_size, Z_SP); // save return value containing the exception oop + __ z_stg(Z_ARG2, 1 * BytesPerWord + frame::z_abi_160_size, Z_SP); // save exception_pc + + // Find exception handler. + __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), + Z_thread, + Z_ARG2); + + // Copy handler's address. + __ z_lgr(Z_R1, Z_RET); + + // Set up the arguments for the exception handler: + // - Z_ARG1: exception oop + // - Z_ARG2: exception pc + __ z_lg(Z_ARG1, 0 * BytesPerWord + frame::z_abi_160_size, Z_SP); // load the exception oop + __ z_lg(Z_ARG2, 1 * BytesPerWord + frame::z_abi_160_size, Z_SP); // load the exception pc + + __ pop_frame(); // pop frame pushed before runtime call + // __ restore_return_pc(); // can be skipped + + __ pop_frame(); // pop enterSpecial frame + __ restore_return_pc(); + + // Jump to exception handler + __ z_br(Z_R1 /*handler address*/); +} + +static void gen_continuation_yield(MacroAssembler* masm, + const VMRegPair* regs, + OopMapSet* oop_maps, + int& frame_complete, + int& framesize_words, + int& compiled_entry_offset) { + const int framesize_bytes = (int)align_up((int)frame::z_abi_160_size, frame::alignment_in_bytes); + framesize_words = framesize_bytes / wordSize; + + Register Rtmp = Z_R1_scratch; + + address start = __ pc(); + compiled_entry_offset = __ pc() - start; + + // Save return pc and push entry frame + __ save_return_pc(); + __ push_frame(framesize_bytes); + + DEBUG_ONLY(__ block_comment("Frame Complete (gen_continuation_yield):")); + frame_complete = __ pc() - start; + address last_java_pc = __ pc(); + + + // This nop must be exactly at the PC we push into the frame info. + // We use this nop for fast CodeBlob lookup, associate the OopMap + // with it right away. + __ post_call_nop(); + OopMap* map = new OopMap(framesize_bytes / VMRegImpl::stack_slot_size, 1); + oop_maps->add_gc_map(last_java_pc - start, map); + + __ z_larl(Rtmp, last_java_pc); + __ set_last_Java_frame(Z_SP, Rtmp); + __ call_VM_leaf(Continuation::freeze_entry(), Z_thread, Z_SP); + __ reset_last_Java_frame(); + + NearLabel L_pinned; + __ z_cij(Z_RET, 0, Assembler::bcondNotEqual, L_pinned); + + // Pop frames of continuation including this stub's frame + __ z_lg(Z_SP, Address(Z_thread, JavaThread::cont_entry_offset())); + // The frame pushed by gen_continuation_enter() is on top now again + continuation_enter_cleanup(masm); + // Pop frame and return + Label L_return; + __ bind(L_return); + __ pop_frame(); + __ restore_return_pc(); + __ z_br(Z_R14); + + // yield failed - continuation is pinned + __ bind(L_pinned); + + // handle pending exception thrown by freeze + __ load_and_test_long(Rtmp, Address(Z_thread, Thread::pending_exception_offset())); + __ z_bre(L_return); // return if no exception is pending + __ pop_frame(); + __ restore_return_pc(); + __ load_const_optimized(Z_R1_scratch, StubRoutines::forward_exception_entry()); + __ z_br(Z_R1_scratch); +} + +void SharedRuntime::continuation_enter_cleanup(MacroAssembler* masm) { + ::continuation_enter_cleanup(masm); +} + nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, const methodHandle& method, int compile_id, @@ -1346,6 +1738,66 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, VMRegPair *in_regs, BasicType ret_type) { int total_in_args = method->size_of_parameters(); + if (method->is_continuation_native_intrinsic()) { + int exception_offset = -1; + OopMapSet* oop_maps = new OopMapSet(); + int frame_complete = -1; + int stack_slots = -1; + int interpreted_entry_offset = -1; + int vep_offset = -1; // verified entry point offset + if (method->is_continuation_enter_intrinsic()) { + gen_continuation_enter(masm, + in_regs, + exception_offset, + oop_maps, + frame_complete, + stack_slots, + interpreted_entry_offset, + vep_offset); + } else if(method->is_continuation_yield_intrinsic()) { + gen_continuation_yield(masm, + in_regs, + oop_maps, + frame_complete, + stack_slots, + vep_offset); + } else { + guarantee(false, "Unknown Continuation native intrinsic"); + } + +#ifdef ASSERT + if (method->is_continuation_enter_intrinsic()) { + assert(interpreted_entry_offset != -1, "Must be set"); + assert(exception_offset != -1, "Must be set"); + } else { + assert(interpreted_entry_offset == -1, "Must be unset"); + assert(exception_offset == -1, "Must be unset"); + } + assert(frame_complete != -1, "Must be set"); + assert(stack_slots != -1, "Must be set"); + assert(vep_offset != -1, "Must be set"); +#endif + + __ flush(); + nmethod* nm = nmethod::new_native_nmethod(method, + compile_id, + masm->code(), + vep_offset, + frame_complete, + stack_slots, + in_ByteSize(-1), + in_ByteSize(-1), + oop_maps, + exception_offset); + if (nm == nullptr) return nm; + if (method->is_continuation_enter_intrinsic()) { + ContinuationEntry::set_enter_code(nm, interpreted_entry_offset); + } else if (method->is_continuation_yield_intrinsic()) { + _cont_doYield_stub = nm; + } + return nm; + } + if (method->is_method_handle_intrinsic()) { vmIntrinsics::ID iid = method->intrinsic_id(); intptr_t start = (intptr_t) __ pc(); @@ -1545,6 +1997,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, unsigned int wrapper_FrameDone; unsigned int wrapper_CRegsSet; Label handle_pending_exception; + Label last_java_pc; //--------------------------------------------------------------------- // Unverified entry point (UEP) @@ -1726,16 +2179,9 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, // So if we must call out we must push a new frame. ////////////////////////////////////////////////////////////////////// - - // Calc the current pc into Z_R10 and into wrapper_CRegsSet. - // Both values represent the same position. - __ get_PC(Z_R10); // PC into register - wrapper_CRegsSet = __ offset(); // and into into variable. - - // Z_R10 now has the pc loaded that we will use when we finally call to native. - - // We use the same pc/oopMap repeatedly when we call out. - oop_maps->add_gc_map((int)(wrapper_CRegsSet-wrapper_CodeStart), map); + // The last java pc will also be used as resume pc if this is the wrapper for wait0. + // For this purpose the precise location matters but not for oopmap lookup. + __ z_larl(Z_R10, last_java_pc); // Lock a synchronized method. @@ -1780,10 +2226,13 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, __ z_lgr(Z_ARG3, Z_thread); __ set_last_Java_frame(oldSP, Z_R10 /* gc map pc */); + assert(Z_R10->is_nonvolatile(), "Z_R10 needs to be preserved accross complete_monitor_locking_C call"); // Do the call. + __ push_cont_fastpath(); __ load_const_optimized(Z_R1_scratch, CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C)); __ call(Z_R1_scratch); + __ pop_cont_fastpath(); __ reset_last_Java_frame(); @@ -1910,6 +2359,23 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, // Transition from _thread_in_native_trans to _thread_in_Java. __ set_thread_state(_thread_in_Java); + // Check preemption for Object.wait() + if (method->is_object_wait0()) { + NearLabel not_preempted; + __ z_ltg(Z_R1_scratch, Address(Z_thread, JavaThread::preempt_alternate_return_offset())); + __ z_brz(not_preempted); // if 0, jump to not_preempted + __ z_mvghi(Address(Z_thread, JavaThread::preempt_alternate_return_offset()), 0); + __ z_br(Z_R1_scratch); + __ bind(not_preempted); + } + __ bind(last_java_pc); + + // Calc the current pc into wrapper_CRegsSet. + wrapper_CRegsSet = __ offset(); // and into into variable. + + // We use the same pc/oopMap repeatedly when we call out. + oop_maps->add_gc_map((int)(wrapper_CRegsSet-wrapper_CodeStart), map); + //-------------------------------------------------------------------- // Reguard any pages if necessary. // Protect native result from being destroyed. @@ -2012,7 +2478,10 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, // Clear "last Java frame" SP and PC. //-------------------------------------------------------------------- - __ reset_last_Java_frame(); + + // Last java frame won't be set if we're resuming after preemption + bool maybe_preempted = method->is_object_wait0(); + __ reset_last_Java_frame(/* check_last_java_sp = */ !maybe_preempted); // Unpack oop result, e.g. JNIHandles::resolve result. if (is_reference_type(ret_type)) { @@ -2317,6 +2786,8 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, } } + __ push_cont_fastpath(); // Set JavaThread::_cont_fastpath to the sp of the oldest interpreted frame we know about + // Jump to the compiled code just as if compiled code was doing it. // load target address from method: __ z_lg(Z_R1_scratch, Address(Z_method, Method::from_compiled_offset())); @@ -2416,8 +2887,7 @@ uint SharedRuntime::out_preserve_stack_slots() { } VMReg SharedRuntime::thread_register() { - Unimplemented(); - return nullptr; + return Z_thread->as_VMReg(); } // @@ -2678,6 +3148,13 @@ void SharedRuntime::generate_deopt_blob() { // stack: (caller_of_deoptee, ...). + // Freezing continuation frames requires that the caller is trimmed to unextended sp if compiled. + // If not compiled the loaded value is equal to the current SP (see frame::initial_deoptimization_info()) + // and the frame is effectively not resized. + Register caller_sp = Z_R1_scratch; + __ z_lg(caller_sp, Address(unroll_block_reg, Deoptimization::UnrollBlock::initial_info_offset())); + __ resize_frame_absolute(caller_sp, Z_R0, true); + // loop through the `UnrollBlock' info and create interpreter frames. push_skeleton_frames(masm, true/*deopt*/, unroll_block_reg, @@ -2809,6 +3286,13 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() { __ zap_from_to(Z_SP, Z_SP, Z_R0_scratch, Z_R1, 500, -1); + // Freezing continuation frames requires that the caller is trimmed to unextended sp if compiled. + // If not compiled the loaded value is equal to the current SP (see frame::initial_deoptimization_info()) + // and the frame is effectively not resized. + Register caller_sp = Z_R1_scratch; + __ z_lg(caller_sp, Address(unroll_block_reg, Deoptimization::UnrollBlock::initial_info_offset())); + __ resize_frame_absolute(caller_sp, Z_R0, true); + // allocate new interpreter frame(s) and possibly resize the caller's frame // (no more adapters !) push_skeleton_frames(masm, false/*deopt*/, @@ -3387,16 +3871,76 @@ int SpinPause() { } #if INCLUDE_JFR + +// For c2: c_rarg0 is junk, call to runtime to write a checkpoint. +// It returns a jobject handle to the event writer. +// The handle is dereferenced and the return value is the event writer oop. RuntimeStub* SharedRuntime::generate_jfr_write_checkpoint() { - if (!Continuations::enabled()) return nullptr; - Unimplemented(); - return nullptr; + const char* name = SharedRuntime::stub_name(StubId::shared_jfr_write_checkpoint_id); + CodeBuffer code(name, 512, 64); + MacroAssembler* masm = new MacroAssembler(&code); + + int framesize = frame::z_abi_160_size / VMRegImpl::stack_slot_size; + address start = __ pc(); + __ save_return_pc(); // save return_pc (Z_R14) + __ push_frame_abi160(0); + int frame_complete = __ pc() - start; + __ set_last_Java_frame(Z_SP, noreg); + + __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::write_checkpoint), Z_thread); + address calls_return_pc = __ last_calls_return_pc(); + __ reset_last_Java_frame(); + + // The handle is dereferenced through a load barrier. + __ resolve_global_jobject(Z_ARG1, Z_tmp_1, Z_tmp_2); + __ pop_frame(); + __ restore_return_pc(); + __ z_br(Z_R14); + + OopMapSet* oop_maps = new OopMapSet(); + OopMap* map = new OopMap(framesize, 0); + oop_maps->add_gc_map(calls_return_pc - start, map); + + RuntimeStub* stub = // codeBlob framesize is in words (not VMRegImpl::slot_size) + RuntimeStub::new_runtime_stub(name, &code, frame_complete, + (framesize >> (LogBytesPerWord - LogBytesPerInt)), + oop_maps, false); + + return stub; } +// For c2: call to return a leased buffer. RuntimeStub* SharedRuntime::generate_jfr_return_lease() { - if (!Continuations::enabled()) return nullptr; - Unimplemented(); - return nullptr; + const char* name = SharedRuntime::stub_name(StubId::shared_jfr_return_lease_id); + CodeBuffer code(name, 512, 64); + MacroAssembler* masm = new MacroAssembler(&code); + + int framesize = frame::z_abi_160_size / VMRegImpl::stack_slot_size; + address start = __ pc(); + __ save_return_pc(); // save return_pc (Z_R14) + __ push_frame_abi160(0); + int frame_complete = __ pc() - start; + __ set_last_Java_frame(Z_SP, noreg); + + __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::return_lease), Z_thread); + address calls_return_pc = __ last_calls_return_pc(); + + __ reset_last_Java_frame(); + + __ pop_frame(); + __ restore_return_pc(); + __ z_br(Z_R14); + + OopMapSet* oop_maps = new OopMapSet(); + OopMap* map = new OopMap(framesize, 0); + oop_maps->add_gc_map(calls_return_pc - start, map); + + RuntimeStub* stub = // codeBlob framesize is in words (not VMRegImpl::slot_size) + RuntimeStub::new_runtime_stub(name, &code, frame_complete, + (framesize >> (LogBytesPerWord - LogBytesPerInt)), + oop_maps, false); + + return stub; } #endif // INCLUDE_JFR diff --git a/src/hotspot/cpu/s390/smallRegisterMap_s390.inline.hpp b/src/hotspot/cpu/s390/smallRegisterMap_s390.inline.hpp index f338fb192ad..630a9516831 100644 --- a/src/hotspot/cpu/s390/smallRegisterMap_s390.inline.hpp +++ b/src/hotspot/cpu/s390/smallRegisterMap_s390.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,7 +30,7 @@ class SmallRegisterMap; -// Java frames don't have callee saved registers (except for rfp), so we can use a smaller SmallRegisterMapType +// Java frames don't have callee saved registers, so we can use a smaller RegisterMap template class SmallRegisterMapType { friend SmallRegisterMap; @@ -39,8 +39,6 @@ class SmallRegisterMapType { ~SmallRegisterMapType() = default; NONCOPYABLE(SmallRegisterMapType); - static void assert_is_rfp(VMReg r) NOT_DEBUG_RETURN - DEBUG_ONLY({ Unimplemented(); }) public: // as_RegisterMap is used when we didn't want to templatize and abstract over RegisterMap type to support SmallRegisterMap // Consider enhancing SmallRegisterMap to support those cases @@ -48,20 +46,21 @@ public: RegisterMap* as_RegisterMap() { return nullptr; } RegisterMap* copy_to_RegisterMap(RegisterMap* map, intptr_t* sp) const { - Unimplemented(); + map->clear(); + map->set_include_argument_oops(this->include_argument_oops()); return map; } inline address location(VMReg reg, intptr_t* sp) const { - Unimplemented(); + assert(false, "Reg: %s", reg->name()); return nullptr; } - inline void set_location(VMReg reg, address loc) { assert_is_rfp(reg); } + inline void set_location(VMReg reg, address loc) { assert(false, "Reg: %s", reg->name()); } JavaThread* thread() const { #ifndef ASSERT - guarantee (false, ""); + guarantee (false, "unreachable"); #endif return nullptr; } @@ -76,7 +75,7 @@ public: #ifdef ASSERT bool should_skip_missing() const { return false; } VMReg find_register_spilled_here(void* p, intptr_t* sp) { - Unimplemented(); + assert(false, "Shouldn't reach here! p:" PTR_FORMAT " sp:" PTR_FORMAT, p2i(p), p2i(p)); return nullptr; } void print() const { print_on(tty); } diff --git a/src/hotspot/cpu/s390/stackChunkFrameStream_s390.inline.hpp b/src/hotspot/cpu/s390/stackChunkFrameStream_s390.inline.hpp index e598117fe7d..3a5b860b7a7 100644 --- a/src/hotspot/cpu/s390/stackChunkFrameStream_s390.inline.hpp +++ b/src/hotspot/cpu/s390/stackChunkFrameStream_s390.inline.hpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2026 IBM Corporation. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,75 +33,120 @@ #ifdef ASSERT template inline bool StackChunkFrameStream::is_in_frame(void* p0) const { - Unimplemented(); - return true; + assert(!is_done(), ""); + assert(is_compiled(), ""); + intptr_t* p = (intptr_t*)p0; + int argsize = (_cb->as_nmethod()->num_stack_arg_slots() * VMRegImpl::stack_slot_size) >> LogBytesPerWord; + int frame_size = _cb->frame_size() + (argsize > 0 ? argsize + frame::metadata_words_at_top : 0); + return (p - unextended_sp()) >= 0 && (p - unextended_sp()) < frame_size; } #endif template inline frame StackChunkFrameStream::to_frame() const { - Unimplemented(); - return frame(); + if (is_done()) { + return frame(_sp, _sp, nullptr, nullptr, nullptr, nullptr, true); + } else { + // Compiled frames on heap don't have back links on s390. The back link is redundant + // and gets computed as unextended_sp + frame_size. In debug builds, FreezeBase::patch_pd() + // explicitly sets it to badAddress. + return frame(sp(), unextended_sp(), Interpreter::contains(pc()) ? fp() : nullptr, pc(), cb(), _oopmap, true); + } } template inline address StackChunkFrameStream::get_pc() const { - Unimplemented(); - return nullptr; + assert(!is_done(), ""); + return (address)((frame::z_common_abi*) _sp)->return_pc; } template inline intptr_t* StackChunkFrameStream::fp() const { - Unimplemented(); - return nullptr; + // See FreezeBase::patch_pd() and frame::setup() + assert((frame_kind == ChunkFrames::Mixed && is_interpreted()), ""); + intptr_t* fp_addr = (intptr_t*)&((frame::z_common_abi*)_sp)->callers_sp; + assert(*(intptr_t**)fp_addr != nullptr, ""); + // derelativize + return fp_addr + *fp_addr; } template inline intptr_t* StackChunkFrameStream::derelativize(int offset) const { - Unimplemented(); - return nullptr; + intptr_t* fp = this->fp(); + assert(fp != nullptr, ""); + return fp + fp[offset]; } template inline intptr_t* StackChunkFrameStream::unextended_sp_for_interpreter_frame() const { - Unimplemented(); - return nullptr; + assert_is_interpreted_and_frame_type_mixed(); + // Compute the unextended SP (stack pointer before any extension for arguments). + // On s390, esp points to the next free slot above the operand stack, so we add 1 + // to get the actual top of the operand stack, then subtract metadata_words to + // account for the frame metadata (callers_sp and return_pc) at the top of the frame. + return derelativize(_z_ijava_idx(esp)) + 1 - frame::metadata_words; } template inline void StackChunkFrameStream::next_for_interpreter_frame() { - Unimplemented(); + assert_is_interpreted_and_frame_type_mixed(); + if (derelativize(_z_ijava_idx(locals)) + 1 >= _end) { + _unextended_sp = _end; + _sp = _end; + } else { + _unextended_sp = derelativize(_z_ijava_idx(sender_sp)); + _sp = this->fp(); + } } template inline int StackChunkFrameStream::interpreter_frame_size() const { - Unimplemented(); - return 0; + assert_is_interpreted_and_frame_type_mixed(); + intptr_t* top = unextended_sp(); // later subtract argsize if callee is interpreted + intptr_t* bottom = derelativize(_z_ijava_idx(locals)) + 1; + return (int)(bottom - top); } +// Size of stack args in words (P0..Pn above). Only valid if the caller is also +// interpreted. The function is also called if the caller is compiled but the +// result is not used in that case (same on x86). +// See also setting of sender_sp in ContinuationHelper::InterpretedFrame::patch_sender_sp() template inline int StackChunkFrameStream::interpreter_frame_stack_argsize() const { - Unimplemented(); - return 0; + assert_is_interpreted_and_frame_type_mixed(); + frame::z_ijava_state* state = (frame::z_ijava_state*)((uintptr_t)fp() - frame::z_ijava_state_size); + int diff = (int)(state->locals - (state->sender_sp + frame::metadata_words_at_top) + 1); + assert(diff == -frame::metadata_words_at_top || ((Method*)state->method)->size_of_parameters() == diff, + "size_of_parameters(): %d diff: %d sp: " PTR_FORMAT " fp:" PTR_FORMAT, + ((Method*)state->method)->size_of_parameters(), diff, p2i(sp()), p2i(fp())); + return diff; } template template inline int StackChunkFrameStream::interpreter_frame_num_oops(RegisterMapT* map) const { - Unimplemented(); - return 0; + assert_is_interpreted_and_frame_type_mixed(); + ResourceMark rm; + frame f = to_frame(); + InterpreterOopCount closure; + f.oops_interpreted_do(&closure, map); + return closure.count(); } template<> template<> inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { - Unimplemented(); + // No register map update needed for s390. + // In the Java calling convention on s390, all registers are volatile (caller-saved), + // so there are no non-volatile (callee-saved) registers that need to be tracked. } template<> template<> inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { - Unimplemented(); + // No register map update needed for s390. + // In the Java calling convention on s390, all registers are volatile (caller-saved), + // so there are no non-volatile (callee-saved) registers that need to be tracked. } template diff --git a/src/hotspot/cpu/s390/stackChunkOop_s390.inline.hpp b/src/hotspot/cpu/s390/stackChunkOop_s390.inline.hpp index dfd3562c9d9..c97751d0d1e 100644 --- a/src/hotspot/cpu/s390/stackChunkOop_s390.inline.hpp +++ b/src/hotspot/cpu/s390/stackChunkOop_s390.inline.hpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2026 IBM Corporation. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,11 +27,15 @@ #define CPU_S390_STACKCHUNKOOP_S390_INLINE_HPP inline void stackChunkOopDesc::relativize_frame_pd(frame& fr) const { - Unimplemented(); + if (fr.is_interpreted_frame()) { + fr.set_offset_fp(relativize_address(fr.fp())); + } } inline void stackChunkOopDesc::derelativize_frame_pd(frame& fr) const { - Unimplemented(); + if (fr.is_interpreted_frame()) { + fr.set_fp(derelativize_address(fr.offset_fp())); + } } #endif // CPU_S390_STACKCHUNKOOP_S390_INLINE_HPP diff --git a/src/hotspot/cpu/s390/stubDeclarations_s390.hpp b/src/hotspot/cpu/s390/stubDeclarations_s390.hpp index d0e26beedab..d773b6ce759 100644 --- a/src/hotspot/cpu/s390/stubDeclarations_s390.hpp +++ b/src/hotspot/cpu/s390/stubDeclarations_s390.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2025, Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -47,7 +47,7 @@ do_arch_entry, \ do_arch_entry_init, \ do_arch_entry_array) \ - do_arch_blob(continuation, 2000) \ + do_arch_blob(continuation, 5000) \ #define STUBGEN_COMPILER_BLOBS_ARCH_DO(do_stub, \ diff --git a/src/hotspot/cpu/s390/stubGenerator_s390.cpp b/src/hotspot/cpu/s390/stubGenerator_s390.cpp index 5309158fc74..d1601d4f147 100644 --- a/src/hotspot/cpu/s390/stubGenerator_s390.cpp +++ b/src/hotspot/cpu/s390/stubGenerator_s390.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016, 2024 SAP SE. All rights reserved. + * Copyright (c) 2026 IBM Corporation. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -37,6 +38,8 @@ #include "oops/oop.inline.hpp" #include "prims/methodHandles.hpp" #include "prims/upcallLinker.hpp" +#include "runtime/continuation.hpp" +#include "runtime/continuationEntry.inline.hpp" #include "runtime/frame.inline.hpp" #include "runtime/handles.inline.hpp" #include "runtime/javaThread.hpp" @@ -330,6 +333,8 @@ class StubGenerator: public StubCodeGenerator { // Pop frame. Done here to minimize stalls. __ pop_frame(); + __ pop_cont_fastpath(); + // Reload some volatile registers which we've spilled before the call // to template interpreter / native entry. // Access all locals via frame pointer, because we know nothing about @@ -3223,28 +3228,180 @@ class StubGenerator: public StubCodeGenerator { return start; } - address generate_cont_thaw(bool return_barrier, bool exception) { + address generate_cont_thaw(StubId stub_id) { if (!Continuations::enabled()) return nullptr; - Unimplemented(); - return nullptr; + + Continuation::thaw_kind kind; + bool return_barrier; + bool return_barrier_exception; + + switch (stub_id) { + case StubId::stubgen_cont_thaw_id: + kind = Continuation::thaw_top; + return_barrier = false; + return_barrier_exception = false; + break; + case StubId::stubgen_cont_returnBarrier_id: + kind = Continuation::thaw_return_barrier; + return_barrier = true; + return_barrier_exception = false; + break; + case StubId::stubgen_cont_returnBarrierExc_id: + kind = Continuation::thaw_return_barrier_exception; + return_barrier = true; + return_barrier_exception = true; + break; + default: + ShouldNotReachHere(); + } + + StubCodeMark mark(this, stub_id); + address start = __ pc(); + + // TODO: Handle Valhalla return types. May require generating different return barriers. + + if (kind == Continuation::thaw_top) { + __ clobber_nonvolatile_registers(); // Except Z_thread + } + + if (return_barrier) { + // Save return values in non-volatile float registers to preserve them across VM calls. + // Z_F8 and Z_F9 are non-volatile (callee-saved) registers on s390 (F8-F15 are non-volatile). + // They are safe to use here because: + // 1. clobber_nonvolatile_registers() is NOT called for return_barrier cases (only for thaw_top) + // 2. These registers are preserved across the VM leaf calls (prepare_thaw, thaw_entry) + __ z_ldgr(Z_F8, Z_RET); // Save integer return value in non-volatile float register + __ z_ldr(Z_F9, Z_FRET); // Save float return value in non-volatile float register + + DEBUG_ONLY(__ z_lg(Z_R1_scratch, _z_common_abi(callers_sp), Z_SP);) + __ z_lg(Z_SP, Address(Z_thread, JavaThread::cont_entry_offset())); +#ifdef ASSERT + __ z_cg(Z_R1_scratch, _z_common_abi(callers_sp), Z_SP); + __ asm_assert(/* check_equal=*/ true, FILE_AND_LINE ": callers sp is corrupt at thaw entry", 69); +#endif + + } + +#ifdef ASSERT + __ z_cg(Z_SP, Address(Z_thread, JavaThread::cont_entry_offset())); + __ asm_assert(/* check_equal=*/ true, FILE_AND_LINE ": incorrect Z_SP", 70); +#endif + + __ z_lghi(Z_ARG2, return_barrier ? 1 : 0); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), Z_thread, Z_ARG2); + +#ifdef ASSERT + __ z_cg(Z_SP, Address(Z_thread, JavaThread::cont_entry_offset())); + __ asm_assert(/* check equal = */ true, FILE_AND_LINE ": incorrect Z_SP after prepare_thaw", 48); +#endif // ASSERT + + // Z_RET contains the size of the frames to thaw, 0 if overflow or no more frames + NearLabel L_thaw_success; + __ z_ltgr(Z_RET, Z_RET); + __ branch_optimized(Assembler::bcondNotEqual, L_thaw_success); + __ load_const_optimized(Z_R1_scratch, (SharedRuntime::throw_StackOverflowError_entry())); + __ call(Z_R1_scratch); + __ bind(L_thaw_success); + + // Make room for the thawed frames and align the stack. + __ add64(Z_RET, frame::z_abi_160_size); + + { // stack alignment + __ z_lcgr(Z_RET, Z_RET); // negate Z_RET value + __ z_nill(Z_RET, -frame::alignment_in_bytes); + } + __ resize_frame( /* offset = */ Z_RET,/* fp = */ Z_R1, /* load_fp = */ true); + + __ z_lghi(Z_ARG2, kind); + __ add64(Z_SP, -frame::z_abi_160_size); // Register save area for Continuation::thaw + __ call_VM_leaf(Continuation::thaw_entry(), Z_thread, Z_ARG2); + __ z_lgr(Z_SP, Z_RET); // Z_RET contains the SP of the thawed top frame + + if (return_barrier) { + // we're now in the caller of the frame that returned to the barrier + // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK) + + __ z_lgdr(Z_RET, Z_F8); // Restore integer return value + __ z_ldr(Z_FRET, Z_F9); // Restore float return value + } else { + // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down) + __ z_lghi(Z_RET, 0); // return 0 (success) from doYield + } + + if (return_barrier_exception) { + Register handler = Z_R1_scratch; + __ z_lg(Z_ARG2, _z_common_abi(return_pc), Z_SP); // exception pc + __ save_return_pc(); + __ push_frame_abi160(0 + 2 * BytesPerWord); + __ z_stg(Z_RET , 0 * BytesPerWord + frame::z_abi_160_size, Z_SP); // save return value containing the exception oop + + __ z_stg(Z_ARG2, 1 * BytesPerWord + frame::z_abi_160_size, Z_SP); // save exception_pc + __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), Z_thread, Z_ARG2); + + // Copy handler's address. + __ z_lgr(handler, Z_RET); + + // Set up the arguments for the exception handler: + // - Z_ARG1: exception oop + // - Z_ARG2: exception pc + __ z_lg(Z_ARG1, 0 * BytesPerWord + frame::z_abi_160_size, Z_SP); // load the exception oop + __ z_lg(Z_ARG2, 1 * BytesPerWord + frame::z_abi_160_size, Z_SP); // load the exception pc + __ pop_frame(); + __ restore_return_pc(); + } else { + // We're "returning" into the topmost thawed frame; see Thaw::push_return_frame + __ z_lg(Z_R1_scratch, _z_common_abi(return_pc), Z_SP); + } + __ z_br(Z_R1_scratch); + + return start; } address generate_cont_thaw() { - if (!Continuations::enabled()) return nullptr; - Unimplemented(); - return nullptr; + return generate_cont_thaw(StubId::stubgen_cont_thaw_id); } address generate_cont_returnBarrier() { - if (!Continuations::enabled()) return nullptr; - Unimplemented(); - return nullptr; + return generate_cont_thaw(StubId::stubgen_cont_returnBarrier_id); } address generate_cont_returnBarrier_exception() { + return generate_cont_thaw(StubId::stubgen_cont_returnBarrierExc_id); + } + + address generate_cont_preempt_stub() { if (!Continuations::enabled()) return nullptr; - Unimplemented(); - return nullptr; + StubId stub_id = StubId::stubgen_cont_preempt_id; + StubCodeMark mark(this, stub_id); + address start = __ pc(); + + __ clobber_nonvolatile_registers(); // Except Z_thread + + __ reset_last_Java_frame(/*check_last_java_sp=*/ false); + + // Set sp to enterSpecial frame, i.e. remove all frames copied into the heap. + __ z_lg(Z_SP, Address(Z_thread, JavaThread::cont_entry_offset())); + + Label preemption_cancelled; + + __ z_cli(in_bytes(JavaThread::preemption_cancelled_offset()), Z_thread, 0); + __ z_brne(preemption_cancelled); + + // Remove enterSpecial frame from the stack and return to Continuation.run() to unmount. + SharedRuntime::continuation_enter_cleanup(_masm); + __ pop_frame(); + __ restore_return_pc(); + __ z_br(Z_R14); + + // We acquired the monitor after freezing the frames so call thaw to continue execution. + __ bind(preemption_cancelled); + __ z_mvi(in_bytes(JavaThread::preemption_cancelled_offset()), Z_thread, 0); + + __ load_const_optimized(Z_R1, ContinuationEntry::thaw_call_pc_address()); + __ z_lg(Z_R1, Address(Z_R1)); + __ z_br(Z_R1); + + return start; } // exception handler for upcall stubs @@ -3327,9 +3484,10 @@ class StubGenerator: public StubCodeGenerator { if (!Continuations::enabled()) return; // Continuation stubs: - StubRoutines::_cont_thaw = generate_cont_thaw(); - StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); + StubRoutines::_cont_thaw = generate_cont_thaw(); + StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); + StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub(); } void generate_final_stubs() { diff --git a/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp b/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp index dba04fc0e85..03470597ab5 100644 --- a/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp +++ b/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016, 2024 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -717,13 +717,31 @@ address TemplateInterpreterGenerator::generate_safept_entry_for (TosState state, address runtime_entry) { address entry = __ pc(); __ push(state); + __ push_cont_fastpath(); __ call_VM(noreg, runtime_entry); + __ pop_cont_fastpath(); __ dispatch_via(vtos, Interpreter::_normal_table.table_for (vtos)); return entry; } address TemplateInterpreterGenerator::generate_cont_resume_interpreter_adapter() { - return nullptr; + if (!Continuations::enabled()) return nullptr; + address start = __ pc(); + __ z_lg(Z_fp, _z_common_abi(callers_sp), Z_SP); + { + Register top_frame_sp = Z_R1_scratch; // anyway going to load it with correct value + __ z_lg(top_frame_sp, Address(Z_fp, _z_ijava_state_neg(top_frame_sp))); + __ z_slag(top_frame_sp, top_frame_sp, Interpreter::logStackElementSize); + __ z_agr(top_frame_sp, Z_fp); + + __ resize_frame_absolute(top_frame_sp, /* temp = */ Z_R0, /* load_fp = */ true); + } + __ restore_bcp(); + __ restore_locals(); + __ restore_esp(); + + __ z_br(Z_R14); + return start; } @@ -1468,8 +1486,13 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { __ bind(call_signature_handler); + bool support_vthread_preemption = Continuations::enabled(); + // We have a TOP_IJAVA_FRAME here, which belongs to us. - __ set_top_ijava_frame_at_SP_as_last_Java_frame(Z_SP, Z_R1/*tmp*/); + Label last_java_pc; + Label *resume_pc = support_vthread_preemption ? &last_java_pc : nullptr; + + __ set_top_ijava_frame_at_SP_as_last_Java_frame(Z_SP, Z_R1/*tmp*/, resume_pc); // Call signature handler and pass locals address in Z_ARG1. __ z_lgr(Z_ARG1, Z_locals); @@ -1526,7 +1549,18 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { // overwritten since "__ call_stub(signature_handler);" (except for // ARG1 and ARG2 for static methods). + if (support_vthread_preemption) { + // Rresult_handler is a nonvolatile register. Its value will be preserved across + // the native call but only if the call isn't preempted. To preserve its value even + // in the case of preemption we save it in the lresult slot. It is restored at + // resume_pc if, and only if the call was preempted. This works because only + // j.l.Object::wait calls are preempted which don't return a result. + + __ z_stg(Rresult_handler, _z_ijava_state_neg(lresult), Z_fp); + } + __ push_cont_fastpath(); __ call_c(Z_R1/*native_method_entry*/); + __ pop_cont_fastpath(); // NOTE: frame::interpreter_frame_result() depends on these stores. __ z_stg(Z_RET, _z_ijava_state_neg(lresult), Z_fp); @@ -1610,6 +1644,32 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { __ z_lg(Z_bcp, Address(Rmethod, Method::const_offset())); // get constMethod __ add2reg(Z_bcp, in_bytes(ConstMethod::codes_offset())); // get codebase + if (support_vthread_preemption) { + // Check preemption for Object.wait() + Label not_preempted; + __ z_ltg(Z_R1_scratch, Address(Z_thread, JavaThread::preempt_alternate_return_offset())); + __ z_brz(not_preempted); // if 0, jump to not_preempted + __ z_mvghi(Address(Z_thread, JavaThread::preempt_alternate_return_offset()), 0); + __ z_br(Z_R1_scratch); + + // Execution will be resumed here when the vthread becomes runnable again. + __ bind(*resume_pc); + __ restore_after_resume(); + // We saved the result handler before the call + __ z_lg(Rresult_handler, _z_ijava_state_neg(lresult), Z_fp); +#ifdef ASSERT + // Clobber result slots. Only native methods returning void can be preemted currently. + __ load_const(Z_RET, UCONST64(0xbad01001)); + __ z_stg(Z_RET, _z_ijava_state_neg(lresult), Z_fp); + __ z_stg(Z_RET, _z_ijava_state_neg(fresult), Z_fp); + // reset_last_Java_frame() below asserts that a last java sp is set + __ asm_assert_mem8_is_zero(in_bytes(JavaThread::last_Java_sp_offset()), + Z_thread, FILE_AND_LINE ": Last java sp should not be set when resuming", 69); + __ z_stg(Z_RET, in_bytes(JavaThread::last_Java_sp_offset()), Z_thread); +#endif + __ bind(not_preempted); + } + if (CheckJNICalls) { // clear_pending_jni_exception_check __ clear_mem(Address(Z_thread, JavaThread::pending_jni_exception_check_fn_offset()), sizeof(oop)); @@ -2030,7 +2090,7 @@ address TemplateInterpreterGenerator::generate_CRC32C_updateBytes_entry(Abstract address TemplateInterpreterGenerator::generate_currentThread() { uint64_t entry_off = __ offset(); - __ z_lg(Z_RET, Address(Z_thread, JavaThread::threadObj_offset())); + __ z_lg(Z_RET, Address(Z_thread, JavaThread::vthread_offset())); __ resolve_oop_handle(Z_RET, Z_R0_scratch, Z_R1_scratch); // Restore caller sp for c2i case. @@ -2176,6 +2236,7 @@ void TemplateInterpreterGenerator::generate_throw_exception() { JavaThread::popframe_force_deopt_reexecution_bit, Z_tmp_1, false); + __ pop_cont_fastpath(); // Continue in deoptimization handler. __ z_br(Z_R14); @@ -2191,6 +2252,7 @@ void TemplateInterpreterGenerator::generate_throw_exception() { false, // install_monitor_exception false); // notify_jvmdi __ z_lg(Z_fp, _z_abi(callers_sp), Z_SP); // Restore frame pointer. + __ pop_cont_fastpath(); { Register top_frame_sp = Z_R1_scratch; __ z_lg(top_frame_sp, Address(Z_fp, _z_ijava_state_neg(top_frame_sp))); @@ -2264,6 +2326,7 @@ void TemplateInterpreterGenerator::generate_throw_exception() { // Remove the activation (without doing throws on illegalMonitorExceptions). __ remove_activation(vtos, noreg/*ret.pc already loaded*/, false/*throw exc*/, true/*install exc*/, false/*notify jvmti*/); __ z_lg(Z_fp, _z_abi(callers_sp), Z_SP); // Restore frame pointer. + __ pop_cont_fastpath(); __ get_vm_result_oop(Z_ARG1); // Restore exception. __ verify_oop(Z_ARG1); diff --git a/src/hotspot/cpu/s390/templateTable_s390.cpp b/src/hotspot/cpu/s390/templateTable_s390.cpp index 3b0929608a3..1da24c0378c 100644 --- a/src/hotspot/cpu/s390/templateTable_s390.cpp +++ b/src/hotspot/cpu/s390/templateTable_s390.cpp @@ -2336,7 +2336,9 @@ void TemplateTable::_return(TosState state) { __ z_tm(poll_byte_addr, SafepointMechanism::poll_bit()); __ z_braz(no_safepoint); __ push(state); + __ push_cont_fastpath(); __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::at_safepoint)); + __ pop_cont_fastpath(); __ pop(state); __ bind(no_safepoint); } @@ -2395,7 +2397,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no, // Class initialization barrier slow path lands here as well. address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache); __ load_const_optimized(Z_ARG2, (int)code); - __ call_VM(noreg, entry, Z_ARG2); + __ call_VM_preemptable(noreg, entry, Z_ARG2); // Update registers with resolved info. __ load_method_entry(Rcache, index); @@ -2445,7 +2447,7 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no, // Class initialization barrier slow path lands here as well. address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache); __ load_const_optimized(Z_ARG2, (int)code); - __ call_VM(noreg, entry, Z_ARG2); + __ call_VM_preemptable(noreg, entry, Z_ARG2); // Update registers with resolved info. __ load_field_entry(cache, index); @@ -4022,7 +4024,7 @@ void TemplateTable::_new() { __ bind(slow_case); __ get_constant_pool(Z_ARG2); __ get_2_byte_integer_at_bcp(Z_ARG3/*dest*/, 1, InterpreterMacroAssembler::Unsigned); - call_VM(Z_tos, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), Z_ARG2, Z_ARG3); + __ call_VM_preemptable(Z_tos, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), Z_ARG2, Z_ARG3); __ verify_oop(Z_tos); // continue diff --git a/src/hotspot/cpu/s390/upcallLinker_s390.cpp b/src/hotspot/cpu/s390/upcallLinker_s390.cpp index 23ac80ddf48..de57e5e0cc4 100644 --- a/src/hotspot/cpu/s390/upcallLinker_s390.cpp +++ b/src/hotspot/cpu/s390/upcallLinker_s390.cpp @@ -220,9 +220,13 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature, __ call(call_target_address); // load taget Method* into Z_method __ block_comment("} load_target"); + __ push_cont_fastpath(); + __ z_lg(call_target_address, Address(Z_method, in_bytes(Method::from_compiled_offset()))); __ call(call_target_address); + __ pop_cont_fastpath(); + // return value shuffle assert(!needs_return_buffer, "unexpected needs_return_buffer"); // CallArranger can pick a return type that goes in the same reg for both CCs. diff --git a/src/hotspot/share/oops/stackChunkOop.inline.hpp b/src/hotspot/share/oops/stackChunkOop.inline.hpp index d0ddbe8dfe6..3ad28190d02 100644 --- a/src/hotspot/share/oops/stackChunkOop.inline.hpp +++ b/src/hotspot/share/oops/stackChunkOop.inline.hpp @@ -369,7 +369,7 @@ inline void stackChunkOopDesc::copy_from_stack_to_chunk(intptr_t* from, intptr_t assert(to >= start_address(), "Chunk underflow"); assert(to + size <= end_address(), "Chunk overflow"); -#if !(defined(AMD64) || defined(AARCH64) || defined(RISCV64) || defined(PPC64)) || defined(ZERO) +#if !(defined(AMD64) || defined(AARCH64) || defined(RISCV64) || defined(PPC64) || defined(S390)) || defined(ZERO) // Suppress compilation warning-as-error on unimplemented architectures // that stub out arch-specific methods. Some compilers are smart enough // to figure out the argument is always null and then warn about it. @@ -388,7 +388,7 @@ inline void stackChunkOopDesc::copy_from_chunk_to_stack(intptr_t* from, intptr_t assert(from >= start_address(), ""); assert(from + size <= end_address(), ""); -#if !(defined(AMD64) || defined(AARCH64) || defined(RISCV64) || defined(PPC64)) || defined(ZERO) +#if !(defined(AMD64) || defined(AARCH64) || defined(RISCV64) || defined(PPC64) || defined(S390)) || defined(ZERO) // Suppress compilation warning-as-error on unimplemented architectures // that stub out arch-specific methods. Some compilers are smart enough // to figure out the argument is always null and then warn about it. diff --git a/src/hotspot/share/runtime/continuation.cpp b/src/hotspot/share/runtime/continuation.cpp index f8af2545c37..e80720072c5 100644 --- a/src/hotspot/share/runtime/continuation.cpp +++ b/src/hotspot/share/runtime/continuation.cpp @@ -317,7 +317,7 @@ frame Continuation::continuation_parent_frame(RegisterMap* map) { map->set_stack_chunk(nullptr); -#if (defined(X86) || defined(AARCH64) || defined(RISCV64) || defined(PPC64)) && !defined(ZERO) +#if (defined(X86) || defined(AARCH64) || defined(RISCV64) || defined(PPC64) || defined(S390)) && !defined(ZERO) frame sender(cont.entrySP(), cont.entryFP(), cont.entryPC()); #else frame sender = frame(); diff --git a/src/hotspot/share/runtime/continuationFreezeThaw.cpp b/src/hotspot/share/runtime/continuationFreezeThaw.cpp index d76652edf36..e9b6325d03b 100644 --- a/src/hotspot/share/runtime/continuationFreezeThaw.cpp +++ b/src/hotspot/share/runtime/continuationFreezeThaw.cpp @@ -220,7 +220,6 @@ template static inline freeze_result freeze_inte static inline int prepare_thaw_internal(JavaThread* thread, bool return_barrier); template static inline intptr_t* thaw_internal(JavaThread* thread, const Continuation::thaw_kind kind); - // Entry point to freeze. Transitions are handled manually // Called from gen_continuation_yield() in sharedRuntime_.cpp through Continuation::freeze_entry(); template @@ -507,13 +506,7 @@ FreezeBase::FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* assert(!Interpreter::contains(_cont.entryPC()), ""); - _bottom_address = _cont.entrySP() - _cont.entry_frame_extension(); -#ifdef _LP64 - if (((intptr_t)_bottom_address & 0xf) != 0) { - _bottom_address--; - } - assert(is_aligned(_bottom_address, frame::frame_alignment), ""); -#endif + _bottom_address = align_down(_cont.entrySP() - _cont.entry_frame_extension(), frame::frame_alignment); log_develop_trace(continuations)("bottom_address: " INTPTR_FORMAT " entrySP: " INTPTR_FORMAT " argsize: " PTR_FORMAT, p2i(_bottom_address), p2i(_cont.entrySP()), (_cont.entrySP() - _bottom_address) << LogBytesPerWord); @@ -523,13 +516,17 @@ FreezeBase::FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* assert(_cont.chunk_invariant(), ""); assert(!Interpreter::contains(_cont.entryPC()), ""); -#if !defined(PPC64) || defined(ZERO) - static const int doYield_stub_frame_size = frame::metadata_words; -#else +#if defined(PPC64) && !defined(ZERO) static const int doYield_stub_frame_size = frame::native_abi_reg_args_size >> LogBytesPerWord; +#elif defined(S390) && !defined(ZERO) + static const int doYield_stub_frame_size = frame::z_abi_160_base_size >> LogBytesPerWord; +#else + static const int doYield_stub_frame_size = frame::metadata_words; #endif // With preemption doYield() might not have been resolved yet - assert(_preempt || SharedRuntime::cont_doYield_stub()->frame_size() == doYield_stub_frame_size, ""); + assert(_preempt || SharedRuntime::cont_doYield_stub()->frame_size() == doYield_stub_frame_size, + "_preempt = %d, cont_doYield_stub()->frame_size() = %d, doYield_stub_frame_size = %d", + (_preempt ? 1 : 0), SharedRuntime::cont_doYield_stub()->frame_size(), doYield_stub_frame_size); if (preempt) { _last_frame = _thread->last_frame(); @@ -2597,7 +2594,13 @@ inline void ThawBase::patch(frame& f, const frame& caller, bool bottom) { } else if (_should_patch_caller_pc) { // Caller was deoptimized during thaw but we've overwritten the return address when copying f from the heap. // Also, on some platforms, if the caller is interpreted but the callee not we also need to patch. - assert(caller.is_deoptimized_frame() PPC64_ONLY(|| caller.is_interpreted_frame()), ""); + +#if defined(PPC64) || defined(S390) + assert(caller.is_deoptimized_frame() || caller.is_interpreted_frame(), ""); +#else + assert(caller.is_deoptimized_frame(), ""); +#endif + ContinuationHelper::Frame::patch_pc(caller, caller.raw_pc()); _should_patch_caller_pc = false; } diff --git a/src/hotspot/share/runtime/frame.cpp b/src/hotspot/share/runtime/frame.cpp index ae04d398043..3e45b6fe310 100644 --- a/src/hotspot/share/runtime/frame.cpp +++ b/src/hotspot/share/runtime/frame.cpp @@ -1685,13 +1685,13 @@ void FrameValues::print_on(outputStream* st, int min_index, int max_index, intpt // 4. Recognize it as being part of the "fixed frame". if (*fv.location != 0 && *fv.location > -100 && *fv.location < 100 && fp != nullptr && *fv.description != '#' -#if !defined(PPC64) +#if !defined(PPC64) && !defined(S390) && (strncmp(fv.description, "interpreter_frame_", 18) == 0 || strstr(fv.description, " method ")) -#else // !defined(PPC64) +#else // !defined(PPC64) && !defined(S390) && (strcmp(fv.description, "sender_sp") == 0 || strcmp(fv.description, "top_frame_sp") == 0 || strcmp(fv.description, "esp") == 0 || strcmp(fv.description, "monitors") == 0 || strcmp(fv.description, "locals") == 0 || strstr(fv.description, " method ")) -#endif //!defined(PPC64) +#endif // !defined(PPC64) && !defined(S390) ) { st->print_cr(" " INTPTR_FORMAT ": " INTPTR_FORMAT " %-32s (relativized: fp%+d)", p2i(fv.location), p2i(&fp[*fv.location]), fv.description, (int)*fv.location); diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 5489735da39..bcb7f5488f5 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -3104,14 +3104,16 @@ void AdapterHandlerLibrary::create_native_wrapper(const methodHandle& method) { struct { double data[20]; } locs_buf; struct { double data[20]; } stubs_locs_buf; buffer.insts()->initialize_shared_locs((relocInfo*)&locs_buf, sizeof(locs_buf) / sizeof(relocInfo)); -#if defined(AARCH64) || defined(PPC64) +#if defined(AARCH64) // On AArch64 with ZGC and nmethod entry barriers, we need all oops to be // in the constant pool to ensure ordering between the barrier and oops // accesses. For native_wrappers we need a constant. - // On PPC64 the continuation enter intrinsic needs the constant pool for the compiled + buffer.initialize_consts_size(8); +#elif defined(PPC64) || defined(S390) + // On PPC64/S390 the continuation enter intrinsic needs the constant pool for the compiled // static java call that is resolved in the runtime. - if (PPC64_ONLY(method->is_continuation_enter_intrinsic() &&) true) { - buffer.initialize_consts_size(8 PPC64_ONLY(+ 24)); + if (method->is_continuation_enter_intrinsic()) { + buffer.initialize_consts_size(8 PPC64_ONLY(+ 24) S390_ONLY(+ 17)); } #endif buffer.stubs()->initialize_shared_locs((relocInfo*)&stubs_locs_buf, sizeof(stubs_locs_buf) / sizeof(relocInfo)); diff --git a/test/hotspot/jtreg/ProblemList.txt b/test/hotspot/jtreg/ProblemList.txt index a9f70fc97a4..e0005bfde07 100644 --- a/test/hotspot/jtreg/ProblemList.txt +++ b/test/hotspot/jtreg/ProblemList.txt @@ -64,7 +64,6 @@ compiler/floatingpoint/TestSubnormalDouble.java 8317810 generic-i586 compiler/codecache/CodeCacheFullCountTest.java 8332954 generic-all compiler/interpreter/Test6833129.java 8335266 generic-i586 -compiler/intrinsics/TestReturnOopSetForJFRWriteCheckpoint.java 8286300 linux-s390x compiler/c2/aarch64/TestStaticCallStub.java 8359963 generic-aarch64 @@ -105,7 +104,6 @@ runtime/ErrorHandling/MachCodeFramesInErrorFile.java 8313315 linux-ppc64le runtime/NMT/VirtualAllocCommitMerge.java 8309698 linux-s390x runtime/Thread/TestAlwaysPreTouchStacks.java 8383372 macosx-aarch64 -applications/ctw/modules/jdk_jfr.java 8286300 linux-s390x applications/jcstress/copy.java 8229852 linux-all containers/docker/TestJFREvents.java 8327723 linux-x64 diff --git a/test/jdk/ProblemList.txt b/test/jdk/ProblemList.txt index fcde1d9c01d..8879aa2e5b6 100644 --- a/test/jdk/ProblemList.txt +++ b/test/jdk/ProblemList.txt @@ -612,24 +612,6 @@ jdk/incubator/vector/LoadJsvmlTest.java 8305390 windows- # jdk_jfr -jdk/jfr/api/consumer/TestRecordingFileWrite.java 8286300 linux-s390x -jdk/jfr/api/consumer/streaming/TestCrossProcessStreaming.java 8286300 linux-s390x -jdk/jfr/api/consumer/streaming/TestFilledChunks.java 8286300 linux-s390x -jdk/jfr/api/consumer/streaming/TestRemovedChunks.java 8286300 linux-s390x -jdk/jfr/api/recording/misc/TestGetStreamWithFailure.java 8286300 linux-s390x -jdk/jfr/api/settings/TestSettingControl.java 8286300 linux-s390x -jdk/jfr/event/runtime/TestBackToBackSensitive.java 8286300 linux-s390x -jdk/jfr/event/runtime/TestSyncOnValueBasedClassEvent.java 8286300 linux-s390x -jdk/jfr/event/tracing/TestMultipleThreads.java 8286300 linux-s390x -jdk/jfr/event/tracing/TestTracedString.java 8286300 linux-s390x -jdk/jfr/javaagent/TestLoadedAgent.java 8286300 linux-s390x -jdk/jfr/javaagent/TestPremainAgent.java 8286300 linux-s390x -jdk/jfr/jmx/streaming/TestClose.java 8286300 linux-s390x -jdk/jfr/jmx/streaming/TestMaxSize.java 8286300 linux-s390x -jdk/jfr/jvm/TestChunkIntegrity.java 8286300 linux-s390x -jdk/jfr/jvm/TestJFRIntrinsic.java 8286300 linux-s390x -jdk/jfr/tool/TestDisassemble.java 8286300 linux-s390x -jdk/jfr/tool/TestScrub.java 8286300 linux-s390x jdk/jfr/event/compiler/TestCodeSweeper.java 8338127 generic-all jdk/jfr/event/oldobject/TestShenandoah.java 8342951 generic-all jdk/jfr/event/runtime/TestResidentSetSizeEvent.java 8309846 aix-ppc64 diff --git a/test/jdk/java/util/concurrent/tck/JSR166TestCase.java b/test/jdk/java/util/concurrent/tck/JSR166TestCase.java index 641fbf2e495..f1f32bee310 100644 --- a/test/jdk/java/util/concurrent/tck/JSR166TestCase.java +++ b/test/jdk/java/util/concurrent/tck/JSR166TestCase.java @@ -37,9 +37,7 @@ /* * @test id=default * @summary Conformance testing variant of JSR-166 tck tests. - * @library /test/lib * @build * - * @build jdk.test.lib.Platform * @modules java.management java.base/jdk.internal.util * @run junit/othervm/timeout=1000 JSR166TestCase */ @@ -48,9 +46,7 @@ * @test id=forkjoinpool-common-parallelism * @summary Test implementation details variant of JSR-166 * tck tests with ForkJoinPool common parallelism. - * @library /test/lib * @build * - * @build jdk.test.lib.Platform * @modules java.management java.base/jdk.internal.util * @run junit/othervm/timeout=1000 * --add-opens java.base/java.util.concurrent=ALL-UNNAMED @@ -72,9 +68,7 @@ * @summary Remaining test implementation details variant of * JSR-166 tck tests apart from ForkJoinPool common * parallelism. - * @library /test/lib * @build * - * @build jdk.test.lib.Platform * @modules java.management java.base/jdk.internal.util * @run junit/othervm/timeout=1000 * --add-opens java.base/java.util.concurrent=ALL-UNNAMED @@ -141,7 +135,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; -import jdk.test.lib.Platform; import junit.framework.Test; import junit.framework.TestCase; import junit.framework.TestResult; @@ -631,13 +624,6 @@ public class JSR166TestCase extends TestCase { "SynchronousQueue20Test", "ReentrantReadWriteLock20Test" }; - - if (Platform.isS390x()) { - java20TestClassNames = new String[] { - "ForkJoinPool20Test", - }; - } - addNamedTestClasses(suite, java20TestClassNames); }