diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index c162baad936..252c277e2c7 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -1796,8 +1796,9 @@ void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const { Label* guard = &dummy_guard; if (!Compile::current()->output()->in_scratch_emit_size()) { // Use real labels from actual stub when not emitting code for the purpose of measuring its size - C2EntryBarrierStub* stub = Compile::current()->output()->entry_barrier_table()->add_entry_barrier(); - slow_path = &stub->slow_path(); + C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub(); + Compile::current()->output()->add_stub(stub); + slow_path = &stub->entry(); continuation = &stub->continuation(); guard = &stub->guard(); } @@ -1879,7 +1880,9 @@ void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const { Label dummy_label; Label* code_stub = &dummy_label; if (!C->output()->in_scratch_emit_size()) { - code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset()); + C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset()); + C->output()->add_stub(stub); + code_stub = &stub->entry(); } __ relocate(relocInfo::poll_return_type); __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */); diff --git a/src/hotspot/cpu/aarch64/c2_safepointPollStubTable_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_CodeStubs_aarch64.cpp similarity index 65% rename from src/hotspot/cpu/aarch64/c2_safepointPollStubTable_aarch64.cpp rename to src/hotspot/cpu/aarch64/c2_CodeStubs_aarch64.cpp index fb36406fbde..009883d6082 100644 --- a/src/hotspot/cpu/aarch64/c2_safepointPollStubTable_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c2_CodeStubs_aarch64.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,24 +23,44 @@ */ #include "precompiled.hpp" -#include "asm/macroAssembler.hpp" -#include "opto/compile.hpp" -#include "opto/node.hpp" -#include "opto/output.hpp" +#include "opto/c2_MacroAssembler.hpp" +#include "opto/c2_CodeStubs.hpp" #include "runtime/sharedRuntime.hpp" +#include "runtime/stubRoutines.hpp" #define __ masm. -void C2SafepointPollStubTable::emit_stub_impl(MacroAssembler& masm, C2SafepointPollStub* entry) const { + +int C2SafepointPollStub::max_size() const { + return 20; +} + +void C2SafepointPollStub::emit(C2_MacroAssembler& masm) { assert(SharedRuntime::polling_page_return_handler_blob() != NULL, "polling page return stub not created yet"); address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point(); RuntimeAddress callback_addr(stub); - __ bind(entry->_stub_label); - InternalAddress safepoint_pc(masm.pc() - masm.offset() + entry->_safepoint_offset); + __ bind(entry()); + InternalAddress safepoint_pc(masm.pc() - masm.offset() + _safepoint_offset); __ adr(rscratch1, safepoint_pc); __ str(rscratch1, Address(rthread, JavaThread::saved_exception_pc_offset())); __ far_jump(callback_addr); } + +int C2EntryBarrierStub::max_size() const { + return 24; +} + +void C2EntryBarrierStub::emit(C2_MacroAssembler& masm) { + __ bind(entry()); + __ movptr(rscratch1, (uintptr_t) StubRoutines::aarch64::method_entry_barrier()); + __ blr(rscratch1); + __ b(continuation()); + + __ bind(guard()); + __ relocate(entry_guard_Relocation::spec()); + __ emit_int32(0); // nmethod guard value +} + #undef __ diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp index 93495e0fc58..0bf4233841d 100644 --- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp @@ -45,21 +45,6 @@ typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr); -void C2_MacroAssembler::emit_entry_barrier_stub(C2EntryBarrierStub* stub) { - bind(stub->slow_path()); - movptr(rscratch1, (uintptr_t) StubRoutines::aarch64::method_entry_barrier()); - blr(rscratch1); - b(stub->continuation()); - - bind(stub->guard()); - relocate(entry_guard_Relocation::spec()); - emit_int32(0); // nmethod guard value -} - -int C2_MacroAssembler::entry_barrier_stub_size() { - return 4 * 6; -} - // Search for str1 in str2 and return index or -1 void C2_MacroAssembler::string_indexof(Register str2, Register str1, Register cnt2, Register cnt1, diff --git a/src/hotspot/cpu/ppc/c2_safepointPollStubTable_ppc.cpp b/src/hotspot/cpu/ppc/c2_CodeStubs_ppc.cpp similarity index 80% rename from src/hotspot/cpu/ppc/c2_safepointPollStubTable_ppc.cpp rename to src/hotspot/cpu/ppc/c2_CodeStubs_ppc.cpp index b65b91df1b8..f99f9e978cf 100644 --- a/src/hotspot/cpu/ppc/c2_safepointPollStubTable_ppc.cpp +++ b/src/hotspot/cpu/ppc/c2_CodeStubs_ppc.cpp @@ -1,6 +1,6 @@ /* - * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2021 SAP SE. All rights reserved. + * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2022, SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,19 +24,22 @@ */ #include "precompiled.hpp" -#include "macroAssembler_ppc.inline.hpp" -#include "opto/compile.hpp" -#include "opto/node.hpp" -#include "opto/output.hpp" +#include "opto/c2_MacroAssembler.hpp" +#include "opto/c2_CodeStubs.hpp" #include "runtime/sharedRuntime.hpp" #define __ masm. -void C2SafepointPollStubTable::emit_stub_impl(MacroAssembler& masm, C2SafepointPollStub* entry) const { + +int C2SafepointPollStub::max_size() const { + return 56; +} + +void C2SafepointPollStub::emit(C2_MacroAssembler& masm) { assert(SharedRuntime::polling_page_return_handler_blob() != NULL, "polling page return stub not created yet"); address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point(); - __ bind(entry->_stub_label); + __ bind(entry()); // Using pc relative address computation. { Label next_pc; @@ -45,7 +48,7 @@ void C2SafepointPollStubTable::emit_stub_impl(MacroAssembler& masm, C2SafepointP } int current_offset = __ offset(); // Code size should not depend on offset: see _stub_size computation in output.cpp - __ load_const32(R12, entry->_safepoint_offset - current_offset); + __ load_const32(R12, _safepoint_offset - current_offset); __ mflr(R0); __ add(R12, R12, R0); __ std(R12, in_bytes(JavaThread::saved_exception_pc_offset()), R16_thread); diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index 296bf160089..32e80028c6f 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -982,6 +982,7 @@ source_hpp %{ source %{ +#include "opto/c2_CodeStubs.hpp" #include "oops/klass.inline.hpp" void PhaseOutput::pd_perform_mach_node_analysis() { @@ -1616,7 +1617,9 @@ void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const { Label dummy_label; Label* code_stub = &dummy_label; if (!UseSIGTRAP && !C->output()->in_scratch_emit_size()) { - code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset()); + C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset()); + C->output()->add_stub(stub); + code_stub = &stub->entry(); __ relocate(relocInfo::poll_return_type); } __ safepoint_poll(*code_stub, temp, true /* at_return */, true /* in_nmethod */); diff --git a/src/hotspot/cpu/riscv/c2_safepointPollStubTable_riscv.cpp b/src/hotspot/cpu/riscv/c2_CodeStubs_riscv.cpp similarity index 63% rename from src/hotspot/cpu/riscv/c2_safepointPollStubTable_riscv.cpp rename to src/hotspot/cpu/riscv/c2_CodeStubs_riscv.cpp index e68b8d0486e..4ddd6f3c2c3 100644 --- a/src/hotspot/cpu/riscv/c2_safepointPollStubTable_riscv.cpp +++ b/src/hotspot/cpu/riscv/c2_CodeStubs_riscv.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -24,21 +24,25 @@ */ #include "precompiled.hpp" -#include "asm/macroAssembler.hpp" -#include "opto/compile.hpp" -#include "opto/node.hpp" -#include "opto/output.hpp" +#include "opto/c2_CodeStubs.hpp" +#include "opto/c2_MacroAssembler.hpp" #include "runtime/sharedRuntime.hpp" +#include "runtime/stubRoutines.hpp" #define __ masm. -void C2SafepointPollStubTable::emit_stub_impl(MacroAssembler& masm, C2SafepointPollStub* entry) const { + +int C2SafepointPollStub::max_size() const { + return 13 * 4; +} + +void C2SafepointPollStub::emit(C2_MacroAssembler& masm) { assert(SharedRuntime::polling_page_return_handler_blob() != NULL, "polling page return stub not created yet"); address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point(); RuntimeAddress callback_addr(stub); - __ bind(entry->_stub_label); - InternalAddress safepoint_pc(__ pc() - __ offset() + entry->_safepoint_offset); + __ bind(entry()); + InternalAddress safepoint_pc(__ pc() - __ offset() + _safepoint_offset); __ relocate(safepoint_pc.rspec(), [&] { int32_t offset; __ la_patchable(t0, safepoint_pc.target(), offset); @@ -47,4 +51,25 @@ void C2SafepointPollStubTable::emit_stub_impl(MacroAssembler& masm, C2SafepointP __ sd(t0, Address(xthread, JavaThread::saved_exception_pc_offset())); __ far_jump(callback_addr); } + +int C2EntryBarrierStub::max_size() const { + // 4 bytes for alignment + return 8 * 4 + 4; +} + +void C2EntryBarrierStub::emit(C2_MacroAssembler& masm) { + __ bind(entry()); + + int32_t offset = 0; + __ movptr(t0, StubRoutines::riscv::method_entry_barrier(), offset); + __ jalr(ra, t0, offset); + __ j(continuation()); + + // make guard value 4-byte aligned so that it can be accessed by atomic instructions on RISC-V + __ align(4); + __ bind(guard()); + __ relocate(entry_guard_Relocation::spec()); + __ emit_int32(0); // nmethod guard value +} + #undef __ diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp index e4c892280d6..b87fe003542 100644 --- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp @@ -243,37 +243,6 @@ void C2_MacroAssembler::string_indexof_char(Register str1, Register cnt1, typedef void (MacroAssembler::* load_chr_insn)(Register rd, const Address &adr, Register temp); -void C2_MacroAssembler::emit_entry_barrier_stub(C2EntryBarrierStub* stub) { - IncompressibleRegion ir(this); // Fixed length: see C2_MacroAssembler::entry_barrier_stub_size() - - // make guard value 4-byte aligned so that it can be accessed by atomic instructions on riscv - int alignment_bytes = align(4); - - bind(stub->slow_path()); - - int32_t offset = 0; - movptr(t0, StubRoutines::riscv::method_entry_barrier(), offset); - jalr(ra, t0, offset); - j(stub->continuation()); - - bind(stub->guard()); - relocate(entry_guard_Relocation::spec()); - assert_alignment(pc()); - emit_int32(0); // nmethod guard value - // make sure the stub with a fixed code size - if (alignment_bytes == 2) { - assert(UseRVC, "bad alignment"); - c_nop(); - } else { - assert(alignment_bytes == 0, "bad alignment"); - nop(); - } -} - -int C2_MacroAssembler::entry_barrier_stub_size() { - return 8 * 4 + 4; // 4 bytes for alignment margin -} - // Search for needle in haystack and return index or -1 // x10: result // x11: haystack @@ -1726,4 +1695,4 @@ void C2_MacroAssembler::rvv_reduce_integral(Register dst, VectorRegister tmp, } vmv_x_s(dst, tmp); -} \ No newline at end of file +} diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index a9693980441..cf9d6bc556f 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -1361,8 +1361,9 @@ void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const { Label* guard = &dummy_guard; if (!Compile::current()->output()->in_scratch_emit_size()) { // Use real labels from actual stub when not emitting code for purpose of measuring its size - C2EntryBarrierStub* stub = Compile::current()->output()->entry_barrier_table()->add_entry_barrier(); - slow_path = &stub->slow_path(); + C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub(); + Compile::current()->output()->add_stub(stub); + slow_path = &stub->entry(); continuation = &stub->continuation(); guard = &stub->guard(); } @@ -1443,7 +1444,9 @@ void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const { Label dummy_label; Label* code_stub = &dummy_label; if (!C->output()->in_scratch_emit_size()) { - code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset()); + C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset()); + C->output()->add_stub(stub); + code_stub = &stub->entry(); } __ relocate(relocInfo::poll_return_type); __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */); diff --git a/src/hotspot/cpu/x86/c2_safepointPollStubTable_x86.cpp b/src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp similarity index 72% rename from src/hotspot/cpu/x86/c2_safepointPollStubTable_x86.cpp rename to src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp index c3d4850a5db..acae574bcf3 100644 --- a/src/hotspot/cpu/x86/c2_safepointPollStubTable_x86.cpp +++ b/src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,22 +23,26 @@ */ #include "precompiled.hpp" -#include "asm/macroAssembler.hpp" -#include "opto/compile.hpp" -#include "opto/node.hpp" -#include "opto/output.hpp" +#include "opto/c2_MacroAssembler.hpp" +#include "opto/c2_CodeStubs.hpp" #include "runtime/sharedRuntime.hpp" +#include "runtime/stubRoutines.hpp" #define __ masm. -void C2SafepointPollStubTable::emit_stub_impl(MacroAssembler& masm, C2SafepointPollStub* entry) const { + +int C2SafepointPollStub::max_size() const { + return 33; +} + +void C2SafepointPollStub::emit(C2_MacroAssembler& masm) { assert(SharedRuntime::polling_page_return_handler_blob() != NULL, "polling page return stub not created yet"); address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point(); RuntimeAddress callback_addr(stub); - __ bind(entry->_stub_label); - InternalAddress safepoint_pc(masm.pc() - masm.offset() + entry->_safepoint_offset); + __ bind(entry()); + InternalAddress safepoint_pc(masm.pc() - masm.offset() + _safepoint_offset); #ifdef _LP64 __ lea(rscratch1, safepoint_pc); __ movptr(Address(r15_thread, JavaThread::saved_exception_pc_offset()), rscratch1); @@ -57,4 +61,15 @@ void C2SafepointPollStubTable::emit_stub_impl(MacroAssembler& masm, C2SafepointP #endif __ jump(callback_addr); } + +int C2EntryBarrierStub::max_size() const { + return 10; +} + +void C2EntryBarrierStub::emit(C2_MacroAssembler& masm) { + __ bind(entry()); + __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier())); + __ jmp(continuation(), false /* maybe_short */); +} + #undef __ diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp index 7436322f878..91949213a20 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp @@ -138,8 +138,9 @@ void C2_MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool Label* continuation = &dummy_continuation; if (!Compile::current()->output()->in_scratch_emit_size()) { // Use real labels from actual stub when not emitting code for the purpose of measuring its size - C2EntryBarrierStub* stub = Compile::current()->output()->entry_barrier_table()->add_entry_barrier(); - slow_path = &stub->slow_path(); + C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub(); + Compile::current()->output()->add_stub(stub); + slow_path = &stub->entry(); continuation = &stub->continuation(); } bs->nmethod_entry_barrier(this, slow_path, continuation); @@ -151,16 +152,6 @@ void C2_MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool } } -void C2_MacroAssembler::emit_entry_barrier_stub(C2EntryBarrierStub* stub) { - bind(stub->slow_path()); - call(RuntimeAddress(StubRoutines::x86::method_entry_barrier())); - jmp(stub->continuation(), false /* maybe_short */); -} - -int C2_MacroAssembler::entry_barrier_stub_size() { - return 10; -} - inline Assembler::AvxVectorLen C2_MacroAssembler::vector_length_encoding(int vlen_in_bytes) { switch (vlen_in_bytes) { case 4: // fall-through diff --git a/src/hotspot/cpu/x86/x86_32.ad b/src/hotspot/cpu/x86/x86_32.ad index 017ce230ba4..979494e4cb1 100644 --- a/src/hotspot/cpu/x86/x86_32.ad +++ b/src/hotspot/cpu/x86/x86_32.ad @@ -708,7 +708,9 @@ void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const { Label dummy_label; Label* code_stub = &dummy_label; if (!C->output()->in_scratch_emit_size()) { - code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset()); + C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset()); + C->output()->add_stub(stub); + code_stub = &stub->entry(); } __ relocate(relocInfo::poll_return_type); __ safepoint_poll(*code_stub, thread, true /* at_return */, true /* in_nmethod */); diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad index e78d416a044..a102fc77358 100644 --- a/src/hotspot/cpu/x86/x86_64.ad +++ b/src/hotspot/cpu/x86/x86_64.ad @@ -1020,7 +1020,9 @@ void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const Label dummy_label; Label* code_stub = &dummy_label; if (!C->output()->in_scratch_emit_size()) { - code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset()); + C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset()); + C->output()->add_stub(stub); + code_stub = &stub->entry(); } __ relocate(relocInfo::poll_return_type); __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */); diff --git a/src/hotspot/share/opto/c2_CodeStubs.cpp b/src/hotspot/share/opto/c2_CodeStubs.cpp new file mode 100644 index 00000000000..52dea559045 --- /dev/null +++ b/src/hotspot/share/opto/c2_CodeStubs.cpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/codeBuffer.hpp" +#include "code/codeBlob.hpp" +#include "opto/c2_CodeStubs.hpp" +#include "opto/c2_MacroAssembler.hpp" +#include "opto/compile.hpp" +#include "opto/output.hpp" + +C2CodeStubList::C2CodeStubList() : + _stubs(Compile::current()->comp_arena(), 2, 0, NULL) {} + +void C2CodeStubList::emit(CodeBuffer& cb) { + C2_MacroAssembler masm(&cb); + for (int i = _stubs.length() - 1; i >= 0; i--) { + C2CodeStub* stub = _stubs.at(i); + int max_size = stub->max_size(); + // Make sure there is enough space in the code buffer + if (cb.insts()->maybe_expand_to_ensure_remaining(max_size) && cb.blob() == NULL) { + ciEnv::current()->record_failure("CodeCache is full"); + return; + } + + DEBUG_ONLY(int size_before = cb.insts_size();) + + stub->emit(masm); + + DEBUG_ONLY(int actual_size = cb.insts_size() - size_before;) + assert(max_size >= actual_size, "Expected stub size (%d) must be larger than or equal to actual stub size (%d)", max_size, actual_size); + } +} diff --git a/src/hotspot/share/opto/c2_CodeStubs.hpp b/src/hotspot/share/opto/c2_CodeStubs.hpp new file mode 100644 index 00000000000..5ef6966656b --- /dev/null +++ b/src/hotspot/share/opto/c2_CodeStubs.hpp @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "asm/assembler.hpp" +#include "asm/codeBuffer.hpp" +#include "memory/allocation.hpp" +#include "opto/c2_MacroAssembler.hpp" +#include "utilities/growableArray.hpp" + +#ifndef SHARE_OPTO_C2_CODESTUBS_HPP +#define SHARE_OPTO_C2_CODESTUBS_HPP + +class C2CodeStub : public ArenaObj { +private: + Label _entry; + Label _continuation; + +protected: + C2CodeStub() : + _entry(), + _continuation() {} + +public: + Label& entry() { return _entry; } + Label& continuation() { return _continuation; } + + virtual void emit(C2_MacroAssembler& masm) = 0; + virtual int max_size() const = 0; +}; + +class C2CodeStubList { +private: + GrowableArray _stubs; + +public: + C2CodeStubList(); + + void add_stub(C2CodeStub* stub) { _stubs.append(stub); } + void emit(CodeBuffer& cb); +}; + +class C2SafepointPollStub : public C2CodeStub { +private: + uintptr_t _safepoint_offset; + +public: + C2SafepointPollStub(uintptr_t safepoint_offset) : + _safepoint_offset(safepoint_offset) {} + int max_size() const; + void emit(C2_MacroAssembler& masm); +}; + +// We move non-hot code of the nmethod entry barrier to an out-of-line stub +class C2EntryBarrierStub: public C2CodeStub { +private: + Label _guard; // Used on AArch64 and RISCV + +public: + C2EntryBarrierStub() : C2CodeStub(), + _guard() {} + + Label& guard() { return _guard; } + + int max_size() const; + void emit(C2_MacroAssembler& masm); +}; + +#endif // SHARE_OPTO_C2_CODESTUBS_HPP diff --git a/src/hotspot/share/opto/c2_MacroAssembler.hpp b/src/hotspot/share/opto/c2_MacroAssembler.hpp index e86edb0798c..41347313a8c 100644 --- a/src/hotspot/share/opto/c2_MacroAssembler.hpp +++ b/src/hotspot/share/opto/c2_MacroAssembler.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,6 +27,7 @@ #include "asm/macroAssembler.hpp" #include "asm/macroAssembler.inline.hpp" +#include "utilities/globalDefinitions.hpp" #include "utilities/macros.hpp" class C2EntryBarrierStub; diff --git a/src/hotspot/share/opto/output.cpp b/src/hotspot/share/opto/output.cpp index 6f756133445..0b5cc251bbd 100644 --- a/src/hotspot/share/opto/output.cpp +++ b/src/hotspot/share/opto/output.cpp @@ -218,118 +218,13 @@ public: }; -volatile int C2SafepointPollStubTable::_stub_size = 0; - -Label& C2SafepointPollStubTable::add_safepoint(uintptr_t safepoint_offset) { - C2SafepointPollStub* entry = new (Compile::current()->comp_arena()) C2SafepointPollStub(safepoint_offset); - _safepoints.append(entry); - return entry->_stub_label; -} - -void C2SafepointPollStubTable::emit(CodeBuffer& cb) { - MacroAssembler masm(&cb); - for (int i = _safepoints.length() - 1; i >= 0; i--) { - // Make sure there is enough space in the code buffer - if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == NULL) { - ciEnv::current()->record_failure("CodeCache is full"); - return; - } - - C2SafepointPollStub* entry = _safepoints.at(i); - emit_stub(masm, entry); - } -} - -int C2SafepointPollStubTable::stub_size_lazy() const { - int size = Atomic::load(&_stub_size); - - if (size != 0) { - return size; - } - - Compile* const C = Compile::current(); - BufferBlob* const blob = C->output()->scratch_buffer_blob(); - CodeBuffer cb(blob->content_begin(), C->output()->scratch_buffer_code_size()); - MacroAssembler masm(&cb); - C2SafepointPollStub* entry = _safepoints.at(0); - emit_stub(masm, entry); - size += cb.insts_size(); - - Atomic::store(&_stub_size, size); - - return size; -} - -int C2SafepointPollStubTable::estimate_stub_size() const { - if (_safepoints.length() == 0) { - return 0; - } - - int result = stub_size_lazy() * _safepoints.length(); - -#ifdef ASSERT - Compile* const C = Compile::current(); - BufferBlob* const blob = C->output()->scratch_buffer_blob(); - int size = 0; - - for (int i = _safepoints.length() - 1; i >= 0; i--) { - CodeBuffer cb(blob->content_begin(), C->output()->scratch_buffer_code_size()); - MacroAssembler masm(&cb); - C2SafepointPollStub* entry = _safepoints.at(i); - emit_stub(masm, entry); - size += cb.insts_size(); - } - assert(size == result, "stubs should not have variable size"); -#endif - - return result; -} - -// Nmethod entry barrier stubs -C2EntryBarrierStub* C2EntryBarrierStubTable::add_entry_barrier() { - assert(_stub == NULL, "There can only be one entry barrier stub"); - _stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub(); - return _stub; -} - -void C2EntryBarrierStubTable::emit(CodeBuffer& cb) { - if (_stub == NULL) { - // No stub - nothing to do - return; - } - - C2_MacroAssembler masm(&cb); - // Make sure there is enough space in the code buffer - if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == NULL) { - ciEnv::current()->record_failure("CodeCache is full"); - return; - } - - intptr_t before = masm.offset(); - masm.emit_entry_barrier_stub(_stub); - intptr_t after = masm.offset(); - int actual_size = (int)(after - before); - int expected_size = masm.entry_barrier_stub_size(); - assert(actual_size == expected_size, "Estimated size is wrong, expected %d, was %d", expected_size, actual_size); -} - -int C2EntryBarrierStubTable::estimate_stub_size() const { - if (BarrierSet::barrier_set()->barrier_set_nmethod() == NULL) { - // No nmethod entry barrier? - return 0; - } - - return C2_MacroAssembler::entry_barrier_stub_size(); -} - PhaseOutput::PhaseOutput() : Phase(Phase::Output), _code_buffer("Compile::Fill_buffer"), _first_block_size(0), _handler_table(), _inc_table(), - _safepoint_poll_table(), - _entry_barrier_table(), + _stub_list(), _oop_map_set(NULL), _scratch_buffer_blob(NULL), _scratch_locs_memory(NULL), @@ -1349,8 +1244,6 @@ CodeBuffer* PhaseOutput::init_buffer() { BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); stub_req += bs->estimate_stub_size(); - stub_req += safepoint_poll_table()->estimate_stub_size(); - stub_req += entry_barrier_table()->estimate_stub_size(); // nmethod and CodeBuffer count stubs & constants as part of method's code. // class HandlerImpl is platform-specific and defined in the *.ad files. @@ -1857,12 +1750,8 @@ void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) { bs->emit_stubs(*cb); if (C->failing()) return; - // Fill in stubs for calling the runtime from safepoint polls. - safepoint_poll_table()->emit(*cb); - if (C->failing()) return; - - // Fill in stubs for calling the runtime from nmethod entries. - entry_barrier_table()->emit(*cb); + // Fill in stubs. + _stub_list.emit(*cb); if (C->failing()) return; #ifndef PRODUCT diff --git a/src/hotspot/share/opto/output.hpp b/src/hotspot/share/opto/output.hpp index 555dcde06b1..09e012d34c3 100644 --- a/src/hotspot/share/opto/output.hpp +++ b/src/hotspot/share/opto/output.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,7 @@ #include "code/exceptionHandlerTable.hpp" #include "metaprogramming/enableIf.hpp" #include "opto/ad.hpp" +#include "opto/c2_CodeStubs.hpp" #include "opto/constantTable.hpp" #include "opto/phase.hpp" #include "runtime/vm_version.hpp" @@ -73,75 +74,6 @@ public: { }; }; -class C2SafepointPollStubTable { -private: - struct C2SafepointPollStub: public ArenaObj { - uintptr_t _safepoint_offset; - Label _stub_label; - Label _trampoline_label; - C2SafepointPollStub(uintptr_t safepoint_offset) : - _safepoint_offset(safepoint_offset), - _stub_label(), - _trampoline_label() {} - }; - - GrowableArray _safepoints; - - static volatile int _stub_size; - - void emit_stub_impl(MacroAssembler& masm, C2SafepointPollStub* entry) const; - - // The selection logic below relieves the need to add dummy files to unsupported platforms. - template - typename EnableIf::type - select_emit_stub(MacroAssembler& masm, C2SafepointPollStub* entry) const { - emit_stub_impl(masm, entry); - } - - template - typename EnableIf::type - select_emit_stub(MacroAssembler& masm, C2SafepointPollStub* entry) const {} - - void emit_stub(MacroAssembler& masm, C2SafepointPollStub* entry) const { - select_emit_stub(masm, entry); - } - - int stub_size_lazy() const; - -public: - Label& add_safepoint(uintptr_t safepoint_offset); - int estimate_stub_size() const; - void emit(CodeBuffer& cb); -}; - -// We move non-hot code of the nmethod entry barrier to an out-of-line stub -class C2EntryBarrierStub: public ArenaObj { - Label _slow_path; - Label _continuation; - Label _guard; // Used on AArch64 and RISCV - -public: - C2EntryBarrierStub() : - _slow_path(), - _continuation(), - _guard() {} - - Label& slow_path() { return _slow_path; } - Label& continuation() { return _continuation; } - Label& guard() { return _guard; } - -}; - -class C2EntryBarrierStubTable { - C2EntryBarrierStub* _stub; - -public: - C2EntryBarrierStubTable() : _stub(NULL) {} - C2EntryBarrierStub* add_entry_barrier(); - int estimate_stub_size() const; - void emit(CodeBuffer& cb); -}; - class PhaseOutput : public Phase { private: // Instruction bits passed off to the VM @@ -150,8 +82,7 @@ private: int _first_block_size; // Size of unvalidated entry point code / OSR poison code ExceptionHandlerTable _handler_table; // Table of native-code exception handlers ImplicitExceptionTable _inc_table; // Table of implicit null checks in native code - C2SafepointPollStubTable _safepoint_poll_table;// Table for safepoint polls - C2EntryBarrierStubTable _entry_barrier_table; // Table for entry barrier stubs + C2CodeStubList _stub_list; // List of code stubs OopMapSet* _oop_map_set; // Table of oop maps (one for each safepoint location) BufferBlob* _scratch_buffer_blob; // For temporary code buffers. relocInfo* _scratch_locs_memory; // For temporary code buffers. @@ -199,11 +130,8 @@ public: // Constant table ConstantTable& constant_table() { return _constant_table; } - // Safepoint poll table - C2SafepointPollStubTable* safepoint_poll_table() { return &_safepoint_poll_table; } - - // Entry barrier table - C2EntryBarrierStubTable* entry_barrier_table() { return &_entry_barrier_table; } + // Code stubs list + void add_stub(C2CodeStub* stub) { _stub_list.add_stub(stub); } // Code emission iterator Block* block() { return _block; }