diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp index 62414611cce..6c2b12fc7a8 100644 --- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp @@ -476,7 +476,7 @@ void SaveLiveRegisters::initialize(BarrierStubC2* stub) { GrowableArray registers; VMReg prev_vm_reg = VMRegImpl::Bad(); - RegMaskIterator rmi(stub->live()); + RegMaskIterator rmi(stub->preserve_set()); while (rmi.has_next()) { OptoReg::Name opto_reg = rmi.next(); VMReg vm_reg = OptoReg::as_VMReg(opto_reg); @@ -491,7 +491,7 @@ void SaveLiveRegisters::initialize(BarrierStubC2* stub) { index = registers.append(reg_data); } } else if (vm_reg->is_FloatRegister()) { - // We have size encoding in OptoReg of stub->live() + // We have size encoding in OptoReg of stub->preserve_set() // After encoding, float/neon/sve register has only one slot in regmask // Decode it to get the actual size VMReg vm_reg_base = vm_reg->as_FloatRegister()->as_VMReg(); @@ -532,12 +532,8 @@ void SaveLiveRegisters::initialize(BarrierStubC2* stub) { } } - // Remove C-ABI SOE registers, scratch regs and _ref register that will be updated - if (stub->result() != noreg) { - _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9, stub->result()); - } else { - _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9); - } + // Remove C-ABI SOE registers and scratch regs + _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9); // Remove C-ABI SOE fp registers _fp_regs -= FloatRegSet::range(v8, v15); diff --git a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp index 3bb205d659f..16de0b6a7dd 100644 --- a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp @@ -282,7 +282,7 @@ OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Na #define __ _masm-> SaveLiveRegisters::SaveLiveRegisters(MacroAssembler *masm, BarrierStubC2 *stub) - : _masm(masm), _reg_mask(stub->live()), _result_reg(stub->result()) { + : _masm(masm), _reg_mask(stub->preserve_set()) { const int register_save_size = iterate_over_register_mask(ACTION_COUNT_ONLY) * BytesPerWord; _frame_size = align_up(register_save_size, frame::alignment_in_bytes) @@ -317,11 +317,6 @@ int SaveLiveRegisters::iterate_over_register_mask(IterationAction action, int of if (vm_reg->is_Register()) { Register std_reg = vm_reg->as_Register(); - // '_result_reg' will hold the end result of the operation. Its content must thus not be preserved. - if (std_reg == _result_reg) { - continue; - } - if (std_reg->encoding() >= R2->encoding() && std_reg->encoding() <= R12->encoding()) { reg_save_index++; diff --git a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp index d70fce77acc..9c1cef5eea4 100644 --- a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp +++ b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp @@ -98,7 +98,6 @@ public: class SaveLiveRegisters { MacroAssembler* _masm; RegMask _reg_mask; - Register _result_reg; int _frame_size; public: diff --git a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp index f411a5572c7..cbf8de7341d 100644 --- a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp @@ -396,7 +396,7 @@ OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Na void SaveLiveRegisters::initialize(BarrierStubC2* stub) { // Record registers that needs to be saved/restored - RegMaskIterator rmi(stub->live()); + RegMaskIterator rmi(stub->preserve_set()); while (rmi.has_next()) { const OptoReg::Name opto_reg = rmi.next(); if (OptoReg::is_reg(opto_reg)) { @@ -414,12 +414,8 @@ void SaveLiveRegisters::initialize(BarrierStubC2* stub) { } } - // Remove C-ABI SOE registers, tmp regs and _ref register that will be updated - if (stub->result() != noreg) { - _gp_regs -= RegSet::range(x18, x27) + RegSet::of(x2) + RegSet::of(x8, x9) + RegSet::of(x5, stub->result()); - } else { - _gp_regs -= RegSet::range(x18, x27) + RegSet::of(x2, x5) + RegSet::of(x8, x9); - } + // Remove C-ABI SOE registers and tmp regs + _gp_regs -= RegSet::range(x18, x27) + RegSet::of(x2, x5) + RegSet::of(x8, x9); } SaveLiveRegisters::SaveLiveRegisters(MacroAssembler* masm, BarrierStubC2* stub) diff --git a/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp index a06fde061f8..a6801f594bd 100644 --- a/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp @@ -613,19 +613,12 @@ void SaveLiveRegisters::initialize(BarrierStubC2* stub) { caller_saved.Insert(OptoReg::as_OptoReg(r10->as_VMReg())); caller_saved.Insert(OptoReg::as_OptoReg(r11->as_VMReg())); - if (stub->result() != noreg) { - caller_saved.Remove(OptoReg::as_OptoReg(stub->result()->as_VMReg())); - } - - // Create mask of live registers - RegMask live = stub->live(); - int gp_spill_size = 0; int opmask_spill_size = 0; int xmm_spill_size = 0; // Record registers that needs to be saved/restored - RegMaskIterator rmi(live); + RegMaskIterator rmi(stub->preserve_set()); while (rmi.has_next()) { const OptoReg::Name opto_reg = rmi.next(); const VMReg vm_reg = OptoReg::as_VMReg(opto_reg); diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp index 3f712c9d391..53e4d7fdf8f 100644 --- a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp +++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp @@ -87,15 +87,16 @@ static BarrierSetC2State* barrier_set_state() { return reinterpret_cast(Compile::current()->barrier_set_state()); } -BarrierStubC2::BarrierStubC2(const MachNode* node) - : _node(node), - _entry(), - _continuation() {} - RegMask& BarrierStubC2::live() const { return *barrier_set_state()->live(_node); } +BarrierStubC2::BarrierStubC2(const MachNode* node) + : _node(node), + _entry(), + _continuation(), + _preserve(live()) {} + Label* BarrierStubC2::entry() { // The _entry will never be bound when in_scratch_emit_size() is true. // However, we still need to return a label that is not bound now, but @@ -108,6 +109,27 @@ Label* BarrierStubC2::continuation() { return &_continuation; } +void BarrierStubC2::preserve(Register r) { + const VMReg vm_reg = r->as_VMReg(); + assert(vm_reg->is_Register(), "r must be a general-purpose register"); + _preserve.Insert(OptoReg::as_OptoReg(vm_reg)); +} + +void BarrierStubC2::dont_preserve(Register r) { + VMReg vm_reg = r->as_VMReg(); + assert(vm_reg->is_Register(), "r must be a general-purpose register"); + // Subtract the given register and all its sub-registers (e.g. {R11, R11_H} + // for r11 in aarch64). + do { + _preserve.Remove(OptoReg::as_OptoReg(vm_reg)); + vm_reg = vm_reg->next(); + } while (vm_reg->is_Register() && !vm_reg->is_concrete()); +} + +const RegMask& BarrierStubC2::preserve_set() const { + return _preserve; +} + Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { DecoratorSet decorators = access.decorators(); @@ -828,6 +850,7 @@ void BarrierSetC2::compute_liveness_at_stubs() const { PhaseRegAlloc* const regalloc = C->regalloc(); RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask)); BarrierSetAssembler* const bs = BarrierSet::barrier_set()->barrier_set_assembler(); + BarrierSetC2State* bs_state = barrier_set_state(); Block_List worklist; for (uint i = 0; i < cfg->number_of_blocks(); ++i) { @@ -850,6 +873,14 @@ void BarrierSetC2::compute_liveness_at_stubs() const { for (int i = block->number_of_nodes() - 1; i >= 0; --i) { const Node* const node = block->get_node(i); + // If this node tracks out-liveness, update it + if (!bs_state->needs_livein_data()) { + RegMask* const regs = bs_state->live(node); + if (regs != nullptr) { + regs->OR(new_live); + } + } + // Remove def bits const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node)); const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node)); @@ -873,10 +904,12 @@ void BarrierSetC2::compute_liveness_at_stubs() const { } } - // If this node tracks liveness, update it - RegMask* const regs = barrier_set_state()->live(node); - if (regs != NULL) { - regs->OR(new_live); + // If this node tracks in-liveness, update it + if (bs_state->needs_livein_data()) { + RegMask* const regs = bs_state->live(node); + if (regs != nullptr) { + regs->OR(new_live); + } } } diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp index aa2df96412d..8787fe83d14 100644 --- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp +++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp @@ -227,6 +227,7 @@ public: } virtual bool needs_liveness_data(const MachNode* mach) const = 0; + virtual bool needs_livein_data() const = 0; }; // This class represents the slow path in a C2 barrier. It is defined by a @@ -238,14 +239,28 @@ protected: const MachNode* _node; Label _entry; Label _continuation; + RegMask _preserve; + + // Registers that are live-in/live-out of the entire memory access + // implementation (possibly including multiple barriers). Whether live-in or + // live-out registers are returned depends on + // BarrierSetC2State::needs_livein_data(). + RegMask& live() const; public: BarrierStubC2(const MachNode* node); - RegMask& live() const; + + // Entry point to the stub. Label* entry(); + // Return point from the stub (typically end of barrier). Label* continuation(); - virtual Register result() const = 0; + // Preserve the value in reg across runtime calls in this barrier. + void preserve(Register reg); + // Do not preserve the value in reg across runtime calls in this barrier. + void dont_preserve(Register reg); + // Set of registers whose value needs to be preserved across runtime calls in this barrier. + const RegMask& preserve_set() const; }; // This is the top-level class for the backend of the Access API in C2. diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp index 85d45553202..4b79b568241 100644 --- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp +++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp @@ -142,6 +142,10 @@ public: return mach->barrier_data() != ZBarrierElided; } + bool needs_livein_data() const { + return true; + } + void inc_trampoline_stubs_count() { assert(_trampoline_stubs_count != INT_MAX, "Overflow"); ++_trampoline_stubs_count; @@ -200,6 +204,9 @@ ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, R _ref(ref) { assert_different_registers(ref, ref_addr.base()); assert_different_registers(ref, ref_addr.index()); + // The runtime call updates the value of ref, so we should not spill and + // reload its outdated value. + dont_preserve(ref); } Address ZLoadBarrierStubC2::ref_addr() const { @@ -210,10 +217,6 @@ Register ZLoadBarrierStubC2::ref() const { return _ref; } -Register ZLoadBarrierStubC2::result() const { - return ref(); -} - address ZLoadBarrierStubC2::slow_path() const { const uint8_t barrier_data = _node->barrier_data(); DecoratorSet decorators = DECORATORS_NONE; @@ -272,10 +275,6 @@ bool ZStoreBarrierStubC2::is_atomic() const { return _is_atomic; } -Register ZStoreBarrierStubC2::result() const { - return noreg; -} - void ZStoreBarrierStubC2::emit_code(MacroAssembler& masm) { ZBarrierSet::assembler()->generate_c2_store_barrier_stub(&masm, static_cast(this)); } diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp index f8892defcff..bf46780226e 100644 --- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp +++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp @@ -51,7 +51,6 @@ static int stubs_start_offset(); ZBarrierStubC2(const MachNode* node); public: - virtual Register result() const = 0; virtual void emit_code(MacroAssembler& masm) = 0; }; @@ -70,7 +69,6 @@ public: Register ref() const; address slow_path() const; - virtual Register result() const; virtual void emit_code(MacroAssembler& masm); }; @@ -94,7 +92,6 @@ public: bool is_native() const; bool is_atomic() const; - virtual Register result() const; virtual void emit_code(MacroAssembler& masm); };