diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp index 9e321391f6c..e83ac3aa628 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -174,24 +174,14 @@ void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec } } -void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm, - Register obj, - Register pre_val, - Register tmp, - bool tosca_live, - bool expand_call) { - - if (ShenandoahSATBBarrier) { - satb_write_barrier_pre(masm, obj, pre_val, tmp, tosca_live, expand_call); - } -} - -void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm, - Register obj, - Register pre_val, - Register tmp, - bool tosca_live, - bool expand_call) { +void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm, + Register obj, + Register pre_val, + Register tmp, + bool tosca_live, + bool expand_call) { + assert(ShenandoahSATBBarrier, "Should be checked by caller"); + // If expand_call is true then we expand the call_VM_leaf macro // directly to skip generating the check by // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. @@ -533,18 +523,18 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d assert_different_registers(dst, tmp1, r15_thread); // Generate the SATB pre-barrier code to log the value of // the referent field in an SATB buffer. - shenandoah_write_barrier_pre(masm /* masm */, - noreg /* obj */, - dst /* pre_val */, - tmp1 /* tmp */, - true /* tosca_live */, - true /* expand_call */); + satb_barrier(masm, + noreg /* obj */, + dst /* pre_val */, + tmp1 /* tmp */, + true /* tosca_live */, + true /* expand_call */); restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true); } } -void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) { +void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj) { assert(ShenandoahCardBarrier, "Should have been checked by caller"); // Does a store check for the oop in register obj. The content of @@ -575,41 +565,41 @@ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register o void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) { - bool on_oop = is_reference_type(type); - bool in_heap = (decorators & IN_HEAP) != 0; - bool as_normal = (decorators & AS_NORMAL) != 0; - if (on_oop && in_heap) { - bool needs_pre_barrier = as_normal; + // 1: non-reference type, no barrier is needed + if (!is_reference_type(type)) { + BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3); + return; + } - // flatten object address if needed - // We do it regardless of precise because we need the registers - if (dst.index() == noreg && dst.disp() == 0) { - if (dst.base() != tmp1) { - __ movptr(tmp1, dst.base()); - } - } else { - __ lea(tmp1, dst); - } + // Flatten object address right away for simplicity: likely needed by barriers + assert_different_registers(val, tmp1, tmp2, tmp3, r15_thread); - assert_different_registers(val, tmp1, tmp2, tmp3, r15_thread); - - if (needs_pre_barrier) { - shenandoah_write_barrier_pre(masm /*masm*/, - tmp1 /* obj */, - tmp2 /* pre_val */, - tmp3 /* tmp */, - val != noreg /* tosca_live */, - false /* expand_call */); - } - - BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg); - if (val != noreg) { - if (ShenandoahCardBarrier) { - store_check(masm, tmp1); - } + if (dst.index() == noreg && dst.disp() == 0) { + if (dst.base() != tmp1) { + __ movptr(tmp1, dst.base()); } } else { - BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3); + __ lea(tmp1, dst); + } + + bool storing_non_null = (val != noreg); + + // 2: pre-barrier: SATB needs the previous value + if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) { + satb_barrier(masm, + tmp1 /* obj */, + tmp2 /* pre_val */, + tmp3 /* tmp */, + storing_non_null /* tosca_live */, + false /* expand_call */); + } + + // Store! + BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg); + + // 3: post-barrier: card barrier needs store address + if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) { + card_barrier(masm, tmp1); } } diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp index b0185f2dbff..b5cc5c8d834 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp @@ -41,21 +41,14 @@ class StubCodeGenerator; class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { private: - void satb_write_barrier_pre(MacroAssembler* masm, - Register obj, - Register pre_val, - Register tmp, - bool tosca_live, - bool expand_call); + void satb_barrier(MacroAssembler* masm, + Register obj, + Register pre_val, + Register tmp, + bool tosca_live, + bool expand_call); - void shenandoah_write_barrier_pre(MacroAssembler* masm, - Register obj, - Register pre_val, - Register tmp, - bool tosca_live, - bool expand_call); - - void store_check(MacroAssembler* masm, Register obj); + void card_barrier(MacroAssembler* masm, Register obj); void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register addr, Register count, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp index 2aa37d7c575..91c790ab875 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -72,13 +72,11 @@ void ShenandoahBarrierSet::print_on(outputStream* st) const { bool ShenandoahBarrierSet::need_load_reference_barrier(DecoratorSet decorators, BasicType type) { if (!ShenandoahLoadRefBarrier) return false; - // Only needed for references return is_reference_type(type); } bool ShenandoahBarrierSet::need_keep_alive_barrier(DecoratorSet decorators, BasicType type) { if (!ShenandoahSATBBarrier) return false; - // Only needed for references if (!is_reference_type(type)) return false; bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; @@ -87,6 +85,20 @@ bool ShenandoahBarrierSet::need_keep_alive_barrier(DecoratorSet decorators, Basi return (on_weak_ref || unknown) && keep_alive; } +bool ShenandoahBarrierSet::need_satb_barrier(DecoratorSet decorators, BasicType type) { + if (!ShenandoahSATBBarrier) return false; + if (!is_reference_type(type)) return false; + bool dest_initialized = (decorators & IS_DEST_UNINITIALIZED) == 0; + return dest_initialized; +} + +bool ShenandoahBarrierSet::need_card_barrier(DecoratorSet decorators, BasicType type) { + if (!ShenandoahCardBarrier) return false; + if (!is_reference_type(type)) return false; + bool in_heap = (decorators & IN_HEAP) != 0; + return in_heap; +} + void ShenandoahBarrierSet::on_slowpath_allocation_exit(JavaThread* thread, oop new_obj) { #if COMPILER2_OR_JVMCI if (ReduceInitialCardMarks && ShenandoahCardBarrier && !ShenandoahHeap::heap()->is_in_young(new_obj)) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp index 28765605267..e7a0ed57740 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp @@ -60,6 +60,8 @@ public: static bool need_load_reference_barrier(DecoratorSet decorators, BasicType type); static bool need_keep_alive_barrier(DecoratorSet decorators, BasicType type); + static bool need_satb_barrier(DecoratorSet decorators, BasicType type); + static bool need_card_barrier(DecoratorSet decorators, BasicType type); static bool is_strong_access(DecoratorSet decorators) { return (decorators & (ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF)) == 0;