From c90d63105ca774c047d5f5a4348aa657efc57953 Mon Sep 17 00:00:00 2001 From: Martin Doerr Date: Fri, 22 Sep 2023 13:21:10 +0000 Subject: [PATCH] 8315750: Update subtype check profile collection on PPC following 8308869 Reviewed-by: rrich, lucy --- src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp | 121 +++++++------------- src/hotspot/cpu/ppc/interp_masm_ppc.hpp | 5 +- src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp | 57 +++------ src/hotspot/cpu/ppc/vm_version_ppc.hpp | 1 - 4 files changed, 60 insertions(+), 124 deletions(-) diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp index ce347fe66d9..4973ec510d5 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp @@ -2382,8 +2382,6 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L } assert_different_registers(obj, k_RInfo, klass_RInfo, Rtmp1); - __ cmpdi(CCR0, obj, 0); - ciMethodData* md = nullptr; ciProfileData* data = nullptr; int mdo_offset_bias = 0; @@ -2395,15 +2393,27 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L Register mdo = k_RInfo; Register data_val = Rtmp1; Label not_null; - __ bne(CCR0, not_null); metadata2reg(md->constant_encoding(), mdo); __ add_const_optimized(mdo, mdo, mdo_offset_bias, R0); + __ cmpdi(CCR0, obj, 0); + __ bne(CCR0, not_null); __ lbz(data_val, md->byte_offset_of_slot(data, DataLayout::flags_offset()) - mdo_offset_bias, mdo); __ ori(data_val, data_val, BitData::null_seen_byte_constant()); __ stb(data_val, md->byte_offset_of_slot(data, DataLayout::flags_offset()) - mdo_offset_bias, mdo); __ b(*obj_is_null); __ bind(not_null); + + Label update_done; + Register recv = klass_RInfo; + __ load_klass(recv, obj); + type_profile_helper(mdo, mdo_offset_bias, md, data, recv, Rtmp1, &update_done); + const int slot_offset = md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias; + __ ld(Rtmp1, slot_offset, mdo); + __ addi(Rtmp1, Rtmp1, DataLayout::counter_increment); + __ std(Rtmp1, slot_offset, mdo); + __ bind(update_done); } else { + __ cmpdi(CCR0, obj, 0); __ beq(CCR0, *obj_is_null); } @@ -2416,20 +2426,11 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L klass2reg_with_patching(k_RInfo, op->info_for_patch()); } - Label profile_cast_failure, failure_restore_obj, profile_cast_success; - Label *failure_target = should_profile ? &profile_cast_failure : failure; - Label *success_target = should_profile ? &profile_cast_success : success; - if (op->fast_check()) { assert_different_registers(klass_RInfo, k_RInfo); __ cmpd(CCR0, k_RInfo, klass_RInfo); - if (should_profile) { - __ bne(CCR0, *failure_target); - // Fall through to success case. - } else { - __ beq(CCR0, *success); - // Fall through to failure case. - } + __ beq(CCR0, *success); + // Fall through to failure case. } else { bool need_slow_path = true; if (k->is_loaded()) { @@ -2437,14 +2438,14 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L need_slow_path = false; } // Perform the fast part of the checking logic. - __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, R0, (need_slow_path ? success_target : nullptr), - failure_target, nullptr, RegisterOrConstant(k->super_check_offset())); + __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, R0, (need_slow_path ? success : nullptr), + failure, nullptr, RegisterOrConstant(k->super_check_offset())); } else { // Perform the fast part of the checking logic. - __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, R0, success_target, failure_target); + __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, R0, success, failure); } if (!need_slow_path) { - if (!should_profile) { __ b(*success); } + __ b(*success); } else { // Call out-of-line instance of __ check_klass_subtype_slow_path(...): address entry = Runtime1::entry_for(Runtime1::slow_subtype_check_id); @@ -2453,7 +2454,6 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L Register original_klass_RInfo = op->tmp2()->as_register(); Register original_Rtmp1 = op->tmp3()->as_register(); bool keep_obj_alive = reg_conflict && (op->code() == lir_checkcast); - bool keep_klass_RInfo_alive = (obj == original_klass_RInfo) && should_profile; if (keep_obj_alive && (obj != original_Rtmp1)) { __ mr(R0, obj); } __ mr_if_needed(original_k_RInfo, k_RInfo); __ mr_if_needed(original_klass_RInfo, klass_RInfo); @@ -2462,39 +2462,12 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L __ calculate_address_from_global_toc(original_Rtmp1, entry, true, true, false); __ mtctr(original_Rtmp1); __ bctrl(); // sets CR0 - if (keep_obj_alive) { - if (keep_klass_RInfo_alive) { __ mr(R0, obj); } - __ mr(obj, dst); - } - if (should_profile) { - __ bne(CCR0, *failure_target); - if (keep_klass_RInfo_alive) { __ mr(klass_RInfo, keep_obj_alive ? R0 : obj); } - // Fall through to success case. - } else { - __ beq(CCR0, *success); - // Fall through to failure case. - } + if (keep_obj_alive) { __ mr(obj, dst); } + __ beq(CCR0, *success); + // Fall through to failure case. } } - if (should_profile) { - Register mdo = k_RInfo, recv = klass_RInfo; - assert_different_registers(mdo, recv, Rtmp1); - __ bind(profile_cast_success); - metadata2reg(md->constant_encoding(), mdo); - __ add_const_optimized(mdo, mdo, mdo_offset_bias, R0); - type_profile_helper(mdo, mdo_offset_bias, md, data, recv, Rtmp1, success); - __ b(*success); - - // Cast failure case. - __ bind(profile_cast_failure); - metadata2reg(md->constant_encoding(), mdo); - __ add_const_optimized(mdo, mdo, mdo_offset_bias, R0); - __ ld(Rtmp1, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias, mdo); - __ addi(Rtmp1, Rtmp1, -DataLayout::counter_increment); - __ std(Rtmp1, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias, mdo); - } - __ bind(*failure); } @@ -2520,23 +2493,34 @@ void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { assert(method != nullptr, "Should have method"); setup_md_access(method, op->profiled_bci(), md, data, mdo_offset_bias); } - Label profile_cast_success, failure, done; - Label *success_target = should_profile ? &profile_cast_success : &done; - __ cmpdi(CCR0, value, 0); + Label done; + if (should_profile) { Label not_null; - __ bne(CCR0, not_null); Register mdo = k_RInfo; Register data_val = Rtmp1; metadata2reg(md->constant_encoding(), mdo); __ add_const_optimized(mdo, mdo, mdo_offset_bias, R0); + __ cmpdi(CCR0, value, 0); + __ bne(CCR0, not_null); __ lbz(data_val, md->byte_offset_of_slot(data, DataLayout::flags_offset()) - mdo_offset_bias, mdo); __ ori(data_val, data_val, BitData::null_seen_byte_constant()); __ stb(data_val, md->byte_offset_of_slot(data, DataLayout::flags_offset()) - mdo_offset_bias, mdo); __ b(done); __ bind(not_null); + + Label update_done; + Register recv = klass_RInfo; + __ load_klass(recv, value); + type_profile_helper(mdo, mdo_offset_bias, md, data, recv, Rtmp1, &update_done); + const int slot_offset = md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias; + __ ld(Rtmp1, slot_offset, mdo); + __ addi(Rtmp1, Rtmp1, DataLayout::counter_increment); + __ std(Rtmp1, slot_offset, mdo); + __ bind(update_done); } else { + __ cmpdi(CCR0, value, 0); __ beq(CCR0, done); } if (!os::zero_page_read_protected() || !ImplicitNullChecks) { @@ -2547,10 +2531,12 @@ void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { __ load_klass(k_RInfo, array); __ load_klass(klass_RInfo, value); + Label failure; + // Get instance klass. __ ld(k_RInfo, in_bytes(ObjArrayKlass::element_klass_offset()), k_RInfo); // Perform the fast part of the checking logic. - __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, R0, success_target, &failure, nullptr); + __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, R0, &done, &failure, nullptr); // Call out-of-line instance of __ check_klass_subtype_slow_path(...): const address slow_path = Runtime1::entry_for(Runtime1::slow_subtype_check_id); @@ -2558,32 +2544,11 @@ void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { __ add_const_optimized(R0, R29_TOC, MacroAssembler::offset_to_global_toc(slow_path)); __ mtctr(R0); __ bctrl(); // sets CR0 - if (!should_profile) { - __ beq(CCR0, done); - __ bind(failure); - } else { - __ bne(CCR0, failure); - // Fall through to the success case. + __ beq(CCR0, done); - Register mdo = klass_RInfo, recv = k_RInfo, tmp1 = Rtmp1; - assert_different_registers(value, mdo, recv, tmp1); - __ bind(profile_cast_success); - metadata2reg(md->constant_encoding(), mdo); - __ add_const_optimized(mdo, mdo, mdo_offset_bias, R0); - __ load_klass(recv, value); - type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, &done); - __ b(done); - - // Cast failure case. - __ bind(failure); - metadata2reg(md->constant_encoding(), mdo); - __ add_const_optimized(mdo, mdo, mdo_offset_bias, R0); - Address data_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias); - __ ld(tmp1, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias, mdo); - __ addi(tmp1, tmp1, -DataLayout::counter_increment); - __ std(tmp1, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias, mdo); - } + __ bind(failure); __ b(*stub->entry()); + __ align(32, 12); __ bind(done); } else if (code == lir_checkcast) { diff --git a/src/hotspot/cpu/ppc/interp_masm_ppc.hpp b/src/hotspot/cpu/ppc/interp_masm_ppc.hpp index c868816b278..16224ab63ed 100644 --- a/src/hotspot/cpu/ppc/interp_masm_ppc.hpp +++ b/src/hotspot/cpu/ppc/interp_masm_ppc.hpp @@ -243,13 +243,12 @@ class InterpreterMacroAssembler: public MacroAssembler { void profile_final_call(Register scratch1, Register scratch2); void profile_virtual_call(Register Rreceiver, Register Rscratch1, Register Rscratch2, bool receiver_can_be_null); void profile_typecheck(Register Rklass, Register Rscratch1, Register Rscratch2); - void profile_typecheck_failed(Register Rscratch1, Register Rscratch2); void profile_ret(TosState state, Register return_bci, Register scratch1, Register scratch2); void profile_switch_default(Register scratch1, Register scratch2); void profile_switch_case(Register index, Register scratch1,Register scratch2, Register scratch3); void profile_null_seen(Register Rscratch1, Register Rscratch2); - void record_klass_in_profile(Register receiver, Register scratch1, Register scratch2, bool is_virtual_call); - void record_klass_in_profile_helper(Register receiver, Register scratch1, Register scratch2, int start_row, Label& done, bool is_virtual_call); + void record_klass_in_profile(Register receiver, Register scratch1, Register scratch2); + void record_klass_in_profile_helper(Register receiver, Register scratch1, Register scratch2, int start_row, Label& done); // Argument and return type profiling. void profile_obj_type(Register obj, Register mdo_addr_base, RegisterOrConstant mdo_addr_offs, Register tmp, Register tmp2); diff --git a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp index 973c596df88..c5243900158 100644 --- a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp +++ b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp @@ -583,7 +583,6 @@ void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass, Register // Profile the not-null value's klass. profile_typecheck(Rsub_klass, Rtmp1, Rtmp2); check_klass_subtype(Rsub_klass, Rsuper_klass, Rtmp1, Rtmp2, ok_is_subtype); - profile_typecheck_failed(Rtmp1, Rtmp2); } // Separate these two to allow for delay slot in middle. @@ -1534,7 +1533,7 @@ void InterpreterMacroAssembler::profile_virtual_call(Register Rreceiver, } // Record the receiver type. - record_klass_in_profile(Rreceiver, Rscratch1, Rscratch2, true); + record_klass_in_profile(Rreceiver, Rscratch1, Rscratch2); bind(skip_receiver_profile); // The method data pointer needs to be updated to reflect the new target. @@ -1554,7 +1553,7 @@ void InterpreterMacroAssembler::profile_typecheck(Register Rklass, Register Rscr mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); // Record the object type. - record_klass_in_profile(Rklass, Rscratch1, Rscratch2, false); + record_klass_in_profile(Rklass, Rscratch1, Rscratch2); } // The method data pointer needs to be updated. @@ -1564,24 +1563,6 @@ void InterpreterMacroAssembler::profile_typecheck(Register Rklass, Register Rscr } } -void InterpreterMacroAssembler::profile_typecheck_failed(Register Rscratch1, Register Rscratch2) { - if (ProfileInterpreter && TypeProfileCasts) { - Label profile_continue; - - // If no method data exists, go to profile_continue. - test_method_data_pointer(profile_continue); - - int count_offset = in_bytes(CounterData::count_offset()); - // Back up the address, since we have already bumped the mdp. - count_offset -= in_bytes(VirtualCallData::virtual_call_data_size()); - - // *Decrement* the counter. We expect to see zero or small negatives. - increment_mdp_data_at(count_offset, Rscratch1, Rscratch2, true); - - bind (profile_continue); - } -} - // Count a ret in the bytecodes. void InterpreterMacroAssembler::profile_ret(TosState state, Register return_bci, Register scratch1, Register scratch2) { @@ -1687,23 +1668,20 @@ void InterpreterMacroAssembler::profile_null_seen(Register Rscratch1, Register R } void InterpreterMacroAssembler::record_klass_in_profile(Register Rreceiver, - Register Rscratch1, Register Rscratch2, - bool is_virtual_call) { + Register Rscratch1, Register Rscratch2) { assert(ProfileInterpreter, "must be profiling"); assert_different_registers(Rreceiver, Rscratch1, Rscratch2); Label done; - record_klass_in_profile_helper(Rreceiver, Rscratch1, Rscratch2, 0, done, is_virtual_call); + record_klass_in_profile_helper(Rreceiver, Rscratch1, Rscratch2, 0, done); bind (done); } void InterpreterMacroAssembler::record_klass_in_profile_helper( Register receiver, Register scratch1, Register scratch2, - int start_row, Label& done, bool is_virtual_call) { + int start_row, Label& done) { if (TypeProfileWidth == 0) { - if (is_virtual_call) { - increment_mdp_data_at(in_bytes(CounterData::count_offset()), scratch1, scratch2); - } + increment_mdp_data_at(in_bytes(CounterData::count_offset()), scratch1, scratch2); return; } @@ -1735,19 +1713,14 @@ void InterpreterMacroAssembler::record_klass_in_profile_helper( // Failed the equality check on receiver[n]... Test for null. if (start_row == last_row) { // The only thing left to do is handle the null case. - if (is_virtual_call) { - // Scratch1 contains test_out from test_mdp_data_at. - cmpdi(CCR0, scratch1, 0); - beq(CCR0, found_null); - // Receiver did not match any saved receiver and there is no empty row for it. - // Increment total counter to indicate polymorphic case. - increment_mdp_data_at(in_bytes(CounterData::count_offset()), scratch1, scratch2); - b(done); - bind(found_null); - } else { - cmpdi(CCR0, scratch1, 0); - bne(CCR0, done); - } + // Scratch1 contains test_out from test_mdp_data_at. + cmpdi(CCR0, scratch1, 0); + beq(CCR0, found_null); + // Receiver did not match any saved receiver and there is no empty row for it. + // Increment total counter to indicate polymorphic case. + increment_mdp_data_at(in_bytes(CounterData::count_offset()), scratch1, scratch2); + b(done); + bind(found_null); break; } // Since null is rare, make it be the branch-taken case. @@ -1755,7 +1728,7 @@ void InterpreterMacroAssembler::record_klass_in_profile_helper( beq(CCR0, found_null); // Put all the "Case 3" tests here. - record_klass_in_profile_helper(receiver, scratch1, scratch2, start_row + 1, done, is_virtual_call); + record_klass_in_profile_helper(receiver, scratch1, scratch2, start_row + 1, done); // Found a null. Keep searching for a matching receiver, // but remember that this is an empty (unused) slot. diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.hpp b/src/hotspot/cpu/ppc/vm_version_ppc.hpp index 78e19288445..b1168ded456 100644 --- a/src/hotspot/cpu/ppc/vm_version_ppc.hpp +++ b/src/hotspot/cpu/ppc/vm_version_ppc.hpp @@ -127,7 +127,6 @@ public: static uint64_t _dscr_val; static void initialize_cpu_information(void); - static bool profile_all_receivers_at_type_check() { return false; } }; #endif // CPU_PPC_VM_VERSION_PPC_HPP