diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index 364db407bd3..fc53c10311b 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -2456,6 +2456,10 @@ bool Matcher::is_reg2reg_move(MachNode* m) { return false; } +bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) { + return false; +} + bool Matcher::is_generic_vector(MachOper* opnd) { return opnd->opcode() == VREG; } diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad index af010caf616..606275d7666 100644 --- a/src/hotspot/cpu/arm/arm.ad +++ b/src/hotspot/cpu/arm/arm.ad @@ -1063,6 +1063,10 @@ bool Matcher::is_reg2reg_move(MachNode* m) { return false; } +bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) { + return false; +} + bool Matcher::is_generic_vector(MachOper* opnd) { ShouldNotReachHere(); // generic vector operands not supported return false; diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index 762536df07f..87fcf112756 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -2383,6 +2383,10 @@ bool Matcher::is_reg2reg_move(MachNode* m) { return false; } +bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) { + return false; +} + bool Matcher::is_generic_vector(MachOper* opnd) { ShouldNotReachHere(); // generic vector operands not supported return false; diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index bb2ed57ef82..3f5dd4ad0ee 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -2053,6 +2053,10 @@ bool Matcher::is_reg2reg_move(MachNode* m) { return false; } +bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) { + return false; +} + bool Matcher::is_generic_vector(MachOper* opnd) { ShouldNotReachHere(); // generic vector operands not supported return false; diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad index 6fe051b55c7..7d3e963a108 100644 --- a/src/hotspot/cpu/s390/s390.ad +++ b/src/hotspot/cpu/s390/s390.ad @@ -1865,6 +1865,10 @@ bool Matcher::is_reg2reg_move(MachNode* m) { return false; } +bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) { + return false; +} + bool Matcher::is_generic_vector(MachOper* opnd) { ShouldNotReachHere(); // generic vector operands not supported return false; diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index be9889b0a99..1d393897bca 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -2633,6 +2633,70 @@ bool Matcher::supports_vector_calling_convention(void) { return EnableVectorSupport; } +static bool is_ndd_demotable(const MachNode* mdef) { + return ((mdef->flags() & Node::PD::Flag_ndd_demotable) != 0); +} + +static bool is_ndd_demotable_commutative(const MachNode* mdef) { + return ((mdef->flags() & Node::PD::Flag_ndd_demotable_commutative) != 0); +} + +static bool is_demotion_candidate(const MachNode* mdef) { + return (is_ndd_demotable(mdef) || is_ndd_demotable_commutative(mdef)); +} + +bool Matcher::is_register_biasing_candidate(const MachNode* mdef, + int oper_index) { + if (mdef == nullptr) { + return false; + } + + if (mdef->num_opnds() <= oper_index || mdef->operand_index(oper_index) < 0 || + mdef->in(mdef->operand_index(oper_index)) == nullptr) { + assert(oper_index != 1 || !is_demotion_candidate(mdef), "%s", mdef->Name()); + assert(oper_index != 2 || !is_ndd_demotable_commutative(mdef), "%s", mdef->Name()); + return false; + } + + // Complex memory operand covers multiple incoming edges needed for + // address computation. Biasing def towards any address component will not + // result in NDD demotion by assembler. + if (mdef->operand_num_edges(oper_index) != 1) { + assert(!is_ndd_demotable(mdef), "%s", mdef->Name()); + return false; + } + + // Demotion candidate must be register mask compatible with definition. + const RegMask& oper_mask = mdef->in_RegMask(mdef->operand_index(oper_index)); + if (!oper_mask.overlap(mdef->out_RegMask())) { + assert(!is_demotion_candidate(mdef), "%s", mdef->Name()); + return false; + } + + switch (oper_index) { + // First operand of MachNode corresponding to Intel APX NDD selection + // pattern can share its assigned register with definition operand if + // their live ranges do not overlap. In such a scenario we can demote + // it to legacy map0/map1 instruction by replacing its 4-byte extended + // EVEX prefix with shorter REX/REX2 encoding. Demotion candidates + // are decorated with a special flag by instruction selector. + case 1: + return is_demotion_candidate(mdef); + + // Definition operand of commutative operation can be biased towards second + // operand. + case 2: + return is_ndd_demotable_commutative(mdef); + + // Current scheme only selects up to two biasing candidates + default: + assert(false, "unhandled operand index: %s", mdef->Name()); + break; + } + + return false; +} + OptoRegPair Matcher::vector_return_value(uint ideal_reg) { assert(EnableVectorSupport, "sanity"); int lo = XMM0_num; @@ -2812,7 +2876,7 @@ static inline bool is_clz_non_subword_predicate_evex(BasicType bt, int vlen_byte class Node::PD { public: - enum NodeFlags { + enum NodeFlags : uint64_t { Flag_intel_jcc_erratum = Node::_last_flag << 1, Flag_sets_carry_flag = Node::_last_flag << 2, Flag_sets_parity_flag = Node::_last_flag << 3, @@ -2824,7 +2888,9 @@ public: Flag_clears_zero_flag = Node::_last_flag << 9, Flag_clears_overflow_flag = Node::_last_flag << 10, Flag_clears_sign_flag = Node::_last_flag << 11, - _last_flag = Flag_clears_sign_flag + Flag_ndd_demotable = Node::_last_flag << 12, + Flag_ndd_demotable_commutative = Node::_last_flag << 13, + _last_flag = Flag_ndd_demotable_commutative }; }; @@ -9801,7 +9867,7 @@ instruct addI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (AddI src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_commutative); format %{ "eaddl $dst, $src1, $src2\t# int ndd" %} ins_encode %{ @@ -9829,7 +9895,7 @@ instruct addI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (AddI src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable); format %{ "eaddl $dst, $src1, $src2\t# int ndd" %} ins_encode %{ @@ -9872,7 +9938,7 @@ instruct addI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr predicate(UseAPX); match(Set dst (AddI src1 (LoadI src2))); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_commutative); ins_cost(150); format %{ "eaddl $dst, $src1, $src2\t# int ndd" %} @@ -9929,6 +9995,7 @@ instruct incI_rReg_ndd(rRegI dst, rRegI src, immI_1 val, rFlagsReg cr) predicate(UseAPX && UseIncDec); match(Set dst (AddI src val)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "eincl $dst, $src\t# int ndd" %} ins_encode %{ @@ -9983,6 +10050,7 @@ instruct decI_rReg_ndd(rRegI dst, rRegI src, immI_M1 val, rFlagsReg cr) predicate(UseAPX && UseIncDec); match(Set dst (AddI src val)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "edecl $dst, $src\t# int ndd" %} ins_encode %{ @@ -10089,7 +10157,7 @@ instruct addL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (AddL src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_commutative); format %{ "eaddq $dst, $src1, $src2\t# long ndd" %} ins_encode %{ @@ -10117,7 +10185,7 @@ instruct addL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr predicate(UseAPX); match(Set dst (AddL src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable); format %{ "eaddq $dst, $src1, $src2\t# long ndd" %} ins_encode %{ @@ -10160,7 +10228,7 @@ instruct addL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr predicate(UseAPX); match(Set dst (AddL src1 (LoadL src2))); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_commutative); ins_cost(150); format %{ "eaddq $dst, $src1, $src2\t# long ndd" %} @@ -10216,6 +10284,7 @@ instruct incL_rReg_ndd(rRegL dst, rRegI src, immL1 val, rFlagsReg cr) predicate(UseAPX && UseIncDec); match(Set dst (AddL src val)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "eincq $dst, $src\t# long ndd" %} ins_encode %{ @@ -10270,6 +10339,7 @@ instruct decL_rReg_ndd(rRegL dst, rRegL src, immL_M1 val, rFlagsReg cr) predicate(UseAPX && UseIncDec); match(Set dst (AddL src val)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "edecq $dst, $src\t# long ndd" %} ins_encode %{ @@ -10984,7 +11054,7 @@ instruct subI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (SubI src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable); format %{ "esubl $dst, $src1, $src2\t# int ndd" %} ins_encode %{ @@ -10998,7 +11068,7 @@ instruct subI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (SubI src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable); format %{ "esubl $dst, $src1, $src2\t# int ndd" %} ins_encode %{ @@ -11041,7 +11111,7 @@ instruct subI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr predicate(UseAPX); match(Set dst (SubI src1 (LoadI src2))); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable); ins_cost(150); format %{ "esubl $dst, $src1, $src2\t# int ndd" %} @@ -11099,7 +11169,7 @@ instruct subL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (SubL src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable); format %{ "esubq $dst, $src1, $src2\t# long ndd" %} ins_encode %{ @@ -11113,7 +11183,7 @@ instruct subL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr predicate(UseAPX); match(Set dst (SubL src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable); format %{ "esubq $dst, $src1, $src2\t# long ndd" %} ins_encode %{ @@ -11156,7 +11226,7 @@ instruct subL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr predicate(UseAPX); match(Set dst (SubL src1 (LoadL src2))); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable); ins_cost(150); format %{ "esubq $dst, $src1, $src2\t# long ndd" %} @@ -11228,7 +11298,7 @@ instruct negI_rReg_ndd(rRegI dst, rRegI src, immI_0 zero, rFlagsReg cr) predicate(UseAPX); match(Set dst (SubI zero src)); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable); format %{ "enegl $dst, $src\t# int ndd" %} ins_encode %{ @@ -11256,7 +11326,7 @@ instruct negI_rReg_2_ndd(rRegI dst, rRegI src, rFlagsReg cr) predicate(UseAPX); match(Set dst (NegI src)); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable); format %{ "enegl $dst, $src\t# int ndd" %} ins_encode %{ @@ -11297,7 +11367,7 @@ instruct negL_rReg_ndd(rRegL dst, rRegL src, immL0 zero, rFlagsReg cr) predicate(UseAPX); match(Set dst (SubL zero src)); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable); format %{ "enegq $dst, $src\t# long ndd" %} ins_encode %{ @@ -11325,7 +11395,7 @@ instruct negL_rReg_2_ndd(rRegL dst, rRegL src, rFlagsReg cr) predicate(UseAPX); match(Set dst (NegL src)); effect(KILL cr); - flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag); + flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable); format %{ "enegq $dst, $src\t# long ndd" %} ins_encode %{ @@ -11370,6 +11440,7 @@ instruct mulI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (MulI src1 src2)); effect(KILL cr); + flag(PD::Flag_ndd_demotable_commutative); ins_cost(300); format %{ "eimull $dst, $src1, $src2\t# int ndd" %} @@ -11411,6 +11482,7 @@ instruct mulI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr predicate(UseAPX); match(Set dst (MulI src1 (LoadI src2))); effect(KILL cr); + flag(PD::Flag_ndd_demotable); ins_cost(350); format %{ "eimull $dst, $src1, $src2\t# int ndd" %} @@ -11462,6 +11534,7 @@ instruct mulL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (MulL src1 src2)); effect(KILL cr); + flag(PD::Flag_ndd_demotable_commutative); ins_cost(300); format %{ "eimulq $dst, $src1, $src2\t# long ndd" %} @@ -11503,6 +11576,7 @@ instruct mulL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr predicate(UseAPX); match(Set dst (MulL src1 (LoadL src2))); effect(KILL cr); + flag(PD::Flag_ndd_demotable_commutative); ins_cost(350); format %{ "eimulq $dst, $src1, $src2 \t# long" %} @@ -11777,6 +11851,7 @@ instruct salI_rReg_immI2_ndd(rRegI dst, rRegI src, immI2 shift, rFlagsReg cr) predicate(UseAPX); match(Set dst (LShiftI src shift)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "esall $dst, $src, $shift\t# int(ndd)" %} ins_encode %{ @@ -11805,6 +11880,7 @@ instruct salI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr) predicate(UseAPX); match(Set dst (LShiftI src shift)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "esall $dst, $src, $shift\t# int (ndd)" %} ins_encode %{ @@ -11911,6 +11987,7 @@ instruct sarI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr) predicate(UseAPX); match(Set dst (RShiftI src shift)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "esarl $dst, $src, $shift\t# int (ndd)" %} ins_encode %{ @@ -12017,6 +12094,7 @@ instruct shrI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr) predicate(UseAPX); match(Set dst (URShiftI src shift)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "eshrl $dst, $src, $shift\t # int (ndd)" %} ins_encode %{ @@ -12124,6 +12202,7 @@ instruct salL_rReg_immI2_ndd(rRegL dst, rRegL src, immI2 shift, rFlagsReg cr) predicate(UseAPX); match(Set dst (LShiftL src shift)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "esalq $dst, $src, $shift\t# long (ndd)" %} ins_encode %{ @@ -12152,6 +12231,7 @@ instruct salL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr) predicate(UseAPX); match(Set dst (LShiftL src shift)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "esalq $dst, $src, $shift\t# long (ndd)" %} ins_encode %{ @@ -12258,6 +12338,7 @@ instruct sarL_rReg_imm_ndd(rRegL dst, rRegL src, immI shift, rFlagsReg cr) predicate(UseAPX); match(Set dst (RShiftL src shift)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "esarq $dst, $src, $shift\t# long (ndd)" %} ins_encode %{ @@ -12364,6 +12445,7 @@ instruct shrL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr) predicate(UseAPX); match(Set dst (URShiftL src shift)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "eshrq $dst, $src, $shift\t# long (ndd)" %} ins_encode %{ @@ -12535,6 +12617,7 @@ instruct rolI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr) predicate(UseAPX && n->bottom_type()->basic_type() == T_INT); match(Set dst (RotateLeft src shift)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "eroll $dst, $src, $shift\t# rotate left (int ndd)" %} ins_encode %{ @@ -12599,6 +12682,7 @@ instruct rorI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr) predicate(UseAPX && n->bottom_type()->basic_type() == T_INT); match(Set dst (RotateRight src shift)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "erorl $dst, $src, $shift\t# rotate right(int ndd)" %} ins_encode %{ @@ -12651,6 +12735,7 @@ instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr) predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG); match(Set dst (RotateLeft dst shift)); effect(KILL cr); + format %{ "rolq $dst, $shift" %} ins_encode %{ __ rolq($dst$$Register); @@ -12664,6 +12749,7 @@ instruct rolL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr) predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG); match(Set dst (RotateLeft src shift)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "erolq $dst, $src, $shift\t# rotate left(long ndd)" %} ins_encode %{ @@ -12728,6 +12814,7 @@ instruct rorL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr) predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG); match(Set dst (RotateRight src shift)); effect(KILL cr); + flag(PD::Flag_ndd_demotable); format %{ "erorq $dst, $src, $shift\t# rotate right(long ndd)" %} ins_encode %{ @@ -12805,7 +12892,7 @@ instruct andI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (AndI src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_commutative); format %{ "eandl $dst, $src1, $src2\t# int ndd" %} ins_encode %{ @@ -12898,7 +12985,7 @@ instruct andI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (AndI src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable); format %{ "eandl $dst, $src1, $src2\t# int ndd" %} ins_encode %{ @@ -12942,7 +13029,7 @@ instruct andI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr predicate(UseAPX); match(Set dst (AndI src1 (LoadI src2))); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_commutative); ins_cost(150); format %{ "eandl $dst, $src1, $src2\t# int ndd" %} @@ -13142,7 +13229,7 @@ instruct orI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (OrI src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_commutative); format %{ "eorl $dst, $src1, $src2\t# int ndd" %} ins_encode %{ @@ -13171,7 +13258,7 @@ instruct orI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (OrI src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable); format %{ "eorl $dst, $src1, $src2\t# int ndd" %} ins_encode %{ @@ -13185,7 +13272,7 @@ instruct orI_rReg_imm_rReg_ndd(rRegI dst, immI src1, rRegI src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (OrI src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable); format %{ "eorl $dst, $src2, $src1\t# int ndd" %} ins_encode %{ @@ -13229,7 +13316,7 @@ instruct orI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (OrI src1 (LoadI src2))); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable); ins_cost(150); format %{ "eorl $dst, $src1, $src2\t# int ndd" %} @@ -13305,7 +13392,7 @@ instruct xorI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (XorI src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_commutative); format %{ "exorl $dst, $src1, $src2\t# int ndd" %} ins_encode %{ @@ -13331,6 +13418,7 @@ instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm) %{ match(Set dst (XorI src imm)); predicate(UseAPX); + flag(PD::Flag_ndd_demotable); format %{ "enotl $dst, $src" %} ins_encode %{ @@ -13361,7 +13449,7 @@ instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr) predicate(UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1); match(Set dst (XorI src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable); format %{ "exorl $dst, $src1, $src2\t# int ndd" %} ins_encode %{ @@ -13407,7 +13495,7 @@ instruct xorI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr predicate(UseAPX); match(Set dst (XorI src1 (LoadI src2))); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable); ins_cost(150); format %{ "exorl $dst, $src1, $src2\t# int ndd" %} @@ -13486,7 +13574,7 @@ instruct andL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (AndL src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_commutative); format %{ "eandq $dst, $src1, $src2\t# long ndd" %} ins_encode %{ @@ -13542,7 +13630,7 @@ instruct andL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr predicate(UseAPX); match(Set dst (AndL src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable); format %{ "eandq $dst, $src1, $src2\t# long ndd" %} ins_encode %{ @@ -13586,7 +13674,7 @@ instruct andL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr predicate(UseAPX); match(Set dst (AndL src1 (LoadL src2))); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_commutative); ins_cost(150); format %{ "eandq $dst, $src1, $src2\t# long ndd" %} @@ -13789,7 +13877,7 @@ instruct orL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (OrL src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_commutative); format %{ "eorq $dst, $src1, $src2\t# long ndd" %} ins_encode %{ @@ -13844,7 +13932,7 @@ instruct orL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (OrL src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable); format %{ "eorq $dst, $src1, $src2\t# long ndd" %} ins_encode %{ @@ -13858,7 +13946,7 @@ instruct orL_rReg_imm_rReg_ndd(rRegL dst, immL32 src1, rRegL src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (OrL src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable); format %{ "eorq $dst, $src2, $src1\t# long ndd" %} ins_encode %{ @@ -13903,7 +13991,7 @@ instruct orL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (OrL src1 (LoadL src2))); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_commutative); ins_cost(150); format %{ "eorq $dst, $src1, $src2\t# long ndd" %} @@ -13982,7 +14070,7 @@ instruct xorL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (XorL src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_commutative); format %{ "exorq $dst, $src1, $src2\t# long ndd" %} ins_encode %{ @@ -14008,6 +14096,7 @@ instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm) %{ predicate(UseAPX); match(Set dst (XorL src imm)); + flag(PD::Flag_ndd_demotable); format %{ "enotq $dst, $src" %} ins_encode %{ @@ -14038,7 +14127,7 @@ instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr) predicate(UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L); match(Set dst (XorL src1 src2)); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable); format %{ "exorq $dst, $src1, $src2\t# long ndd" %} ins_encode %{ @@ -14084,7 +14173,7 @@ instruct xorL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr predicate(UseAPX); match(Set dst (XorL src1 (LoadL src2))); effect(KILL cr); - flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_commutative); ins_cost(150); format %{ "exorq $dst, $src1, $src2\t# long ndd" %} @@ -16539,6 +16628,7 @@ instruct minI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2) predicate(UseAPX); match(Set dst (MinI src1 src2)); effect(DEF dst, USE src1, USE src2); + flag(PD::Flag_ndd_demotable); ins_cost(200); expand %{ @@ -16590,6 +16680,7 @@ instruct maxI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2) predicate(UseAPX); match(Set dst (MaxI src1 src2)); effect(DEF dst, USE src1, USE src2); + flag(PD::Flag_ndd_demotable); ins_cost(200); expand %{ diff --git a/src/hotspot/share/opto/chaitin.cpp b/src/hotspot/share/opto/chaitin.cpp index 524dee6e06a..667270d96b4 100644 --- a/src/hotspot/share/opto/chaitin.cpp +++ b/src/hotspot/share/opto/chaitin.cpp @@ -1471,6 +1471,65 @@ static OptoReg::Name find_first_set(LRG& lrg, RegMask& mask) { return assigned; } +OptoReg::Name PhaseChaitin::select_bias_lrg_color(LRG& lrg) { + uint bias_lrg1_idx = _lrg_map.find(lrg._copy_bias); + uint bias_lrg2_idx = _lrg_map.find(lrg._copy_bias2); + + // If bias_lrg1 has a color + if (bias_lrg1_idx != 0 && !_ifg->_yanked->test(bias_lrg1_idx)) { + OptoReg::Name reg = lrgs(bias_lrg1_idx).reg(); + // and it is legal for lrg + if (is_legal_reg(lrg, reg)) { + return reg; + } + } + + // If bias_lrg2 has a color + if (bias_lrg2_idx != 0 && !_ifg->_yanked->test(bias_lrg2_idx)) { + OptoReg::Name reg = lrgs(bias_lrg2_idx).reg(); + // and it is legal for lrg + if (is_legal_reg(lrg, reg)) { + return reg; + } + } + + uint bias_lrg_idx = 0; + if (bias_lrg1_idx != 0 && bias_lrg2_idx != 0) { + // Since none of the bias live ranges are part of the IFG yet, constrain the + // definition mask with the bias live range with the least degrees of + // freedom. This will increase the chances of register sharing once the bias + // live range becomes part of the IFG. + lrgs(bias_lrg1_idx).compute_set_mask_size(); + lrgs(bias_lrg2_idx).compute_set_mask_size(); + bias_lrg_idx = lrgs(bias_lrg1_idx).degrees_of_freedom() > + lrgs(bias_lrg2_idx).degrees_of_freedom() + ? bias_lrg2_idx + : bias_lrg1_idx; + } else if (bias_lrg1_idx != 0) { + bias_lrg_idx = bias_lrg1_idx; + } else if (bias_lrg2_idx != 0) { + bias_lrg_idx = bias_lrg2_idx; + } + + // Register masks with offset excludes all mask bits before the offset. + // Such masks are mainly used for allocation from stack slots. Constrain the + // register mask of definition live range using bias mask only if + // both masks have zero offset. + if (bias_lrg_idx != 0 && !lrg.mask().is_offset() && + !lrgs(bias_lrg_idx).mask().is_offset()) { + // Choose a color which is legal for bias_lrg + ResourceMark rm(C->regmask_arena()); + RegMask tempmask(lrg.mask(), C->regmask_arena()); + tempmask.and_with(lrgs(bias_lrg_idx).mask()); + tempmask.clear_to_sets(lrg.num_regs()); + OptoReg::Name reg = find_first_set(lrg, tempmask); + if (OptoReg::is_valid(reg)) { + return reg; + } + } + return OptoReg::Bad; +} + // Choose a color using the biasing heuristic OptoReg::Name PhaseChaitin::bias_color(LRG& lrg) { @@ -1492,25 +1551,10 @@ OptoReg::Name PhaseChaitin::bias_color(LRG& lrg) { } } - uint copy_lrg = _lrg_map.find(lrg._copy_bias); - if (copy_lrg != 0) { - // If he has a color, - if(!_ifg->_yanked->test(copy_lrg)) { - OptoReg::Name reg = lrgs(copy_lrg).reg(); - // And it is legal for you, - if (is_legal_reg(lrg, reg)) { - return reg; - } - } else if (!lrg.mask().is_offset()) { - // Choose a color which is legal for him - ResourceMark rm(C->regmask_arena()); - RegMask tempmask(lrg.mask(), C->regmask_arena()); - tempmask.and_with(lrgs(copy_lrg).mask()); - tempmask.clear_to_sets(lrg.num_regs()); - OptoReg::Name reg = find_first_set(lrg, tempmask); - if (OptoReg::is_valid(reg)) - return reg; - } + // Try biasing the color with non-interfering bias live range[s]. + OptoReg::Name reg = select_bias_lrg_color(lrg); + if (OptoReg::is_valid(reg)) { + return reg; } // If no bias info exists, just go with the register selection ordering @@ -1524,7 +1568,7 @@ OptoReg::Name PhaseChaitin::bias_color(LRG& lrg) { // CNC - Fun hack. Alternate 1st and 2nd selection. Enables post-allocate // copy removal to remove many more copies, by preventing a just-assigned // register from being repeatedly assigned. - OptoReg::Name reg = lrg.mask().find_first_elem(); + reg = lrg.mask().find_first_elem(); if( (++_alternate & 1) && OptoReg::is_valid(reg) ) { // This 'Remove; find; Insert' idiom is an expensive way to find the // SECOND element in the mask. @@ -1640,6 +1684,27 @@ uint PhaseChaitin::Select( ) { } } } + + Node* def = lrg->_def; + if (lrg->is_singledef() && !lrg->_is_bound && def->is_Mach()) { + MachNode* mdef = def->as_Mach(); + if (Matcher::is_register_biasing_candidate(mdef, 1)) { + Node* in1 = mdef->in(mdef->operand_index(1)); + if (in1 != nullptr && lrg->_copy_bias == 0) { + lrg->_copy_bias = _lrg_map.find(in1); + } + } + + // For commutative operations, def allocation can also be + // biased towards LRG of second input's def. + if (Matcher::is_register_biasing_candidate(mdef, 2)) { + Node* in2 = mdef->in(mdef->operand_index(2)); + if (in2 != nullptr && lrg->_copy_bias2 == 0) { + lrg->_copy_bias2 = _lrg_map.find(in2); + } + } + } + //assert(is_infinite_stack == lrg->mask().is_infinite_stack(), "nbrs must not change InfiniteStackedness"); // Aligned pairs need aligned masks assert(!lrg->_is_vector || !lrg->_fat_proj, "sanity"); diff --git a/src/hotspot/share/opto/chaitin.hpp b/src/hotspot/share/opto/chaitin.hpp index b477c54fcae..ac072e94e2b 100644 --- a/src/hotspot/share/opto/chaitin.hpp +++ b/src/hotspot/share/opto/chaitin.hpp @@ -63,6 +63,7 @@ public: uint _risk_bias; // Index of LRG which we want to avoid color uint _copy_bias; // Index of LRG which we want to share color + uint _copy_bias2; // Index of second LRG which we want to share color uint _next; // Index of next LRG in linked list uint _prev; // Index of prev LRG in linked list @@ -703,6 +704,8 @@ private: OptoReg::Name choose_color(LRG& lrg); // Helper function which implements biasing heuristic OptoReg::Name bias_color(LRG& lrg); + // Helper function which implements color biasing + OptoReg::Name select_bias_lrg_color(LRG& lrg); // Split uncolorable live ranges // Return new number of live ranges diff --git a/src/hotspot/share/opto/idealGraphPrinter.cpp b/src/hotspot/share/opto/idealGraphPrinter.cpp index b28949e27c2..5070a9f00e1 100644 --- a/src/hotspot/share/opto/idealGraphPrinter.cpp +++ b/src/hotspot/share/opto/idealGraphPrinter.cpp @@ -88,6 +88,7 @@ void PrintProperties::print_lrg_properties(const LRG &lrg, const char *buffer) { print_property(true, "score", lrg.score()); print_property((lrg._risk_bias != 0), "risk_bias", lrg._risk_bias); print_property((lrg._copy_bias != 0), "copy_bias", lrg._copy_bias); + print_property((lrg._copy_bias2 != 0), "copy_bias2", lrg._copy_bias2); print_property(lrg.is_singledef(), "is_singledef"); print_property(lrg.is_multidef(), "is_multidef"); print_property(lrg._is_oop, "is_oop"); diff --git a/src/hotspot/share/opto/machnode.cpp b/src/hotspot/share/opto/machnode.cpp index e58befd8032..ec861865ff5 100644 --- a/src/hotspot/share/opto/machnode.cpp +++ b/src/hotspot/share/opto/machnode.cpp @@ -460,6 +460,13 @@ int MachNode::operand_index(Node* def) const { return -1; } +int MachNode::operand_num_edges(uint oper_index) const { + if (num_opnds() > oper_index) { + return _opnds[oper_index]->num_edges(); + } + return 0; +} + //------------------------------peephole--------------------------------------- // Apply peephole rule(s) to this instruction int MachNode::peephole(Block *block, int block_index, PhaseCFG* cfg_, PhaseRegAlloc *ra_) { diff --git a/src/hotspot/share/opto/machnode.hpp b/src/hotspot/share/opto/machnode.hpp index 093f466678c..b60313b7f75 100644 --- a/src/hotspot/share/opto/machnode.hpp +++ b/src/hotspot/share/opto/machnode.hpp @@ -266,6 +266,7 @@ public: int operand_index(uint operand) const; int operand_index(const MachOper *oper) const; int operand_index(Node* m) const; + int operand_num_edges(uint operand) const; // Register class input is expected in virtual const RegMask &in_RegMask(uint) const; diff --git a/src/hotspot/share/opto/matcher.hpp b/src/hotspot/share/opto/matcher.hpp index 01f11b1fdc9..ca13d0166a1 100644 --- a/src/hotspot/share/opto/matcher.hpp +++ b/src/hotspot/share/opto/matcher.hpp @@ -512,6 +512,8 @@ public: DEBUG_ONLY( bool verify_after_postselect_cleanup(); ) public: + static bool is_register_biasing_candidate(const MachNode* mdef, int oper_index); + // This routine is run whenever a graph fails to match. // If it returns, the compiler should bailout to interpreter without error. // In non-product mode, SoftMatchFailure is false to detect non-canonical diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp index ec80fb6a0ab..2e19d1d247b 100644 --- a/src/hotspot/share/opto/node.hpp +++ b/src/hotspot/share/opto/node.hpp @@ -828,26 +828,26 @@ public: #undef DEFINE_CLASS_ID // Flags are sorted by usage frequency. - enum NodeFlags { - Flag_is_Copy = 1 << 0, // should be first bit to avoid shift - Flag_rematerialize = 1 << 1, - Flag_needs_anti_dependence_check = 1 << 2, - Flag_is_macro = 1 << 3, - Flag_is_Con = 1 << 4, - Flag_is_cisc_alternate = 1 << 5, - Flag_is_dead_loop_safe = 1 << 6, - Flag_may_be_short_branch = 1 << 7, - Flag_avoid_back_to_back_before = 1 << 8, - Flag_avoid_back_to_back_after = 1 << 9, - Flag_has_call = 1 << 10, - Flag_has_swapped_edges = 1 << 11, - Flag_is_scheduled = 1 << 12, - Flag_is_expensive = 1 << 13, - Flag_is_predicated_vector = 1 << 14, - Flag_for_post_loop_opts_igvn = 1 << 15, - Flag_for_merge_stores_igvn = 1 << 16, - Flag_is_removed_by_peephole = 1 << 17, - Flag_is_predicated_using_blend = 1 << 18, + enum NodeFlags : uint64_t { + Flag_is_Copy = 1ULL << 0, // should be first bit to avoid shift + Flag_rematerialize = 1ULL << 1, + Flag_needs_anti_dependence_check = 1ULL << 2, + Flag_is_macro = 1ULL << 3, + Flag_is_Con = 1ULL << 4, + Flag_is_cisc_alternate = 1ULL << 5, + Flag_is_dead_loop_safe = 1ULL << 6, + Flag_may_be_short_branch = 1ULL << 7, + Flag_avoid_back_to_back_before = 1ULL << 8, + Flag_avoid_back_to_back_after = 1ULL << 9, + Flag_has_call = 1ULL << 10, + Flag_has_swapped_edges = 1ULL << 11, + Flag_is_scheduled = 1ULL << 12, + Flag_is_expensive = 1ULL << 13, + Flag_is_predicated_vector = 1ULL << 14, + Flag_for_post_loop_opts_igvn = 1ULL << 15, + Flag_for_merge_stores_igvn = 1ULL << 16, + Flag_is_removed_by_peephole = 1ULL << 17, + Flag_is_predicated_using_blend = 1ULL << 18, _last_flag = Flag_is_predicated_using_blend };