diff --git a/src/hotspot/cpu/aarch64/aarch64_vector.ad b/src/hotspot/cpu/aarch64/aarch64_vector.ad index d2780db0702..420194559a2 100644 --- a/src/hotspot/cpu/aarch64/aarch64_vector.ad +++ b/src/hotspot/cpu/aarch64/aarch64_vector.ad @@ -317,6 +317,13 @@ source %{ return false; // NEON only, since SLI/USHR are not available in SVE } break; + case Op_VectorBitwiseBlend: + // Use NEON BSL when UseSVE < 2; SVE1 has no BSL so larger vectors are + // not supported on UseSVE == 1 machines. + if (UseSVE < 2 && length_in_bytes > 16) { + return false; + } + break; default: break; } @@ -7053,76 +7060,25 @@ instruct vblend_sve(vReg dst, vReg src1, vReg src2, pReg pg) %{ // ------------------------------ Vector bitwise blend ------------------------- -instruct vbitwise_blend_neon_sve1(vReg dst_src1, vReg src2, vReg src3) %{ +instruct vbitwise_blend_neon_sve1(vReg src1, vReg src2, vReg dst_src3) %{ predicate(UseSVE < 2 && VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n))); - match(Set dst_src1 (XorV src3 (AndV dst_src1 (XorV src2 src3)))); - // Second form: inner XorV may have operands (src3, src2) after Ideal/GVN. - // adlc only auto-swaps commutative ops when at least one operand is a subtree, - // not when both sides are leaves, so both shapes need explicit match rules. - match(Set dst_src1 (XorV src3 (AndV dst_src1 (XorV src3 src2)))); - format %{ "vbitwise_blend_neon_sve1 $dst_src1, $src2, $src3" %} + match(Set dst_src3 (VectorBitwiseBlend (Binary src1 src2) dst_src3)); + format %{ "vbitwise_blend_neon_sve1 $src1, $src2, $dst_src3" %} ins_encode %{ uint length_in_bytes = Matcher::vector_length_in_bytes(this); Assembler::SIMD_Arrangement T = length_in_bytes == 16 ? __ T16B : __ T8B; - __ bsl($dst_src1$$FloatRegister, T, $src2$$FloatRegister, $src3$$FloatRegister); + __ bsl($dst_src3$$FloatRegister, T, $src2$$FloatRegister, $src1$$FloatRegister); %} ins_pipe(pipe_slow); %} -instruct vbitwise_blend_sve2(vReg dst_src1, vReg src2, vReg src3) %{ +instruct vbitwise_blend_sve2(vReg src1, vReg dst_src2, vReg src3) %{ predicate(UseSVE == 2); - match(Set dst_src1 (XorV src2 (AndV src3 (XorV dst_src1 src2)))); - // Second form: inner XorV may have operands (src2, dst_src1) after Ideal/GVN. - // adlc only auto-swaps commutative ops when at least one operand is a subtree, - // not when both sides are leaves, so both shapes need explicit match rules. - match(Set dst_src1 (XorV src2 (AndV src3 (XorV src2 dst_src1)))); - format %{ "vbitwise_blend_sve2 $dst_src1, $src2, $src3" %} + match(Set dst_src2 (VectorBitwiseBlend (Binary src1 dst_src2) src3)); + format %{ "vbitwise_blend_sve2 $src1, $dst_src2, $src3" %} ins_encode %{ - __ sve_bsl($dst_src1$$FloatRegister, $src2$$FloatRegister, $src3$$FloatRegister); - %} - ins_pipe(pipe_slow); -%} - -instruct vbitwise_blend_masked_sve1(vReg dst_src1, vReg src2, vReg src3, pReg pg) %{ - predicate(UseSVE == 1 && - Matcher::vector_length_in_bytes(n) <= 16); - match(Set dst_src1 (XorV (Binary src3 (AndV dst_src1 (XorV src2 src3))) pg)); - // Second form: inner XorV may have operands (src3, src2) after Ideal/GVN. - // adlc only auto-swaps commutative ops when at least one operand is a subtree, - // not when both sides are leaves, so both shapes need explicit match rules. - match(Set dst_src1 (XorV (Binary src3 (AndV dst_src1 (XorV src3 src2))) pg)); - format %{ "vbitwise_blend_masked_sve1 $dst_src1, $pg, $src2, $src3" %} - ins_encode %{ - uint length_in_bytes = Matcher::vector_length_in_bytes(this); - Assembler::SIMD_Arrangement T = length_in_bytes == 16 ? __ T16B : __ T8B; - BasicType bt = Matcher::vector_element_basic_type(this); - __ bsl($dst_src1$$FloatRegister, T, $src2$$FloatRegister, $src3$$FloatRegister); - // sve_sel reads src3 after bsl overwrites dst_src1. - assert($dst_src1$$FloatRegister != $src3$$FloatRegister, - "dst_src1 and src3 must not alias"); - __ sve_sel($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt), - $pg$$PRegister, $dst_src1$$FloatRegister, $src3$$FloatRegister); - %} - ins_pipe(pipe_slow); -%} - -instruct vbitwise_blend_masked_sve2(vReg dst_src1, vReg src2, vReg src3, pReg pg) %{ - predicate(UseSVE == 2); - match(Set dst_src1 (XorV (Binary src2 (AndV src3 (XorV dst_src1 src2))) pg)); - // Second form: inner XorV may have operands (src2, dst_src1) after Ideal/GVN. - // adlc only auto-swaps commutative ops when at least one operand is a subtree, - // not when both sides are leaves, so both shapes need explicit match rules. - match(Set dst_src1 (XorV (Binary src2 (AndV src3 (XorV src2 dst_src1))) pg)); - format %{ "vbitwise_blend_masked_sve2 $dst_src1, $pg, $src2, $src3" %} - ins_encode %{ - BasicType bt = Matcher::vector_element_basic_type(this); - __ sve_bsl($dst_src1$$FloatRegister, $src2$$FloatRegister, $src3$$FloatRegister); - // sve_sel reads src2 after sve_bsl overwrites dst_src1. - assert($dst_src1$$FloatRegister != $src2$$FloatRegister, - "dst_src1 and src2 must not alias"); - __ sve_sel($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt), - $pg$$PRegister, $dst_src1$$FloatRegister, $src2$$FloatRegister); + __ sve_bsl($dst_src2$$FloatRegister, $src1$$FloatRegister, $src3$$FloatRegister); %} ins_pipe(pipe_slow); %} diff --git a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 index 1e8f9507edc..0d86eb2fef3 100644 --- a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 +++ b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 @@ -307,6 +307,13 @@ source %{ return false; // NEON only, since SLI/USHR are not available in SVE } break; + case Op_VectorBitwiseBlend: + // Use NEON BSL when UseSVE < 2; SVE1 has no BSL so larger vectors are + // not supported on UseSVE == 1 machines. + if (UseSVE < 2 && length_in_bytes > 16) { + return false; + } + break; default: break; } @@ -4756,76 +4763,25 @@ instruct vblend_sve(vReg dst, vReg src1, vReg src2, pReg pg) %{ // ------------------------------ Vector bitwise blend ------------------------- -instruct vbitwise_blend_neon_sve1(vReg dst_src1, vReg src2, vReg src3) %{ +instruct vbitwise_blend_neon_sve1(vReg src1, vReg src2, vReg dst_src3) %{ predicate(UseSVE < 2 && VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n))); - match(Set dst_src1 (XorV src3 (AndV dst_src1 (XorV src2 src3)))); - // Second form: inner XorV may have operands (src3, src2) after Ideal/GVN. - // adlc only auto-swaps commutative ops when at least one operand is a subtree, - // not when both sides are leaves, so both shapes need explicit match rules. - match(Set dst_src1 (XorV src3 (AndV dst_src1 (XorV src3 src2)))); - format %{ "vbitwise_blend_neon_sve1 $dst_src1, $src2, $src3" %} + match(Set dst_src3 (VectorBitwiseBlend (Binary src1 src2) dst_src3)); + format %{ "vbitwise_blend_neon_sve1 $src1, $src2, $dst_src3" %} ins_encode %{ uint length_in_bytes = Matcher::vector_length_in_bytes(this); Assembler::SIMD_Arrangement T = length_in_bytes == 16 ? __ T16B : __ T8B; - __ bsl($dst_src1$$FloatRegister, T, $src2$$FloatRegister, $src3$$FloatRegister); + __ bsl($dst_src3$$FloatRegister, T, $src2$$FloatRegister, $src1$$FloatRegister); %} ins_pipe(pipe_slow); %} -instruct vbitwise_blend_sve2(vReg dst_src1, vReg src2, vReg src3) %{ +instruct vbitwise_blend_sve2(vReg src1, vReg dst_src2, vReg src3) %{ predicate(UseSVE == 2); - match(Set dst_src1 (XorV src2 (AndV src3 (XorV dst_src1 src2)))); - // Second form: inner XorV may have operands (src2, dst_src1) after Ideal/GVN. - // adlc only auto-swaps commutative ops when at least one operand is a subtree, - // not when both sides are leaves, so both shapes need explicit match rules. - match(Set dst_src1 (XorV src2 (AndV src3 (XorV src2 dst_src1)))); - format %{ "vbitwise_blend_sve2 $dst_src1, $src2, $src3" %} + match(Set dst_src2 (VectorBitwiseBlend (Binary src1 dst_src2) src3)); + format %{ "vbitwise_blend_sve2 $src1, $dst_src2, $src3" %} ins_encode %{ - __ sve_bsl($dst_src1$$FloatRegister, $src2$$FloatRegister, $src3$$FloatRegister); - %} - ins_pipe(pipe_slow); -%} - -instruct vbitwise_blend_masked_sve1(vReg dst_src1, vReg src2, vReg src3, pReg pg) %{ - predicate(UseSVE == 1 && - Matcher::vector_length_in_bytes(n) <= 16); - match(Set dst_src1 (XorV (Binary src3 (AndV dst_src1 (XorV src2 src3))) pg)); - // Second form: inner XorV may have operands (src3, src2) after Ideal/GVN. - // adlc only auto-swaps commutative ops when at least one operand is a subtree, - // not when both sides are leaves, so both shapes need explicit match rules. - match(Set dst_src1 (XorV (Binary src3 (AndV dst_src1 (XorV src3 src2))) pg)); - format %{ "vbitwise_blend_masked_sve1 $dst_src1, $pg, $src2, $src3" %} - ins_encode %{ - uint length_in_bytes = Matcher::vector_length_in_bytes(this); - Assembler::SIMD_Arrangement T = length_in_bytes == 16 ? __ T16B : __ T8B; - BasicType bt = Matcher::vector_element_basic_type(this); - __ bsl($dst_src1$$FloatRegister, T, $src2$$FloatRegister, $src3$$FloatRegister); - // sve_sel reads src3 after bsl overwrites dst_src1. - assert($dst_src1$$FloatRegister != $src3$$FloatRegister, - "dst_src1 and src3 must not alias"); - __ sve_sel($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt), - $pg$$PRegister, $dst_src1$$FloatRegister, $src3$$FloatRegister); - %} - ins_pipe(pipe_slow); -%} - -instruct vbitwise_blend_masked_sve2(vReg dst_src1, vReg src2, vReg src3, pReg pg) %{ - predicate(UseSVE == 2); - match(Set dst_src1 (XorV (Binary src2 (AndV src3 (XorV dst_src1 src2))) pg)); - // Second form: inner XorV may have operands (src2, dst_src1) after Ideal/GVN. - // adlc only auto-swaps commutative ops when at least one operand is a subtree, - // not when both sides are leaves, so both shapes need explicit match rules. - match(Set dst_src1 (XorV (Binary src2 (AndV src3 (XorV src2 dst_src1))) pg)); - format %{ "vbitwise_blend_masked_sve2 $dst_src1, $pg, $src2, $src3" %} - ins_encode %{ - BasicType bt = Matcher::vector_element_basic_type(this); - __ sve_bsl($dst_src1$$FloatRegister, $src2$$FloatRegister, $src3$$FloatRegister); - // sve_sel reads src2 after sve_bsl overwrites dst_src1. - assert($dst_src1$$FloatRegister != $src2$$FloatRegister, - "dst_src1 and src2 must not alias"); - __ sve_sel($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt), - $pg$$PRegister, $dst_src1$$FloatRegister, $src2$$FloatRegister); + __ sve_bsl($dst_src2$$FloatRegister, $src1$$FloatRegister, $src3$$FloatRegister); %} ins_pipe(pipe_slow); %} diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp index 4ac5c31789f..7033dad211c 100644 --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -512,6 +512,7 @@ macro(VectorMaskWrapper) macro(VectorMaskCmp) macro(VectorMaskCast) macro(VectorTest) +macro(VectorBitwiseBlend) macro(VectorBlend) macro(VectorRearrange) macro(VectorLoadMask) diff --git a/src/hotspot/share/opto/matcher.cpp b/src/hotspot/share/opto/matcher.cpp index d2a9250b3ee..57fa2de05df 100644 --- a/src/hotspot/share/opto/matcher.cpp +++ b/src/hotspot/share/opto/matcher.cpp @@ -2385,7 +2385,8 @@ void Matcher::find_shared_post_visit(Node* n, uint opcode) { break; } case Op_VectorBlend: - case Op_VectorInsert: { + case Op_VectorInsert: + case Op_VectorBitwiseBlend: { Node* pair = new BinaryNode(n->in(1), n->in(2)); n->set_req(1, pair); n->set_req(2, n->in(3)); diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index dd49d88ce96..a00d8669c61 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -874,6 +874,7 @@ VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, Node* n3, const TypeV case Op_SignumVD: return new SignumVDNode(n1, n2, n3, vt); case Op_SignumVF: return new SignumVFNode(n1, n2, n3, vt); case Op_VectorBlend: return new VectorBlendNode(n1, n2, n3); + case Op_VectorBitwiseBlend: return new VectorBitwiseBlendNode(n1, n2, n3, vt); default: fatal("Missed vector creation for '%s'", NodeClassNames[vopc]); return nullptr; @@ -2768,6 +2769,73 @@ Node* XorVNode::Ideal_XorV_VectorMaskCmp(PhaseGVN* phase, bool can_reshape) { return res; } +// XorV(a, AndV(sel, XorV(a, b))) => VectorBitwiseBlend(a, b, sel) +// XorV(a, AndV(sel, XorV(a, b)), mask) => +// VectorBlend(a, VectorBitwiseBlend(a, b, sel), mask) +Node* XorVNode::Ideal_XorV_to_VectorBitwiseBlend(PhaseGVN* phase, bool can_reshape) { + const TypeVect* vt = vect_type(); + BasicType bt = vt->element_basic_type(); + uint vlen = vt->length(); + if (!Matcher::match_rule_supported_vector(Op_VectorBitwiseBlend, vlen, bt)) { + return nullptr; + } + + bool is_masked = is_predicated_vector(); + if (is_masked && + !Matcher::match_rule_supported_vector(Op_VectorBlend, vlen, bt)) { + return nullptr; + } + + // For the predicated case in(1) is fixed as the merge source. Otherwise the + // outer XorV is commutative. + Node* a = nullptr; + Node* andv = nullptr; + if (is_masked) { + a = in(1); + andv = in(2); + } else if (in(2)->Opcode() == Op_AndV) { + andv = in(2); + a = in(1); + } else { + andv = in(1); + a = in(2); + } + if (andv->Opcode() != Op_AndV || andv->is_predicated_vector()) { + return nullptr; + } + + Node* sel = nullptr; + Node* inner_xor = nullptr; + if (andv->in(2)->Opcode() == Op_XorV) { + inner_xor = andv->in(2); + sel = andv->in(1); + } else if (andv->in(1)->Opcode() == Op_XorV) { + inner_xor = andv->in(1); + sel = andv->in(2); + } else { + return nullptr; + } + if (inner_xor->is_predicated_vector()) { + return nullptr; + } + + Node* b = nullptr; + if (inner_xor->in(1) == a) { + b = inner_xor->in(2); + } else if (inner_xor->in(2) == a) { + b = inner_xor->in(1); + } else { + return nullptr; + } + + Node* blend = new VectorBitwiseBlendNode(a, b, sel, vt); + if (!is_masked) { + return blend; + } + blend = phase->transform(blend); + return new VectorBlendNode(a, blend, in(3)); +} + Node* XorVNode::Ideal(PhaseGVN* phase, bool can_reshape) { // (XorV src src) => (Replicate zero) // (XorVMask src src) => (MaskAll zero) @@ -2786,6 +2854,11 @@ Node* XorVNode::Ideal(PhaseGVN* phase, bool can_reshape) { if (res != nullptr) { return res; } + + res = Ideal_XorV_to_VectorBitwiseBlend(phase, can_reshape); + if (res != nullptr) { + return res; + } return VectorNode::Ideal(phase, can_reshape); } diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index de077015bca..13c370c7fcf 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -1075,6 +1075,7 @@ class XorVNode : public VectorNode { virtual int Opcode() const; virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); Node* Ideal_XorV_VectorMaskCmp(PhaseGVN* phase, bool can_reshape); + Node* Ideal_XorV_to_VectorBitwiseBlend(PhaseGVN* phase, bool can_reshape); }; // Vector xor byte, short, int, long as a reduction @@ -1802,6 +1803,14 @@ class VectorBlendNode : public VectorNode { Node* vec_mask() const { return in(3); } }; +// Vector bitwise blend (bit-select): (sel & vec_true) | (~sel & vec_false). +class VectorBitwiseBlendNode : public VectorNode { + public: + VectorBitwiseBlendNode(Node* vec_false, Node* vec_true, Node* sel, const TypeVect* vt) + : VectorNode(vec_false, vec_true, sel, vt) {} + virtual int Opcode() const; +}; + // Rearrange lane elements from a source vector under the control of a shuffle // (indexes) vector. Each lane in the shuffle vector specifies which lane from // the source vector to select for the corresponding output lane. All indexes diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java index 2b0961eb1d0..a042fb2889c 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java @@ -2392,26 +2392,6 @@ public class IRNode { machOnlyNameRegex(VAND_NOT_L_MASKED, "vand_notL_masked"); } - public static final String VBITWISE_BLEND_NEON_SVE1 = PREFIX + "VBITWISE_BLEND_NEON_SVE1" + POSTFIX; - static { - machOnlyNameRegex(VBITWISE_BLEND_NEON_SVE1, "vbitwise_blend_neon_sve1"); - } - - public static final String VBITWISE_BLEND_SVE2 = PREFIX + "VBITWISE_BLEND_SVE2" + POSTFIX; - static { - machOnlyNameRegex(VBITWISE_BLEND_SVE2, "vbitwise_blend_sve2"); - } - - public static final String VBITWISE_BLEND_MASKED_SVE1 = PREFIX + "VBITWISE_BLEND_MASKED_SVE1" + POSTFIX; - static { - machOnlyNameRegex(VBITWISE_BLEND_MASKED_SVE1, "vbitwise_blend_masked_sve1"); - } - - public static final String VBITWISE_BLEND_MASKED_SVE2 = PREFIX + "VBITWISE_BLEND_MASKED_SVE2" + POSTFIX; - static { - machOnlyNameRegex(VBITWISE_BLEND_MASKED_SVE2, "vbitwise_blend_masked_sve2"); - } - public static final String RISCV_VAND_NOTI_VX = PREFIX + "RISCV_VAND_NOTI_VX" + POSTFIX; static { machOnlyNameRegex(RISCV_VAND_NOTI_VX, "vand_notI_vx"); @@ -2462,6 +2442,12 @@ public class IRNode { vectorNode(VECTOR_BLEND_D, "VectorBlend", TYPE_DOUBLE); } + public static final String VECTOR_BITWISE_BLEND = PREFIX + "VECTOR_BITWISE_BLEND" + POSTFIX; + static { + String regex = START + "VectorBitwiseBlend" + MID + END; + afterBarrierExpansionToBeforeMatching(VECTOR_BITWISE_BLEND, regex); + } + public static final String VECTOR_MASK_CMP_I = VECTOR_PREFIX + "VECTOR_MASK_CMP_I" + POSTFIX; static { vectorNode(VECTOR_MASK_CMP_I, "VectorMaskCmp", TYPE_INT); diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorBitwiseBlendTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorBitwiseBlendTest.java index 1ae76e0415c..bc345075aac 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorBitwiseBlendTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorBitwiseBlendTest.java @@ -93,10 +93,10 @@ public class VectorBitwiseBlendTest { } @Test - @IR(counts = { IRNode.VBITWISE_BLEND_NEON_SVE1, "= 1" }, + @IR(counts = { IRNode.VECTOR_BITWISE_BLEND, "= 1" }, applyIfCPUFeatureAnd = { "asimd", "true", "sve2", "false" }, applyIf = { "MaxVectorSize", "<= 16" }) - @IR(counts = { IRNode.VBITWISE_BLEND_SVE2, "= 1" }, + @IR(counts = { IRNode.VECTOR_BITWISE_BLEND, "= 1" }, applyIfCPUFeature = { "sve2", "true" }) public static void testUnmaskedBlendByte() { ByteVector va = ByteVector.fromArray(B_SPECIES, ba, 0); @@ -106,10 +106,10 @@ public class VectorBitwiseBlendTest { } @Test - @IR(counts = { IRNode.VBITWISE_BLEND_NEON_SVE1, "= 1" }, + @IR(counts = { IRNode.VECTOR_BITWISE_BLEND, "= 1" }, applyIfCPUFeatureAnd = { "asimd", "true", "sve2", "false" }, applyIf = { "MaxVectorSize", "<= 16" }) - @IR(counts = { IRNode.VBITWISE_BLEND_SVE2, "= 1" }, + @IR(counts = { IRNode.VECTOR_BITWISE_BLEND, "= 1" }, applyIfCPUFeature = { "sve2", "true" }) public static void testUnmaskedBlendShort() { ShortVector va = ShortVector.fromArray(S_SPECIES, sa, 0); @@ -119,10 +119,10 @@ public class VectorBitwiseBlendTest { } @Test - @IR(counts = { IRNode.VBITWISE_BLEND_NEON_SVE1, "= 1" }, + @IR(counts = { IRNode.VECTOR_BITWISE_BLEND, "= 1" }, applyIfCPUFeatureAnd = { "asimd", "true", "sve2", "false" }, applyIf = { "MaxVectorSize", "<= 16" }) - @IR(counts = { IRNode.VBITWISE_BLEND_SVE2, "= 1" }, + @IR(counts = { IRNode.VECTOR_BITWISE_BLEND, "= 1" }, applyIfCPUFeature = { "sve2", "true" }) public static void testUnmaskedBlendInt() { IntVector va = IntVector.fromArray(I_SPECIES, ia, 0); @@ -132,10 +132,10 @@ public class VectorBitwiseBlendTest { } @Test - @IR(counts = { IRNode.VBITWISE_BLEND_NEON_SVE1, "= 1" }, + @IR(counts = { IRNode.VECTOR_BITWISE_BLEND, "= 1" }, applyIfCPUFeatureAnd = { "asimd", "true", "sve2", "false" }, applyIf = { "MaxVectorSize", "<= 16" }) - @IR(counts = { IRNode.VBITWISE_BLEND_SVE2, "= 1" }, + @IR(counts = { IRNode.VECTOR_BITWISE_BLEND, "= 1" }, applyIfCPUFeature = { "sve2", "true" }) public static void testUnmaskedBlendLong() { LongVector va = LongVector.fromArray(L_SPECIES, la, 0); @@ -145,13 +145,12 @@ public class VectorBitwiseBlendTest { } @Test - @IR(counts = { IRNode.VBITWISE_BLEND_NEON_SVE1, "= 1" }, - applyIfCPUFeatureAnd = { "asimd", "true", "sve", "false" }, + @IR(counts = { IRNode.VECTOR_BLEND_B, "= 1", + IRNode.VECTOR_BITWISE_BLEND, "= 1" }, + applyIfCPUFeatureAnd = { "asimd", "true", "sve2", "false" }, applyIf = { "MaxVectorSize", "<= 16" }) - @IR(counts = { IRNode.VBITWISE_BLEND_MASKED_SVE1, "= 1" }, - applyIfCPUFeatureAnd = { "sve", "true", "sve2", "false" }, - applyIf = { "MaxVectorSize", "<= 16" }) - @IR(counts = { IRNode.VBITWISE_BLEND_MASKED_SVE2, "= 1" }, + @IR(counts = { IRNode.VECTOR_BLEND_B, "= 1", + IRNode.VECTOR_BITWISE_BLEND, "= 1" }, applyIfCPUFeature = { "sve2", "true" }) public static void testMaskedBlendByte() { VectorMask mask = VectorMask.fromArray(B_SPECIES, mask_arr, 0); @@ -162,13 +161,12 @@ public class VectorBitwiseBlendTest { } @Test - @IR(counts = { IRNode.VBITWISE_BLEND_NEON_SVE1, "= 1" }, - applyIfCPUFeatureAnd = { "asimd", "true", "sve", "false" }, + @IR(counts = { IRNode.VECTOR_BLEND_S, "= 1", + IRNode.VECTOR_BITWISE_BLEND, "= 1" }, + applyIfCPUFeatureAnd = { "asimd", "true", "sve2", "false" }, applyIf = { "MaxVectorSize", "<= 16" }) - @IR(counts = { IRNode.VBITWISE_BLEND_MASKED_SVE1, "= 1" }, - applyIfCPUFeatureAnd = { "sve", "true", "sve2", "false" }, - applyIf = { "MaxVectorSize", "<= 16" }) - @IR(counts = { IRNode.VBITWISE_BLEND_MASKED_SVE2, "= 1" }, + @IR(counts = { IRNode.VECTOR_BLEND_S, "= 1", + IRNode.VECTOR_BITWISE_BLEND, "= 1" }, applyIfCPUFeature = { "sve2", "true" }) public static void testMaskedBlendShort() { VectorMask mask = VectorMask.fromArray(S_SPECIES, mask_arr, 0); @@ -179,13 +177,12 @@ public class VectorBitwiseBlendTest { } @Test - @IR(counts = { IRNode.VBITWISE_BLEND_NEON_SVE1, "= 1" }, - applyIfCPUFeatureAnd = { "asimd", "true", "sve", "false" }, + @IR(counts = { IRNode.VECTOR_BLEND_I, "= 1", + IRNode.VECTOR_BITWISE_BLEND, "= 1" }, + applyIfCPUFeatureAnd = { "asimd", "true", "sve2", "false" }, applyIf = { "MaxVectorSize", "<= 16" }) - @IR(counts = { IRNode.VBITWISE_BLEND_MASKED_SVE1, "= 1" }, - applyIfCPUFeatureAnd = { "sve", "true", "sve2", "false" }, - applyIf = { "MaxVectorSize", "<= 16" }) - @IR(counts = { IRNode.VBITWISE_BLEND_MASKED_SVE2, "= 1" }, + @IR(counts = { IRNode.VECTOR_BLEND_I, "= 1", + IRNode.VECTOR_BITWISE_BLEND, "= 1" }, applyIfCPUFeature = { "sve2", "true" }) public static void testMaskedBlendInt() { VectorMask mask = VectorMask.fromArray(I_SPECIES, mask_arr, 0); @@ -196,13 +193,12 @@ public class VectorBitwiseBlendTest { } @Test - @IR(counts = { IRNode.VBITWISE_BLEND_NEON_SVE1, "= 1" }, - applyIfCPUFeatureAnd = { "asimd", "true", "sve", "false" }, + @IR(counts = { IRNode.VECTOR_BLEND_L, "= 1", + IRNode.VECTOR_BITWISE_BLEND, "= 1" }, + applyIfCPUFeatureAnd = { "asimd", "true", "sve2", "false" }, applyIf = { "MaxVectorSize", "<= 16" }) - @IR(counts = { IRNode.VBITWISE_BLEND_MASKED_SVE1, "= 1" }, - applyIfCPUFeatureAnd = { "sve", "true", "sve2", "false" }, - applyIf = { "MaxVectorSize", "<= 16" }) - @IR(counts = { IRNode.VBITWISE_BLEND_MASKED_SVE2, "= 1" }, + @IR(counts = { IRNode.VECTOR_BLEND_L, "= 1", + IRNode.VECTOR_BITWISE_BLEND, "= 1" }, applyIfCPUFeature = { "sve2", "true" }) public static void testMaskedBlendLong() { VectorMask mask = VectorMask.fromArray(L_SPECIES, mask_arr, 0);