diff --git a/src/hotspot/cpu/aarch64/aarch64_vector.ad b/src/hotspot/cpu/aarch64/aarch64_vector.ad index 12f98bb8549..c06c8b856b7 100644 --- a/src/hotspot/cpu/aarch64/aarch64_vector.ad +++ b/src/hotspot/cpu/aarch64/aarch64_vector.ad @@ -1157,7 +1157,8 @@ instruct vmulI_sve(vReg dst_src1, vReg src2) %{ // vector mul - LONG instruct vmulL_neon(vReg dst, vReg src1, vReg src2) %{ - predicate(UseSVE == 0); + predicate(UseSVE == 0 && !n->as_MulVL()->has_int_inputs() && + !n->as_MulVL()->has_uint_inputs()); match(Set dst (MulVL src1 src2)); format %{ "vmulL_neon $dst, $src1, $src2\t# 2L" %} ins_encode %{ @@ -1175,8 +1176,75 @@ instruct vmulL_neon(vReg dst, vReg src1, vReg src2) %{ ins_pipe(pipe_slow); %} +// Specialization of vmulL_int_neon when both inputs are the same IR node +// (e.g. v * v). Avoids one redundant xtn and saves one temporary register. +instruct vmulL_int_neon_same(vReg dst, vReg src, vReg tmp) %{ + predicate(UseSVE == 0 && n->as_MulVL()->has_int_inputs() && + n->in(1) == n->in(2)); + match(Set dst (MulVL src src)); + effect(TEMP tmp); + format %{ "vmulL_int_neon_same $dst, $src, $src\t# 2L. KILL $tmp" %} + ins_encode %{ + uint length_in_bytes = Matcher::vector_length_in_bytes(this); + assert(length_in_bytes == 16, "must be"); + __ xtn($tmp$$FloatRegister, __ T2S, $src$$FloatRegister, __ T2D); + __ smullv($dst$$FloatRegister, __ T2S, $tmp$$FloatRegister, $tmp$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + +instruct vmulL_int_neon(vReg dst, vReg src1, vReg src2, vReg tmp1, vReg tmp2) %{ + predicate(UseSVE == 0 && n->as_MulVL()->has_int_inputs() && + n->in(1) != n->in(2)); + match(Set dst (MulVL src1 src2)); + effect(TEMP tmp1, TEMP tmp2); + format %{ "vmulL_int_neon $dst, $src1, $src2\t# 2L. KILL $tmp1, $tmp2" %} + ins_encode %{ + uint length_in_bytes = Matcher::vector_length_in_bytes(this); + assert(length_in_bytes == 16, "must be"); + __ xtn($tmp1$$FloatRegister, __ T2S, $src1$$FloatRegister, __ T2D); + __ xtn($tmp2$$FloatRegister, __ T2S, $src2$$FloatRegister, __ T2D); + __ smullv($dst$$FloatRegister, __ T2S, $tmp1$$FloatRegister, $tmp2$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + +// Specialization of vmulL_uint_neon when both inputs are the same IR node +// (e.g. v * v). Avoids one redundant xtn and saves one temporary register. +instruct vmulL_uint_neon_same(vReg dst, vReg src, vReg tmp) %{ + predicate(UseSVE == 0 && n->as_MulVL()->has_uint_inputs() && + n->in(1) == n->in(2)); + match(Set dst (MulVL src src)); + effect(TEMP tmp); + format %{ "vmulL_uint_neon_same $dst, $src, $src\t# 2L. KILL $tmp" %} + ins_encode %{ + uint length_in_bytes = Matcher::vector_length_in_bytes(this); + assert(length_in_bytes == 16, "must be"); + __ xtn($tmp$$FloatRegister, __ T2S, $src$$FloatRegister, __ T2D); + __ umullv($dst$$FloatRegister, __ T2S, $tmp$$FloatRegister, $tmp$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + +instruct vmulL_uint_neon(vReg dst, vReg src1, vReg src2, vReg tmp1, vReg tmp2) %{ + predicate(UseSVE == 0 && n->as_MulVL()->has_uint_inputs() && + n->in(1) != n->in(2)); + match(Set dst (MulVL src1 src2)); + effect(TEMP tmp1, TEMP tmp2); + format %{ "vmulL_uint_neon $dst, $src1, $src2\t# 2L. KILL $tmp1, $tmp2" %} + ins_encode %{ + uint length_in_bytes = Matcher::vector_length_in_bytes(this); + assert(length_in_bytes == 16, "must be"); + __ xtn($tmp1$$FloatRegister, __ T2S, $src1$$FloatRegister, __ T2D); + __ xtn($tmp2$$FloatRegister, __ T2S, $src2$$FloatRegister, __ T2D); + __ umullv($dst$$FloatRegister, __ T2S, $tmp1$$FloatRegister, $tmp2$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + instruct vmulL_sve(vReg dst_src1, vReg src2) %{ - predicate(UseSVE > 0); + predicate(UseSVE == 1 || (UseSVE == 2 && !n->as_MulVL()->has_int_inputs() && + !n->as_MulVL()->has_uint_inputs())); match(Set dst_src1 (MulVL dst_src1 src2)); format %{ "vmulL_sve $dst_src1, $dst_src1, $src2" %} ins_encode %{ @@ -1185,6 +1253,26 @@ instruct vmulL_sve(vReg dst_src1, vReg src2) %{ ins_pipe(pipe_slow); %} +instruct vmulL_int_sve2(vReg dst, vReg src1, vReg src2) %{ + predicate(UseSVE == 2 && n->as_MulVL()->has_int_inputs()); + match(Set dst (MulVL src1 src2)); + format %{ "vmulL_int_sve2 $dst, $src1, $src2" %} + ins_encode %{ + __ sve_smullb($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + +instruct vmulL_uint_sve2(vReg dst, vReg src1, vReg src2) %{ + predicate(UseSVE == 2 && n->as_MulVL()->has_uint_inputs()); + match(Set dst (MulVL src1 src2)); + format %{ "vmulL_uint_sve2 $dst, $src1, $src2" %} + ins_encode %{ + __ sve_umullb($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + // vector mul - floating-point instruct vmulHF(vReg dst, vReg src1, vReg src2) %{ diff --git a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 index 68c407bc9af..b749647ae1e 100644 --- a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 +++ b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 @@ -736,7 +736,8 @@ BINARY_OP_NEON_SVE_PAIRWISE(vmulI, MulVI, mulv, sve_mul, S) // vector mul - LONG instruct vmulL_neon(vReg dst, vReg src1, vReg src2) %{ - predicate(UseSVE == 0); + predicate(UseSVE == 0 && !n->as_MulVL()->has_int_inputs() && + !n->as_MulVL()->has_uint_inputs()); match(Set dst (MulVL src1 src2)); format %{ "vmulL_neon $dst, $src1, $src2\t# 2L" %} ins_encode %{ @@ -754,8 +755,47 @@ instruct vmulL_neon(vReg dst, vReg src1, vReg src2) %{ ins_pipe(pipe_slow); %} +dnl VMUL_L_NEON($1, $2 ) +dnl VMUL_L_NEON(kind, insn ) +define(`VMUL_L_NEON', `dnl +// Specialization of vmulL_$1_neon when both inputs are the same IR node +// (e.g. v * v). Avoids one redundant xtn and saves one temporary register. +instruct vmulL_$1_neon_same(vReg dst, vReg src, vReg tmp) %{ + predicate(UseSVE == 0 && n->as_MulVL()->has_$1_inputs() && + n->in(1) == n->in(2)); + match(Set dst (MulVL src src)); + effect(TEMP tmp); + format %{ "vmulL_$1_neon_same $dst, $src, $src\t# 2L. KILL $tmp" %} + ins_encode %{ + uint length_in_bytes = Matcher::vector_length_in_bytes(this); + assert(length_in_bytes == 16, "must be"); + __ xtn($tmp$$FloatRegister, __ T2S, $src$$FloatRegister, __ T2D); + __ $2($dst$$FloatRegister, __ T2S, $tmp$$FloatRegister, $tmp$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + +instruct vmulL_$1_neon(vReg dst, vReg src1, vReg src2, vReg tmp1, vReg tmp2) %{ + predicate(UseSVE == 0 && n->as_MulVL()->has_$1_inputs() && + n->in(1) != n->in(2)); + match(Set dst (MulVL src1 src2)); + effect(TEMP tmp1, TEMP tmp2); + format %{ "vmulL_$1_neon $dst, $src1, $src2\t# 2L. KILL $tmp1, $tmp2" %} + ins_encode %{ + uint length_in_bytes = Matcher::vector_length_in_bytes(this); + assert(length_in_bytes == 16, "must be"); + __ xtn($tmp1$$FloatRegister, __ T2S, $src1$$FloatRegister, __ T2D); + __ xtn($tmp2$$FloatRegister, __ T2S, $src2$$FloatRegister, __ T2D); + __ $2($dst$$FloatRegister, __ T2S, $tmp1$$FloatRegister, $tmp2$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} +')dnl +VMUL_L_NEON(int, smullv) +VMUL_L_NEON(uint, umullv) instruct vmulL_sve(vReg dst_src1, vReg src2) %{ - predicate(UseSVE > 0); + predicate(UseSVE == 1 || (UseSVE == 2 && !n->as_MulVL()->has_int_inputs() && + !n->as_MulVL()->has_uint_inputs())); match(Set dst_src1 (MulVL dst_src1 src2)); format %{ "vmulL_sve $dst_src1, $dst_src1, $src2" %} ins_encode %{ @@ -764,6 +804,21 @@ instruct vmulL_sve(vReg dst_src1, vReg src2) %{ ins_pipe(pipe_slow); %} +dnl VMUL_L_SVE2($1, $2 ) +dnl VMUL_L_SVE2(kind, sve2_insn ) +define(`VMUL_L_SVE2', `dnl +instruct vmulL_$1_sve2(vReg dst, vReg src1, vReg src2) %{ + predicate(UseSVE == 2 && n->as_MulVL()->has_$1_inputs()); + match(Set dst (MulVL src1 src2)); + format %{ "vmulL_$1_sve2 $dst, $src1, $src2" %} + ins_encode %{ + __ $2($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} +')dnl +VMUL_L_SVE2(int, sve_smullb) +VMUL_L_SVE2(uint, sve_umullb) // vector mul - floating-point BINARY_OP(vmulHF, MulVHF, fmul, sve_fmul, H) BINARY_OP(vmulF, MulVF, fmul, sve_fmul, S) diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp index ae2b9ac9bf7..a81213c5ae4 100644 --- a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp @@ -4331,6 +4331,22 @@ public: INSN(sve_bsl, 0b001, 0b1); // Bitwise select #undef INSN +// SVE2 widening integer multiply - vector +#define INSN(NAME, is_unsigned, is_top) \ + void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, FloatRegister Zm) { \ + starti; \ + assert(T != B && T != Q, "invalid size"); \ + int op = 0b011100 | (is_unsigned ? 0b10 : 0) | (is_top ? 0b1 : 0); \ + f(0b01000101, 31, 24), f(T, 23, 22), f(0, 21), rf(Zm, 16); \ + f(op, 15, 10), rf(Zn, 5), rf(Zd, 0); \ + } + + INSN(sve_umullb, /* is_unsigned */ true, /* is_top */ false); // Unsigned widening multiply of bottom elements + INSN(sve_umullt, /* is_unsigned */ true, /* is_top */ true ); // Unsigned widening multiply of top elements + INSN(sve_smullb, /* is_unsigned */ false, /* is_top */ false); // Signed widening multiply of bottom elements + INSN(sve_smullt, /* is_unsigned */ false, /* is_top */ true ); // Signed widening multiply of top elements +#undef INSN + // SVE2 saturating operations - predicate #define INSN(NAME, op1, op2) \ void NAME(FloatRegister Zdn, SIMD_RegVariant T, PRegister Pg, FloatRegister Znm) { \ diff --git a/test/hotspot/gtest/aarch64/aarch64-asmtest.py b/test/hotspot/gtest/aarch64/aarch64-asmtest.py index 04088bb0dc8..b5386d47a73 100644 --- a/test/hotspot/gtest/aarch64/aarch64-asmtest.py +++ b/test/hotspot/gtest/aarch64/aarch64-asmtest.py @@ -2163,6 +2163,10 @@ generate(SpecialCases, [["ccmn", "__ ccmn(zr, zr, 3u, Assembler::LE);", # SVE2 instructions ["histcnt", "__ sve_histcnt(z16, __ S, p0, z16, z16);", "histcnt\tz16.s, p0/z, z16.s, z16.s"], ["histcnt", "__ sve_histcnt(z17, __ D, p0, z17, z17);", "histcnt\tz17.d, p0/z, z17.d, z17.d"], + ["umullb", "__ sve_umullb(z16, __ H, z17, z18);", "umullb\tz16.h, z17.b, z18.b"], + ["umullt", "__ sve_umullt(z19, __ S, z20, z21);", "umullt\tz19.s, z20.h, z21.h"], + ["smullb", "__ sve_smullb(z22, __ D, z23, z24);", "smullb\tz22.d, z23.s, z24.s"], + ["smullt", "__ sve_smullt(z25, __ H, z26, z27);", "smullt\tz25.h, z26.b, z27.b"], ]) print "\n// FloatImmediateOp" diff --git a/test/hotspot/gtest/aarch64/asmtest.out.h b/test/hotspot/gtest/aarch64/asmtest.out.h index bad9825af9b..95832a1faf6 100644 --- a/test/hotspot/gtest/aarch64/asmtest.out.h +++ b/test/hotspot/gtest/aarch64/asmtest.out.h @@ -1180,6 +1180,10 @@ __ sve_splice(z0, __ D, p0, z1); // splice z0.d, p0, z0.d, z1.d __ sve_histcnt(z16, __ S, p0, z16, z16); // histcnt z16.s, p0/z, z16.s, z16.s __ sve_histcnt(z17, __ D, p0, z17, z17); // histcnt z17.d, p0/z, z17.d, z17.d + __ sve_umullb(z16, __ H, z17, z18); // umullb z16.h, z17.b, z18.b + __ sve_umullt(z19, __ S, z20, z21); // umullt z19.s, z20.h, z21.h + __ sve_smullb(z22, __ D, z23, z24); // smullb z22.d, z23.s, z24.s + __ sve_smullt(z25, __ H, z26, z27); // smullt z25.h, z26.b, z27.b // FloatImmediateOp __ fmovd(v0, 2.0); // fmov d0, #2.0 @@ -1470,30 +1474,30 @@ 0x9101a1a0, 0xb10a5cc8, 0xd10810aa, 0xf10fd061, 0x120cb166, 0x321764bc, 0x52174681, 0x720c0227, 0x9241018e, 0xb25a2969, 0xd278b411, 0xf26aad01, - 0x14000000, 0x17ffffd7, 0x140004cc, 0x94000000, - 0x97ffffd4, 0x940004c9, 0x3400000a, 0x34fffa2a, - 0x340098ca, 0x35000008, 0x35fff9c8, 0x35009868, - 0xb400000b, 0xb4fff96b, 0xb400980b, 0xb500001d, - 0xb5fff91d, 0xb50097bd, 0x10000013, 0x10fff8b3, - 0x10009753, 0x90000013, 0x36300016, 0x3637f836, - 0x363096d6, 0x3758000c, 0x375ff7cc, 0x3758966c, + 0x14000000, 0x17ffffd7, 0x140004d0, 0x94000000, + 0x97ffffd4, 0x940004cd, 0x3400000a, 0x34fffa2a, + 0x3400994a, 0x35000008, 0x35fff9c8, 0x350098e8, + 0xb400000b, 0xb4fff96b, 0xb400988b, 0xb500001d, + 0xb5fff91d, 0xb500983d, 0x10000013, 0x10fff8b3, + 0x100097d3, 0x90000013, 0x36300016, 0x3637f836, + 0x36309756, 0x3758000c, 0x375ff7cc, 0x375896ec, 0x128313a0, 0x528a32c7, 0x7289173b, 0x92ab3acc, 0xd2a0bf94, 0xf2c285e8, 0x9358722f, 0x330e652f, 0x53067f3b, 0x93577c53, 0xb34a1aac, 0xd35a4016, 0x13946c63, 0x93c3dbc8, 0x54000000, 0x54fff5a0, - 0x54009440, 0x54000001, 0x54fff541, 0x540093e1, - 0x54000002, 0x54fff4e2, 0x54009382, 0x54000002, - 0x54fff482, 0x54009322, 0x54000003, 0x54fff423, - 0x540092c3, 0x54000003, 0x54fff3c3, 0x54009263, - 0x54000004, 0x54fff364, 0x54009204, 0x54000005, - 0x54fff305, 0x540091a5, 0x54000006, 0x54fff2a6, - 0x54009146, 0x54000007, 0x54fff247, 0x540090e7, - 0x54000008, 0x54fff1e8, 0x54009088, 0x54000009, - 0x54fff189, 0x54009029, 0x5400000a, 0x54fff12a, - 0x54008fca, 0x5400000b, 0x54fff0cb, 0x54008f6b, - 0x5400000c, 0x54fff06c, 0x54008f0c, 0x5400000d, - 0x54fff00d, 0x54008ead, 0x5400000e, 0x54ffefae, - 0x54008e4e, 0x5400000f, 0x54ffef4f, 0x54008def, + 0x540094c0, 0x54000001, 0x54fff541, 0x54009461, + 0x54000002, 0x54fff4e2, 0x54009402, 0x54000002, + 0x54fff482, 0x540093a2, 0x54000003, 0x54fff423, + 0x54009343, 0x54000003, 0x54fff3c3, 0x540092e3, + 0x54000004, 0x54fff364, 0x54009284, 0x54000005, + 0x54fff305, 0x54009225, 0x54000006, 0x54fff2a6, + 0x540091c6, 0x54000007, 0x54fff247, 0x54009167, + 0x54000008, 0x54fff1e8, 0x54009108, 0x54000009, + 0x54fff189, 0x540090a9, 0x5400000a, 0x54fff12a, + 0x5400904a, 0x5400000b, 0x54fff0cb, 0x54008feb, + 0x5400000c, 0x54fff06c, 0x54008f8c, 0x5400000d, + 0x54fff00d, 0x54008f2d, 0x5400000e, 0x54ffefae, + 0x54008ece, 0x5400000f, 0x54ffef4f, 0x54008e6f, 0xd40658e1, 0xd4014d22, 0xd4046543, 0xd4273f60, 0xd44cad80, 0xd503201f, 0xd503203f, 0xd503205f, 0xd503209f, 0xd50320bf, 0xd503219f, 0xd50323bf, @@ -1536,7 +1540,7 @@ 0x39598921, 0x795d3077, 0x399d0675, 0x7998d8f3, 0x79dbd02a, 0xb99d068a, 0xfd5d11a0, 0xbd58d76b, 0xfd1ac72d, 0xbd1d9c14, 0x5800001a, 0x18ffda33, - 0xf8991100, 0xd80078a0, 0xf8a758e0, 0xf9989d80, + 0xf8991100, 0xd8007920, 0xf8a758e0, 0xf9989d80, 0x1a0b0298, 0x3a1c01a0, 0x5a0400ea, 0x7a02020f, 0x9a1d028c, 0xba0e01ad, 0xda140186, 0xfa19022c, 0x0b2b877e, 0x2b21c8ee, 0xcb3ba47d, 0x6b3ae9a0, @@ -1719,7 +1723,8 @@ 0x0420bc31, 0x05271e11, 0x6545e891, 0x6585e891, 0x65c5e891, 0x6545c891, 0x6585c891, 0x65c5c891, 0x052c8020, 0x056c8020, 0x05ac8020, 0x05ec8020, - 0x45b0c210, 0x45f1c231, 0x1e601000, 0x1e603000, + 0x45b0c210, 0x45f1c231, 0x45527a30, 0x45957e93, + 0x45d872f6, 0x455b7759, 0x1e601000, 0x1e603000, 0x1e621000, 0x1e623000, 0x1e641000, 0x1e643000, 0x1e661000, 0x1e663000, 0x1e681000, 0x1e683000, 0x1e6a1000, 0x1e6a3000, 0x1e6c1000, 0x1e6c3000, diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java index a76853016d9..249e73fa54b 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java @@ -2863,6 +2863,26 @@ public class IRNode { machOnlyNameRegex(X86_VMULDQ_REG, "vmuldq_reg"); } + public static final String AARCH64_VMULL_UINT_SVE2 = PREFIX + "AARCH64_VMULL_UINT_SVE2" + POSTFIX; + static { + machOnlyNameRegex(AARCH64_VMULL_UINT_SVE2, "vmulL_uint_sve2"); + } + + public static final String AARCH64_VMULL_INT_SVE2 = PREFIX + "AARCH64_VMULL_INT_SVE2" + POSTFIX; + static { + machOnlyNameRegex(AARCH64_VMULL_INT_SVE2, "vmulL_int_sve2"); + } + + public static final String AARCH64_VMULL_UINT_NEON = PREFIX + "AARCH64_VMULL_UINT_NEON" + POSTFIX; + static { + machOnlyNameRegex(AARCH64_VMULL_UINT_NEON, "vmulL_uint_neon"); + } + + public static final String AARCH64_VMULL_INT_NEON = PREFIX + "AARCH64_VMULL_INT_NEON" + POSTFIX; + static { + machOnlyNameRegex(AARCH64_VMULL_INT_NEON, "vmulL_int_neon"); + } + public static final String X86_SCONV_D2I = PREFIX + "X86_SCONV_D2I" + POSTFIX; static { machOnlyNameRegex(X86_SCONV_D2I, "convD2I_reg_reg"); diff --git a/test/hotspot/jtreg/compiler/vectorapi/TestVectorMulLongToSignedUnsignedInt.java b/test/hotspot/jtreg/compiler/vectorapi/TestVectorMulLongToSignedUnsignedInt.java index d5b4771d3e1..e7745b5e88c 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/TestVectorMulLongToSignedUnsignedInt.java +++ b/test/hotspot/jtreg/compiler/vectorapi/TestVectorMulLongToSignedUnsignedInt.java @@ -34,7 +34,7 @@ import compiler.lib.verify.*; /* * @test - * @bug 8384963 + * @bug 8384963 8383905 * @key randomness * @summary C2: Incorrect uint constant match mishandles negative values in vectors * @modules jdk.incubator.vector @@ -87,8 +87,18 @@ public class TestVectorMulLongToSignedUnsignedInt { // Case 1: Negative mask (-2L = 0xFFFF_FFFF_FFFF_FFFE). @Test - @IR(counts = {IRNode.AND_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) - @IR(failOn = {IRNode.X86_VMULUDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"}) + @IR(counts = {IRNode.AND_VL, " >0 ", + IRNode.MUL_VL, " >0 "}, + applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"}) + @IR(failOn = {IRNode.X86_VMULUDQ_REG}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"avx", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_UINT_SVE2}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"sve2", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_UINT_NEON}, + phase = CompilePhase.MATCHING, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void testNegativeMask() { LongVector v1 = LongVector.fromArray(SPECIES, src1, 0); LongVector v2 = LongVector.fromArray(SPECIES, src2, 0); @@ -107,8 +117,16 @@ public class TestVectorMulLongToSignedUnsignedInt { // Case 3: Mask = 0x1_0000_0000L (bit 32 set, exceeds uint range). @Test - @IR(counts = {IRNode.AND_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) - @IR(failOn = {IRNode.X86_VMULUDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"}) + @IR(counts = {IRNode.AND_VL, " >0 ", + IRNode.MUL_VL, " >0 "}, + applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"}) + @IR(failOn = {IRNode.X86_VMULUDQ_REG}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"avx", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_UINT_SVE2}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"sve2", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_UINT_NEON}, phase = CompilePhase.MATCHING, applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void testBit32SetMask() { LongVector v1 = LongVector.fromArray(SPECIES, src1, 0); LongVector v2 = LongVector.fromArray(SPECIES, src2, 0); @@ -127,8 +145,18 @@ public class TestVectorMulLongToSignedUnsignedInt { // Case 4: Mask = Long.MIN_VALUE (0x8000_0000_0000_0000). @Test - @IR(counts = {IRNode.AND_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) - @IR(failOn = {IRNode.X86_VMULUDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"}) + @IR(counts = {IRNode.AND_VL, " >0 ", + IRNode.MUL_VL, " >0 "}, + applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"}) + @IR(failOn = {IRNode.X86_VMULUDQ_REG}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"avx", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_UINT_SVE2}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"sve2", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_UINT_NEON}, + phase = CompilePhase.MATCHING, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void testMinValueMask() { LongVector v1 = LongVector.fromArray(SPECIES, src1, 0); LongVector v2 = LongVector.fromArray(SPECIES, src2, 0); @@ -147,8 +175,17 @@ public class TestVectorMulLongToSignedUnsignedInt { // Case 5: Mask = 0xFFFF_FFFFL (exactly uint max, boundary valid case). @Test - @IR(counts = {IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) - @IR(counts = {IRNode.X86_VMULUDQ_REG, " >0 "}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"}) + @IR(counts = {IRNode.MUL_VL, " >0 "}, + applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"}) + @IR(counts = {IRNode.X86_VMULUDQ_REG, " >0 "}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"avx", "true"}) + @IR(counts = {IRNode.AARCH64_VMULL_UINT_SVE2, " >0 "}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"sve2", "true"}) + @IR(counts = {IRNode.AARCH64_VMULL_UINT_NEON, " >0 "}, + phase = CompilePhase.MATCHING, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void testUintMaxMask() { LongVector v1 = LongVector.fromArray(SPECIES, src1, 0); LongVector v2 = LongVector.fromArray(SPECIES, src2, 0); @@ -167,8 +204,17 @@ public class TestVectorMulLongToSignedUnsignedInt { // Case 6: Small mask (0xFFFFL), clearly fits in uint. @Test - @IR(counts = {IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) - @IR(counts = {IRNode.X86_VMULUDQ_REG, " >0 "}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"}) + @IR(counts = {IRNode.MUL_VL, " >0 "}, + applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"}) + @IR(counts = {IRNode.X86_VMULUDQ_REG, " >0 "}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"avx", "true"}) + @IR(counts = {IRNode.AARCH64_VMULL_UINT_SVE2, " >0 "}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"sve2", "true"}) + @IR(counts = {IRNode.AARCH64_VMULL_UINT_NEON, " >0 "}, + phase = CompilePhase.MATCHING, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void testSmallMask() { LongVector v1 = LongVector.fromArray(SPECIES, src1, 0); LongVector v2 = LongVector.fromArray(SPECIES, src2, 0); @@ -187,8 +233,18 @@ public class TestVectorMulLongToSignedUnsignedInt { // Case 7: URShift by 32 clears upper doubleword. @Test - @IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.URSHIFT_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) - @IR(counts = {IRNode.X86_VMULUDQ_REG, " >0 "}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"}) + @IR(counts = {IRNode.MUL_VL, " >0 ", + IRNode.URSHIFT_VL, " >0 "}, + applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"}) + @IR(counts = {IRNode.X86_VMULUDQ_REG, " >0 "}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"avx", "true"}) + @IR(counts = {IRNode.AARCH64_VMULL_UINT_SVE2, " >0 "}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"sve2", "true"}) + @IR(counts = {IRNode.AARCH64_VMULL_UINT_NEON, " >0 "}, + phase = CompilePhase.MATCHING, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void testURShift32() { LongVector v1 = LongVector.fromArray(SPECIES, src1, 0); LongVector v2 = LongVector.fromArray(SPECIES, src2, 0); @@ -207,8 +263,18 @@ public class TestVectorMulLongToSignedUnsignedInt { // Case 8: Asymmetric — one input valid uint mask, other negative mask. @Test - @IR(counts = {IRNode.AND_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) - @IR(failOn = {IRNode.X86_VMULUDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"}) + @IR(counts = {IRNode.AND_VL, " >0 ", + IRNode.MUL_VL, " >0 "}, + applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"}) + @IR(failOn = {IRNode.X86_VMULUDQ_REG}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"avx", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_UINT_SVE2}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"sve2", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_UINT_NEON}, + phase = CompilePhase.MATCHING, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void testAsymmetricMask() { LongVector v1 = LongVector.fromArray(SPECIES, src1, 0); LongVector v2 = LongVector.fromArray(SPECIES, src2, 0); @@ -228,8 +294,19 @@ public class TestVectorMulLongToSignedUnsignedInt { // Case 9: Mixed — one input URShift (valid), other negative mask (invalid). // Note: -2L is used (not -1L) since AND with -1L is identity and gets folded. @Test - @IR(counts = {IRNode.URSHIFT_VL, " >0 ", IRNode.AND_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) - @IR(failOn = {IRNode.X86_VMULUDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"}) + @IR(counts = {IRNode.URSHIFT_VL, " >0 ", + IRNode.AND_VL, " >0 ", + IRNode.MUL_VL, " >0 "}, + applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"}) + @IR(failOn = {IRNode.X86_VMULUDQ_REG}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"avx", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_UINT_SVE2}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"sve2", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_UINT_NEON}, + phase = CompilePhase.MATCHING, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void testMixedURShiftAndNegMask() { LongVector v1 = LongVector.fromArray(SPECIES, src1, 0); LongVector v2 = LongVector.fromArray(SPECIES, src2, 0); @@ -248,8 +325,18 @@ public class TestVectorMulLongToSignedUnsignedInt { // Case 10: Predicated AndV (uint path). Inactive lanes preserves destination with non-zero upper 32 bits. @Test - @IR(counts = {IRNode.AND_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx512f", "true"}) - @IR(failOn = {IRNode.X86_VMULUDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx512f", "true"}) + @IR(counts = {IRNode.AND_VL, " >0 ", + IRNode.MUL_VL, " >0 "}, + applyIfCPUFeatureOr = {"avx512f", "true", "sve", "true"}) + @IR(failOn = {IRNode.X86_VMULUDQ_REG}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"avx512f", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_UINT_SVE2}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"sve2", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_UINT_NEON}, + phase = CompilePhase.MATCHING, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void testPredicatedAndMask() { LongVector v1 = LongVector.fromArray(SPECIES, src1, 0); LongVector v2 = LongVector.fromArray(SPECIES, src2, 0); @@ -270,8 +357,18 @@ public class TestVectorMulLongToSignedUnsignedInt { // Case 11: Predicated URShiftVL by 32 (uint path). Inactive lanes preserves destination with non-zero upper 32 bits. @Test - @IR(counts = {IRNode.URSHIFT_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx512f", "true"}) - @IR(failOn = {IRNode.X86_VMULUDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx512f", "true"}) + @IR(counts = {IRNode.URSHIFT_VL, " >0 ", + IRNode.MUL_VL, " >0 "}, + applyIfCPUFeatureOr = {"avx512f", "true", "sve", "true"}) + @IR(failOn = {IRNode.X86_VMULUDQ_REG}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"avx512f", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_UINT_SVE2}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"sve2", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_UINT_NEON}, + phase = CompilePhase.MATCHING, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void testPredicatedURShift32() { LongVector v1 = LongVector.fromArray(SPECIES, src1, 0); LongVector v2 = LongVector.fromArray(SPECIES, src2, 0); @@ -292,8 +389,18 @@ public class TestVectorMulLongToSignedUnsignedInt { // Case 12: Predicated RShiftVL (arithmetic) by 32. @Test - @IR(counts = {IRNode.RSHIFT_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx512f", "true"}) - @IR(failOn = {IRNode.X86_VMULDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx512f", "true"}) + @IR(counts = {IRNode.RSHIFT_VL, " >0 ", + IRNode.MUL_VL, " >0 "}, + applyIfCPUFeatureOr = {"avx512f", "true", "sve", "true"}) + @IR(failOn = {IRNode.X86_VMULDQ_REG}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"avx512f", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_INT_SVE2}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"sve2", "true"}) + @IR(failOn = {IRNode.AARCH64_VMULL_INT_NEON}, + phase = CompilePhase.MATCHING, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void testPredicatedRShift32() { LongVector v1 = LongVector.fromArray(SPECIES, src1, 0); LongVector v2 = LongVector.fromArray(SPECIES, src2, 0); diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorMultiplyOpt.java b/test/hotspot/jtreg/compiler/vectorapi/VectorMultiplyOpt.java index a8394f41f8a..68ac9249ebf 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorMultiplyOpt.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorMultiplyOpt.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,9 +31,9 @@ import java.lang.reflect.Array; /** * @test - * @bug 8341137 + * @bug 8341137 8383905 * @key randomness - * @summary Optimize long vector multiplication using x86 VPMUL[U]DQ instruction. + * @summary Optimize long vector multiplication. * @modules jdk.incubator.vector * @library /test/lib / * @run driver compiler.vectorapi.VectorMultiplyOpt @@ -80,7 +80,7 @@ public class VectorMultiplyOpt { public static void main(String[] args) { TestFramework testFramework = new TestFramework(); - testFramework.setDefaultWarmup(5000) + testFramework.setDefaultWarmup(10000) .addFlags("--add-modules=jdk.incubator.vector") .start(); System.out.println("PASSED"); @@ -109,7 +109,12 @@ public class VectorMultiplyOpt { @Test @IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.AND_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @IR(counts = {"vmuludq", " >0 "}, phase = CompilePhase.FINAL_CODE, applyIfCPUFeature = {"avx", "true"}) - @Warmup(value = 10000) + @IR(counts = {"vmulL_uint_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeature = {"sve2", "true"}) + @IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"}) + @IR(counts = {"vmulL_uint_neon", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void test_pattern1() { int i = 0; for (; i < LSP.loopBound(res.length); i += LSP.length()) { @@ -132,7 +137,12 @@ public class VectorMultiplyOpt { @Test @IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.AND_VL, " >0 ", IRNode.URSHIFT_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @IR(counts = {"vmuludq", " >0 "}, phase = CompilePhase.FINAL_CODE, applyIfCPUFeature = {"avx", "true"}) - @Warmup(value = 10000) + @IR(counts = {"vmulL_uint_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeature = {"sve2", "true"}) + @IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"}) + @IR(counts = {"vmulL_uint_neon", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void test_pattern2() { int i = 0; for (; i < LSP.loopBound(res.length); i += LSP.length()) { @@ -155,7 +165,12 @@ public class VectorMultiplyOpt { @Test @IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.URSHIFT_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @IR(counts = {"vmuludq", " >0 "}, phase = CompilePhase.FINAL_CODE, applyIfCPUFeature = {"avx", "true"}) - @Warmup(value = 10000) + @IR(counts = {"vmulL_uint_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeature = {"sve2", "true"}) + @IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"}) + @IR(counts = {"vmulL_uint_neon", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void test_pattern3() { int i = 0; for (; i < LSP.loopBound(res.length); i += LSP.length()) { @@ -178,7 +193,12 @@ public class VectorMultiplyOpt { @Test @IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.URSHIFT_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @IR(counts = {"vmuludq", " >0 "}, applyIfCPUFeature = {"avx", "true"}, phase = CompilePhase.FINAL_CODE) - @Warmup(value = 10000) + @IR(counts = {"vmulL_uint_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeature = {"sve2", "true"}) + @IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"}) + @IR(counts = {"vmulL_uint_neon", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void test_pattern4() { int i = 0; for (; i < LSP.loopBound(res.length); i += LSP.length()) { @@ -201,7 +221,12 @@ public class VectorMultiplyOpt { @Test @IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.VECTOR_CAST_I2L, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @IR(counts = {"vmuldq", " >0 "}, applyIfCPUFeature = {"avx", "true"}, phase = CompilePhase.FINAL_CODE) - @Warmup(value = 10000) + @IR(counts = {"vmulL_int_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeature = {"sve2", "true"}) + @IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"}) + @IR(counts = {"vmulL_int_neon", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void test_pattern5() { int i = 0; for (; i < LSP.loopBound(res.length); i += LSP.length()) { @@ -227,7 +252,12 @@ public class VectorMultiplyOpt { @Test @IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.RSHIFT_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @IR(counts = {"vmuldq", " >0 "}, applyIfCPUFeature = {"avx", "true"}, phase = CompilePhase.FINAL_CODE) - @Warmup(value = 10000) + @IR(counts = {"vmulL_int_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeature = {"sve2", "true"}) + @IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"}) + @IR(counts = {"vmulL_int_neon", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) public static void test_pattern6() { int i = 0; for (; i < LSP.loopBound(res.length); i += LSP.length()) { @@ -247,4 +277,61 @@ public class VectorMultiplyOpt { validate("pattern6 ", res, lsrc1, lsrc2, (l1, l2) -> (l1 >> shift5) * (l2 >> shift5)); } + // Same-operand multiplication (v * v) where v has zero-extended high bits. + // On NEON this should map to the dedicated rule that emits a single xtn. + @Test + @IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.AND_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) + @IR(counts = {"vmuludq", " >0 "}, phase = CompilePhase.FINAL_CODE, applyIfCPUFeature = {"avx", "true"}) + @IR(counts = {"vmulL_uint_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeature = {"sve2", "true"}) + @IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"}) + @IR(counts = {"vmulL_uint_neon_same", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) + public static void test_pattern7() { + int i = 0; + for (; i < LSP.loopBound(res.length); i += LSP.length()) { + LongVector vsrc = LongVector.fromArray(LSP, lsrc1, i) + .lanewise(VectorOperators.AND, mask1); + vsrc.lanewise(VectorOperators.MUL, vsrc).intoArray(res, i); + } + for (; i < res.length; i++) { + long x = lsrc1[i] & mask1; + res[i] = x * x; + } + } + + @Check(test = "test_pattern7") + public void test_pattern7_validate() { + validate("pattern7 ", res, lsrc1, lsrc1, (l1, l2) -> { long x = l1 & mask1; return x * x; }); + } + + // Same-operand multiplication (v * v) where v has sign-extended high bits. + // On NEON this should map to the dedicated rule that emits a single xtn. + @Test + @IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.VECTOR_CAST_I2L, " >0 "}, applyIfCPUFeature = {"avx", "true"}) + @IR(counts = {"vmuldq", " >0 "}, applyIfCPUFeature = {"avx", "true"}, phase = CompilePhase.FINAL_CODE) + @IR(counts = {"vmulL_int_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeature = {"sve2", "true"}) + @IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"}) + @IR(counts = {"vmulL_int_neon_same", " >0 "}, phase = CompilePhase.FINAL_CODE, + applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"}) + public static void test_pattern8() { + int i = 0; + for (; i < LSP.loopBound(res.length); i += LSP.length()) { + LongVector vsrc = IntVector.fromArray(ISP, isrc1, i) + .convert(VectorOperators.I2L, 0) + .reinterpretAsLongs(); + vsrc.lanewise(VectorOperators.MUL, vsrc).intoArray(res, i); + } + for (; i < res.length; i++) { + res[i] = Math.multiplyFull(isrc1[i], isrc1[i]); + } + } + + @Check(test = "test_pattern8") + public void test_pattern8_validate() { + validate("pattern8 ", res, isrc1, isrc1, (i1, i2) -> Math.multiplyFull((int)i1, (int)i1)); + } }