mirror of
https://github.com/openjdk/jdk.git
synced 2026-07-08 01:59:15 +00:00
8383905: AArch64: Improve code generation for long vector multiply
Reviewed-by: aph, xgong
This commit is contained in:
Eric Fang
Jatin Bhateja
parent
92b0565b00
commit
6e7e6f0bbf
@ -1157,7 +1157,8 @@ instruct vmulI_sve(vReg dst_src1, vReg src2) %{
|
||||
// vector mul - LONG
|
||||
|
||||
instruct vmulL_neon(vReg dst, vReg src1, vReg src2) %{
|
||||
predicate(UseSVE == 0);
|
||||
predicate(UseSVE == 0 && !n->as_MulVL()->has_int_inputs() &&
|
||||
!n->as_MulVL()->has_uint_inputs());
|
||||
match(Set dst (MulVL src1 src2));
|
||||
format %{ "vmulL_neon $dst, $src1, $src2\t# 2L" %}
|
||||
ins_encode %{
|
||||
@ -1175,8 +1176,75 @@ instruct vmulL_neon(vReg dst, vReg src1, vReg src2) %{
|
||||
ins_pipe(pipe_slow);
|
||||
%}
|
||||
|
||||
// Specialization of vmulL_int_neon when both inputs are the same IR node
|
||||
// (e.g. v * v). Avoids one redundant xtn and saves one temporary register.
|
||||
instruct vmulL_int_neon_same(vReg dst, vReg src, vReg tmp) %{
|
||||
predicate(UseSVE == 0 && n->as_MulVL()->has_int_inputs() &&
|
||||
n->in(1) == n->in(2));
|
||||
match(Set dst (MulVL src src));
|
||||
effect(TEMP tmp);
|
||||
format %{ "vmulL_int_neon_same $dst, $src, $src\t# 2L. KILL $tmp" %}
|
||||
ins_encode %{
|
||||
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
||||
assert(length_in_bytes == 16, "must be");
|
||||
__ xtn($tmp$$FloatRegister, __ T2S, $src$$FloatRegister, __ T2D);
|
||||
__ smullv($dst$$FloatRegister, __ T2S, $tmp$$FloatRegister, $tmp$$FloatRegister);
|
||||
%}
|
||||
ins_pipe(pipe_slow);
|
||||
%}
|
||||
|
||||
instruct vmulL_int_neon(vReg dst, vReg src1, vReg src2, vReg tmp1, vReg tmp2) %{
|
||||
predicate(UseSVE == 0 && n->as_MulVL()->has_int_inputs() &&
|
||||
n->in(1) != n->in(2));
|
||||
match(Set dst (MulVL src1 src2));
|
||||
effect(TEMP tmp1, TEMP tmp2);
|
||||
format %{ "vmulL_int_neon $dst, $src1, $src2\t# 2L. KILL $tmp1, $tmp2" %}
|
||||
ins_encode %{
|
||||
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
||||
assert(length_in_bytes == 16, "must be");
|
||||
__ xtn($tmp1$$FloatRegister, __ T2S, $src1$$FloatRegister, __ T2D);
|
||||
__ xtn($tmp2$$FloatRegister, __ T2S, $src2$$FloatRegister, __ T2D);
|
||||
__ smullv($dst$$FloatRegister, __ T2S, $tmp1$$FloatRegister, $tmp2$$FloatRegister);
|
||||
%}
|
||||
ins_pipe(pipe_slow);
|
||||
%}
|
||||
|
||||
// Specialization of vmulL_uint_neon when both inputs are the same IR node
|
||||
// (e.g. v * v). Avoids one redundant xtn and saves one temporary register.
|
||||
instruct vmulL_uint_neon_same(vReg dst, vReg src, vReg tmp) %{
|
||||
predicate(UseSVE == 0 && n->as_MulVL()->has_uint_inputs() &&
|
||||
n->in(1) == n->in(2));
|
||||
match(Set dst (MulVL src src));
|
||||
effect(TEMP tmp);
|
||||
format %{ "vmulL_uint_neon_same $dst, $src, $src\t# 2L. KILL $tmp" %}
|
||||
ins_encode %{
|
||||
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
||||
assert(length_in_bytes == 16, "must be");
|
||||
__ xtn($tmp$$FloatRegister, __ T2S, $src$$FloatRegister, __ T2D);
|
||||
__ umullv($dst$$FloatRegister, __ T2S, $tmp$$FloatRegister, $tmp$$FloatRegister);
|
||||
%}
|
||||
ins_pipe(pipe_slow);
|
||||
%}
|
||||
|
||||
instruct vmulL_uint_neon(vReg dst, vReg src1, vReg src2, vReg tmp1, vReg tmp2) %{
|
||||
predicate(UseSVE == 0 && n->as_MulVL()->has_uint_inputs() &&
|
||||
n->in(1) != n->in(2));
|
||||
match(Set dst (MulVL src1 src2));
|
||||
effect(TEMP tmp1, TEMP tmp2);
|
||||
format %{ "vmulL_uint_neon $dst, $src1, $src2\t# 2L. KILL $tmp1, $tmp2" %}
|
||||
ins_encode %{
|
||||
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
||||
assert(length_in_bytes == 16, "must be");
|
||||
__ xtn($tmp1$$FloatRegister, __ T2S, $src1$$FloatRegister, __ T2D);
|
||||
__ xtn($tmp2$$FloatRegister, __ T2S, $src2$$FloatRegister, __ T2D);
|
||||
__ umullv($dst$$FloatRegister, __ T2S, $tmp1$$FloatRegister, $tmp2$$FloatRegister);
|
||||
%}
|
||||
ins_pipe(pipe_slow);
|
||||
%}
|
||||
|
||||
instruct vmulL_sve(vReg dst_src1, vReg src2) %{
|
||||
predicate(UseSVE > 0);
|
||||
predicate(UseSVE == 1 || (UseSVE == 2 && !n->as_MulVL()->has_int_inputs() &&
|
||||
!n->as_MulVL()->has_uint_inputs()));
|
||||
match(Set dst_src1 (MulVL dst_src1 src2));
|
||||
format %{ "vmulL_sve $dst_src1, $dst_src1, $src2" %}
|
||||
ins_encode %{
|
||||
@ -1185,6 +1253,26 @@ instruct vmulL_sve(vReg dst_src1, vReg src2) %{
|
||||
ins_pipe(pipe_slow);
|
||||
%}
|
||||
|
||||
instruct vmulL_int_sve2(vReg dst, vReg src1, vReg src2) %{
|
||||
predicate(UseSVE == 2 && n->as_MulVL()->has_int_inputs());
|
||||
match(Set dst (MulVL src1 src2));
|
||||
format %{ "vmulL_int_sve2 $dst, $src1, $src2" %}
|
||||
ins_encode %{
|
||||
__ sve_smullb($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister);
|
||||
%}
|
||||
ins_pipe(pipe_slow);
|
||||
%}
|
||||
|
||||
instruct vmulL_uint_sve2(vReg dst, vReg src1, vReg src2) %{
|
||||
predicate(UseSVE == 2 && n->as_MulVL()->has_uint_inputs());
|
||||
match(Set dst (MulVL src1 src2));
|
||||
format %{ "vmulL_uint_sve2 $dst, $src1, $src2" %}
|
||||
ins_encode %{
|
||||
__ sve_umullb($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister);
|
||||
%}
|
||||
ins_pipe(pipe_slow);
|
||||
%}
|
||||
|
||||
// vector mul - floating-point
|
||||
|
||||
instruct vmulHF(vReg dst, vReg src1, vReg src2) %{
|
||||
|
||||
@ -736,7 +736,8 @@ BINARY_OP_NEON_SVE_PAIRWISE(vmulI, MulVI, mulv, sve_mul, S)
|
||||
// vector mul - LONG
|
||||
|
||||
instruct vmulL_neon(vReg dst, vReg src1, vReg src2) %{
|
||||
predicate(UseSVE == 0);
|
||||
predicate(UseSVE == 0 && !n->as_MulVL()->has_int_inputs() &&
|
||||
!n->as_MulVL()->has_uint_inputs());
|
||||
match(Set dst (MulVL src1 src2));
|
||||
format %{ "vmulL_neon $dst, $src1, $src2\t# 2L" %}
|
||||
ins_encode %{
|
||||
@ -754,8 +755,47 @@ instruct vmulL_neon(vReg dst, vReg src1, vReg src2) %{
|
||||
ins_pipe(pipe_slow);
|
||||
%}
|
||||
|
||||
dnl VMUL_L_NEON($1, $2 )
|
||||
dnl VMUL_L_NEON(kind, insn )
|
||||
define(`VMUL_L_NEON', `dnl
|
||||
// Specialization of vmulL_$1_neon when both inputs are the same IR node
|
||||
// (e.g. v * v). Avoids one redundant xtn and saves one temporary register.
|
||||
instruct vmulL_$1_neon_same(vReg dst, vReg src, vReg tmp) %{
|
||||
predicate(UseSVE == 0 && n->as_MulVL()->has_$1_inputs() &&
|
||||
n->in(1) == n->in(2));
|
||||
match(Set dst (MulVL src src));
|
||||
effect(TEMP tmp);
|
||||
format %{ "vmulL_$1_neon_same $dst, $src, $src\t# 2L. KILL $tmp" %}
|
||||
ins_encode %{
|
||||
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
||||
assert(length_in_bytes == 16, "must be");
|
||||
__ xtn($tmp$$FloatRegister, __ T2S, $src$$FloatRegister, __ T2D);
|
||||
__ $2($dst$$FloatRegister, __ T2S, $tmp$$FloatRegister, $tmp$$FloatRegister);
|
||||
%}
|
||||
ins_pipe(pipe_slow);
|
||||
%}
|
||||
|
||||
instruct vmulL_$1_neon(vReg dst, vReg src1, vReg src2, vReg tmp1, vReg tmp2) %{
|
||||
predicate(UseSVE == 0 && n->as_MulVL()->has_$1_inputs() &&
|
||||
n->in(1) != n->in(2));
|
||||
match(Set dst (MulVL src1 src2));
|
||||
effect(TEMP tmp1, TEMP tmp2);
|
||||
format %{ "vmulL_$1_neon $dst, $src1, $src2\t# 2L. KILL $tmp1, $tmp2" %}
|
||||
ins_encode %{
|
||||
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
||||
assert(length_in_bytes == 16, "must be");
|
||||
__ xtn($tmp1$$FloatRegister, __ T2S, $src1$$FloatRegister, __ T2D);
|
||||
__ xtn($tmp2$$FloatRegister, __ T2S, $src2$$FloatRegister, __ T2D);
|
||||
__ $2($dst$$FloatRegister, __ T2S, $tmp1$$FloatRegister, $tmp2$$FloatRegister);
|
||||
%}
|
||||
ins_pipe(pipe_slow);
|
||||
%}
|
||||
')dnl
|
||||
VMUL_L_NEON(int, smullv)
|
||||
VMUL_L_NEON(uint, umullv)
|
||||
instruct vmulL_sve(vReg dst_src1, vReg src2) %{
|
||||
predicate(UseSVE > 0);
|
||||
predicate(UseSVE == 1 || (UseSVE == 2 && !n->as_MulVL()->has_int_inputs() &&
|
||||
!n->as_MulVL()->has_uint_inputs()));
|
||||
match(Set dst_src1 (MulVL dst_src1 src2));
|
||||
format %{ "vmulL_sve $dst_src1, $dst_src1, $src2" %}
|
||||
ins_encode %{
|
||||
@ -764,6 +804,21 @@ instruct vmulL_sve(vReg dst_src1, vReg src2) %{
|
||||
ins_pipe(pipe_slow);
|
||||
%}
|
||||
|
||||
dnl VMUL_L_SVE2($1, $2 )
|
||||
dnl VMUL_L_SVE2(kind, sve2_insn )
|
||||
define(`VMUL_L_SVE2', `dnl
|
||||
instruct vmulL_$1_sve2(vReg dst, vReg src1, vReg src2) %{
|
||||
predicate(UseSVE == 2 && n->as_MulVL()->has_$1_inputs());
|
||||
match(Set dst (MulVL src1 src2));
|
||||
format %{ "vmulL_$1_sve2 $dst, $src1, $src2" %}
|
||||
ins_encode %{
|
||||
__ $2($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister);
|
||||
%}
|
||||
ins_pipe(pipe_slow);
|
||||
%}
|
||||
')dnl
|
||||
VMUL_L_SVE2(int, sve_smullb)
|
||||
VMUL_L_SVE2(uint, sve_umullb)
|
||||
// vector mul - floating-point
|
||||
BINARY_OP(vmulHF, MulVHF, fmul, sve_fmul, H)
|
||||
BINARY_OP(vmulF, MulVF, fmul, sve_fmul, S)
|
||||
|
||||
@ -4331,6 +4331,22 @@ public:
|
||||
INSN(sve_bsl, 0b001, 0b1); // Bitwise select
|
||||
#undef INSN
|
||||
|
||||
// SVE2 widening integer multiply - vector
|
||||
#define INSN(NAME, is_unsigned, is_top) \
|
||||
void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, FloatRegister Zm) { \
|
||||
starti; \
|
||||
assert(T != B && T != Q, "invalid size"); \
|
||||
int op = 0b011100 | (is_unsigned ? 0b10 : 0) | (is_top ? 0b1 : 0); \
|
||||
f(0b01000101, 31, 24), f(T, 23, 22), f(0, 21), rf(Zm, 16); \
|
||||
f(op, 15, 10), rf(Zn, 5), rf(Zd, 0); \
|
||||
}
|
||||
|
||||
INSN(sve_umullb, /* is_unsigned */ true, /* is_top */ false); // Unsigned widening multiply of bottom elements
|
||||
INSN(sve_umullt, /* is_unsigned */ true, /* is_top */ true ); // Unsigned widening multiply of top elements
|
||||
INSN(sve_smullb, /* is_unsigned */ false, /* is_top */ false); // Signed widening multiply of bottom elements
|
||||
INSN(sve_smullt, /* is_unsigned */ false, /* is_top */ true ); // Signed widening multiply of top elements
|
||||
#undef INSN
|
||||
|
||||
// SVE2 saturating operations - predicate
|
||||
#define INSN(NAME, op1, op2) \
|
||||
void NAME(FloatRegister Zdn, SIMD_RegVariant T, PRegister Pg, FloatRegister Znm) { \
|
||||
|
||||
@ -2163,6 +2163,10 @@ generate(SpecialCases, [["ccmn", "__ ccmn(zr, zr, 3u, Assembler::LE);",
|
||||
# SVE2 instructions
|
||||
["histcnt", "__ sve_histcnt(z16, __ S, p0, z16, z16);", "histcnt\tz16.s, p0/z, z16.s, z16.s"],
|
||||
["histcnt", "__ sve_histcnt(z17, __ D, p0, z17, z17);", "histcnt\tz17.d, p0/z, z17.d, z17.d"],
|
||||
["umullb", "__ sve_umullb(z16, __ H, z17, z18);", "umullb\tz16.h, z17.b, z18.b"],
|
||||
["umullt", "__ sve_umullt(z19, __ S, z20, z21);", "umullt\tz19.s, z20.h, z21.h"],
|
||||
["smullb", "__ sve_smullb(z22, __ D, z23, z24);", "smullb\tz22.d, z23.s, z24.s"],
|
||||
["smullt", "__ sve_smullt(z25, __ H, z26, z27);", "smullt\tz25.h, z26.b, z27.b"],
|
||||
])
|
||||
|
||||
print "\n// FloatImmediateOp"
|
||||
|
||||
@ -1180,6 +1180,10 @@
|
||||
__ sve_splice(z0, __ D, p0, z1); // splice z0.d, p0, z0.d, z1.d
|
||||
__ sve_histcnt(z16, __ S, p0, z16, z16); // histcnt z16.s, p0/z, z16.s, z16.s
|
||||
__ sve_histcnt(z17, __ D, p0, z17, z17); // histcnt z17.d, p0/z, z17.d, z17.d
|
||||
__ sve_umullb(z16, __ H, z17, z18); // umullb z16.h, z17.b, z18.b
|
||||
__ sve_umullt(z19, __ S, z20, z21); // umullt z19.s, z20.h, z21.h
|
||||
__ sve_smullb(z22, __ D, z23, z24); // smullb z22.d, z23.s, z24.s
|
||||
__ sve_smullt(z25, __ H, z26, z27); // smullt z25.h, z26.b, z27.b
|
||||
|
||||
// FloatImmediateOp
|
||||
__ fmovd(v0, 2.0); // fmov d0, #2.0
|
||||
@ -1470,30 +1474,30 @@
|
||||
0x9101a1a0, 0xb10a5cc8, 0xd10810aa, 0xf10fd061,
|
||||
0x120cb166, 0x321764bc, 0x52174681, 0x720c0227,
|
||||
0x9241018e, 0xb25a2969, 0xd278b411, 0xf26aad01,
|
||||
0x14000000, 0x17ffffd7, 0x140004cc, 0x94000000,
|
||||
0x97ffffd4, 0x940004c9, 0x3400000a, 0x34fffa2a,
|
||||
0x340098ca, 0x35000008, 0x35fff9c8, 0x35009868,
|
||||
0xb400000b, 0xb4fff96b, 0xb400980b, 0xb500001d,
|
||||
0xb5fff91d, 0xb50097bd, 0x10000013, 0x10fff8b3,
|
||||
0x10009753, 0x90000013, 0x36300016, 0x3637f836,
|
||||
0x363096d6, 0x3758000c, 0x375ff7cc, 0x3758966c,
|
||||
0x14000000, 0x17ffffd7, 0x140004d0, 0x94000000,
|
||||
0x97ffffd4, 0x940004cd, 0x3400000a, 0x34fffa2a,
|
||||
0x3400994a, 0x35000008, 0x35fff9c8, 0x350098e8,
|
||||
0xb400000b, 0xb4fff96b, 0xb400988b, 0xb500001d,
|
||||
0xb5fff91d, 0xb500983d, 0x10000013, 0x10fff8b3,
|
||||
0x100097d3, 0x90000013, 0x36300016, 0x3637f836,
|
||||
0x36309756, 0x3758000c, 0x375ff7cc, 0x375896ec,
|
||||
0x128313a0, 0x528a32c7, 0x7289173b, 0x92ab3acc,
|
||||
0xd2a0bf94, 0xf2c285e8, 0x9358722f, 0x330e652f,
|
||||
0x53067f3b, 0x93577c53, 0xb34a1aac, 0xd35a4016,
|
||||
0x13946c63, 0x93c3dbc8, 0x54000000, 0x54fff5a0,
|
||||
0x54009440, 0x54000001, 0x54fff541, 0x540093e1,
|
||||
0x54000002, 0x54fff4e2, 0x54009382, 0x54000002,
|
||||
0x54fff482, 0x54009322, 0x54000003, 0x54fff423,
|
||||
0x540092c3, 0x54000003, 0x54fff3c3, 0x54009263,
|
||||
0x54000004, 0x54fff364, 0x54009204, 0x54000005,
|
||||
0x54fff305, 0x540091a5, 0x54000006, 0x54fff2a6,
|
||||
0x54009146, 0x54000007, 0x54fff247, 0x540090e7,
|
||||
0x54000008, 0x54fff1e8, 0x54009088, 0x54000009,
|
||||
0x54fff189, 0x54009029, 0x5400000a, 0x54fff12a,
|
||||
0x54008fca, 0x5400000b, 0x54fff0cb, 0x54008f6b,
|
||||
0x5400000c, 0x54fff06c, 0x54008f0c, 0x5400000d,
|
||||
0x54fff00d, 0x54008ead, 0x5400000e, 0x54ffefae,
|
||||
0x54008e4e, 0x5400000f, 0x54ffef4f, 0x54008def,
|
||||
0x540094c0, 0x54000001, 0x54fff541, 0x54009461,
|
||||
0x54000002, 0x54fff4e2, 0x54009402, 0x54000002,
|
||||
0x54fff482, 0x540093a2, 0x54000003, 0x54fff423,
|
||||
0x54009343, 0x54000003, 0x54fff3c3, 0x540092e3,
|
||||
0x54000004, 0x54fff364, 0x54009284, 0x54000005,
|
||||
0x54fff305, 0x54009225, 0x54000006, 0x54fff2a6,
|
||||
0x540091c6, 0x54000007, 0x54fff247, 0x54009167,
|
||||
0x54000008, 0x54fff1e8, 0x54009108, 0x54000009,
|
||||
0x54fff189, 0x540090a9, 0x5400000a, 0x54fff12a,
|
||||
0x5400904a, 0x5400000b, 0x54fff0cb, 0x54008feb,
|
||||
0x5400000c, 0x54fff06c, 0x54008f8c, 0x5400000d,
|
||||
0x54fff00d, 0x54008f2d, 0x5400000e, 0x54ffefae,
|
||||
0x54008ece, 0x5400000f, 0x54ffef4f, 0x54008e6f,
|
||||
0xd40658e1, 0xd4014d22, 0xd4046543, 0xd4273f60,
|
||||
0xd44cad80, 0xd503201f, 0xd503203f, 0xd503205f,
|
||||
0xd503209f, 0xd50320bf, 0xd503219f, 0xd50323bf,
|
||||
@ -1536,7 +1540,7 @@
|
||||
0x39598921, 0x795d3077, 0x399d0675, 0x7998d8f3,
|
||||
0x79dbd02a, 0xb99d068a, 0xfd5d11a0, 0xbd58d76b,
|
||||
0xfd1ac72d, 0xbd1d9c14, 0x5800001a, 0x18ffda33,
|
||||
0xf8991100, 0xd80078a0, 0xf8a758e0, 0xf9989d80,
|
||||
0xf8991100, 0xd8007920, 0xf8a758e0, 0xf9989d80,
|
||||
0x1a0b0298, 0x3a1c01a0, 0x5a0400ea, 0x7a02020f,
|
||||
0x9a1d028c, 0xba0e01ad, 0xda140186, 0xfa19022c,
|
||||
0x0b2b877e, 0x2b21c8ee, 0xcb3ba47d, 0x6b3ae9a0,
|
||||
@ -1719,7 +1723,8 @@
|
||||
0x0420bc31, 0x05271e11, 0x6545e891, 0x6585e891,
|
||||
0x65c5e891, 0x6545c891, 0x6585c891, 0x65c5c891,
|
||||
0x052c8020, 0x056c8020, 0x05ac8020, 0x05ec8020,
|
||||
0x45b0c210, 0x45f1c231, 0x1e601000, 0x1e603000,
|
||||
0x45b0c210, 0x45f1c231, 0x45527a30, 0x45957e93,
|
||||
0x45d872f6, 0x455b7759, 0x1e601000, 0x1e603000,
|
||||
0x1e621000, 0x1e623000, 0x1e641000, 0x1e643000,
|
||||
0x1e661000, 0x1e663000, 0x1e681000, 0x1e683000,
|
||||
0x1e6a1000, 0x1e6a3000, 0x1e6c1000, 0x1e6c3000,
|
||||
|
||||
@ -2863,6 +2863,26 @@ public class IRNode {
|
||||
machOnlyNameRegex(X86_VMULDQ_REG, "vmuldq_reg");
|
||||
}
|
||||
|
||||
public static final String AARCH64_VMULL_UINT_SVE2 = PREFIX + "AARCH64_VMULL_UINT_SVE2" + POSTFIX;
|
||||
static {
|
||||
machOnlyNameRegex(AARCH64_VMULL_UINT_SVE2, "vmulL_uint_sve2");
|
||||
}
|
||||
|
||||
public static final String AARCH64_VMULL_INT_SVE2 = PREFIX + "AARCH64_VMULL_INT_SVE2" + POSTFIX;
|
||||
static {
|
||||
machOnlyNameRegex(AARCH64_VMULL_INT_SVE2, "vmulL_int_sve2");
|
||||
}
|
||||
|
||||
public static final String AARCH64_VMULL_UINT_NEON = PREFIX + "AARCH64_VMULL_UINT_NEON" + POSTFIX;
|
||||
static {
|
||||
machOnlyNameRegex(AARCH64_VMULL_UINT_NEON, "vmulL_uint_neon");
|
||||
}
|
||||
|
||||
public static final String AARCH64_VMULL_INT_NEON = PREFIX + "AARCH64_VMULL_INT_NEON" + POSTFIX;
|
||||
static {
|
||||
machOnlyNameRegex(AARCH64_VMULL_INT_NEON, "vmulL_int_neon");
|
||||
}
|
||||
|
||||
public static final String X86_SCONV_D2I = PREFIX + "X86_SCONV_D2I" + POSTFIX;
|
||||
static {
|
||||
machOnlyNameRegex(X86_SCONV_D2I, "convD2I_reg_reg");
|
||||
|
||||
@ -34,7 +34,7 @@ import compiler.lib.verify.*;
|
||||
|
||||
/*
|
||||
* @test
|
||||
* @bug 8384963
|
||||
* @bug 8384963 8383905
|
||||
* @key randomness
|
||||
* @summary C2: Incorrect uint constant match mishandles negative values in vectors
|
||||
* @modules jdk.incubator.vector
|
||||
@ -87,8 +87,18 @@ public class TestVectorMulLongToSignedUnsignedInt {
|
||||
|
||||
// Case 1: Negative mask (-2L = 0xFFFF_FFFF_FFFF_FFFE).
|
||||
@Test
|
||||
@IR(counts = {IRNode.AND_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULUDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {IRNode.AND_VL, " >0 ",
|
||||
IRNode.MUL_VL, " >0 "},
|
||||
applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULUDQ_REG},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_UINT_SVE2},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_UINT_NEON},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void testNegativeMask() {
|
||||
LongVector v1 = LongVector.fromArray(SPECIES, src1, 0);
|
||||
LongVector v2 = LongVector.fromArray(SPECIES, src2, 0);
|
||||
@ -107,8 +117,16 @@ public class TestVectorMulLongToSignedUnsignedInt {
|
||||
|
||||
// Case 3: Mask = 0x1_0000_0000L (bit 32 set, exceeds uint range).
|
||||
@Test
|
||||
@IR(counts = {IRNode.AND_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULUDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {IRNode.AND_VL, " >0 ",
|
||||
IRNode.MUL_VL, " >0 "},
|
||||
applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULUDQ_REG},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_UINT_SVE2},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_UINT_NEON}, phase = CompilePhase.MATCHING, applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void testBit32SetMask() {
|
||||
LongVector v1 = LongVector.fromArray(SPECIES, src1, 0);
|
||||
LongVector v2 = LongVector.fromArray(SPECIES, src2, 0);
|
||||
@ -127,8 +145,18 @@ public class TestVectorMulLongToSignedUnsignedInt {
|
||||
|
||||
// Case 4: Mask = Long.MIN_VALUE (0x8000_0000_0000_0000).
|
||||
@Test
|
||||
@IR(counts = {IRNode.AND_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULUDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {IRNode.AND_VL, " >0 ",
|
||||
IRNode.MUL_VL, " >0 "},
|
||||
applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULUDQ_REG},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_UINT_SVE2},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_UINT_NEON},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void testMinValueMask() {
|
||||
LongVector v1 = LongVector.fromArray(SPECIES, src1, 0);
|
||||
LongVector v2 = LongVector.fromArray(SPECIES, src2, 0);
|
||||
@ -147,8 +175,17 @@ public class TestVectorMulLongToSignedUnsignedInt {
|
||||
|
||||
// Case 5: Mask = 0xFFFF_FFFFL (exactly uint max, boundary valid case).
|
||||
@Test
|
||||
@IR(counts = {IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {IRNode.X86_VMULUDQ_REG, " >0 "}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {IRNode.MUL_VL, " >0 "},
|
||||
applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"})
|
||||
@IR(counts = {IRNode.X86_VMULUDQ_REG, " >0 "},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {IRNode.AARCH64_VMULL_UINT_SVE2, " >0 "},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(counts = {IRNode.AARCH64_VMULL_UINT_NEON, " >0 "},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void testUintMaxMask() {
|
||||
LongVector v1 = LongVector.fromArray(SPECIES, src1, 0);
|
||||
LongVector v2 = LongVector.fromArray(SPECIES, src2, 0);
|
||||
@ -167,8 +204,17 @@ public class TestVectorMulLongToSignedUnsignedInt {
|
||||
|
||||
// Case 6: Small mask (0xFFFFL), clearly fits in uint.
|
||||
@Test
|
||||
@IR(counts = {IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {IRNode.X86_VMULUDQ_REG, " >0 "}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {IRNode.MUL_VL, " >0 "},
|
||||
applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"})
|
||||
@IR(counts = {IRNode.X86_VMULUDQ_REG, " >0 "},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {IRNode.AARCH64_VMULL_UINT_SVE2, " >0 "},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(counts = {IRNode.AARCH64_VMULL_UINT_NEON, " >0 "},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void testSmallMask() {
|
||||
LongVector v1 = LongVector.fromArray(SPECIES, src1, 0);
|
||||
LongVector v2 = LongVector.fromArray(SPECIES, src2, 0);
|
||||
@ -187,8 +233,18 @@ public class TestVectorMulLongToSignedUnsignedInt {
|
||||
|
||||
// Case 7: URShift by 32 clears upper doubleword.
|
||||
@Test
|
||||
@IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.URSHIFT_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {IRNode.X86_VMULUDQ_REG, " >0 "}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {IRNode.MUL_VL, " >0 ",
|
||||
IRNode.URSHIFT_VL, " >0 "},
|
||||
applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"})
|
||||
@IR(counts = {IRNode.X86_VMULUDQ_REG, " >0 "},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {IRNode.AARCH64_VMULL_UINT_SVE2, " >0 "},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(counts = {IRNode.AARCH64_VMULL_UINT_NEON, " >0 "},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void testURShift32() {
|
||||
LongVector v1 = LongVector.fromArray(SPECIES, src1, 0);
|
||||
LongVector v2 = LongVector.fromArray(SPECIES, src2, 0);
|
||||
@ -207,8 +263,18 @@ public class TestVectorMulLongToSignedUnsignedInt {
|
||||
|
||||
// Case 8: Asymmetric — one input valid uint mask, other negative mask.
|
||||
@Test
|
||||
@IR(counts = {IRNode.AND_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULUDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {IRNode.AND_VL, " >0 ",
|
||||
IRNode.MUL_VL, " >0 "},
|
||||
applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULUDQ_REG},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_UINT_SVE2},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_UINT_NEON},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void testAsymmetricMask() {
|
||||
LongVector v1 = LongVector.fromArray(SPECIES, src1, 0);
|
||||
LongVector v2 = LongVector.fromArray(SPECIES, src2, 0);
|
||||
@ -228,8 +294,19 @@ public class TestVectorMulLongToSignedUnsignedInt {
|
||||
// Case 9: Mixed — one input URShift (valid), other negative mask (invalid).
|
||||
// Note: -2L is used (not -1L) since AND with -1L is identity and gets folded.
|
||||
@Test
|
||||
@IR(counts = {IRNode.URSHIFT_VL, " >0 ", IRNode.AND_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULUDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {IRNode.URSHIFT_VL, " >0 ",
|
||||
IRNode.AND_VL, " >0 ",
|
||||
IRNode.MUL_VL, " >0 "},
|
||||
applyIfCPUFeatureOr = {"avx", "true", "asimd", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULUDQ_REG},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_UINT_SVE2},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_UINT_NEON},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void testMixedURShiftAndNegMask() {
|
||||
LongVector v1 = LongVector.fromArray(SPECIES, src1, 0);
|
||||
LongVector v2 = LongVector.fromArray(SPECIES, src2, 0);
|
||||
@ -248,8 +325,18 @@ public class TestVectorMulLongToSignedUnsignedInt {
|
||||
|
||||
// Case 10: Predicated AndV (uint path). Inactive lanes preserves destination with non-zero upper 32 bits.
|
||||
@Test
|
||||
@IR(counts = {IRNode.AND_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx512f", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULUDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx512f", "true"})
|
||||
@IR(counts = {IRNode.AND_VL, " >0 ",
|
||||
IRNode.MUL_VL, " >0 "},
|
||||
applyIfCPUFeatureOr = {"avx512f", "true", "sve", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULUDQ_REG},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"avx512f", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_UINT_SVE2},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_UINT_NEON},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void testPredicatedAndMask() {
|
||||
LongVector v1 = LongVector.fromArray(SPECIES, src1, 0);
|
||||
LongVector v2 = LongVector.fromArray(SPECIES, src2, 0);
|
||||
@ -270,8 +357,18 @@ public class TestVectorMulLongToSignedUnsignedInt {
|
||||
|
||||
// Case 11: Predicated URShiftVL by 32 (uint path). Inactive lanes preserves destination with non-zero upper 32 bits.
|
||||
@Test
|
||||
@IR(counts = {IRNode.URSHIFT_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx512f", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULUDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx512f", "true"})
|
||||
@IR(counts = {IRNode.URSHIFT_VL, " >0 ",
|
||||
IRNode.MUL_VL, " >0 "},
|
||||
applyIfCPUFeatureOr = {"avx512f", "true", "sve", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULUDQ_REG},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"avx512f", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_UINT_SVE2},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_UINT_NEON},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void testPredicatedURShift32() {
|
||||
LongVector v1 = LongVector.fromArray(SPECIES, src1, 0);
|
||||
LongVector v2 = LongVector.fromArray(SPECIES, src2, 0);
|
||||
@ -292,8 +389,18 @@ public class TestVectorMulLongToSignedUnsignedInt {
|
||||
|
||||
// Case 12: Predicated RShiftVL (arithmetic) by 32.
|
||||
@Test
|
||||
@IR(counts = {IRNode.RSHIFT_VL, " >0 ", IRNode.MUL_VL, " >0 "}, applyIfCPUFeature = {"avx512f", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULDQ_REG}, phase = CompilePhase.MATCHING, applyIfCPUFeature = {"avx512f", "true"})
|
||||
@IR(counts = {IRNode.RSHIFT_VL, " >0 ",
|
||||
IRNode.MUL_VL, " >0 "},
|
||||
applyIfCPUFeatureOr = {"avx512f", "true", "sve", "true"})
|
||||
@IR(failOn = {IRNode.X86_VMULDQ_REG},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"avx512f", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_INT_SVE2},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(failOn = {IRNode.AARCH64_VMULL_INT_NEON},
|
||||
phase = CompilePhase.MATCHING,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void testPredicatedRShift32() {
|
||||
LongVector v1 = LongVector.fromArray(SPECIES, src1, 0);
|
||||
LongVector v2 = LongVector.fromArray(SPECIES, src2, 0);
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
|
||||
* Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
|
||||
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
|
||||
*
|
||||
* This code is free software; you can redistribute it and/or modify it
|
||||
@ -31,9 +31,9 @@ import java.lang.reflect.Array;
|
||||
|
||||
/**
|
||||
* @test
|
||||
* @bug 8341137
|
||||
* @bug 8341137 8383905
|
||||
* @key randomness
|
||||
* @summary Optimize long vector multiplication using x86 VPMUL[U]DQ instruction.
|
||||
* @summary Optimize long vector multiplication.
|
||||
* @modules jdk.incubator.vector
|
||||
* @library /test/lib /
|
||||
* @run driver compiler.vectorapi.VectorMultiplyOpt
|
||||
@ -80,7 +80,7 @@ public class VectorMultiplyOpt {
|
||||
|
||||
public static void main(String[] args) {
|
||||
TestFramework testFramework = new TestFramework();
|
||||
testFramework.setDefaultWarmup(5000)
|
||||
testFramework.setDefaultWarmup(10000)
|
||||
.addFlags("--add-modules=jdk.incubator.vector")
|
||||
.start();
|
||||
System.out.println("PASSED");
|
||||
@ -109,7 +109,12 @@ public class VectorMultiplyOpt {
|
||||
@Test
|
||||
@IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.AND_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {"vmuludq", " >0 "}, phase = CompilePhase.FINAL_CODE, applyIfCPUFeature = {"avx", "true"})
|
||||
@Warmup(value = 10000)
|
||||
@IR(counts = {"vmulL_uint_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"})
|
||||
@IR(counts = {"vmulL_uint_neon", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void test_pattern1() {
|
||||
int i = 0;
|
||||
for (; i < LSP.loopBound(res.length); i += LSP.length()) {
|
||||
@ -132,7 +137,12 @@ public class VectorMultiplyOpt {
|
||||
@Test
|
||||
@IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.AND_VL, " >0 ", IRNode.URSHIFT_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {"vmuludq", " >0 "}, phase = CompilePhase.FINAL_CODE, applyIfCPUFeature = {"avx", "true"})
|
||||
@Warmup(value = 10000)
|
||||
@IR(counts = {"vmulL_uint_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"})
|
||||
@IR(counts = {"vmulL_uint_neon", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void test_pattern2() {
|
||||
int i = 0;
|
||||
for (; i < LSP.loopBound(res.length); i += LSP.length()) {
|
||||
@ -155,7 +165,12 @@ public class VectorMultiplyOpt {
|
||||
@Test
|
||||
@IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.URSHIFT_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {"vmuludq", " >0 "}, phase = CompilePhase.FINAL_CODE, applyIfCPUFeature = {"avx", "true"})
|
||||
@Warmup(value = 10000)
|
||||
@IR(counts = {"vmulL_uint_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"})
|
||||
@IR(counts = {"vmulL_uint_neon", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void test_pattern3() {
|
||||
int i = 0;
|
||||
for (; i < LSP.loopBound(res.length); i += LSP.length()) {
|
||||
@ -178,7 +193,12 @@ public class VectorMultiplyOpt {
|
||||
@Test
|
||||
@IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.URSHIFT_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {"vmuludq", " >0 "}, applyIfCPUFeature = {"avx", "true"}, phase = CompilePhase.FINAL_CODE)
|
||||
@Warmup(value = 10000)
|
||||
@IR(counts = {"vmulL_uint_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"})
|
||||
@IR(counts = {"vmulL_uint_neon", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void test_pattern4() {
|
||||
int i = 0;
|
||||
for (; i < LSP.loopBound(res.length); i += LSP.length()) {
|
||||
@ -201,7 +221,12 @@ public class VectorMultiplyOpt {
|
||||
@Test
|
||||
@IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.VECTOR_CAST_I2L, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {"vmuldq", " >0 "}, applyIfCPUFeature = {"avx", "true"}, phase = CompilePhase.FINAL_CODE)
|
||||
@Warmup(value = 10000)
|
||||
@IR(counts = {"vmulL_int_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"})
|
||||
@IR(counts = {"vmulL_int_neon", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void test_pattern5() {
|
||||
int i = 0;
|
||||
for (; i < LSP.loopBound(res.length); i += LSP.length()) {
|
||||
@ -227,7 +252,12 @@ public class VectorMultiplyOpt {
|
||||
@Test
|
||||
@IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.RSHIFT_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {"vmuldq", " >0 "}, applyIfCPUFeature = {"avx", "true"}, phase = CompilePhase.FINAL_CODE)
|
||||
@Warmup(value = 10000)
|
||||
@IR(counts = {"vmulL_int_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"})
|
||||
@IR(counts = {"vmulL_int_neon", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void test_pattern6() {
|
||||
int i = 0;
|
||||
for (; i < LSP.loopBound(res.length); i += LSP.length()) {
|
||||
@ -247,4 +277,61 @@ public class VectorMultiplyOpt {
|
||||
validate("pattern6 ", res, lsrc1, lsrc2, (l1, l2) -> (l1 >> shift5) * (l2 >> shift5));
|
||||
}
|
||||
|
||||
// Same-operand multiplication (v * v) where v has zero-extended high bits.
|
||||
// On NEON this should map to the dedicated rule that emits a single xtn.
|
||||
@Test
|
||||
@IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.AND_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {"vmuludq", " >0 "}, phase = CompilePhase.FINAL_CODE, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {"vmulL_uint_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"})
|
||||
@IR(counts = {"vmulL_uint_neon_same", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void test_pattern7() {
|
||||
int i = 0;
|
||||
for (; i < LSP.loopBound(res.length); i += LSP.length()) {
|
||||
LongVector vsrc = LongVector.fromArray(LSP, lsrc1, i)
|
||||
.lanewise(VectorOperators.AND, mask1);
|
||||
vsrc.lanewise(VectorOperators.MUL, vsrc).intoArray(res, i);
|
||||
}
|
||||
for (; i < res.length; i++) {
|
||||
long x = lsrc1[i] & mask1;
|
||||
res[i] = x * x;
|
||||
}
|
||||
}
|
||||
|
||||
@Check(test = "test_pattern7")
|
||||
public void test_pattern7_validate() {
|
||||
validate("pattern7 ", res, lsrc1, lsrc1, (l1, l2) -> { long x = l1 & mask1; return x * x; });
|
||||
}
|
||||
|
||||
// Same-operand multiplication (v * v) where v has sign-extended high bits.
|
||||
// On NEON this should map to the dedicated rule that emits a single xtn.
|
||||
@Test
|
||||
@IR(counts = {IRNode.MUL_VL, " >0 ", IRNode.VECTOR_CAST_I2L, " >0 "}, applyIfCPUFeature = {"avx", "true"})
|
||||
@IR(counts = {"vmuldq", " >0 "}, applyIfCPUFeature = {"avx", "true"}, phase = CompilePhase.FINAL_CODE)
|
||||
@IR(counts = {"vmulL_int_sve2", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeature = {"sve2", "true"})
|
||||
@IR(counts = {"vmulL_sve", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"sve", "true", "sve2", "false"})
|
||||
@IR(counts = {"vmulL_int_neon_same", " >0 "}, phase = CompilePhase.FINAL_CODE,
|
||||
applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
|
||||
public static void test_pattern8() {
|
||||
int i = 0;
|
||||
for (; i < LSP.loopBound(res.length); i += LSP.length()) {
|
||||
LongVector vsrc = IntVector.fromArray(ISP, isrc1, i)
|
||||
.convert(VectorOperators.I2L, 0)
|
||||
.reinterpretAsLongs();
|
||||
vsrc.lanewise(VectorOperators.MUL, vsrc).intoArray(res, i);
|
||||
}
|
||||
for (; i < res.length; i++) {
|
||||
res[i] = Math.multiplyFull(isrc1[i], isrc1[i]);
|
||||
}
|
||||
}
|
||||
|
||||
@Check(test = "test_pattern8")
|
||||
public void test_pattern8_validate() {
|
||||
validate("pattern8 ", res, isrc1, isrc1, (i1, i2) -> Math.multiplyFull((int)i1, (int)i1));
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user