diff --git a/src/hotspot/cpu/aarch64/aarch64_vector.ad b/src/hotspot/cpu/aarch64/aarch64_vector.ad index c06c8b856b7..b844ac561cc 100644 --- a/src/hotspot/cpu/aarch64/aarch64_vector.ad +++ b/src/hotspot/cpu/aarch64/aarch64_vector.ad @@ -324,6 +324,17 @@ source %{ return false; } break; + case Op_DivVB: + case Op_DivVS: + case Op_DivVI: + case Op_DivVL: + // Integer vector divide is only available on SVE (SDIV for 32-bit and + // 64-bit elements). NEON has no integer vector divide instruction. + // BYTE/SHORT are widened to 32-bit, divided, and narrowed back. + if (UseSVE == 0) { + return false; + } + break; default: break; } @@ -348,6 +359,11 @@ source %{ case Op_CompressBitsV: case Op_ExpandBitsV: case Op_VectorBitwiseBlend: + // There is no native SVE divide for BYTE/SHORT elements (these are + // emulated by widening to 32-bit), so the masked variants are handled + // by an unpredicated divide combined with a VectorBlend. + case Op_DivVB: + case Op_DivVS: return false; case Op_SaturatingAddV: case Op_SaturatingSubV: @@ -1477,6 +1493,83 @@ instruct vdivD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ ins_pipe(pipe_slow); %} +// ------------------------------ Vector integer div --------------------------- + +// BYTE and SHORT have no native integer divide on SVE (SDIV only supports 32-bit +// and 64-bit elements), so they are emulated by widening each element to 32-bit, +// performing SDIV, and narrowing the result back. + +instruct vdivB_sve(vReg dst_src1, vReg src2, vReg vtmp1, vReg vtmp2, vReg vtmp3, vReg vtmp4) %{ + predicate(UseSVE > 0); + match(Set dst_src1 (DivVB dst_src1 src2)); + effect(TEMP_DEF dst_src1, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4); + format %{ "vdivB_sve $dst_src1, $dst_src1, $src2" %} + ins_encode %{ + __ sve_sdiv_byte($dst_src1$$FloatRegister, $src2$$FloatRegister, + $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, + $vtmp3$$FloatRegister, $vtmp4$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + +instruct vdivS_sve(vReg dst_src1, vReg src2, vReg vtmp1, vReg vtmp2) %{ + predicate(UseSVE > 0); + match(Set dst_src1 (DivVS dst_src1 src2)); + effect(TEMP_DEF dst_src1, TEMP vtmp1, TEMP vtmp2); + format %{ "vdivS_sve $dst_src1, $dst_src1, $src2" %} + ins_encode %{ + __ sve_sdiv_short($dst_src1$$FloatRegister, $src2$$FloatRegister, + $vtmp1$$FloatRegister, $vtmp2$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + +instruct vdivI_sve(vReg dst_src1, vReg src2) %{ + predicate(UseSVE > 0); + match(Set dst_src1 (DivVI dst_src1 src2)); + format %{ "vdivI_sve $dst_src1, $dst_src1, $src2" %} + ins_encode %{ + __ sve_sdiv($dst_src1$$FloatRegister, __ S, ptrue, $src2$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + +instruct vdivL_sve(vReg dst_src1, vReg src2) %{ + predicate(UseSVE > 0); + match(Set dst_src1 (DivVL dst_src1 src2)); + format %{ "vdivL_sve $dst_src1, $dst_src1, $src2" %} + ins_encode %{ + __ sve_sdiv($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + +// Vector integer div - predicated +// +// There is no native SVE divide for BYTE/SHORT elements (these are emulated by +// widening to 32-bit), so the masked variants are handled by an unpredicated +// divide combined with a VectorBlend. + +instruct vdivI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ + predicate(UseSVE > 0); + match(Set dst_src1 (DivVI (Binary dst_src1 src2) pg)); + format %{ "vdivI_masked $dst_src1, $pg, $dst_src1, $src2" %} + ins_encode %{ + __ sve_sdiv($dst_src1$$FloatRegister, __ S, $pg$$PRegister, $src2$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + +instruct vdivL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ + predicate(UseSVE > 0); + match(Set dst_src1 (DivVL (Binary dst_src1 src2) pg)); + format %{ "vdivL_masked $dst_src1, $pg, $dst_src1, $src2" %} + ins_encode %{ + __ sve_sdiv($dst_src1$$FloatRegister, __ D, $pg$$PRegister, $src2$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + // ------------------------------ Vector and ----------------------------------- // vector and diff --git a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 index b749647ae1e..70d27e5248a 100644 --- a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 +++ b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 @@ -314,6 +314,17 @@ source %{ return false; } break; + case Op_DivVB: + case Op_DivVS: + case Op_DivVI: + case Op_DivVL: + // Integer vector divide is only available on SVE (SDIV for 32-bit and + // 64-bit elements). NEON has no integer vector divide instruction. + // BYTE/SHORT are widened to 32-bit, divided, and narrowed back. + if (UseSVE == 0) { + return false; + } + break; default: break; } @@ -338,6 +349,11 @@ source %{ case Op_CompressBitsV: case Op_ExpandBitsV: case Op_VectorBitwiseBlend: + // There is no native SVE divide for BYTE/SHORT elements (these are + // emulated by widening to 32-bit), so the masked variants are handled + // by an unpredicated divide combined with a VectorBlend. + case Op_DivVB: + case Op_DivVS: return false; case Op_SaturatingAddV: case Op_SaturatingSubV: @@ -842,6 +858,61 @@ BINARY_OP_NEON_SVE_PAIRWISE(vdivD, DivVD, fdiv, sve_fdiv, D) // vector float div - predicated BINARY_OP_PREDICATE(vdivF, DivVF, sve_fdiv, S) BINARY_OP_PREDICATE(vdivD, DivVD, sve_fdiv, D) + +dnl +dnl BINARY_OP_SVE_ONLY($1, $2, $3, $4 ) +dnl BINARY_OP_SVE_ONLY(rule_name, op_name, insn, size) +define(`BINARY_OP_SVE_ONLY', ` +instruct $1_sve(vReg dst_src1, vReg src2) %{ + predicate(UseSVE > 0); + match(Set dst_src1 ($2 dst_src1 src2)); + format %{ "$1_sve $dst_src1, $dst_src1, $src2" %} + ins_encode %{ + __ $3($dst_src1$$FloatRegister, __ $4, ptrue, $src2$$FloatRegister); + %} + ins_pipe(pipe_slow); +%}')dnl +dnl +// ------------------------------ Vector integer div --------------------------- + +// BYTE and SHORT have no native integer divide on SVE (SDIV only supports 32-bit +// and 64-bit elements), so they are emulated by widening each element to 32-bit, +// performing SDIV, and narrowing the result back. + +instruct vdivB_sve(vReg dst_src1, vReg src2, vReg vtmp1, vReg vtmp2, vReg vtmp3, vReg vtmp4) %{ + predicate(UseSVE > 0); + match(Set dst_src1 (DivVB dst_src1 src2)); + effect(TEMP_DEF dst_src1, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4); + format %{ "vdivB_sve $dst_src1, $dst_src1, $src2" %} + ins_encode %{ + __ sve_sdiv_byte($dst_src1$$FloatRegister, $src2$$FloatRegister, + $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, + $vtmp3$$FloatRegister, $vtmp4$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + +instruct vdivS_sve(vReg dst_src1, vReg src2, vReg vtmp1, vReg vtmp2) %{ + predicate(UseSVE > 0); + match(Set dst_src1 (DivVS dst_src1 src2)); + effect(TEMP_DEF dst_src1, TEMP vtmp1, TEMP vtmp2); + format %{ "vdivS_sve $dst_src1, $dst_src1, $src2" %} + ins_encode %{ + __ sve_sdiv_short($dst_src1$$FloatRegister, $src2$$FloatRegister, + $vtmp1$$FloatRegister, $vtmp2$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} +BINARY_OP_SVE_ONLY(vdivI, DivVI, sve_sdiv, S) +BINARY_OP_SVE_ONLY(vdivL, DivVL, sve_sdiv, D) + +// Vector integer div - predicated +// +// There is no native SVE divide for BYTE/SHORT elements (these are emulated by +// widening to 32-bit), so the masked variants are handled by an unpredicated +// divide combined with a VectorBlend. +BINARY_OP_PREDICATE(vdivI, DivVI, sve_sdiv, S) +BINARY_OP_PREDICATE(vdivL, DivVL, sve_sdiv, D) dnl dnl BITWISE_OP_IMM($1, $2, $3, $4, $5, $6 ) dnl BITWISE_OP_IMM(rule_name, type, op_name, insn, size, basic_type) diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp index a81213c5ae4..89f3dd63f42 100644 --- a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp @@ -3475,13 +3475,15 @@ private: public: -// SVE integer arithmetic - predicate +// SVE Arithmetic - Predicated #define INSN(NAME, op1, op2) \ void NAME(FloatRegister Zdn_or_Zd_or_Vd, SIMD_RegVariant T, PRegister Pg, FloatRegister Znm_or_Vn) { \ - assert(T != Q, "invalid register variant"); \ + assert(ALLOWED, "invalid register variant"); \ sve_predicate_reg_insn(op1, op2, Zdn_or_Zd_or_Vd, T, Pg, Znm_or_Vn); \ } +// SVE Integer Arithmetic - Predicated (B/H/S/D element sizes). +#define ALLOWED (T != Q) INSN(sve_abs, 0b00000100, 0b010110101); // vector abs, unary INSN(sve_add, 0b00000100, 0b000000000); // vector add INSN(sve_and, 0b00000100, 0b011010000); // vector and @@ -3511,15 +3513,16 @@ public: INSN(sve_umaxv, 0b00000100, 0b001001001); // unsigned maximum reduction to scalar INSN(sve_umin, 0b00000100, 0b001011000); // unsigned minimum vectors INSN(sve_uminv, 0b00000100, 0b001011001); // unsigned minimum reduction to scalar -#undef INSN +#undef ALLOWED -// SVE floating-point arithmetic - predicate -#define INSN(NAME, op1, op2) \ - void NAME(FloatRegister Zd_or_Zdn_or_Vd, SIMD_RegVariant T, PRegister Pg, FloatRegister Zn_or_Zm) { \ - assert(T == H || T == S || T == D, "invalid register variant"); \ - sve_predicate_reg_insn(op1, op2, Zd_or_Zdn_or_Vd, T, Pg, Zn_or_Zm); \ - } +// SVE Integer Binary Arithmetic - Predicated (S/D element sizes). +#define ALLOWED (T == S || T == D) + INSN(sve_sdiv, 0b00000100, 0b010100000); // signed divide + INSN(sve_udiv, 0b00000100, 0b010101000); // unsigned divide +#undef ALLOWED +// SVE Floating-point Arithmetic - Predicated (H/S/D element sizes). +#define ALLOWED (T == H || T == S || T == D) INSN(sve_fabd, 0b01100101, 0b001000100); // floating-point absolute difference INSN(sve_fabs, 0b00000100, 0b011100101); INSN(sve_fadd, 0b01100101, 0b000000100); @@ -3537,9 +3540,18 @@ public: INSN(sve_frintp, 0b01100101, 0b000001101); // floating-point round to integral value, toward plus infinity INSN(sve_fsqrt, 0b01100101, 0b001101101); INSN(sve_fsub, 0b01100101, 0b000001100); +#undef ALLOWED + +// SVE2 Signed/Unsigned Saturating Add/Sub - Predicated (B/H/S/D element sizes). +#define ALLOWED (T != Q) + INSN(sve_sqadd, 0b01000100, 0b011000100); // signed saturating add + INSN(sve_sqsub, 0b01000100, 0b011010100); // signed saturating sub + INSN(sve_uqadd, 0b01000100, 0b011001100); // unsigned saturating add + INSN(sve_uqsub, 0b01000100, 0b011011100); // unsigned saturating sub +#undef ALLOWED #undef INSN - // SVE multiple-add/sub - predicated +// SVE multiple-add/sub - predicated #define INSN(NAME, op0, op1, op2) \ void NAME(FloatRegister Zda, SIMD_RegVariant T, PRegister Pg, FloatRegister Zn, FloatRegister Zm) { \ starti; \ @@ -4347,20 +4359,6 @@ public: INSN(sve_smullt, /* is_unsigned */ false, /* is_top */ true ); // Signed widening multiply of top elements #undef INSN -// SVE2 saturating operations - predicate -#define INSN(NAME, op1, op2) \ - void NAME(FloatRegister Zdn, SIMD_RegVariant T, PRegister Pg, FloatRegister Znm) { \ - assert(T != Q, "invalid register variant"); \ - sve_predicate_reg_insn(op1, op2, Zdn, T, Pg, Znm); \ - } - - INSN(sve_sqadd, 0b01000100, 0b011000100); // signed saturating add - INSN(sve_sqsub, 0b01000100, 0b011010100); // signed saturating sub - INSN(sve_uqadd, 0b01000100, 0b011001100); // unsigned saturating add - INSN(sve_uqsub, 0b01000100, 0b011011100); // unsigned saturating sub - -#undef INSN - Assembler(CodeBuffer* code) : AbstractAssembler(code) { MACOS_AARCH64_ONLY(os::thread_wx_enable_write()); } diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp index 3321fcc4edb..7d490a99eaa 100644 --- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp @@ -2970,3 +2970,42 @@ int C2_MacroAssembler::vector_iota_entry_index(BasicType bt) { ShouldNotReachHere(); } } + +// Vector integer division for BYTE elements. Each BYTE is widened to SHORT for +// the low and high halves of the register, divided using the SHORT helper +// (which widens further to INT), and the two SHORT result halves are narrowed +// back to BYTE. +void C2_MacroAssembler::sve_sdiv_byte(FloatRegister dst_src1, FloatRegister src2, + FloatRegister vtmp1, FloatRegister vtmp2, + FloatRegister vtmp3, FloatRegister vtmp4) { + assert_different_registers(dst_src1, src2, vtmp1, vtmp2, vtmp3, vtmp4); + FloatRegister src1 = dst_src1; + // Low half of the bytes -> SHORT, then divide (result SHORT in vtmp1). + sve_sunpklo(vtmp1, H, src1); + sve_sunpklo(vtmp2, H, src2); + sve_sdiv_short(vtmp1, vtmp2, vtmp3, vtmp4); + // High half of the bytes -> SHORT, then divide (result SHORT in src1). + sve_sunpkhi(src1, H, src1); + sve_sunpkhi(vtmp2, H, src2); + sve_sdiv_short(src1, vtmp2, vtmp3, vtmp4); + // Narrow the two SHORT result halves back to BYTE. + sve_uzp1(dst_src1, B, vtmp1, src1); +} + +// Vector integer division for SHORT elements, implemented by widening each +// element to 32 bits, performing SDIV, and narrowing back. +void C2_MacroAssembler::sve_sdiv_short(FloatRegister dst_src1, FloatRegister src2, + FloatRegister vtmp1, FloatRegister vtmp2) { + assert_different_registers(dst_src1, src2, vtmp1, vtmp2); + FloatRegister src1 = dst_src1; + // Low half: SHORT -> INT, then divide. + sve_sunpklo(vtmp1, S, src1); + sve_sunpklo(vtmp2, S, src2); + sve_sdiv(vtmp1, S, ptrue, vtmp2); + // High half: SHORT -> INT, then divide. + sve_sunpkhi(src1, S, src1); + sve_sunpkhi(vtmp2, S, src2); + sve_sdiv(src1, S, ptrue, vtmp2); + // Narrow the two INT result halves back to SHORT. + sve_uzp1(dst_src1, H, vtmp1, src1); +} \ No newline at end of file diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp index f96d3ffb863..778f5e46589 100644 --- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp @@ -253,4 +253,9 @@ void sve_cpy(FloatRegister dst, SIMD_RegVariant T, PRegister pg, int imm8, bool isMerge); int vector_iota_entry_index(BasicType bt); + + void sve_sdiv_byte(FloatRegister dst_src1, FloatRegister src2, FloatRegister vtmp1, + FloatRegister vtmp2, FloatRegister vtmp3, FloatRegister vtmp4); + void sve_sdiv_short(FloatRegister dst_src1, FloatRegister src2, + FloatRegister vtmp1, FloatRegister vtmp2); #endif // CPU_AARCH64_C2_MACROASSEMBLER_AARCH64_HPP diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp index 6dfdde51ac5..df331ed02d3 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp @@ -1807,6 +1807,8 @@ public: SVE_DESTRUCTIVE_BINARY_5(sve_fmul, sve_fsub, sve_lsl, sve_lsr, sve_mul) SVE_DESTRUCTIVE_BINARY_5(sve_orr, sve_smax, sve_smin, sve_sqadd, sve_sqsub) SVE_DESTRUCTIVE_BINARY_5(sve_sub, sve_uqadd, sve_uqsub, sve_umax, sve_umin) + SVE_DESTRUCTIVE_BINARY_INS(sve_sdiv); + SVE_DESTRUCTIVE_BINARY_INS(sve_udiv); #undef SVE_DESTRUCTIVE_BINARY_INS #undef SVE_DESTRUCTIVE_BINARY_5 diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp index 53a72f979db..3527c6dad37 100644 --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -414,6 +414,10 @@ macro(MulAddVS2VI) macro(FmaVD) macro(FmaVF) macro(FmaVHF) +macro(DivVB) +macro(DivVS) +macro(DivVI) +macro(DivVL) macro(DivVHF) macro(DivVF) macro(DivVD) diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 20857eed35c..b1251562ca8 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -96,6 +96,15 @@ int VectorNode::opcode(int sopc, BasicType bt) { return (bt == T_DOUBLE ? Op_VectorBlend : 0); case Op_Bool: return Op_VectorMaskCmp; + case Op_DivI: + switch (bt) { + case T_BYTE: return Op_DivVB; + case T_SHORT: return Op_DivVS; + case T_INT: return Op_DivVI; + default: return 0; + } + case Op_DivL: + return (bt == T_LONG ? Op_DivVL : 0); case Op_DivHF: return (bt == T_SHORT ? Op_DivVHF : 0); case Op_DivF: @@ -333,6 +342,12 @@ int VectorNode::scalar_opcode(int vopc, BasicType bt) { case Op_MulVD: return Op_MulD; + case Op_DivVB: + case Op_DivVS: + case Op_DivVI: + return Op_DivI; + case Op_DivVL: + return Op_DivL; case Op_DivVF: return Op_DivF; case Op_DivVD: @@ -680,7 +695,7 @@ void VectorNode::vector_operands(Node* n, uint* start, uint* end) { case Op_AddI: case Op_AddL: case Op_AddHF: case Op_AddF: case Op_AddD: case Op_SubI: case Op_SubL: case Op_SubHF: case Op_SubF: case Op_SubD: case Op_MulI: case Op_MulL: case Op_MulHF: case Op_MulF: case Op_MulD: - case Op_DivHF: case Op_DivF: case Op_DivD: + case Op_DivI: case Op_DivL: case Op_DivHF: case Op_DivF: case Op_DivD: case Op_AndI: case Op_AndL: case Op_OrI: case Op_OrL: case Op_XorI: case Op_XorL: @@ -759,6 +774,10 @@ VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, const TypeVect* vt, b case Op_MulVF: return new MulVFNode(n1, n2, vt); case Op_MulVD: return new MulVDNode(n1, n2, vt); + case Op_DivVB: return new DivVBNode(n1, n2, vt); + case Op_DivVS: return new DivVSNode(n1, n2, vt); + case Op_DivVI: return new DivVINode(n1, n2, vt); + case Op_DivVL: return new DivVLNode(n1, n2, vt); case Op_DivVHF: return new DivVHFNode(n1, n2, vt); case Op_DivVF: return new DivVFNode(n1, n2, vt); case Op_DivVD: return new DivVDNode(n1, n2, vt); diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index d013bbc25d6..31faad5826e 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -696,6 +696,34 @@ public: virtual uint size_of() const { return sizeof(*this); } }; +// Vector divide byte +class DivVBNode : public VectorNode { +public: + DivVBNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {} + virtual int Opcode() const; +}; + +// Vector divide short +class DivVSNode : public VectorNode { +public: + DivVSNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {} + virtual int Opcode() const; +}; + +// Vector divide int +class DivVINode : public VectorNode { +public: + DivVINode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {} + virtual int Opcode() const; +}; + +// Vector divide long +class DivVLNode : public VectorNode { +public: + DivVLNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {} + virtual int Opcode() const; +}; + // Vector divide half float class DivVHFNode : public VectorNode { public: diff --git a/test/hotspot/gtest/aarch64/aarch64-asmtest.py b/test/hotspot/gtest/aarch64/aarch64-asmtest.py index b5386d47a73..f9eb3f7e7cc 100644 --- a/test/hotspot/gtest/aarch64/aarch64-asmtest.py +++ b/test/hotspot/gtest/aarch64/aarch64-asmtest.py @@ -1121,6 +1121,8 @@ class SVEVectorOp(Instruction): self._bitwiseop = False if name[0] == 'f': self._width = RegVariant(2, 3) + elif name in ["sdiv", "udiv"]: + self._width = RegVariant(2, 3) elif not self._isPredicated and (name in ["and", "bic", "bsl", "eor", "eor3", "orr"]): self._width = RegVariant(3, 3) self._bitwiseop = True @@ -2220,6 +2222,8 @@ generate(SVEVectorOp, [["add", "ZZZ"], ["lsl", "ZPZ", "m", "dn"], ["lsr", "ZPZ", "m", "dn"], ["mul", "ZPZ", "m", "dn"], + ["sdiv", "ZPZ", "m", "dn"], + ["udiv", "ZPZ", "m", "dn"], ["neg", "ZPZ", "m"], ["not", "ZPZ", "m"], ["orr", "ZPZ", "m", "dn"], diff --git a/test/hotspot/gtest/aarch64/asmtest.out.h b/test/hotspot/gtest/aarch64/asmtest.out.h index 95832a1faf6..070e8376d3b 100644 --- a/test/hotspot/gtest/aarch64/asmtest.out.h +++ b/test/hotspot/gtest/aarch64/asmtest.out.h @@ -1382,80 +1382,82 @@ __ sve_lsl(z17, __ D, p1, z11); // lsl z17.d, p1/m, z17.d, z11.d __ sve_lsr(z16, __ B, p0, z16); // lsr z16.b, p0/m, z16.b, z16.b __ sve_mul(z28, __ D, p1, z23); // mul z28.d, p1/m, z28.d, z23.d - __ sve_neg(z28, __ S, p4, z10); // neg z28.s, p4/m, z10.s - __ sve_not(z17, __ S, p7, z7); // not z17.s, p7/m, z7.s - __ sve_orr(z4, __ H, p3, z24); // orr z4.h, p3/m, z4.h, z24.h - __ sve_rbit(z9, __ B, p2, z11); // rbit z9.b, p2/m, z11.b - __ sve_revb(z4, __ S, p5, z22); // revb z4.s, p5/m, z22.s - __ sve_smax(z4, __ H, p0, z15); // smax z4.h, p0/m, z4.h, z15.h - __ sve_smin(z4, __ D, p7, z26); // smin z4.d, p7/m, z4.d, z26.d - __ sve_umax(z5, __ H, p5, z26); // umax z5.h, p5/m, z5.h, z26.h - __ sve_umin(z31, __ B, p0, z25); // umin z31.b, p0/m, z31.b, z25.b - __ sve_sub(z8, __ S, p1, z3); // sub z8.s, p1/m, z8.s, z3.s - __ sve_fabs(z7, __ D, p6, z24); // fabs z7.d, p6/m, z24.d - __ sve_fadd(z24, __ S, p7, z17); // fadd z24.s, p7/m, z24.s, z17.s - __ sve_fdiv(z10, __ S, p3, z30); // fdiv z10.s, p3/m, z10.s, z30.s - __ sve_fmax(z8, __ S, p6, z29); // fmax z8.s, p6/m, z8.s, z29.s - __ sve_fmin(z31, __ D, p5, z31); // fmin z31.d, p5/m, z31.d, z31.d - __ sve_fmul(z0, __ D, p5, z7); // fmul z0.d, p5/m, z0.d, z7.d - __ sve_fneg(z29, __ S, p6, z22); // fneg z29.s, p6/m, z22.s - __ sve_frintm(z29, __ S, p6, z20); // frintm z29.s, p6/m, z20.s - __ sve_frintn(z6, __ S, p4, z18); // frintn z6.s, p4/m, z18.s - __ sve_frintp(z26, __ S, p5, z8); // frintp z26.s, p5/m, z8.s - __ sve_fsqrt(z19, __ S, p2, z28); // fsqrt z19.s, p2/m, z28.s - __ sve_fsub(z17, __ D, p1, z30); // fsub z17.d, p1/m, z17.d, z30.d - __ sve_fmad(z24, __ S, p7, z14, z17); // fmad z24.s, p7/m, z14.s, z17.s - __ sve_fmla(z19, __ D, p2, z26, z11); // fmla z19.d, p2/m, z26.d, z11.d - __ sve_fmls(z0, __ D, p2, z15, z28); // fmls z0.d, p2/m, z15.d, z28.d - __ sve_fmsb(z23, __ D, p5, z28, z23); // fmsb z23.d, p5/m, z28.d, z23.d - __ sve_fnmad(z29, __ S, p6, z0, z27); // fnmad z29.s, p6/m, z0.s, z27.s - __ sve_fnmsb(z23, __ S, p3, z12, z4); // fnmsb z23.s, p3/m, z12.s, z4.s - __ sve_fnmla(z31, __ S, p6, z23, z20); // fnmla z31.s, p6/m, z23.s, z20.s - __ sve_fnmls(z2, __ D, p7, z29, z0); // fnmls z2.d, p7/m, z29.d, z0.d - __ sve_mla(z23, __ H, p0, z4, z5); // mla z23.h, p0/m, z4.h, z5.h - __ sve_mls(z28, __ H, p3, z17, z13); // mls z28.h, p3/m, z17.h, z13.h - __ sve_and(z8, z10, z8); // and z8.d, z10.d, z8.d - __ sve_eor(z19, z0, z29); // eor z19.d, z0.d, z29.d - __ sve_orr(z16, z13, z23); // orr z16.d, z13.d, z23.d - __ sve_bic(z23, z30, z13); // bic z23.d, z30.d, z13.d - __ sve_uzp1(z25, __ H, z22, z0); // uzp1 z25.h, z22.h, z0.h - __ sve_uzp2(z25, __ H, z30, z11); // uzp2 z25.h, z30.h, z11.h - __ sve_fabd(z14, __ S, p5, z22); // fabd z14.s, p5/m, z14.s, z22.s - __ sve_bext(z5, __ H, z18, z0); // bext z5.h, z18.h, z0.h - __ sve_bdep(z9, __ D, z2, z3); // bdep z9.d, z2.d, z3.d - __ sve_bsl(z14, z4, z29); // bsl z14.d, z14.d, z4.d, z29.d - __ sve_eor3(z14, z22, z4); // eor3 z14.d, z14.d, z22.d, z4.d - __ sve_sqadd(z27, __ S, p3, z22); // sqadd z27.s, p3/m, z27.s, z22.s - __ sve_sqsub(z31, __ S, p6, z11); // sqsub z31.s, p6/m, z31.s, z11.s - __ sve_uqadd(z12, __ B, p4, z28); // uqadd z12.b, p4/m, z12.b, z28.b - __ sve_uqsub(z28, __ D, p4, z4); // uqsub z28.d, p4/m, z28.d, z4.d + __ sve_sdiv(z28, __ D, p4, z10); // sdiv z28.d, p4/m, z28.d, z10.d + __ sve_udiv(z17, __ D, p7, z7); // udiv z17.d, p7/m, z17.d, z7.d + __ sve_neg(z4, __ H, p3, z24); // neg z4.h, p3/m, z24.h + __ sve_not(z9, __ B, p2, z11); // not z9.b, p2/m, z11.b + __ sve_orr(z4, __ S, p5, z22); // orr z4.s, p5/m, z4.s, z22.s + __ sve_rbit(z4, __ H, p0, z15); // rbit z4.h, p0/m, z15.h + __ sve_revb(z4, __ D, p7, z26); // revb z4.d, p7/m, z26.d + __ sve_smax(z5, __ H, p5, z26); // smax z5.h, p5/m, z5.h, z26.h + __ sve_smin(z31, __ B, p0, z25); // smin z31.b, p0/m, z31.b, z25.b + __ sve_umax(z8, __ S, p1, z3); // umax z8.s, p1/m, z8.s, z3.s + __ sve_umin(z7, __ D, p6, z24); // umin z7.d, p6/m, z7.d, z24.d + __ sve_sub(z24, __ B, p7, z17); // sub z24.b, p7/m, z24.b, z17.b + __ sve_fabs(z10, __ S, p3, z30); // fabs z10.s, p3/m, z30.s + __ sve_fadd(z8, __ S, p6, z29); // fadd z8.s, p6/m, z8.s, z29.s + __ sve_fdiv(z31, __ D, p5, z31); // fdiv z31.d, p5/m, z31.d, z31.d + __ sve_fmax(z0, __ D, p5, z7); // fmax z0.d, p5/m, z0.d, z7.d + __ sve_fmin(z29, __ S, p6, z22); // fmin z29.s, p6/m, z29.s, z22.s + __ sve_fmul(z29, __ S, p6, z20); // fmul z29.s, p6/m, z29.s, z20.s + __ sve_fneg(z6, __ S, p4, z18); // fneg z6.s, p4/m, z18.s + __ sve_frintm(z26, __ S, p5, z8); // frintm z26.s, p5/m, z8.s + __ sve_frintn(z19, __ S, p2, z28); // frintn z19.s, p2/m, z28.s + __ sve_frintp(z17, __ D, p1, z30); // frintp z17.d, p1/m, z30.d + __ sve_fsqrt(z24, __ D, p7, z14); // fsqrt z24.d, p7/m, z14.d + __ sve_fsub(z14, __ D, p4, z10); // fsub z14.d, p4/m, z14.d, z10.d + __ sve_fmad(z11, __ S, p6, z0, z11); // fmad z11.s, p6/m, z0.s, z11.s + __ sve_fmla(z28, __ D, p5, z23, z20); // fmla z28.d, p5/m, z23.d, z20.d + __ sve_fmls(z23, __ S, p5, z29, z24); // fmls z23.s, p5/m, z29.s, z24.s + __ sve_fmsb(z27, __ S, p1, z23, z13); // fmsb z27.s, p1/m, z23.s, z13.s + __ sve_fnmad(z4, __ D, p3, z31, z26); // fnmad z4.d, p3/m, z31.d, z26.d + __ sve_fnmsb(z20, __ D, p1, z2, z29); // fnmsb z20.d, p1/m, z2.d, z29.d + __ sve_fnmla(z0, __ S, p7, z23, z3); // fnmla z0.s, p7/m, z23.s, z3.s + __ sve_fnmls(z5, __ D, p2, z28, z13); // fnmls z5.d, p2/m, z28.d, z13.d + __ sve_mla(z13, __ H, p3, z8, z10); // mla z13.h, p3/m, z8.h, z10.h + __ sve_mls(z9, __ S, p4, z0, z29); // mls z9.s, p4/m, z0.s, z29.s + __ sve_and(z16, z13, z23); // and z16.d, z13.d, z23.d + __ sve_eor(z23, z30, z13); // eor z23.d, z30.d, z13.d + __ sve_orr(z25, z22, z0); // orr z25.d, z22.d, z0.d + __ sve_bic(z25, z30, z11); // bic z25.d, z30.d, z11.d + __ sve_uzp1(z14, __ H, z23, z22); // uzp1 z14.h, z23.h, z22.h + __ sve_uzp2(z5, __ H, z18, z0); // uzp2 z5.h, z18.h, z0.h + __ sve_fabd(z9, __ D, p0, z3); // fabd z9.d, p0/m, z9.d, z3.d + __ sve_bext(z14, __ H, z4, z29); // bext z14.h, z4.h, z29.h + __ sve_bdep(z14, __ D, z22, z4); // bdep z14.d, z22.d, z4.d + __ sve_bsl(z27, z15, z22); // bsl z27.d, z27.d, z15.d, z22.d + __ sve_eor3(z31, z24, z11); // eor3 z31.d, z31.d, z24.d, z11.d + __ sve_sqadd(z12, __ B, p4, z28); // sqadd z12.b, p4/m, z12.b, z28.b + __ sve_sqsub(z28, __ D, p4, z4); // sqsub z28.d, p4/m, z28.d, z4.d + __ sve_uqadd(z6, __ S, p0, z15); // uqadd z6.s, p0/m, z6.s, z15.s + __ sve_uqsub(z1, __ S, p5, z18); // uqsub z1.s, p5/m, z1.s, z18.s // SVEReductionOp - __ sve_andv(v6, __ S, p0, z15); // andv s6, p0, z15.s - __ sve_orv(v1, __ S, p5, z18); // orv s1, p5, z18.s - __ sve_eorv(v2, __ H, p2, z4); // eorv h2, p2, z4.h - __ sve_smaxv(v11, __ S, p2, z28); // smaxv s11, p2, z28.s - __ sve_sminv(v3, __ H, p5, z31); // sminv h3, p5, z31.h - __ sve_umaxv(v24, __ H, p5, z15); // umaxv h24, p5, z15.h - __ sve_uminv(v6, __ H, p3, z8); // uminv h6, p3, z8.h - __ sve_fminv(v21, __ D, p7, z4); // fminv d21, p7, z4.d - __ sve_fmaxv(v24, __ S, p5, z6); // fmaxv s24, p5, z6.s - __ sve_fadda(v4, __ D, p2, z9); // fadda d4, p2, d4, z9.d - __ sve_uaddv(v10, __ S, p1, z31); // uaddv d10, p1, z31.s + __ sve_andv(v2, __ H, p2, z4); // andv h2, p2, z4.h + __ sve_orv(v11, __ S, p2, z28); // orv s11, p2, z28.s + __ sve_eorv(v3, __ H, p5, z31); // eorv h3, p5, z31.h + __ sve_smaxv(v24, __ H, p5, z15); // smaxv h24, p5, z15.h + __ sve_sminv(v6, __ H, p3, z8); // sminv h6, p3, z8.h + __ sve_umaxv(v21, __ D, p7, z4); // umaxv d21, p7, z4.d + __ sve_uminv(v24, __ B, p5, z6); // uminv b24, p5, z6.b + __ sve_fminv(v4, __ D, p2, z9); // fminv d4, p2, z9.d + __ sve_fmaxv(v10, __ D, p1, z31); // fmaxv d10, p1, z31.d + __ sve_fadda(v25, __ D, p3, z3); // fadda d25, p3, d25, z3.d + __ sve_uaddv(v14, __ H, p2, z2); // uaddv d14, p2, z2.h // AddWideNEONOp - __ saddwv(v25, v26, __ T8H, v27, __ T8B); // saddw v25.8H, v26.8H, v27.8B - __ saddwv2(v15, v16, __ T8H, v17, __ T16B); // saddw2 v15.8H, v16.8H, v17.16B - __ saddwv(v3, v4, __ T4S, v5, __ T4H); // saddw v3.4S, v4.4S, v5.4H - __ saddwv2(v18, v19, __ T4S, v20, __ T8H); // saddw2 v18.4S, v19.4S, v20.8H - __ saddwv(v14, v15, __ T2D, v16, __ T2S); // saddw v14.2D, v15.2D, v16.2S - __ saddwv2(v10, v11, __ T2D, v12, __ T4S); // saddw2 v10.2D, v11.2D, v12.4S - __ uaddwv(v2, v3, __ T8H, v4, __ T8B); // uaddw v2.8H, v3.8H, v4.8B - __ uaddwv2(v10, v11, __ T8H, v12, __ T16B); // uaddw2 v10.8H, v11.8H, v12.16B - __ uaddwv(v8, v9, __ T4S, v10, __ T4H); // uaddw v8.4S, v9.4S, v10.4H - __ uaddwv2(v11, v12, __ T4S, v13, __ T8H); // uaddw2 v11.4S, v12.4S, v13.8H - __ uaddwv(v22, v23, __ T2D, v24, __ T2S); // uaddw v22.2D, v23.2D, v24.2S - __ uaddwv2(v3, v4, __ T2D, v5, __ T4S); // uaddw2 v3.2D, v4.2D, v5.4S + __ saddwv(v8, v9, __ T8H, v10, __ T8B); // saddw v8.8H, v9.8H, v10.8B + __ saddwv2(v11, v12, __ T8H, v13, __ T16B); // saddw2 v11.8H, v12.8H, v13.16B + __ saddwv(v22, v23, __ T4S, v24, __ T4H); // saddw v22.4S, v23.4S, v24.4H + __ saddwv2(v3, v4, __ T4S, v5, __ T8H); // saddw2 v3.4S, v4.4S, v5.8H + __ saddwv(v12, v13, __ T2D, v14, __ T2S); // saddw v12.2D, v13.2D, v14.2S + __ saddwv2(v24, v25, __ T2D, v26, __ T4S); // saddw2 v24.2D, v25.2D, v26.4S + __ uaddwv(v9, v10, __ T8H, v11, __ T8B); // uaddw v9.8H, v10.8H, v11.8B + __ uaddwv2(v27, v28, __ T8H, v29, __ T16B); // uaddw2 v27.8H, v28.8H, v29.16B + __ uaddwv(v27, v28, __ T4S, v29, __ T4H); // uaddw v27.4S, v28.4S, v29.4H + __ uaddwv2(v6, v7, __ T4S, v8, __ T8H); // uaddw2 v6.4S, v7.4S, v8.8H + __ uaddwv(v20, v21, __ T2D, v22, __ T2S); // uaddw v20.2D, v21.2D, v22.2S + __ uaddwv2(v20, v21, __ T2D, v22, __ T4S); // uaddw2 v20.2D, v21.2D, v22.4S __ bind(forth); @@ -1474,30 +1476,30 @@ 0x9101a1a0, 0xb10a5cc8, 0xd10810aa, 0xf10fd061, 0x120cb166, 0x321764bc, 0x52174681, 0x720c0227, 0x9241018e, 0xb25a2969, 0xd278b411, 0xf26aad01, - 0x14000000, 0x17ffffd7, 0x140004d0, 0x94000000, - 0x97ffffd4, 0x940004cd, 0x3400000a, 0x34fffa2a, - 0x3400994a, 0x35000008, 0x35fff9c8, 0x350098e8, - 0xb400000b, 0xb4fff96b, 0xb400988b, 0xb500001d, - 0xb5fff91d, 0xb500983d, 0x10000013, 0x10fff8b3, - 0x100097d3, 0x90000013, 0x36300016, 0x3637f836, - 0x36309756, 0x3758000c, 0x375ff7cc, 0x375896ec, + 0x14000000, 0x17ffffd7, 0x140004d2, 0x94000000, + 0x97ffffd4, 0x940004cf, 0x3400000a, 0x34fffa2a, + 0x3400998a, 0x35000008, 0x35fff9c8, 0x35009928, + 0xb400000b, 0xb4fff96b, 0xb40098cb, 0xb500001d, + 0xb5fff91d, 0xb500987d, 0x10000013, 0x10fff8b3, + 0x10009813, 0x90000013, 0x36300016, 0x3637f836, + 0x36309796, 0x3758000c, 0x375ff7cc, 0x3758972c, 0x128313a0, 0x528a32c7, 0x7289173b, 0x92ab3acc, 0xd2a0bf94, 0xf2c285e8, 0x9358722f, 0x330e652f, 0x53067f3b, 0x93577c53, 0xb34a1aac, 0xd35a4016, 0x13946c63, 0x93c3dbc8, 0x54000000, 0x54fff5a0, - 0x540094c0, 0x54000001, 0x54fff541, 0x54009461, - 0x54000002, 0x54fff4e2, 0x54009402, 0x54000002, - 0x54fff482, 0x540093a2, 0x54000003, 0x54fff423, - 0x54009343, 0x54000003, 0x54fff3c3, 0x540092e3, - 0x54000004, 0x54fff364, 0x54009284, 0x54000005, - 0x54fff305, 0x54009225, 0x54000006, 0x54fff2a6, - 0x540091c6, 0x54000007, 0x54fff247, 0x54009167, - 0x54000008, 0x54fff1e8, 0x54009108, 0x54000009, - 0x54fff189, 0x540090a9, 0x5400000a, 0x54fff12a, - 0x5400904a, 0x5400000b, 0x54fff0cb, 0x54008feb, - 0x5400000c, 0x54fff06c, 0x54008f8c, 0x5400000d, - 0x54fff00d, 0x54008f2d, 0x5400000e, 0x54ffefae, - 0x54008ece, 0x5400000f, 0x54ffef4f, 0x54008e6f, + 0x54009500, 0x54000001, 0x54fff541, 0x540094a1, + 0x54000002, 0x54fff4e2, 0x54009442, 0x54000002, + 0x54fff482, 0x540093e2, 0x54000003, 0x54fff423, + 0x54009383, 0x54000003, 0x54fff3c3, 0x54009323, + 0x54000004, 0x54fff364, 0x540092c4, 0x54000005, + 0x54fff305, 0x54009265, 0x54000006, 0x54fff2a6, + 0x54009206, 0x54000007, 0x54fff247, 0x540091a7, + 0x54000008, 0x54fff1e8, 0x54009148, 0x54000009, + 0x54fff189, 0x540090e9, 0x5400000a, 0x54fff12a, + 0x5400908a, 0x5400000b, 0x54fff0cb, 0x5400902b, + 0x5400000c, 0x54fff06c, 0x54008fcc, 0x5400000d, + 0x54fff00d, 0x54008f6d, 0x5400000e, 0x54ffefae, + 0x54008f0e, 0x5400000f, 0x54ffef4f, 0x54008eaf, 0xd40658e1, 0xd4014d22, 0xd4046543, 0xd4273f60, 0xd44cad80, 0xd503201f, 0xd503203f, 0xd503205f, 0xd503209f, 0xd50320bf, 0xd503219f, 0xd50323bf, @@ -1540,7 +1542,7 @@ 0x39598921, 0x795d3077, 0x399d0675, 0x7998d8f3, 0x79dbd02a, 0xb99d068a, 0xfd5d11a0, 0xbd58d76b, 0xfd1ac72d, 0xbd1d9c14, 0x5800001a, 0x18ffda33, - 0xf8991100, 0xd8007920, 0xf8a758e0, 0xf9989d80, + 0xf8991100, 0xd8007960, 0xf8a758e0, 0xf9989d80, 0x1a0b0298, 0x3a1c01a0, 0x5a0400ea, 0x7a02020f, 0x9a1d028c, 0xba0e01ad, 0xda140186, 0xfa19022c, 0x0b2b877e, 0x2b21c8ee, 0xcb3ba47d, 0x6b3ae9a0, @@ -1765,23 +1767,24 @@ 0x04e71f54, 0x04d6b0d4, 0x044003ad, 0x041a0029, 0x041099fb, 0x04db1e24, 0x0419a302, 0x041abdba, 0x04d90e16, 0x04d38571, 0x04118210, 0x04d006fc, - 0x0497b15c, 0x049ebcf1, 0x04580f04, 0x05278969, - 0x05a496c4, 0x044801e4, 0x04ca1f44, 0x04491745, - 0x040b033f, 0x04810468, 0x04dcbb07, 0x65809e38, - 0x658d8fca, 0x65869ba8, 0x65c797ff, 0x65c294e0, - 0x049dbadd, 0x6582ba9d, 0x6580b246, 0x6581b51a, - 0x658dab93, 0x65c187d1, 0x65b19dd8, 0x65eb0b53, - 0x65fc29e0, 0x65f7b797, 0x65bbd81d, 0x65a4ed97, - 0x65b45aff, 0x65e07fa2, 0x04454097, 0x044d6e3c, - 0x04283148, 0x04bd3013, 0x047731b0, 0x04ed33d7, - 0x05606ad9, 0x056b6fd9, 0x658896ce, 0x4540b245, - 0x45c3b449, 0x04243fae, 0x0436388e, 0x44988edb, - 0x449a997f, 0x4419938c, 0x44db909c, 0x049a21e6, - 0x04983641, 0x04592882, 0x04882b8b, 0x044a37e3, - 0x044935f8, 0x044b2d06, 0x65c73c95, 0x658634d8, - 0x65d82924, 0x048127ea, 0x0e3b1359, 0x4e31120f, - 0x0e651083, 0x4e741272, 0x0eb011ee, 0x4eac116a, - 0x2e241062, 0x6e2c116a, 0x2e6a1128, 0x6e6d118b, - 0x2eb812f6, 0x6ea51083, + 0x04d4115c, 0x04d51cf1, 0x0457af04, 0x041ea969, + 0x049816c4, 0x056781e4, 0x05e49f44, 0x04481745, + 0x040a033f, 0x04890468, 0x04cb1b07, 0x04011e38, + 0x049cafca, 0x65809ba8, 0x65cd97ff, 0x65c694e0, + 0x65879add, 0x65829a9d, 0x049db246, 0x6582b51a, + 0x6580ab93, 0x65c1a7d1, 0x65cdbdd8, 0x65c1914e, + 0x65ab980b, 0x65f416fc, 0x65b837b7, 0x65ada6fb, + 0x65facfe4, 0x65fde454, 0x65a35ee0, 0x65ed6b85, + 0x044a4d0d, 0x049d7009, 0x043731b0, 0x04ad33d7, + 0x046032d9, 0x04eb33d9, 0x05766aee, 0x05606e45, + 0x65c88069, 0x455db08e, 0x45c4b6ce, 0x042f3edb, + 0x0438397f, 0x4418938c, 0x44da909c, 0x449981e6, + 0x449b9641, 0x045a2882, 0x04982b8b, 0x045937e3, + 0x044835f8, 0x044a2d06, 0x04c93c95, 0x040b34d8, + 0x65c72924, 0x65c627ea, 0x65d82c79, 0x0441284e, + 0x0e2a1128, 0x4e2d118b, 0x0e7812f6, 0x4e651083, + 0x0eae11ac, 0x4eba1338, 0x2e2b1149, 0x6e3d139b, + 0x2e7d139b, 0x6e6810e6, 0x2eb612b4, 0x6eb612b4, + }; // END Generated code -- do not edit diff --git a/test/hotspot/jtreg/compiler/c2/cr7200264/TestIntVect.java b/test/hotspot/jtreg/compiler/c2/cr7200264/TestIntVect.java index 336abf32480..8bf8c9846ec 100644 --- a/test/hotspot/jtreg/compiler/c2/cr7200264/TestIntVect.java +++ b/test/hotspot/jtreg/compiler/c2/cr7200264/TestIntVect.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -555,12 +555,16 @@ public class TestIntVect { } } - // Not vectorized: no vector div. NOTE: This check does not document the - // _desired_ behavior of the system but the current behavior (no - // vectorization) + // Not vectorized: On AArch64 SVE, vectorization for this example results in + // DivVI nodes. @Test + @IR(counts = { IRNode.LOAD_VECTOR_I, "> 0", + IRNode.STORE_VECTOR, "> 0", + IRNode.DIV_VI, "> 0" }, + applyIfCPUFeature = {"sve", "true"}) @IR(counts = { IRNode.LOAD_VECTOR_I, "= 0", - IRNode.STORE_VECTOR, "= 0" }) + IRNode.STORE_VECTOR, "= 0" }, + applyIfCPUFeature = {"sve", "false"}) void test_divv(int[] a0, int[] a1, int b) { for (int i = 0; i < a0.length; i+=1) { a0[i] = (int)(a1[i]/b); diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java index 249e73fa54b..a3ac0b78a11 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java @@ -763,6 +763,26 @@ public class IRNode { beforeMatchingNameRegex(DIV_MOD_L, "DivModL"); } + public static final String DIV_VB = VECTOR_PREFIX + "DIV_VB" + POSTFIX; + static { + vectorNode(DIV_VB, "DivVB", TYPE_BYTE); + } + + public static final String DIV_VS = VECTOR_PREFIX + "DIV_VS" + POSTFIX; + static { + vectorNode(DIV_VS, "DivVS", TYPE_SHORT); + } + + public static final String DIV_VI = VECTOR_PREFIX + "DIV_VI" + POSTFIX; + static { + vectorNode(DIV_VI, "DivVI", TYPE_INT); + } + + public static final String DIV_VL = VECTOR_PREFIX + "DIV_VL" + POSTFIX; + static { + vectorNode(DIV_VL, "DivVL", TYPE_LONG); + } + public static final String DIV_VHF = VECTOR_PREFIX + "DIV_VHF" + POSTFIX; static { vectorNode(DIV_VHF, "DivVHF", TYPE_SHORT); diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorDivTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorDivTest.java new file mode 100644 index 00000000000..cfa51b9434c --- /dev/null +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorDivTest.java @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @bug 8387594 + * @key randomness + * @library /test/lib / + * @summary IR tests for Vector API lanewise DIV + * @modules jdk.incubator.vector + * + * @run driver ${test.main.class} + */ + +package compiler.vectorapi; + +import compiler.lib.generators.*; +import compiler.lib.ir_framework.*; +import jdk.incubator.vector.*; + +public class VectorDivTest { + private static final Generators RD = Generators.G; + + private static final VectorSpecies B_SPECIES = ByteVector.SPECIES_MAX; + private static final VectorSpecies S_SPECIES = ShortVector.SPECIES_MAX; + private static final VectorSpecies I_SPECIES = IntVector.SPECIES_MAX; + private static final VectorSpecies L_SPECIES = LongVector.SPECIES_MAX; + private static final VectorSpecies F_SPECIES = FloatVector.SPECIES_MAX; + private static final VectorSpecies D_SPECIES = DoubleVector.SPECIES_MAX; + + private static final int BUF_LEN = 256; + + private static final byte[] ba = new byte[BUF_LEN]; + private static final byte[] bb = new byte[BUF_LEN]; + private static final byte[] br = new byte[BUF_LEN]; + + private static final short[] sa = new short[BUF_LEN]; + private static final short[] sb = new short[BUF_LEN]; + private static final short[] sr = new short[BUF_LEN]; + + private static final int[] ia = new int[BUF_LEN]; + private static final int[] ib = new int[BUF_LEN]; + private static final int[] ir = new int[BUF_LEN]; + + private static final long[] la = new long[BUF_LEN]; + private static final long[] lb = new long[BUF_LEN]; + private static final long[] lr = new long[BUF_LEN]; + + private static final float[] fa = new float[BUF_LEN]; + private static final float[] fb = new float[BUF_LEN]; + private static final float[] fr = new float[BUF_LEN]; + + private static final double[] da = new double[BUF_LEN]; + private static final double[] db = new double[BUF_LEN]; + private static final double[] dr = new double[BUF_LEN]; + + private static final boolean[] mask_arr = new boolean[BUF_LEN]; + + static { + Generator iGen = RD.ints(); + Generator lGen = RD.longs(); + Generator fGen = RD.floats(); + Generator dGen = RD.doubles(); + + for (int i = 0; i < BUF_LEN; i++) { + mask_arr[i] = (i & 1) != 0; + ba[i] = iGen.next().byteValue(); + // Integer divisors must be non-zero, otherwise lanewise DIV throws. + bb[i] = nonZeroByte(iGen.next().byteValue()); + sa[i] = iGen.next().shortValue(); + sb[i] = nonZeroShort(iGen.next().shortValue()); + ib[i] = nonZeroInt(iGen.next()); + lb[i] = nonZeroLong(lGen.next()); + } + + RD.fill(iGen, ia); + RD.fill(lGen, la); + RD.fill(fGen, fa); + // Floating-point division has no divide-by-zero exception. + RD.fill(fGen, fb); + RD.fill(dGen, da); + RD.fill(dGen, db); + } + + private static byte nonZeroByte(byte v) { return v == 0 ? (byte) 1 : v; } + private static short nonZeroShort(short v) { return v == 0 ? (short) 1 : v; } + private static int nonZeroInt(int v) { return v == 0 ? 1 : v; } + private static long nonZeroLong(long v) { return v == 0 ? 1L : v; } + + // Unmasked lanewise DIV. + + @Test + @IR(counts = { IRNode.DIV_VB, ">= 1" }, + applyIfCPUFeature = { "sve", "true" }) + public static void testDivByte() { + ByteVector va = ByteVector.fromArray(B_SPECIES, ba, 0); + ByteVector vb = ByteVector.fromArray(B_SPECIES, bb, 0); + va.lanewise(VectorOperators.DIV, vb).intoArray(br, 0); + } + + @Test + @IR(counts = { IRNode.DIV_VS, ">= 1" }, + applyIfCPUFeature = { "sve", "true" }) + public static void testDivShort() { + ShortVector va = ShortVector.fromArray(S_SPECIES, sa, 0); + ShortVector vb = ShortVector.fromArray(S_SPECIES, sb, 0); + va.lanewise(VectorOperators.DIV, vb).intoArray(sr, 0); + } + + @Test + @IR(counts = { IRNode.DIV_VI, ">= 1" }, + applyIfCPUFeature = { "sve", "true" }) + public static void testDivInt() { + IntVector va = IntVector.fromArray(I_SPECIES, ia, 0); + IntVector vb = IntVector.fromArray(I_SPECIES, ib, 0); + va.lanewise(VectorOperators.DIV, vb).intoArray(ir, 0); + } + + @Test + @IR(counts = { IRNode.DIV_VL, ">= 1" }, + applyIfCPUFeature = { "sve", "true" }) + public static void testDivLong() { + LongVector va = LongVector.fromArray(L_SPECIES, la, 0); + LongVector vb = LongVector.fromArray(L_SPECIES, lb, 0); + va.lanewise(VectorOperators.DIV, vb).intoArray(lr, 0); + } + + @Test + @IR(counts = { IRNode.DIV_VF, ">= 1" }, + applyIfCPUFeature = { "asimd", "true" }) + public static void testDivFloat() { + FloatVector va = FloatVector.fromArray(F_SPECIES, fa, 0); + FloatVector vb = FloatVector.fromArray(F_SPECIES, fb, 0); + va.lanewise(VectorOperators.DIV, vb).intoArray(fr, 0); + } + + @Test + @IR(counts = { IRNode.DIV_VD, ">= 1" }, + applyIfCPUFeature = { "asimd", "true" }) + public static void testDivDouble() { + DoubleVector va = DoubleVector.fromArray(D_SPECIES, da, 0); + DoubleVector vb = DoubleVector.fromArray(D_SPECIES, db, 0); + va.lanewise(VectorOperators.DIV, vb).intoArray(dr, 0); + } + + // Masked lanewise DIV. On AArch64, BYTE/SHORT have no native predicated + // divide, so they are lowered to an unpredicated divide combined with a + // VectorBlend. + + @Test + @IR(counts = { IRNode.DIV_VB, ">= 1", + IRNode.VECTOR_BLEND_B, ">= 1" }, + applyIfCPUFeature = { "sve", "true" }) + public static void testMaskedDivByte() { + VectorMask mask = VectorMask.fromArray(B_SPECIES, mask_arr, 0); + ByteVector va = ByteVector.fromArray(B_SPECIES, ba, 0); + ByteVector vb = ByteVector.fromArray(B_SPECIES, bb, 0); + va.lanewise(VectorOperators.DIV, vb, mask).intoArray(br, 0); + } + + @Test + @IR(counts = { IRNode.DIV_VS, ">= 1", + IRNode.VECTOR_BLEND_S, ">= 1" }, + applyIfCPUFeature = { "sve", "true" }) + public static void testMaskedDivShort() { + VectorMask mask = VectorMask.fromArray(S_SPECIES, mask_arr, 0); + ShortVector va = ShortVector.fromArray(S_SPECIES, sa, 0); + ShortVector vb = ShortVector.fromArray(S_SPECIES, sb, 0); + va.lanewise(VectorOperators.DIV, vb, mask).intoArray(sr, 0); + } + + @Test + @IR(counts = { IRNode.DIV_VI, ">= 1" }, + applyIfCPUFeature = { "sve", "true" }) + public static void testMaskedDivInt() { + VectorMask mask = VectorMask.fromArray(I_SPECIES, mask_arr, 0); + IntVector va = IntVector.fromArray(I_SPECIES, ia, 0); + IntVector vb = IntVector.fromArray(I_SPECIES, ib, 0); + va.lanewise(VectorOperators.DIV, vb, mask).intoArray(ir, 0); + } + + @Test + @IR(counts = { IRNode.DIV_VL, ">= 1" }, + applyIfCPUFeature = { "sve", "true" }) + public static void testMaskedDivLong() { + VectorMask mask = VectorMask.fromArray(L_SPECIES, mask_arr, 0); + LongVector va = LongVector.fromArray(L_SPECIES, la, 0); + LongVector vb = LongVector.fromArray(L_SPECIES, lb, 0); + va.lanewise(VectorOperators.DIV, vb, mask).intoArray(lr, 0); + } + + @Test + @IR(counts = { IRNode.DIV_VF, ">= 1" }, + applyIfCPUFeature = { "sve", "true" }) + public static void testMaskedDivFloat() { + VectorMask mask = VectorMask.fromArray(F_SPECIES, mask_arr, 0); + FloatVector va = FloatVector.fromArray(F_SPECIES, fa, 0); + FloatVector vb = FloatVector.fromArray(F_SPECIES, fb, 0); + va.lanewise(VectorOperators.DIV, vb, mask).intoArray(fr, 0); + } + + @Test + @IR(counts = { IRNode.DIV_VD, ">= 1" }, + applyIfCPUFeature = { "sve", "true" }) + public static void testMaskedDivDouble() { + VectorMask mask = VectorMask.fromArray(D_SPECIES, mask_arr, 0); + DoubleVector va = DoubleVector.fromArray(D_SPECIES, da, 0); + DoubleVector vb = DoubleVector.fromArray(D_SPECIES, db, 0); + va.lanewise(VectorOperators.DIV, vb, mask).intoArray(dr, 0); + } + + public static void main(String[] args) { + TestFramework testFramework = new TestFramework(); + testFramework.setDefaultWarmup(10000) + .addFlags("--add-modules=jdk.incubator.vector") + .start(); + } +} diff --git a/test/micro/org/openjdk/bench/vm/compiler/VectorIntegerDiv.java b/test/micro/org/openjdk/bench/vm/compiler/VectorIntegerDiv.java new file mode 100644 index 00000000000..110e984dcfe --- /dev/null +++ b/test/micro/org/openjdk/bench/vm/compiler/VectorIntegerDiv.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.bench.vm.compiler; + +import org.openjdk.jmh.annotations.*; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +// Measures auto-vectorization of integer division by a loop-invariant divisor. + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Thread) +@Warmup(iterations = 5, time = 2) +@Measurement(iterations = 5, time = 2) +@Fork(value = 3) +public class VectorIntegerDiv { + + @Param({"1024"}) + private int size; + + // Loop-invariant divisors. These are non-final fields, so the JIT sees a + // runtime value and cannot strength-reduce the division to a multiply. + private int iDivisor; + private long lDivisor; + + private int[] ia, ir; + private long[] la, lr; + + @Setup + public void setup() { + Random r = new Random(42); + iDivisor = r.nextInt() | 1; + lDivisor = r.nextLong() | 1L; + + ia = new int[size]; + ir = new int[size]; + la = new long[size]; + lr = new long[size]; + for (int i = 0; i < size; i++) { + ia[i] = r.nextInt(); + la[i] = r.nextLong(); + } + } + + @Benchmark + public void intDiv() { + for (int i = 0; i < ia.length; i++) { + ir[i] = ia[i] / iDivisor; + } + } + + @Benchmark + public void longDiv() { + for (int i = 0; i < la.length; i++) { + lr[i] = la[i] / lDivisor; + } + } +}