mirror of
https://github.com/openjdk/jdk.git
synced 2026-01-28 12:09:14 +00:00
7350 lines
280 KiB
Plaintext
7350 lines
280 KiB
Plaintext
//
|
|
// Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
|
|
// Copyright (c) 2020, 2025, Arm Limited. All rights reserved.
|
|
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
|
|
//
|
|
// This code is free software; you can redistribute it and/or modify it
|
|
// under the terms of the GNU General Public License version 2 only, as
|
|
// published by the Free Software Foundation.
|
|
//
|
|
// This code is distributed in the hope that it will be useful, but WITHOUT
|
|
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
// version 2 for more details (a copy is included in the LICENSE file that
|
|
// accompanied this code).
|
|
//
|
|
// You should have received a copy of the GNU General Public License version
|
|
// 2 along with this work; if not, write to the Free Software Foundation,
|
|
// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
//
|
|
// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
|
|
// or visit www.oracle.com if you need additional information or have any
|
|
// questions.
|
|
//
|
|
//
|
|
|
|
// This file is automatically generated by running "m4 aarch64_vector_ad.m4". Do not edit!
|
|
|
|
// AArch64 VECTOR Architecture Description File
|
|
|
|
|
|
// 4 bit signed offset -- for predicated load/store
|
|
|
|
operand vmemA_immIOffset4() %{
|
|
// (esize / msize) = 1
|
|
predicate(Address::offset_ok_for_sve_immed(n->get_int(), 4,
|
|
Matcher::scalable_vector_reg_size(T_BYTE)));
|
|
match(ConI);
|
|
op_cost(0);
|
|
format %{ %}
|
|
interface(CONST_INTER);
|
|
%}
|
|
|
|
operand vmemA_immLOffset4() %{
|
|
// (esize / msize) = 1
|
|
predicate(Address::offset_ok_for_sve_immed(n->get_long(), 4,
|
|
Matcher::scalable_vector_reg_size(T_BYTE)));
|
|
match(ConL);
|
|
op_cost(0);
|
|
format %{ %}
|
|
interface(CONST_INTER);
|
|
%}
|
|
|
|
operand vmemA_indOffI4(iRegP reg, vmemA_immIOffset4 off) %{
|
|
constraint(ALLOC_IN_RC(ptr_reg));
|
|
match(AddP reg off);
|
|
op_cost(0);
|
|
format %{ "[$reg, $off]" %}
|
|
interface(MEMORY_INTER) %{
|
|
base($reg);
|
|
index(0xffffffff);
|
|
scale(0x0);
|
|
disp($off);
|
|
%}
|
|
%}
|
|
|
|
operand vmemA_indOffL4(iRegP reg, vmemA_immLOffset4 off) %{
|
|
constraint(ALLOC_IN_RC(ptr_reg));
|
|
match(AddP reg off);
|
|
op_cost(0);
|
|
format %{ "[$reg, $off]" %}
|
|
interface(MEMORY_INTER) %{
|
|
base($reg);
|
|
index(0xffffffff);
|
|
scale(0x0);
|
|
disp($off);
|
|
%}
|
|
%}
|
|
|
|
// The indOff of vmemA is valid only when the vector element (load to/store from)
|
|
// size equals to memory element (load from/store to) size.
|
|
opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
|
|
|
|
source_hpp %{
|
|
// Assert that the given node is not a variable shift.
|
|
bool assert_not_var_shift(const Node* n);
|
|
|
|
Assembler::SIMD_Arrangement get_arrangement(const Node* n);
|
|
Assembler::SIMD_RegVariant get_reg_variant(const Node* n);
|
|
%}
|
|
|
|
source %{
|
|
|
|
typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
|
|
PRegister Pg, const Address &adr);
|
|
|
|
// Predicated load/store, with optional ptrue to all elements of given predicate register.
|
|
static void loadStoreA_predicated(C2_MacroAssembler* masm, bool is_store, FloatRegister reg,
|
|
PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
|
|
int opcode, Register base, int index, int size, int disp) {
|
|
sve_mem_insn_predicate insn;
|
|
int mesize = type2aelembytes(mem_elem_bt);
|
|
if (index == -1) {
|
|
assert(size == 0, "unsupported address mode: scale size = %d", size);
|
|
switch(mesize) {
|
|
case 1:
|
|
insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
|
|
break;
|
|
case 2:
|
|
insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
|
|
break;
|
|
case 4:
|
|
insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
|
|
break;
|
|
case 8:
|
|
insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
|
|
break;
|
|
default:
|
|
assert(false, "unsupported");
|
|
ShouldNotReachHere();
|
|
}
|
|
int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
|
|
(masm->*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
|
|
} else {
|
|
assert(false, "unimplemented");
|
|
ShouldNotReachHere();
|
|
}
|
|
}
|
|
|
|
bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
|
|
if (UseSVE == 0) {
|
|
// These operations are not profitable to be vectorized on NEON, because no direct
|
|
// NEON instructions support them. They use multiple instructions which is more
|
|
// expensive in almost all cases where we would auto vectorize.
|
|
// But the match rule support for them is profitable for Vector API intrinsics.
|
|
if ((opcode == Op_VectorCastD2X && (bt == T_INT || bt == T_SHORT)) ||
|
|
(opcode == Op_VectorCastL2X && bt == T_FLOAT) ||
|
|
(opcode == Op_CountLeadingZerosV && bt == T_LONG) ||
|
|
(opcode == Op_CountTrailingZerosV && bt == T_LONG) ||
|
|
opcode == Op_MulVL ||
|
|
// The implementations of Op_AddReductionVD/F in Neon are for the Vector API only.
|
|
// They are not suitable for auto-vectorization because the result would not conform
|
|
// to the JLS, Section Evaluation Order.
|
|
// Note: we could implement sequential reductions for these reduction operators, but
|
|
// this will still almost never lead to speedups, because the sequential
|
|
// reductions are latency limited along the reduction chain, and not
|
|
// throughput limited. This is unlike unordered reductions (associative op)
|
|
// and element-wise ops which are usually throughput limited.
|
|
opcode == Op_AddReductionVD || opcode == Op_AddReductionVF ||
|
|
opcode == Op_MulReductionVD || opcode == Op_MulReductionVF) {
|
|
return false;
|
|
}
|
|
}
|
|
return match_rule_supported_vector(opcode, vlen, bt);
|
|
}
|
|
|
|
// Identify extra cases that we might want to provide match rules for vector nodes and
|
|
// other intrinsics guarded with vector length (vlen) and element type (bt).
|
|
bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
|
|
if (!match_rule_supported(opcode)) {
|
|
return false;
|
|
}
|
|
|
|
int length_in_bytes = vlen * type2aelembytes(bt);
|
|
if (UseSVE == 0 && length_in_bytes > FloatRegister::neon_vl) {
|
|
return false;
|
|
}
|
|
|
|
// Check whether specific Op is supported.
|
|
// Fail fast, otherwise fall through to common vector_size_supported() check.
|
|
switch (opcode) {
|
|
case Op_AndVMask:
|
|
case Op_OrVMask:
|
|
case Op_XorVMask:
|
|
case Op_MaskAll:
|
|
case Op_VectorMaskGen:
|
|
case Op_LoadVectorMasked:
|
|
case Op_StoreVectorMasked:
|
|
case Op_StoreVectorScatter:
|
|
case Op_StoreVectorScatterMasked:
|
|
case Op_PopulateIndex:
|
|
case Op_CompressM:
|
|
case Op_CompressV:
|
|
if (UseSVE == 0) {
|
|
return false;
|
|
}
|
|
break;
|
|
case Op_LoadVectorGather:
|
|
case Op_LoadVectorGatherMasked:
|
|
if (UseSVE == 0 || is_subword_type(bt)) {
|
|
return false;
|
|
}
|
|
break;
|
|
case Op_MulAddVS2VI:
|
|
if (length_in_bytes != 16) {
|
|
return false;
|
|
}
|
|
break;
|
|
case Op_AddReductionVI:
|
|
case Op_AndReductionV:
|
|
case Op_OrReductionV:
|
|
case Op_XorReductionV:
|
|
case Op_MinReductionV:
|
|
case Op_MaxReductionV:
|
|
// Reductions with less than 8 bytes vector length are
|
|
// not supported.
|
|
if (length_in_bytes < 8) {
|
|
return false;
|
|
}
|
|
break;
|
|
case Op_MulReductionVD:
|
|
case Op_MulReductionVF:
|
|
case Op_MulReductionVI:
|
|
case Op_MulReductionVL:
|
|
// No vector multiply reduction instructions, but we do
|
|
// emit scalar instructions for 64/128-bit vectors.
|
|
if (length_in_bytes != 8 && length_in_bytes != 16) {
|
|
return false;
|
|
}
|
|
break;
|
|
case Op_VectorMaskCmp:
|
|
if (length_in_bytes < 8) {
|
|
return false;
|
|
}
|
|
break;
|
|
case Op_VectorMaskToLong:
|
|
if (UseSVE > 0 && vlen > 64) {
|
|
return false;
|
|
}
|
|
break;
|
|
case Op_VectorLongToMask:
|
|
if (vlen > 64 || !VM_Version::supports_svebitperm()) {
|
|
return false;
|
|
}
|
|
break;
|
|
case Op_CompressBitsV:
|
|
case Op_ExpandBitsV:
|
|
if (!VM_Version::supports_svebitperm()) {
|
|
return false;
|
|
}
|
|
break;
|
|
case Op_AddVHF:
|
|
case Op_SubVHF:
|
|
case Op_MulVHF:
|
|
case Op_DivVHF:
|
|
case Op_MinVHF:
|
|
case Op_MaxVHF:
|
|
case Op_SqrtVHF:
|
|
// FEAT_FP16 is enabled if both "fphp" and "asimdhp" features are supported.
|
|
// Only the Neon instructions need this check. SVE supports half-precision floats
|
|
// by default.
|
|
if (UseSVE == 0 && !is_feat_fp16_supported()) {
|
|
return false;
|
|
}
|
|
break;
|
|
case Op_FmaVHF:
|
|
// UseFMA flag needs to be checked along with FEAT_FP16
|
|
if (!UseFMA || (UseSVE == 0 && !is_feat_fp16_supported())) {
|
|
return false;
|
|
}
|
|
break;
|
|
case Op_SelectFromTwoVector:
|
|
// The "tbl" instruction for two vector table is supported only in Neon and SVE2. Return
|
|
// false if vector length > 16B but supported SVE version < 2.
|
|
// For vector length of 16B, generate SVE2 "tbl" instruction if SVE2 is supported, else
|
|
// generate Neon "tbl" instruction to select from two vectors.
|
|
// This operation is disabled for doubles and longs on machines with SVE < 2 and instead
|
|
// the default VectorRearrange + VectorBlend is generated because the performance of the default
|
|
// implementation was better than or equal to the implementation for SelectFromTwoVector.
|
|
if (UseSVE < 2 && (type2aelembytes(bt) == 8 || length_in_bytes > 16)) {
|
|
return false;
|
|
}
|
|
|
|
// Because the SVE2 "tbl" instruction is unpredicated and partial operations cannot be generated
|
|
// using masks, we disable this operation on machines where length_in_bytes < MaxVectorSize
|
|
// on that machine with the only exception of 8B vector length. This is because at the time of
|
|
// writing this, there is no SVE2 machine available with length_in_bytes > 8 and
|
|
// length_in_bytes < MaxVectorSize to test this operation on (for example - there isn't an
|
|
// SVE2 machine available with MaxVectorSize = 32 to test a case with length_in_bytes = 16).
|
|
if (UseSVE == 2 && length_in_bytes > 8 && length_in_bytes < MaxVectorSize) {
|
|
return false;
|
|
}
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
return vector_size_supported(bt, vlen);
|
|
}
|
|
|
|
bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
|
|
// Only SVE supports masked operations.
|
|
if (UseSVE == 0) {
|
|
return false;
|
|
}
|
|
|
|
// If an opcode does not support the masked version,
|
|
// unpredicated node with VectorBlend node will be used instead.
|
|
switch(opcode) {
|
|
case Op_VectorRearrange:
|
|
case Op_MulReductionVD:
|
|
case Op_MulReductionVF:
|
|
case Op_MulReductionVI:
|
|
case Op_MulReductionVL:
|
|
case Op_CompressBitsV:
|
|
case Op_ExpandBitsV:
|
|
return false;
|
|
case Op_SaturatingAddV:
|
|
case Op_SaturatingSubV:
|
|
// Only SVE2 supports the predicated saturating instructions.
|
|
if (UseSVE < 2) {
|
|
return false;
|
|
}
|
|
break;
|
|
// We use Op_LoadVectorMasked to implement the predicated Op_LoadVector.
|
|
// Hence we turn to check whether Op_LoadVectorMasked is supported. The
|
|
// same as vector store/gather/scatter.
|
|
case Op_LoadVector:
|
|
opcode = Op_LoadVectorMasked;
|
|
break;
|
|
case Op_StoreVector:
|
|
opcode = Op_StoreVectorMasked;
|
|
break;
|
|
case Op_LoadVectorGather:
|
|
opcode = Op_LoadVectorGatherMasked;
|
|
break;
|
|
case Op_StoreVectorScatter:
|
|
opcode = Op_StoreVectorScatterMasked;
|
|
break;
|
|
// Currently, the masked versions of the following 8 Float16 operations are disabled.
|
|
// When the support for Float16 vector classes is added in VectorAPI and the masked
|
|
// Float16 IR can be generated, these masked operations will be enabled and relevant
|
|
// backend support added.
|
|
case Op_AddVHF:
|
|
case Op_SubVHF:
|
|
case Op_MulVHF:
|
|
case Op_DivVHF:
|
|
case Op_MaxVHF:
|
|
case Op_MinVHF:
|
|
case Op_SqrtVHF:
|
|
case Op_FmaVHF:
|
|
return false;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return match_rule_supported_vector(opcode, vlen, bt);
|
|
}
|
|
|
|
bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* vt) {
|
|
// 1. Only SVE requires partial vector operations.
|
|
// 2. The vector size in bytes must be smaller than MaxVectorSize.
|
|
// 3. Predicated vectors have a mask input, which guarantees that
|
|
// out-of-bounds lanes remain inactive.
|
|
int length_in_bytes = vt->length_in_bytes();
|
|
if (UseSVE == 0 ||
|
|
length_in_bytes == MaxVectorSize ||
|
|
node->is_predicated_vector()) {
|
|
return false;
|
|
}
|
|
|
|
switch(node->Opcode()) {
|
|
case Op_VectorLoadMask:
|
|
case Op_VectorMaskCmp:
|
|
case Op_LoadVectorGather:
|
|
case Op_StoreVectorScatter:
|
|
case Op_AddReductionVF:
|
|
case Op_AddReductionVD:
|
|
case Op_AndReductionV:
|
|
case Op_OrReductionV:
|
|
case Op_XorReductionV:
|
|
// Mask is needed for partial Op_VectorMaskFirstTrue, because when the
|
|
// input predicate is all-false, the result should be the vector length
|
|
// instead of the vector register size.
|
|
case Op_VectorMaskFirstTrue:
|
|
return true;
|
|
case Op_MaskAll:
|
|
return !node->in(1)->is_Con();
|
|
case Op_LoadVector:
|
|
case Op_StoreVector:
|
|
case Op_AddReductionVI:
|
|
case Op_AddReductionVL:
|
|
// For these ops, we prefer using NEON instructions rather than SVE
|
|
// predicated instructions for better performance.
|
|
return !VM_Version::use_neon_for_vector(length_in_bytes);
|
|
case Op_MinReductionV:
|
|
case Op_MaxReductionV:
|
|
// For BYTE/SHORT/INT/FLOAT/DOUBLE types, we prefer using NEON
|
|
// instructions rather than SVE predicated instructions for
|
|
// better performance.
|
|
return vt->element_basic_type() == T_LONG ||
|
|
!VM_Version::use_neon_for_vector(length_in_bytes);
|
|
default:
|
|
// For other ops whose vector size is smaller than the max vector
|
|
// size, a full-sized unpredicated operation does not impact the
|
|
// vector result.
|
|
return false;
|
|
}
|
|
}
|
|
|
|
bool Matcher::vector_rearrange_requires_load_shuffle(BasicType elem_bt, int vlen) {
|
|
return false;
|
|
}
|
|
|
|
bool Matcher::mask_op_prefers_predicate(int opcode, const TypeVect* vt) {
|
|
// Only SVE supports the predicate feature.
|
|
if (UseSVE == 0) {
|
|
// On architectures that do not support predicate, masks are stored in
|
|
// general vector registers (TypeVect) with sizes ranging from TypeVectA
|
|
// to TypeVectX based on the vector size in bytes.
|
|
assert(vt->isa_vectmask() == nullptr, "mask type is not matched");
|
|
return false;
|
|
}
|
|
|
|
assert(vt->isa_vectmask() != nullptr, "expected TypeVectMask on SVE");
|
|
switch (opcode) {
|
|
case Op_VectorMaskToLong:
|
|
case Op_VectorLongToMask:
|
|
// These operations lack native SVE predicate instructions and are
|
|
// implemented using general vector instructions instead. Use vector
|
|
// registers rather than predicate registers to save the mask for
|
|
// better performance.
|
|
return false;
|
|
default:
|
|
// By default, the mask operations are implemented with predicate
|
|
// instructions with a predicate input/output.
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Assert that the given node is not a variable shift.
|
|
bool assert_not_var_shift(const Node* n) {
|
|
assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift");
|
|
return true;
|
|
}
|
|
|
|
Assembler::SIMD_Arrangement get_arrangement(const Node* n) {
|
|
BasicType bt = Matcher::vector_element_basic_type(n);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(n);
|
|
return Assembler::esize2arrangement((uint)type2aelembytes(bt),
|
|
/* isQ */ length_in_bytes == 16);
|
|
}
|
|
|
|
Assembler::SIMD_RegVariant get_reg_variant(const Node* n) {
|
|
BasicType bt = Matcher::vector_element_basic_type(n);
|
|
return Assembler::elemType_to_regVariant(bt);
|
|
}
|
|
%}
|
|
|
|
|
|
// All VECTOR instructions
|
|
|
|
// ------------------------------ Vector load/store ----------------------------
|
|
|
|
// Load Vector (16 bits)
|
|
instruct loadV2(vReg dst, vmem2 mem) %{
|
|
predicate(n->as_LoadVector()->memory_size() == 2);
|
|
match(Set dst (LoadVector mem));
|
|
format %{ "loadV2 $dst, $mem\t# vector (16 bits)" %}
|
|
ins_encode( aarch64_enc_ldrvH(dst, mem) );
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Store Vector (16 bits)
|
|
instruct storeV2(vReg src, vmem2 mem) %{
|
|
predicate(n->as_StoreVector()->memory_size() == 2);
|
|
match(Set mem (StoreVector mem src));
|
|
format %{ "storeV2 $mem, $src\t# vector (16 bits)" %}
|
|
ins_encode( aarch64_enc_strvH(src, mem) );
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Load Vector (32 bits)
|
|
instruct loadV4(vReg dst, vmem4 mem) %{
|
|
predicate(n->as_LoadVector()->memory_size() == 4);
|
|
match(Set dst (LoadVector mem));
|
|
format %{ "loadV4 $dst, $mem\t# vector (32 bits)" %}
|
|
ins_encode( aarch64_enc_ldrvS(dst, mem) );
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Store Vector (32 bits)
|
|
instruct storeV4(vReg src, vmem4 mem) %{
|
|
predicate(n->as_StoreVector()->memory_size() == 4);
|
|
match(Set mem (StoreVector mem src));
|
|
format %{ "storeV4 $mem, $src\t# vector (32 bits)" %}
|
|
ins_encode( aarch64_enc_strvS(src, mem) );
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Load Vector (64 bits)
|
|
instruct loadV8(vReg dst, vmem8 mem) %{
|
|
predicate(n->as_LoadVector()->memory_size() == 8);
|
|
match(Set dst (LoadVector mem));
|
|
format %{ "loadV8 $dst, $mem\t# vector (64 bits)" %}
|
|
ins_encode( aarch64_enc_ldrvD(dst, mem) );
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Store Vector (64 bits)
|
|
instruct storeV8(vReg src, vmem8 mem) %{
|
|
predicate(n->as_StoreVector()->memory_size() == 8);
|
|
match(Set mem (StoreVector mem src));
|
|
format %{ "storeV8 $mem, $src\t# vector (64 bits)" %}
|
|
ins_encode( aarch64_enc_strvD(src, mem) );
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Load Vector (128 bits)
|
|
instruct loadV16(vReg dst, vmem16 mem) %{
|
|
predicate(n->as_LoadVector()->memory_size() == 16);
|
|
match(Set dst (LoadVector mem));
|
|
format %{ "loadV16 $dst, $mem\t# vector (128 bits)" %}
|
|
ins_encode( aarch64_enc_ldrvQ(dst, mem) );
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Store Vector (128 bits)
|
|
instruct storeV16(vReg src, vmem16 mem) %{
|
|
predicate(n->as_StoreVector()->memory_size() == 16);
|
|
match(Set mem (StoreVector mem src));
|
|
format %{ "storeV16 $mem, $src\t# vector (128 bits)" %}
|
|
ins_encode( aarch64_enc_strvQ(src, mem) );
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Load Vector (> 128 bits)
|
|
instruct loadV(vReg dst, vmemA mem) %{
|
|
predicate(n->as_LoadVector()->memory_size() > 16);
|
|
match(Set dst (LoadVector mem));
|
|
format %{ "loadV $dst, $mem\t# vector (sve)" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
loadStoreA_predicated(masm, /* is_store */ false,
|
|
$dst$$FloatRegister, ptrue, bt, bt, $mem->opcode(),
|
|
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Store Vector (> 128 bits)
|
|
instruct storeV(vReg src, vmemA mem) %{
|
|
predicate(n->as_StoreVector()->memory_size() > 16);
|
|
match(Set mem (StoreVector mem src));
|
|
format %{ "storeV $mem, $src\t# vector (sve)" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $src);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
loadStoreA_predicated(masm, /* is_store */ true,
|
|
$src$$FloatRegister, ptrue, bt, bt, $mem->opcode(),
|
|
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector load/store - predicated
|
|
|
|
instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (LoadVectorMasked mem pg));
|
|
format %{ "loadV_masked $dst, $pg, $mem" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
loadStoreA_predicated(masm, /* is_store */ false, $dst$$FloatRegister,
|
|
$pg$$PRegister, bt, bt, $mem->opcode(),
|
|
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set mem (StoreVectorMasked mem (Binary src pg)));
|
|
format %{ "storeV_masked $mem, $pg, $src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $src);
|
|
loadStoreA_predicated(masm, /* is_store */ true, $src$$FloatRegister,
|
|
$pg$$PRegister, bt, bt, $mem->opcode(),
|
|
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector load const
|
|
|
|
instruct vloadcon(vReg dst, immI0 src) %{
|
|
match(Set dst (VectorLoadConst src));
|
|
format %{ "vloadcon $dst, $src\t# load/generate iota indices" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
if (UseSVE == 0) {
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes <= 16, "must be");
|
|
// The iota indices are ordered by type B/S/I/L/F/D, and the offset between two types is 16.
|
|
int offset = exact_log2(type2aelembytes(bt)) << 4;
|
|
if (is_floating_point_type(bt)) {
|
|
offset += 32;
|
|
}
|
|
__ lea(rscratch1, ExternalAddress(StubRoutines::aarch64::vector_iota_indices() + offset));
|
|
if (length_in_bytes == 16) {
|
|
__ ldrq($dst$$FloatRegister, rscratch1);
|
|
} else {
|
|
__ ldrd($dst$$FloatRegister, rscratch1);
|
|
}
|
|
} else {
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_index($dst$$FloatRegister, size, 0, 1);
|
|
if (is_floating_point_type(bt)) {
|
|
__ sve_scvtf($dst$$FloatRegister, size, ptrue, $dst$$FloatRegister, size);
|
|
}
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector add -----------------------------------
|
|
|
|
// vector add
|
|
|
|
instruct vaddB(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (AddVB src1 src2));
|
|
format %{ "vaddB $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ addv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_add($dst$$FloatRegister, __ B, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vaddS(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (AddVS src1 src2));
|
|
format %{ "vaddS $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ addv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_add($dst$$FloatRegister, __ H, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vaddI(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (AddVI src1 src2));
|
|
format %{ "vaddI $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ addv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_add($dst$$FloatRegister, __ S, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vaddL(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (AddVL src1 src2));
|
|
format %{ "vaddL $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ addv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_add($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vaddHF(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (AddVHF src1 src2));
|
|
format %{ "vaddHF $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fadd($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fadd($dst$$FloatRegister, __ H, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vaddF(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (AddVF src1 src2));
|
|
format %{ "vaddF $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fadd($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fadd($dst$$FloatRegister, __ S, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vaddD(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (AddVD src1 src2));
|
|
format %{ "vaddD $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fadd($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fadd($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector add - predicated
|
|
|
|
instruct vaddB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (AddVB (Binary dst_src1 src2) pg));
|
|
format %{ "vaddB_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_add($dst_src1$$FloatRegister, __ B, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vaddS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (AddVS (Binary dst_src1 src2) pg));
|
|
format %{ "vaddS_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_add($dst_src1$$FloatRegister, __ H, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vaddI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (AddVI (Binary dst_src1 src2) pg));
|
|
format %{ "vaddI_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_add($dst_src1$$FloatRegister, __ S, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vaddL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (AddVL (Binary dst_src1 src2) pg));
|
|
format %{ "vaddL_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_add($dst_src1$$FloatRegister, __ D, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vaddF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (AddVF (Binary dst_src1 src2) pg));
|
|
format %{ "vaddF_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_fadd($dst_src1$$FloatRegister, __ S, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vaddD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (AddVD (Binary dst_src1 src2) pg));
|
|
format %{ "vaddD_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_fadd($dst_src1$$FloatRegister, __ D, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector add reg imm (unpredicated)
|
|
|
|
instruct vaddImmB(vReg dst_src, immBAddSubV con) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (AddVB dst_src (Replicate con)));
|
|
format %{ "vaddImmB $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
int val = (int)$con$$constant;
|
|
if (val > 0) {
|
|
__ sve_add($dst_src$$FloatRegister, __ B, val);
|
|
} else {
|
|
__ sve_sub($dst_src$$FloatRegister, __ B, -val);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vaddImmS(vReg dst_src, immIAddSubV con) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (AddVS dst_src (Replicate con)));
|
|
format %{ "vaddImmS $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
int val = (int)$con$$constant;
|
|
if (val > 0) {
|
|
__ sve_add($dst_src$$FloatRegister, __ H, val);
|
|
} else {
|
|
__ sve_sub($dst_src$$FloatRegister, __ H, -val);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vaddImmI(vReg dst_src, immIAddSubV con) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (AddVI dst_src (Replicate con)));
|
|
format %{ "vaddImmI $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
int val = (int)$con$$constant;
|
|
if (val > 0) {
|
|
__ sve_add($dst_src$$FloatRegister, __ S, val);
|
|
} else {
|
|
__ sve_sub($dst_src$$FloatRegister, __ S, -val);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vaddImmL(vReg dst_src, immLAddSubV con) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (AddVL dst_src (Replicate con)));
|
|
format %{ "vaddImmL $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
int val = (int)$con$$constant;
|
|
if (val > 0) {
|
|
__ sve_add($dst_src$$FloatRegister, __ D, val);
|
|
} else {
|
|
__ sve_sub($dst_src$$FloatRegister, __ D, -val);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector sub -----------------------------------
|
|
|
|
// vector sub
|
|
|
|
instruct vsubB(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (SubVB src1 src2));
|
|
format %{ "vsubB $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ subv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_sub($dst$$FloatRegister, __ B, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsubS(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (SubVS src1 src2));
|
|
format %{ "vsubS $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ subv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_sub($dst$$FloatRegister, __ H, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsubI(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (SubVI src1 src2));
|
|
format %{ "vsubI $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ subv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_sub($dst$$FloatRegister, __ S, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsubL(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (SubVL src1 src2));
|
|
format %{ "vsubL $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ subv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_sub($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsubHF(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (SubVHF src1 src2));
|
|
format %{ "vsubHF $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fsub($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fsub($dst$$FloatRegister, __ H, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsubF(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (SubVF src1 src2));
|
|
format %{ "vsubF $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fsub($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fsub($dst$$FloatRegister, __ S, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsubD(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (SubVD src1 src2));
|
|
format %{ "vsubD $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fsub($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fsub($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector sub - predicated
|
|
|
|
instruct vsubB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (SubVB (Binary dst_src1 src2) pg));
|
|
format %{ "vsubB_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_sub($dst_src1$$FloatRegister, __ B, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsubS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (SubVS (Binary dst_src1 src2) pg));
|
|
format %{ "vsubS_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_sub($dst_src1$$FloatRegister, __ H, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsubI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (SubVI (Binary dst_src1 src2) pg));
|
|
format %{ "vsubI_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_sub($dst_src1$$FloatRegister, __ S, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsubL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (SubVL (Binary dst_src1 src2) pg));
|
|
format %{ "vsubL_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_sub($dst_src1$$FloatRegister, __ D, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsubF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (SubVF (Binary dst_src1 src2) pg));
|
|
format %{ "vsubF_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_fsub($dst_src1$$FloatRegister, __ S, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsubD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (SubVD (Binary dst_src1 src2) pg));
|
|
format %{ "vsubD_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_fsub($dst_src1$$FloatRegister, __ D, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector mul -----------------------------------
|
|
|
|
// vector mul - BYTE, CHAR, SHORT, INT
|
|
|
|
instruct vmulB_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst (MulVB src1 src2));
|
|
format %{ "vmulB_neon $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
__ mulv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmulB_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst_src1 (MulVB dst_src1 src2));
|
|
format %{ "vmulB_sve $dst_src1, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_mul($dst_src1$$FloatRegister, __ B, ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmulS_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst (MulVS src1 src2));
|
|
format %{ "vmulS_neon $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
__ mulv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmulS_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst_src1 (MulVS dst_src1 src2));
|
|
format %{ "vmulS_sve $dst_src1, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_mul($dst_src1$$FloatRegister, __ H, ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmulI_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst (MulVI src1 src2));
|
|
format %{ "vmulI_neon $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
__ mulv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmulI_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst_src1 (MulVI dst_src1 src2));
|
|
format %{ "vmulI_sve $dst_src1, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_mul($dst_src1$$FloatRegister, __ S, ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector mul - LONG
|
|
|
|
instruct vmulL_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(UseSVE == 0);
|
|
match(Set dst (MulVL src1 src2));
|
|
format %{ "vmulL_neon $dst, $src1, $src2\t# 2L" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == 16, "must be");
|
|
__ umov(rscratch1, $src1$$FloatRegister, __ D, 0);
|
|
__ umov(rscratch2, $src2$$FloatRegister, __ D, 0);
|
|
__ mul(rscratch2, rscratch2, rscratch1);
|
|
__ mov($dst$$FloatRegister, __ D, 0, rscratch2);
|
|
__ umov(rscratch1, $src1$$FloatRegister, __ D, 1);
|
|
__ umov(rscratch2, $src2$$FloatRegister, __ D, 1);
|
|
__ mul(rscratch2, rscratch2, rscratch1);
|
|
__ mov($dst$$FloatRegister, __ D, 1, rscratch2);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmulL_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (MulVL dst_src1 src2));
|
|
format %{ "vmulL_sve $dst_src1, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_mul($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector mul - floating-point
|
|
|
|
instruct vmulHF(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (MulVHF src1 src2));
|
|
format %{ "vmulHF $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fmul($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fmul($dst$$FloatRegister, __ H, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmulF(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (MulVF src1 src2));
|
|
format %{ "vmulF $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fmul($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fmul($dst$$FloatRegister, __ S, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmulD(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (MulVD src1 src2));
|
|
format %{ "vmulD $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fmul($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fmul($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector mul - predicated
|
|
|
|
instruct vmulB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (MulVB (Binary dst_src1 src2) pg));
|
|
format %{ "vmulB_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_mul($dst_src1$$FloatRegister, __ B, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmulS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (MulVS (Binary dst_src1 src2) pg));
|
|
format %{ "vmulS_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_mul($dst_src1$$FloatRegister, __ H, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmulI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (MulVI (Binary dst_src1 src2) pg));
|
|
format %{ "vmulI_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_mul($dst_src1$$FloatRegister, __ S, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmulL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (MulVL (Binary dst_src1 src2) pg));
|
|
format %{ "vmulL_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_mul($dst_src1$$FloatRegister, __ D, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmulF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (MulVF (Binary dst_src1 src2) pg));
|
|
format %{ "vmulF_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_fmul($dst_src1$$FloatRegister, __ S, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmulD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (MulVD (Binary dst_src1 src2) pg));
|
|
format %{ "vmulD_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_fmul($dst_src1$$FloatRegister, __ D, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector float div -----------------------------
|
|
|
|
// vector float div
|
|
|
|
instruct vdivHF_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst (DivVHF src1 src2));
|
|
format %{ "vdivHF_neon $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
__ fdiv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vdivHF_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst_src1 (DivVHF dst_src1 src2));
|
|
format %{ "vdivHF_sve $dst_src1, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fdiv($dst_src1$$FloatRegister, __ H, ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vdivF_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst (DivVF src1 src2));
|
|
format %{ "vdivF_neon $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
__ fdiv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vdivF_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst_src1 (DivVF dst_src1 src2));
|
|
format %{ "vdivF_sve $dst_src1, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fdiv($dst_src1$$FloatRegister, __ S, ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vdivD_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst (DivVD src1 src2));
|
|
format %{ "vdivD_neon $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
__ fdiv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vdivD_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst_src1 (DivVD dst_src1 src2));
|
|
format %{ "vdivD_sve $dst_src1, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fdiv($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector float div - predicated
|
|
|
|
instruct vdivF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (DivVF (Binary dst_src1 src2) pg));
|
|
format %{ "vdivF_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_fdiv($dst_src1$$FloatRegister, __ S, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vdivD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (DivVD (Binary dst_src1 src2) pg));
|
|
format %{ "vdivD_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_fdiv($dst_src1$$FloatRegister, __ D, $pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector and -----------------------------------
|
|
|
|
// vector and
|
|
|
|
instruct vand(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (AndV src1 src2));
|
|
format %{ "vand $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ andr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_and($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector and - predicated
|
|
|
|
instruct vand_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (AndV (Binary dst_src1 src2) pg));
|
|
format %{ "vand_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_and($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector and reg imm (unpredicated)
|
|
|
|
instruct vandImmB(vReg dst_src, immBLog con) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
|
|
match(Set dst_src (AndV dst_src (Replicate con)));
|
|
format %{ "vandImmB $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
__ sve_and($dst_src$$FloatRegister, __ B, (uint64_t)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vandImmS(vReg dst_src, immSLog con) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_SHORT);
|
|
match(Set dst_src (AndV dst_src (Replicate con)));
|
|
format %{ "vandImmS $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
__ sve_and($dst_src$$FloatRegister, __ H, (uint64_t)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vandImmI(vReg dst_src, immILog con) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_INT);
|
|
match(Set dst_src (AndV dst_src (Replicate con)));
|
|
format %{ "vandImmI $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
__ sve_and($dst_src$$FloatRegister, __ S, (uint64_t)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vandImmL(vReg dst_src, immLLog con) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
|
|
match(Set dst_src (AndV dst_src (Replicate con)));
|
|
format %{ "vandImmL $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
__ sve_and($dst_src$$FloatRegister, __ D, (uint64_t)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector or ------------------------------------
|
|
|
|
// vector or
|
|
|
|
instruct vor(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (OrV src1 src2));
|
|
format %{ "vor $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_orr($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector or - predicated
|
|
|
|
instruct vor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (OrV (Binary dst_src1 src2) pg));
|
|
format %{ "vor_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_orr($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector or reg imm (unpredicated)
|
|
|
|
instruct vorImmB(vReg dst_src, immBLog con) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
|
|
match(Set dst_src (OrV dst_src (Replicate con)));
|
|
format %{ "vorImmB $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
__ sve_orr($dst_src$$FloatRegister, __ B, (uint64_t)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vorImmS(vReg dst_src, immSLog con) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_SHORT);
|
|
match(Set dst_src (OrV dst_src (Replicate con)));
|
|
format %{ "vorImmS $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
__ sve_orr($dst_src$$FloatRegister, __ H, (uint64_t)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vorImmI(vReg dst_src, immILog con) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_INT);
|
|
match(Set dst_src (OrV dst_src (Replicate con)));
|
|
format %{ "vorImmI $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
__ sve_orr($dst_src$$FloatRegister, __ S, (uint64_t)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vorImmL(vReg dst_src, immLLog con) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
|
|
match(Set dst_src (OrV dst_src (Replicate con)));
|
|
format %{ "vorImmL $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
__ sve_orr($dst_src$$FloatRegister, __ D, (uint64_t)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector xor -----------------------------------
|
|
|
|
// vector xor
|
|
|
|
instruct vxor(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (XorV src1 src2));
|
|
format %{ "vxor $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ eor($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_eor($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector xor - predicated
|
|
|
|
instruct vxor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (XorV (Binary dst_src1 src2) pg));
|
|
format %{ "vxor_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_eor($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector xor reg imm (unpredicated)
|
|
|
|
instruct vxorImmB(vReg dst_src, immBLog con) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
|
|
match(Set dst_src (XorV dst_src (Replicate con)));
|
|
format %{ "vxorImmB $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
__ sve_eor($dst_src$$FloatRegister, __ B, (uint64_t)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vxorImmS(vReg dst_src, immSLog con) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_SHORT);
|
|
match(Set dst_src (XorV dst_src (Replicate con)));
|
|
format %{ "vxorImmS $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
__ sve_eor($dst_src$$FloatRegister, __ H, (uint64_t)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vxorImmI(vReg dst_src, immILog con) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_INT);
|
|
match(Set dst_src (XorV dst_src (Replicate con)));
|
|
format %{ "vxorImmI $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
__ sve_eor($dst_src$$FloatRegister, __ S, (uint64_t)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vxorImmL(vReg dst_src, immLLog con) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
|
|
match(Set dst_src (XorV dst_src (Replicate con)));
|
|
format %{ "vxorImmL $dst_src, $dst_src, $con" %}
|
|
ins_encode %{
|
|
__ sve_eor($dst_src$$FloatRegister, __ D, (uint64_t)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector eor3 (unpredicated)
|
|
|
|
instruct veor3_neon(vReg dst, vReg src1, vReg src2, vReg src3) %{
|
|
predicate(VM_Version::supports_sha3() &&
|
|
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst (XorV src1 (XorV src2 src3)));
|
|
format %{ "veor3_neon $dst, $src1, $src2, $src3" %}
|
|
ins_encode %{
|
|
__ eor3($dst$$FloatRegister, __ T16B, $src1$$FloatRegister,
|
|
$src2$$FloatRegister, $src3$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct veor3_sve(vReg dst_src1, vReg src2, vReg src3) %{
|
|
predicate(UseSVE == 2 && !VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst_src1 (XorV dst_src1 (XorV src2 src3)));
|
|
format %{ "veor3_sve $dst_src1, $dst_src1, $src2, $src3" %}
|
|
ins_encode %{
|
|
__ sve_eor3($dst_src1$$FloatRegister, $src2$$FloatRegister, $src3$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector not -----------------------------------
|
|
|
|
// vector not
|
|
|
|
instruct vnotI(vReg dst, vReg src, immI_M1 m1) %{
|
|
match(Set dst (XorV src (Replicate m1)));
|
|
format %{ "vnotI $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ notr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_not($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{
|
|
match(Set dst (XorV src (Replicate m1)));
|
|
format %{ "vnotL $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ notr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_not($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector not - predicated
|
|
|
|
instruct vnotI_masked(vReg dst_src, immI_M1 m1, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (XorV (Binary dst_src (Replicate m1)) pg));
|
|
format %{ "vnotI_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
__ sve_not($dst_src$$FloatRegister, get_reg_variant(this),
|
|
$pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vnotL_masked(vReg dst_src, immL_M1 m1, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (XorV (Binary dst_src (Replicate m1)) pg));
|
|
format %{ "vnotL_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
__ sve_not($dst_src$$FloatRegister, get_reg_variant(this),
|
|
$pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector and_not -------------------------------
|
|
|
|
// vector and_not
|
|
|
|
instruct vand_notI(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
|
|
match(Set dst (AndV src1 (XorV src2 (Replicate m1))));
|
|
format %{ "vand_notI $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ bic($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_bic($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vand_notL(vReg dst, vReg src1, vReg src2, immL_M1 m1) %{
|
|
match(Set dst (AndV src1 (XorV src2 (Replicate m1))));
|
|
format %{ "vand_notL $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ bic($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_bic($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector and_not - predicated
|
|
|
|
instruct vand_notI_masked(vReg dst_src1, vReg src2, immI_M1 m1, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (Replicate m1))) pg));
|
|
format %{ "vand_notI_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_bic($dst_src1$$FloatRegister, get_reg_variant(this),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vand_notL_masked(vReg dst_src1, vReg src2, immL_M1 m1, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (Replicate m1))) pg));
|
|
format %{ "vand_notL_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_bic($dst_src1$$FloatRegister, get_reg_variant(this),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------- Vector saturating add -----------------------------
|
|
|
|
// Signed saturating add
|
|
|
|
instruct vsqadd(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(!n->as_SaturatingVector()->is_unsigned());
|
|
match(Set dst (SaturatingAddV src1 src2));
|
|
format %{ "vsqadd $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ sqaddv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_sqadd($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsqadd_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE == 2 && !n->as_SaturatingVector()->is_unsigned());
|
|
match(Set dst_src1 (SaturatingAddV (Binary dst_src1 src2) pg));
|
|
format %{ "vsqadd_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_sqadd($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Unsigned saturating add
|
|
|
|
instruct vuqadd(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(n->as_SaturatingVector()->is_unsigned());
|
|
match(Set dst (SaturatingAddV src1 src2));
|
|
format %{ "vuqadd $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ uqaddv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_uqadd($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vuqadd_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE == 2 && n->as_SaturatingVector()->is_unsigned());
|
|
match(Set dst_src1 (SaturatingAddV (Binary dst_src1 src2) pg));
|
|
format %{ "vuqadd_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_uqadd($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------- Vector saturating sub -----------------------------
|
|
|
|
// Signed saturating sub
|
|
|
|
instruct vsqsub(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(!n->as_SaturatingVector()->is_unsigned());
|
|
match(Set dst (SaturatingSubV src1 src2));
|
|
format %{ "vsqsub $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ sqsubv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_sqsub($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsqsub_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE == 2 && !n->as_SaturatingVector()->is_unsigned());
|
|
match(Set dst_src1 (SaturatingSubV (Binary dst_src1 src2) pg));
|
|
format %{ "vsqsub_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_sqsub($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Unsigned saturating sub
|
|
|
|
instruct vuqsub(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(n->as_SaturatingVector()->is_unsigned());
|
|
match(Set dst (SaturatingSubV src1 src2));
|
|
format %{ "vuqsub $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ uqsubv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_uqsub($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vuqsub_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE == 2 && n->as_SaturatingVector()->is_unsigned());
|
|
match(Set dst_src1 (SaturatingSubV (Binary dst_src1 src2) pg));
|
|
format %{ "vuqsub_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_uqsub($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector abs -----------------------------------
|
|
|
|
// vector abs
|
|
|
|
instruct vabsB(vReg dst, vReg src) %{
|
|
match(Set dst (AbsVB src));
|
|
format %{ "vabsB $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ absr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_abs($dst$$FloatRegister, __ B, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vabsS(vReg dst, vReg src) %{
|
|
match(Set dst (AbsVS src));
|
|
format %{ "vabsS $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ absr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_abs($dst$$FloatRegister, __ H, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vabsI(vReg dst, vReg src) %{
|
|
match(Set dst (AbsVI src));
|
|
format %{ "vabsI $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ absr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_abs($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vabsL(vReg dst, vReg src) %{
|
|
match(Set dst (AbsVL src));
|
|
format %{ "vabsL $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ absr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_abs($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vabsF(vReg dst, vReg src) %{
|
|
match(Set dst (AbsVF src));
|
|
format %{ "vabsF $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fabs($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fabs($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vabsD(vReg dst, vReg src) %{
|
|
match(Set dst (AbsVD src));
|
|
format %{ "vabsD $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fabs($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fabs($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector abs - predicated
|
|
|
|
instruct vabsB_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (AbsVB dst_src pg));
|
|
format %{ "vabsB_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
__ sve_abs($dst_src$$FloatRegister, __ B, $pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vabsS_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (AbsVS dst_src pg));
|
|
format %{ "vabsS_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
__ sve_abs($dst_src$$FloatRegister, __ H, $pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vabsI_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (AbsVI dst_src pg));
|
|
format %{ "vabsI_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
__ sve_abs($dst_src$$FloatRegister, __ S, $pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vabsL_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (AbsVL dst_src pg));
|
|
format %{ "vabsL_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
__ sve_abs($dst_src$$FloatRegister, __ D, $pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vabsF_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (AbsVF dst_src pg));
|
|
format %{ "vabsF_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
__ sve_fabs($dst_src$$FloatRegister, __ S, $pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vabsD_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (AbsVD dst_src pg));
|
|
format %{ "vabsD_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
__ sve_fabs($dst_src$$FloatRegister, __ D, $pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector fabd ----------------------------------
|
|
|
|
// vector fabs diff
|
|
|
|
instruct vfabd_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst (AbsVF (SubVF src1 src2)));
|
|
match(Set dst (AbsVD (SubVD src1 src2)));
|
|
format %{ "vfabd_neon $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
__ fabd($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vfabd_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst_src1 (AbsVF (SubVF dst_src1 src2)));
|
|
match(Set dst_src1 (AbsVD (SubVD dst_src1 src2)));
|
|
format %{ "vfabd_sve $dst_src1, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_fabd($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector fabs diff - predicated
|
|
|
|
instruct vfabd_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (AbsVF (SubVF (Binary dst_src1 src2) pg) pg));
|
|
match(Set dst_src1 (AbsVD (SubVD (Binary dst_src1 src2) pg) pg));
|
|
format %{ "vfabd_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_fabd($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector neg -----------------------------------
|
|
|
|
// vector neg
|
|
|
|
instruct vnegI(vReg dst, vReg src) %{
|
|
match(Set dst (NegVI src));
|
|
format %{ "vnegI $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ negr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_neg($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vnegL(vReg dst, vReg src) %{
|
|
match(Set dst (NegVL src));
|
|
format %{ "vnegL $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ negr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_neg($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vnegF(vReg dst, vReg src) %{
|
|
match(Set dst (NegVF src));
|
|
format %{ "vnegF $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fneg($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fneg($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vnegD(vReg dst, vReg src) %{
|
|
match(Set dst (NegVD src));
|
|
format %{ "vnegD $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fneg($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fneg($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector neg - predicated
|
|
|
|
instruct vnegI_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (NegVI dst_src pg));
|
|
format %{ "vnegI_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_neg($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vnegL_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (NegVL dst_src pg));
|
|
format %{ "vnegL_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
__ sve_neg($dst_src$$FloatRegister, __ D, $pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vnegF_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (NegVF dst_src pg));
|
|
format %{ "vnegF_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
__ sve_fneg($dst_src$$FloatRegister, __ S, $pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vnegD_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (NegVD dst_src pg));
|
|
format %{ "vnegD_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
__ sve_fneg($dst_src$$FloatRegister, __ D, $pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector sqrt ----------------------------------
|
|
|
|
// vector sqrt
|
|
|
|
instruct vsqrtHF(vReg dst, vReg src) %{
|
|
match(Set dst (SqrtVHF src));
|
|
format %{ "vsqrtHF $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fsqrt($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fsqrt($dst$$FloatRegister, __ H, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsqrtF(vReg dst, vReg src) %{
|
|
match(Set dst (SqrtVF src));
|
|
format %{ "vsqrtF $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fsqrt($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fsqrt($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsqrtD(vReg dst, vReg src) %{
|
|
match(Set dst (SqrtVD src));
|
|
format %{ "vsqrtD $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fsqrt($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fsqrt($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector sqrt - predicated
|
|
|
|
instruct vsqrtF_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (SqrtVF dst_src pg));
|
|
format %{ "vsqrtF_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
__ sve_fsqrt($dst_src$$FloatRegister, __ S, $pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsqrtD_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (SqrtVD dst_src pg));
|
|
format %{ "vsqrtD_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
__ sve_fsqrt($dst_src$$FloatRegister, __ D, $pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector min -----------------------------------
|
|
|
|
// vector min - LONG
|
|
|
|
instruct vminL_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) == T_LONG);
|
|
match(Set dst (MinV src1 src2));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "vminL_neon $dst, $src1, $src2\t# 2L" %}
|
|
ins_encode %{
|
|
__ cm(Assembler::GT, $dst$$FloatRegister, __ T2D, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
__ bsl($dst$$FloatRegister, __ T16B, $src2$$FloatRegister, $src1$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vminL_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
|
|
match(Set dst_src1 (MinV dst_src1 src2));
|
|
format %{ "vminL_sve $dst_src1, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_smin($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector min - B/S/I/HF/F/D
|
|
|
|
instruct vmin_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
|
|
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst (MinV src1 src2));
|
|
format %{ "vmin_neon $dst, $src1, $src2\t# B/S/I/F/D" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
if (is_floating_point_type(bt)) {
|
|
__ fmin($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
|
|
__ minv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmin_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
|
|
!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst_src1 (MinV dst_src1 src2));
|
|
format %{ "vmin_sve $dst_src1, $dst_src1, $src2\t# B/S/I/F/D" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
if (is_floating_point_type(bt)) {
|
|
__ sve_fmin($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src2$$FloatRegister);
|
|
} else {
|
|
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
|
|
__ sve_smin($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmin_HF_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst (MinVHF src1 src2));
|
|
format %{ "vmin_HF_neon $dst, $src1, $src2\t# Half float" %}
|
|
ins_encode %{
|
|
__ fmin($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmin_HF_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst_src1 (MinVHF dst_src1 src2));
|
|
format %{ "vmin_HF_sve $dst_src1, $dst_src1, $src2\t# Half float" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fmin($dst_src1$$FloatRegister, __ H,
|
|
ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector min - predicated
|
|
|
|
instruct vmin_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (MinV (Binary dst_src1 src2) pg));
|
|
format %{ "vmin_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
if (is_floating_point_type(bt)) {
|
|
__ sve_fmin($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(is_integral_type(bt), "unsupported type");
|
|
__ sve_smin($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector unsigned min - LONG
|
|
|
|
instruct vuminL_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) == T_LONG);
|
|
match(Set dst (UMinV src1 src2));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "vuminL_neon $dst, $src1, $src2\t# 2L" %}
|
|
ins_encode %{
|
|
__ cm(Assembler::HI, $dst$$FloatRegister, __ T2D, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
__ bsl($dst$$FloatRegister, __ T16B, $src2$$FloatRegister, $src1$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vuminL_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
|
|
match(Set dst_src1 (UMinV dst_src1 src2));
|
|
format %{ "vuminL_sve $dst_src1, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_umin($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector unsigned min - B/S/I
|
|
|
|
instruct vumin_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
|
|
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst (UMinV src1 src2));
|
|
format %{ "vumin_neon $dst, $src1, $src2\t# B/S/I" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
|
|
__ uminv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vumin_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
|
|
!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst_src1 (UMinV dst_src1 src2));
|
|
format %{ "vumin_sve $dst_src1, $dst_src1, $src2\t# B/S/I" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
|
|
__ sve_umin($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector unsigned min - predicated
|
|
|
|
instruct vumin_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (UMinV (Binary dst_src1 src2) pg));
|
|
format %{ "vumin_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
assert(is_integral_type(bt), "unsupported type");
|
|
__ sve_umin($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector max -----------------------------------
|
|
|
|
// vector max - LONG
|
|
|
|
instruct vmaxL_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) == T_LONG);
|
|
match(Set dst (MaxV src1 src2));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "vmaxL_neon $dst, $src1, $src2\t# 2L" %}
|
|
ins_encode %{
|
|
__ cm(Assembler::GT, $dst$$FloatRegister, __ T2D, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
__ bsl($dst$$FloatRegister, __ T16B, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaxL_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
|
|
match(Set dst_src1 (MaxV dst_src1 src2));
|
|
format %{ "vmaxL_sve $dst_src1, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_smax($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector max - B/S/I/HF/F/D
|
|
|
|
instruct vmax_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
|
|
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst (MaxV src1 src2));
|
|
format %{ "vmax_neon $dst, $src1, $src2\t# B/S/I/F/D" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
if (is_floating_point_type(bt)) {
|
|
__ fmax($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
|
|
__ maxv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmax_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
|
|
!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst_src1 (MaxV dst_src1 src2));
|
|
format %{ "vmax_sve $dst_src1, $dst_src1, $src2\t# B/S/I/F/D" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
if (is_floating_point_type(bt)) {
|
|
__ sve_fmax($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src2$$FloatRegister);
|
|
} else {
|
|
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
|
|
__ sve_smax($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmax_HF_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst (MaxVHF src1 src2));
|
|
format %{ "vmax_HF_neon $dst, $src1, $src2\t# Half float" %}
|
|
ins_encode %{
|
|
__ fmax($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmax_HF_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst_src1 (MaxVHF dst_src1 src2));
|
|
format %{ "vmax_HF_sve $dst_src1, $dst_src1, $src2\t# Half float" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fmax($dst_src1$$FloatRegister, __ H,
|
|
ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector max - predicated
|
|
|
|
instruct vmax_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (MaxV (Binary dst_src1 src2) pg));
|
|
format %{ "vmax_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
if (is_floating_point_type(bt)) {
|
|
__ sve_fmax($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
} else {
|
|
assert(is_integral_type(bt), "unsupported type");
|
|
__ sve_smax($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector unsigned max - LONG
|
|
|
|
instruct vumaxL_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) == T_LONG);
|
|
match(Set dst (UMaxV src1 src2));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "vumaxL_neon $dst, $src1, $src2\t# 2L" %}
|
|
ins_encode %{
|
|
__ cm(Assembler::HI, $dst$$FloatRegister, __ T2D, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
__ bsl($dst$$FloatRegister, __ T16B, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vumaxL_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
|
|
match(Set dst_src1 (UMaxV dst_src1 src2));
|
|
format %{ "vumaxL_sve $dst_src1, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_umax($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector unsigned max - B/S/I
|
|
|
|
instruct vumax_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
|
|
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst (UMaxV src1 src2));
|
|
format %{ "vumax_neon $dst, $src1, $src2\t# B/S/I" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
|
|
__ umaxv($dst$$FloatRegister, get_arrangement(this),
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vumax_sve(vReg dst_src1, vReg src2) %{
|
|
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
|
|
!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst_src1 (UMaxV dst_src1 src2));
|
|
format %{ "vumax_sve $dst_src1, $dst_src1, $src2\t# B/S/I" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
|
|
__ sve_umax($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector unsigned max - predicated
|
|
|
|
instruct vumax_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (UMaxV (Binary dst_src1 src2) pg));
|
|
format %{ "vumax_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
assert(is_integral_type(bt), "unsupported type");
|
|
__ sve_umax($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ MLA RELATED ----------------------------------
|
|
|
|
// vector mla
|
|
// dst_src1 = dst_src1 + src2 * src3
|
|
|
|
instruct vmla(vReg dst_src1, vReg src2, vReg src3) %{
|
|
match(Set dst_src1 (AddVB dst_src1 (MulVB src2 src3)));
|
|
match(Set dst_src1 (AddVS dst_src1 (MulVS src2 src3)));
|
|
match(Set dst_src1 (AddVI dst_src1 (MulVI src2 src3)));
|
|
format %{ "vmla $dst_src1, $src2, $src3" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ mlav($dst_src1$$FloatRegister, get_arrangement(this),
|
|
$src2$$FloatRegister, $src3$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_mla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmlaL(vReg dst_src1, vReg src2, vReg src3) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (AddVL dst_src1 (MulVL src2 src3)));
|
|
format %{ "vmlaL $dst_src1, $src2, $src3" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_mla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmla_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (AddVB (Binary dst_src1 (MulVB src2 src3)) pg));
|
|
match(Set dst_src1 (AddVS (Binary dst_src1 (MulVS src2 src3)) pg));
|
|
match(Set dst_src1 (AddVI (Binary dst_src1 (MulVI src2 src3)) pg));
|
|
match(Set dst_src1 (AddVL (Binary dst_src1 (MulVL src2 src3)) pg));
|
|
format %{ "vmla_masked $dst_src1, $pg, $src2, $src3" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_mla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector fmla
|
|
// dst_src1 = src2 * src3 + dst_src1
|
|
|
|
instruct vfmla(vReg dst_src1, vReg src2, vReg src3) %{
|
|
match(Set dst_src1 (FmaVHF dst_src1 (Binary src2 src3)));
|
|
match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
|
|
match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
|
|
format %{ "vfmla $dst_src1, $src2, $src3" %}
|
|
ins_encode %{
|
|
assert(UseFMA, "Needs FMA instructions support.");
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fmla($dst_src1$$FloatRegister, get_arrangement(this),
|
|
$src2$$FloatRegister, $src3$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_fmla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector fmad - predicated
|
|
// dst_src1 = dst_src1 * src2 + src3
|
|
|
|
instruct vfmad_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary src3 pg)));
|
|
match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary src3 pg)));
|
|
format %{ "vfmad_masked $dst_src1, $pg, $src2, $src3" %}
|
|
ins_encode %{
|
|
assert(UseFMA, "Needs FMA instructions support.");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_fmad($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector mls
|
|
// dst_src1 = dst_src1 - src2 * src3
|
|
|
|
instruct vmls(vReg dst_src1, vReg src2, vReg src3) %{
|
|
match(Set dst_src1 (SubVB dst_src1 (MulVB src2 src3)));
|
|
match(Set dst_src1 (SubVS dst_src1 (MulVS src2 src3)));
|
|
match(Set dst_src1 (SubVI dst_src1 (MulVI src2 src3)));
|
|
format %{ "vmls $dst_src1, $src2, $src3" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ mlsv($dst_src1$$FloatRegister, get_arrangement(this),
|
|
$src2$$FloatRegister, $src3$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_mls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmlsL(vReg dst_src1, vReg src2, vReg src3) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3)));
|
|
format %{ "vmlsL $dst_src1, $src2, $src3" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_mls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmls_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (SubVB (Binary dst_src1 (MulVB src2 src3)) pg));
|
|
match(Set dst_src1 (SubVS (Binary dst_src1 (MulVS src2 src3)) pg));
|
|
match(Set dst_src1 (SubVI (Binary dst_src1 (MulVI src2 src3)) pg));
|
|
match(Set dst_src1 (SubVL (Binary dst_src1 (MulVL src2 src3)) pg));
|
|
format %{ "vmls_masked $dst_src1, $pg, $src2, $src3" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_mls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector fmls
|
|
|
|
// dst_src1 = src2 * (-src3) + dst_src1
|
|
// "(-src2) * src3 + dst_src1" has been idealized to "src3 * (-src2) + dst_src1"
|
|
instruct vfmls(vReg dst_src1, vReg src2, vReg src3) %{
|
|
match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
|
|
match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3))));
|
|
format %{ "vfmls $dst_src1, $src2, $src3" %}
|
|
ins_encode %{
|
|
assert(UseFMA, "Needs FMA instructions support.");
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fmls($dst_src1$$FloatRegister, get_arrangement(this),
|
|
$src2$$FloatRegister, $src3$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_fmls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector fmsb - predicated
|
|
|
|
// dst_src1 = dst_src1 * (-src2) + src3
|
|
instruct vfmsb_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary src3 pg)));
|
|
match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary src3 pg)));
|
|
format %{ "vfmsb_masked $dst_src1, $pg, $src2, $src3" %}
|
|
ins_encode %{
|
|
assert(UseFMA, "Needs FMA instructions support.");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_fmsb($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector fnmla (sve)
|
|
|
|
// dst_src1 = src2 * (-src3) - dst_src1
|
|
// "(-src2) * src3 - dst_src1" has been idealized to "src3 * (-src2) - dst_src1"
|
|
instruct vfnmla(vReg dst_src1, vReg src2, vReg src3) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3))));
|
|
match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3))));
|
|
format %{ "vfnmla $dst_src1, $src2, $src3" %}
|
|
ins_encode %{
|
|
assert(UseFMA, "Needs FMA instructions support.");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_fnmla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector fnmad - predicated
|
|
|
|
// dst_src1 = dst_src1 * (-src2) - src3
|
|
instruct vfnmad_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary (NegVF src3) pg)));
|
|
match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary (NegVD src3) pg)));
|
|
format %{ "vfnmad_masked $dst_src1, $pg, $src2, $src3" %}
|
|
ins_encode %{
|
|
assert(UseFMA, "Needs FMA instructions support.");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_fnmad($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector fnmls (sve)
|
|
|
|
// dst_src1 = src2 * src3 - dst_src1
|
|
instruct vfnmls(vReg dst_src1, vReg src2, vReg src3) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3)));
|
|
match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3)));
|
|
format %{ "vfnmls $dst_src1, $src2, $src3" %}
|
|
ins_encode %{
|
|
assert(UseFMA, "Needs FMA instructions support.");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_fnmls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector fnmsb - predicated
|
|
|
|
// dst_src1 = dst_src1 * src2 - src3
|
|
instruct vfnmsb_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary (NegVF src3) pg)));
|
|
match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary (NegVD src3) pg)));
|
|
format %{ "vfnmsb_masked $dst_src1, $pg, $src2, $src3" %}
|
|
ins_encode %{
|
|
assert(UseFMA, "Needs FMA instructions support.");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_fnmsb($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// MulAddVS2VI
|
|
// Vector Multiply-Add Shorts into Integer
|
|
|
|
instruct vmuladdS2I(vReg dst, vReg src1, vReg src2, vReg tmp) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) == 16 &&
|
|
Matcher::vector_element_basic_type(n->in(1)) == T_SHORT);
|
|
match(Set dst (MulAddVS2VI src1 src2));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "vmuladdS2I $dst, $src1, $src2\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
__ smullv($tmp$$FloatRegister, __ T4H, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
__ smullv($dst$$FloatRegister, __ T8H, $src1$$FloatRegister, $src2$$FloatRegister);
|
|
__ addpv($dst$$FloatRegister, __ T4S, $tmp$$FloatRegister, $dst$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector shift ---------------------------------
|
|
|
|
// Vector right shift in AArch64 ASIMD
|
|
//
|
|
// Right shifts with vector shift count on AArch64 ASIMD are implemented
|
|
// as left shift by negative shift count.
|
|
// There are two cases for vector shift count.
|
|
//
|
|
// Case 1: The vector shift count is from replication.
|
|
// | |
|
|
// LoadVector RShiftCntV
|
|
// | /
|
|
// RShiftVI
|
|
//
|
|
// Case 2: The vector shift count is from loading.
|
|
// This case isn't supported by middle-end now. But it's supported by
|
|
// panama/vectorIntrinsics(JEP 338: Vector API).
|
|
// | |
|
|
// LoadVector LoadVector
|
|
// | /
|
|
// RShiftVI
|
|
//
|
|
// The negate is conducted in RShiftCntV rule for case 1, whereas it's done in
|
|
// RShiftV* rules for case 2. Because there exists an optimization opportunity
|
|
// for case 1, that is, multiple neg instructions in inner loop can be hoisted
|
|
// to outer loop and merged into one neg instruction.
|
|
//
|
|
// Note that ShiftVNode::is_var_shift() indicates whether the vector shift
|
|
// count is a variable vector(case 2) or not(a vector generated by RShiftCntV,
|
|
// i.e. case 1).
|
|
|
|
// vector shift count
|
|
|
|
instruct vshiftcntL(vReg dst, iRegIorL2I cnt) %{
|
|
match(Set dst (LShiftCntV cnt));
|
|
format %{ "vshiftcntL $dst, $cnt" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ dup($dst$$FloatRegister, get_arrangement(this), $cnt$$Register);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_dup($dst$$FloatRegister, __ elemType_to_regVariant(bt), $cnt$$Register);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vshiftcntR(vReg dst, iRegIorL2I cnt) %{
|
|
match(Set dst (RShiftCntV cnt));
|
|
format %{ "vshiftcntR $dst, $cnt" %}
|
|
ins_encode %{
|
|
if (UseSVE == 0) {
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes <= 16, "must be");
|
|
__ negw(rscratch1, $cnt$$Register);
|
|
__ dup($dst$$FloatRegister, get_arrangement(this), rscratch1);
|
|
} else {
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_dup($dst$$FloatRegister, __ elemType_to_regVariant(bt), $cnt$$Register);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector shift left
|
|
|
|
instruct vlsl_neon(vReg dst, vReg src, vReg shift) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst (LShiftVB src shift));
|
|
match(Set dst (LShiftVS src shift));
|
|
match(Set dst (LShiftVI src shift));
|
|
match(Set dst (LShiftVL src shift));
|
|
format %{ "vlsl_neon $dst, $src, $shift" %}
|
|
ins_encode %{
|
|
__ sshl($dst$$FloatRegister, get_arrangement(this),
|
|
$src$$FloatRegister, $shift$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vlsl_sve(vReg dst_src, vReg shift) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
|
|
match(Set dst_src (LShiftVB dst_src shift));
|
|
match(Set dst_src (LShiftVS dst_src shift));
|
|
match(Set dst_src (LShiftVI dst_src shift));
|
|
match(Set dst_src (LShiftVL dst_src shift));
|
|
format %{ "vlsl_sve $dst_src, $dst_src, $shift" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_lsl($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $shift$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector shift right (arithmetic)
|
|
|
|
instruct vasr_neon(vReg dst, vReg src, vReg shift) %{
|
|
predicate(UseSVE == 0 && !n->as_ShiftV()->is_var_shift());
|
|
match(Set dst (RShiftVB src shift));
|
|
match(Set dst (RShiftVS src shift));
|
|
match(Set dst (RShiftVI src shift));
|
|
match(Set dst (RShiftVL src shift));
|
|
format %{ "vasr_neon $dst, $src, $shift\t# not variable shift" %}
|
|
ins_encode %{
|
|
__ sshl($dst$$FloatRegister, get_arrangement(this),
|
|
$src$$FloatRegister, $shift$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vasr_neon_var(vReg dst, vReg src, vReg shift) %{
|
|
predicate(UseSVE == 0 && n->as_ShiftV()->is_var_shift());
|
|
match(Set dst (RShiftVB src shift));
|
|
match(Set dst (RShiftVS src shift));
|
|
match(Set dst (RShiftVI src shift));
|
|
match(Set dst (RShiftVL src shift));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "vasr_neon_var $dst, $src, $shift\t# variable shift" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
__ negr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$shift$$FloatRegister);
|
|
__ sshl($dst$$FloatRegister, get_arrangement(this),
|
|
$src$$FloatRegister, $dst$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vasr_sve(vReg dst_src, vReg shift) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (RShiftVB dst_src shift));
|
|
match(Set dst_src (RShiftVS dst_src shift));
|
|
match(Set dst_src (RShiftVI dst_src shift));
|
|
match(Set dst_src (RShiftVL dst_src shift));
|
|
format %{ "vasr_sve $dst_src, $dst_src, $shift" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_asr($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $shift$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector shift right (logical)
|
|
|
|
instruct vlsr_neon(vReg dst, vReg src, vReg shift) %{
|
|
predicate(UseSVE == 0 && !n->as_ShiftV()->is_var_shift());
|
|
match(Set dst (URShiftVB src shift));
|
|
match(Set dst (URShiftVS src shift));
|
|
match(Set dst (URShiftVI src shift));
|
|
match(Set dst (URShiftVL src shift));
|
|
format %{ "vlsr_neon $dst, $src, $shift\t# not variable shift" %}
|
|
ins_encode %{
|
|
__ ushl($dst$$FloatRegister, get_arrangement(this),
|
|
$src$$FloatRegister, $shift$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vlsr_neon_var(vReg dst, vReg src, vReg shift) %{
|
|
predicate(UseSVE == 0 && n->as_ShiftV()->is_var_shift());
|
|
match(Set dst (URShiftVB src shift));
|
|
match(Set dst (URShiftVS src shift));
|
|
match(Set dst (URShiftVI src shift));
|
|
match(Set dst (URShiftVL src shift));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "vlsr_neon_var $dst, $src, $shift\t# variable shift" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
__ negr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$shift$$FloatRegister);
|
|
__ ushl($dst$$FloatRegister, get_arrangement(this),
|
|
$src$$FloatRegister, $dst$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vlsr_sve(vReg dst_src, vReg shift) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (URShiftVB dst_src shift));
|
|
match(Set dst_src (URShiftVS dst_src shift));
|
|
match(Set dst_src (URShiftVI dst_src shift));
|
|
match(Set dst_src (URShiftVL dst_src shift));
|
|
format %{ "vlsr_sve $dst_src, $dst_src, $shift" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_lsr($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $shift$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector shift with imm
|
|
|
|
instruct vlsl_imm(vReg dst, vReg src, immI shift) %{
|
|
predicate(assert_not_var_shift(n));
|
|
match(Set dst (LShiftVB src (LShiftCntV shift)));
|
|
match(Set dst (LShiftVS src (LShiftCntV shift)));
|
|
match(Set dst (LShiftVI src (LShiftCntV shift)));
|
|
match(Set dst (LShiftVL src (LShiftCntV shift)));
|
|
format %{ "vlsl_imm $dst, $src, $shift" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
int con = (int)$shift$$constant;
|
|
if (is_subword_type(bt)) {
|
|
// Optimize for B/S
|
|
int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
|
|
if (con >= esize_in_bits) {
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ eor($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src$$FloatRegister, $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_eor($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ shl($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_lsl($dst$$FloatRegister, __ elemType_to_regVariant(bt), $src$$FloatRegister, con);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vasr_imm(vReg dst, vReg src, immI_positive shift) %{
|
|
predicate(assert_not_var_shift(n));
|
|
match(Set dst (RShiftVB src (RShiftCntV shift)));
|
|
match(Set dst (RShiftVS src (RShiftCntV shift)));
|
|
match(Set dst (RShiftVI src (RShiftCntV shift)));
|
|
match(Set dst (RShiftVL src (RShiftCntV shift)));
|
|
format %{ "vasr_imm $dst, $src, $shift" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
int con = (int)$shift$$constant;
|
|
if (is_subword_type(bt)) {
|
|
// Refine con for B/S
|
|
int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
|
|
if (con >= esize_in_bits) con = esize_in_bits - 1;
|
|
}
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ sshr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_asr($dst$$FloatRegister, __ elemType_to_regVariant(bt), $src$$FloatRegister, con);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vlsr_imm(vReg dst, vReg src, immI_positive shift) %{
|
|
predicate(assert_not_var_shift(n));
|
|
match(Set dst (URShiftVB src (RShiftCntV shift)));
|
|
match(Set dst (URShiftVS src (RShiftCntV shift)));
|
|
match(Set dst (URShiftVI src (RShiftCntV shift)));
|
|
match(Set dst (URShiftVL src (RShiftCntV shift)));
|
|
format %{ "vlsr_imm $dst, $src, $shift" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
int con = (int)$shift$$constant;
|
|
if (is_subword_type(bt)) {
|
|
// Optimize for B/S
|
|
int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
|
|
if (con >= esize_in_bits) {
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ eor($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src$$FloatRegister, $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_eor($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ ushr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_lsr($dst$$FloatRegister, __ elemType_to_regVariant(bt), $src$$FloatRegister, con);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// shift right add with imm (vector length <= 128 bits only)
|
|
|
|
instruct vasra_imm(vReg dst, vReg src, immI_positive shift) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) <= 16);
|
|
match(Set dst (AddVB dst (RShiftVB src (RShiftCntV shift))));
|
|
match(Set dst (AddVS dst (RShiftVS src (RShiftCntV shift))));
|
|
match(Set dst (AddVI dst (RShiftVI src (RShiftCntV shift))));
|
|
match(Set dst (AddVL dst (RShiftVL src (RShiftCntV shift))));
|
|
format %{ "vasra_imm $dst, $src, $shift" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
int con = (int)$shift$$constant;
|
|
if (is_subword_type(bt)) {
|
|
// Refine con for B/S
|
|
int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
|
|
if (con >= esize_in_bits) con = esize_in_bits - 1;
|
|
}
|
|
__ ssra($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vlsra_imm(vReg dst, vReg src, immI_positive shift) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) <= 16);
|
|
match(Set dst (AddVB dst (URShiftVB src (RShiftCntV shift))));
|
|
match(Set dst (AddVS dst (URShiftVS src (RShiftCntV shift))));
|
|
match(Set dst (AddVI dst (URShiftVI src (RShiftCntV shift))));
|
|
match(Set dst (AddVL dst (URShiftVL src (RShiftCntV shift))));
|
|
format %{ "vlsra_imm $dst, $src, $shift" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
int con = (int)$shift$$constant;
|
|
if (is_subword_type(bt)) { // for B/H
|
|
int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
|
|
if (con < esize_in_bits) {
|
|
__ usra($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
|
|
}
|
|
} else { // for S/D
|
|
assert(type2aelembytes(bt) == 4 || type2aelembytes(bt) == 8, "unsupported type");
|
|
__ usra($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector shift - predicated
|
|
|
|
instruct vlsl_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (LShiftVB (Binary dst_src1 src2) pg));
|
|
match(Set dst_src1 (LShiftVS (Binary dst_src1 src2) pg));
|
|
match(Set dst_src1 (LShiftVI (Binary dst_src1 src2) pg));
|
|
match(Set dst_src1 (LShiftVL (Binary dst_src1 src2) pg));
|
|
format %{ "vlsl_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_lsl($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vasr_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (RShiftVB (Binary dst_src1 src2) pg));
|
|
match(Set dst_src1 (RShiftVS (Binary dst_src1 src2) pg));
|
|
match(Set dst_src1 (RShiftVI (Binary dst_src1 src2) pg));
|
|
match(Set dst_src1 (RShiftVL (Binary dst_src1 src2) pg));
|
|
format %{ "vasr_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_asr($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vlsr_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (URShiftVB (Binary dst_src1 src2) pg));
|
|
match(Set dst_src1 (URShiftVS (Binary dst_src1 src2) pg));
|
|
match(Set dst_src1 (URShiftVI (Binary dst_src1 src2) pg));
|
|
match(Set dst_src1 (URShiftVL (Binary dst_src1 src2) pg));
|
|
format %{ "vlsr_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_lsr($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector shift with imm - predicated
|
|
|
|
instruct vlsl_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (LShiftVB (Binary dst_src (LShiftCntV shift)) pg));
|
|
match(Set dst_src (LShiftVS (Binary dst_src (LShiftCntV shift)) pg));
|
|
match(Set dst_src (LShiftVI (Binary dst_src (LShiftCntV shift)) pg));
|
|
match(Set dst_src (LShiftVL (Binary dst_src (LShiftCntV shift)) pg));
|
|
format %{ "vlsl_imm_masked $dst_src, $pg, $dst_src, $shift" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
|
|
int con = (int)$shift$$constant;
|
|
assert(con >= 0 && con < esize_in_bits, "invalid shift immediate");
|
|
__ sve_lsl($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, con);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vasr_imm_masked(vReg dst_src, immI_positive shift, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (RShiftVB (Binary dst_src (RShiftCntV shift)) pg));
|
|
match(Set dst_src (RShiftVS (Binary dst_src (RShiftCntV shift)) pg));
|
|
match(Set dst_src (RShiftVI (Binary dst_src (RShiftCntV shift)) pg));
|
|
match(Set dst_src (RShiftVL (Binary dst_src (RShiftCntV shift)) pg));
|
|
format %{ "vasr_imm_masked $dst_src, $pg, $dst_src, $shift" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
|
|
int con = (int)$shift$$constant;
|
|
assert(con > 0 && con < esize_in_bits, "invalid shift immediate");
|
|
__ sve_asr($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, con);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vlsr_imm_masked(vReg dst_src, immI_positive shift, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (URShiftVB (Binary dst_src (RShiftCntV shift)) pg));
|
|
match(Set dst_src (URShiftVS (Binary dst_src (RShiftCntV shift)) pg));
|
|
match(Set dst_src (URShiftVI (Binary dst_src (RShiftCntV shift)) pg));
|
|
match(Set dst_src (URShiftVL (Binary dst_src (RShiftCntV shift)) pg));
|
|
format %{ "vlsr_imm_masked $dst_src, $pg, $dst_src, $shift" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
|
|
int con = (int)$shift$$constant;
|
|
assert(con > 0 && con < esize_in_bits, "invalid shift immediate");
|
|
__ sve_lsr($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, con);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector reduction add -------------------------
|
|
|
|
// reduction addI
|
|
|
|
instruct reduce_addI_neon(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, vReg tmp) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))));
|
|
match(Set dst (AddReductionVI isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_addI_neon $dst, $isrc, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
__ neon_reduce_add_integral($dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
length_in_bytes, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_addI_sve(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, vRegD tmp) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))));
|
|
match(Set dst (AddReductionVI isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_addI_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
ptrue, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction addL
|
|
|
|
instruct reduce_addL_neon(iRegLNoSp dst, iRegL isrc, vReg vsrc, vReg tmp) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))));
|
|
match(Set dst (AddReductionVL isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_addL_neon $dst, $isrc, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
__ neon_reduce_add_integral($dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
length_in_bytes, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_addL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc, vRegD tmp) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))));
|
|
match(Set dst (AddReductionVL isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_addL_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
ptrue, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction addF
|
|
|
|
instruct reduce_non_strict_order_add2F_neon(vRegF dst, vRegF fsrc, vReg vsrc) %{
|
|
// Non-strictly ordered floating-point add reduction for a 64-bits-long vector. This rule is
|
|
// intended for the VectorAPI (which allows for non-strictly ordered add reduction).
|
|
predicate(Matcher::vector_length(n->in(2)) == 2 && !n->as_Reduction()->requires_strict_order());
|
|
match(Set dst (AddReductionVF fsrc vsrc));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_non_strict_order_add2F_neon $dst, $fsrc, $vsrc" %}
|
|
ins_encode %{
|
|
__ faddp($dst$$FloatRegister, $vsrc$$FloatRegister, __ S);
|
|
__ fadds($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_non_strict_order_add4F_neon(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
|
|
// Non-strictly ordered floating-point add reduction for 128-bits-long vector. This rule is
|
|
// intended for the VectorAPI (which allows for non-strictly ordered add reduction).
|
|
predicate(Matcher::vector_length(n->in(2)) == 4 && !n->as_Reduction()->requires_strict_order());
|
|
match(Set dst (AddReductionVF fsrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_non_strict_order_add4F_neon $dst, $fsrc, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
__ faddp($tmp$$FloatRegister, __ T4S, $vsrc$$FloatRegister, $vsrc$$FloatRegister);
|
|
__ faddp($dst$$FloatRegister, $tmp$$FloatRegister, __ S);
|
|
__ fadds($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// This rule calculates the reduction result in strict order. Two cases will
|
|
// reach here:
|
|
// 1. Non strictly-ordered AddReductionVF when vector size > 128-bits. For example -
|
|
// AddReductionVF generated by Vector API. For vector size > 128-bits, it is more
|
|
// beneficial performance-wise to generate direct SVE instruction even if it is
|
|
// strictly ordered.
|
|
// 2. Strictly-ordered AddReductionVF. For example - AddReductionVF generated by
|
|
// auto-vectorization on SVE machine.
|
|
instruct reduce_addF_sve(vRegF dst_src1, vReg src2) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) ||
|
|
n->as_Reduction()->requires_strict_order());
|
|
match(Set dst_src1 (AddReductionVF dst_src1 src2));
|
|
format %{ "reduce_addF_sve $dst_src1, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src2);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_fadda($dst_src1$$FloatRegister, __ S, ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction addD
|
|
|
|
instruct reduce_non_strict_order_add2D_neon(vRegD dst, vRegD dsrc, vReg vsrc) %{
|
|
// Non-strictly ordered floating-point add reduction for doubles. This rule is
|
|
// intended for the VectorAPI (which allows for non-strictly ordered add reduction).
|
|
predicate(!n->as_Reduction()->requires_strict_order());
|
|
match(Set dst (AddReductionVD dsrc vsrc));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_non_strict_order_add2D_neon $dst, $dsrc, $vsrc\t# 2D" %}
|
|
ins_encode %{
|
|
__ faddp($dst$$FloatRegister, $vsrc$$FloatRegister, __ D);
|
|
__ faddd($dst$$FloatRegister, $dst$$FloatRegister, $dsrc$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// This rule calculates the reduction result in strict order. Two cases will
|
|
// reach here:
|
|
// 1. Non strictly-ordered AddReductionVD when vector size > 128-bits. For example -
|
|
// AddReductionVD generated by Vector API. For vector size > 128-bits, it is more
|
|
// beneficial performance-wise to generate direct SVE instruction even if it is
|
|
// strictly ordered.
|
|
// 2. Strictly-ordered AddReductionVD. For example - AddReductionVD generated by
|
|
// auto-vectorization on SVE machine.
|
|
instruct reduce_addD_sve(vRegD dst_src1, vReg src2) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) ||
|
|
n->as_Reduction()->requires_strict_order());
|
|
match(Set dst_src1 (AddReductionVD dst_src1 src2));
|
|
format %{ "reduce_addD_sve $dst_src1, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src2);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_fadda($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction add - predicated
|
|
|
|
instruct reduce_addI_masked(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (AddReductionVI (Binary isrc vsrc) pg));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_addI_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
$pg$$PRegister, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_addL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (AddReductionVL (Binary isrc vsrc) pg));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_addL_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
$pg$$PRegister, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_addF_masked(vRegF dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (AddReductionVF (Binary dst_src1 src2) pg));
|
|
format %{ "reduce_addF_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_fadda($dst_src1$$FloatRegister, __ S,
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_addD_masked(vRegD dst_src1, vReg src2, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src1 (AddReductionVD (Binary dst_src1 src2) pg));
|
|
format %{ "reduce_addD_masked $dst_src1, $pg, $dst_src1, $src2" %}
|
|
ins_encode %{
|
|
__ sve_fadda($dst_src1$$FloatRegister, __ D,
|
|
$pg$$PRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector reduction mul -------------------------
|
|
|
|
instruct reduce_mulI(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
|
|
vReg tmp1, vReg tmp2) %{
|
|
predicate(Matcher::vector_length_in_bytes(n->in(2)) == 8 ||
|
|
Matcher::vector_length_in_bytes(n->in(2)) == 16);
|
|
match(Set dst (MulReductionVI isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
|
|
format %{ "reduce_mulI $dst, $isrc, $vsrc\t# vector (64/128 bits). KILL $tmp1, $tmp2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
__ neon_reduce_mul_integral($dst$$Register, bt, $isrc$$Register,
|
|
$vsrc$$FloatRegister, length_in_bytes,
|
|
$tmp1$$FloatRegister, $tmp2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_mulL(iRegLNoSp dst, iRegL isrc, vReg vsrc) %{
|
|
predicate(Matcher::vector_length_in_bytes(n->in(2)) == 16);
|
|
match(Set dst (MulReductionVL isrc vsrc));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_mulL $dst, $isrc, $vsrc\t# 2L" %}
|
|
ins_encode %{
|
|
__ neon_reduce_mul_integral($dst$$Register, T_LONG, $isrc$$Register,
|
|
$vsrc$$FloatRegister, 16, fnoreg, fnoreg);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_mulF(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
|
|
predicate(Matcher::vector_length_in_bytes(n->in(2)) <= 16);
|
|
match(Set dst (MulReductionVF fsrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_mulF $dst, $fsrc, $vsrc\t# 2F/4F. KILL $tmp" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
__ neon_reduce_mul_fp($dst$$FloatRegister, T_FLOAT, $fsrc$$FloatRegister,
|
|
$vsrc$$FloatRegister, length_in_bytes, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_mulD(vRegD dst, vRegD dsrc, vReg vsrc, vReg tmp) %{
|
|
predicate(Matcher::vector_length_in_bytes(n->in(2)) == 16);
|
|
match(Set dst (MulReductionVD dsrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_mulD $dst, $dsrc, $vsrc\t# 2D. KILL $tmp" %}
|
|
ins_encode %{
|
|
__ neon_reduce_mul_fp($dst$$FloatRegister, T_DOUBLE, $dsrc$$FloatRegister,
|
|
$vsrc$$FloatRegister, 16, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector reduction and -------------------------
|
|
|
|
// reduction andI
|
|
|
|
instruct reduce_andI_neon(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc) %{
|
|
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) != T_LONG);
|
|
match(Set dst (AndReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_andI_neon $dst, $isrc, $vsrc" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
__ neon_reduce_logical(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_andI_sve(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, vRegD tmp) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) != T_LONG);
|
|
match(Set dst (AndReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_andI_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
ptrue, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction andL
|
|
|
|
instruct reduce_andL_neon(iRegLNoSp dst, iRegL isrc, vReg vsrc) %{
|
|
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
|
|
match(Set dst (AndReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_andL_neon $dst, $isrc, $vsrc" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
__ neon_reduce_logical(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_andL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc, vRegD tmp) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
|
|
match(Set dst (AndReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_andL_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
ptrue, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction and - predicated
|
|
|
|
instruct reduce_andI_masked(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) != T_LONG);
|
|
match(Set dst (AndReductionV (Binary isrc vsrc) pg));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_andI_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
$pg$$PRegister, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_andL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_LONG);
|
|
match(Set dst (AndReductionV (Binary isrc vsrc) pg));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_andL_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
$pg$$PRegister, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector reduction or --------------------------
|
|
|
|
// reduction orI
|
|
|
|
instruct reduce_orI_neon(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc) %{
|
|
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) != T_LONG);
|
|
match(Set dst (OrReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_orI_neon $dst, $isrc, $vsrc" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
__ neon_reduce_logical(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_orI_sve(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, vRegD tmp) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) != T_LONG);
|
|
match(Set dst (OrReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_orI_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
ptrue, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction orL
|
|
|
|
instruct reduce_orL_neon(iRegLNoSp dst, iRegL isrc, vReg vsrc) %{
|
|
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
|
|
match(Set dst (OrReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_orL_neon $dst, $isrc, $vsrc" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
__ neon_reduce_logical(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_orL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc, vRegD tmp) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
|
|
match(Set dst (OrReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_orL_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
ptrue, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction or - predicated
|
|
|
|
instruct reduce_orI_masked(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) != T_LONG);
|
|
match(Set dst (OrReductionV (Binary isrc vsrc) pg));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_orI_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
$pg$$PRegister, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_orL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_LONG);
|
|
match(Set dst (OrReductionV (Binary isrc vsrc) pg));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_orL_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
$pg$$PRegister, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector reduction xor -------------------------
|
|
|
|
// reduction xorI
|
|
|
|
instruct reduce_xorI_neon(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc) %{
|
|
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) != T_LONG);
|
|
match(Set dst (XorReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_xorI_neon $dst, $isrc, $vsrc" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
__ neon_reduce_logical(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_xorI_sve(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, vRegD tmp) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) != T_LONG);
|
|
match(Set dst (XorReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_xorI_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
ptrue, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction xorL
|
|
|
|
instruct reduce_xorL_neon(iRegLNoSp dst, iRegL isrc, vReg vsrc) %{
|
|
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
|
|
match(Set dst (XorReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_xorL_neon $dst, $isrc, $vsrc" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
__ neon_reduce_logical(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_xorL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc, vRegD tmp) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
|
|
match(Set dst (XorReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_xorL_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
ptrue, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction xor - predicated
|
|
|
|
instruct reduce_xorI_masked(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) != T_LONG);
|
|
match(Set dst (XorReductionV (Binary isrc vsrc) pg));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_xorI_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
$pg$$PRegister, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_xorL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_LONG);
|
|
match(Set dst (XorReductionV (Binary isrc vsrc) pg));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "reduce_xorL_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
$pg$$PRegister, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector reduction max -------------------------
|
|
|
|
// reduction maxI
|
|
|
|
instruct reduce_maxI_neon(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
|
|
vReg tmp, rFlagsReg cr) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) &&
|
|
(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
|
|
Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
|
|
Matcher::vector_element_basic_type(n->in(2)) == T_INT));
|
|
match(Set dst (MaxReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
|
|
format %{ "reduce_maxI_neon $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
__ neon_reduce_minmax_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
length_in_bytes, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_maxI_sve(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
|
|
vRegD tmp, rFlagsReg cr) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) &&
|
|
(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
|
|
Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
|
|
Matcher::vector_element_basic_type(n->in(2)) == T_INT));
|
|
match(Set dst (MaxReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
|
|
format %{ "reduce_maxI_sve $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
ptrue, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction maxL
|
|
|
|
instruct reduce_maxL_neon(iRegLNoSp dst, iRegL isrc, vReg vsrc, rFlagsReg cr) %{
|
|
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
|
|
match(Set dst (MaxReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst, KILL cr);
|
|
format %{ "reduce_maxL_neon $dst, $isrc, $vsrc\t# 2L. KILL cr" %}
|
|
ins_encode %{
|
|
__ neon_reduce_minmax_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
/* vector_length_in_bytes */ 16, fnoreg);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_maxL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc,
|
|
vRegD tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
|
|
match(Set dst (MaxReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
|
|
format %{ "reduce_maxL_sve $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
ptrue, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction maxF
|
|
|
|
instruct reduce_maxF(vRegF dst, vRegF fsrc, vReg vsrc) %{
|
|
predicate(Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT);
|
|
match(Set dst (MaxReductionV fsrc vsrc));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_maxF $dst, $fsrc, $vsrc" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
if (length_in_bytes == 8) {
|
|
__ fmaxp($dst$$FloatRegister, $vsrc$$FloatRegister, __ S);
|
|
} else {
|
|
__ fmaxv($dst$$FloatRegister, __ T4S, $vsrc$$FloatRegister);
|
|
}
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_fmaxv($dst$$FloatRegister, __ S, ptrue, $vsrc$$FloatRegister);
|
|
}
|
|
__ fmaxs($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction maxD
|
|
|
|
instruct reduce_maxD(vRegD dst, vRegD dsrc, vReg vsrc) %{
|
|
predicate(Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE);
|
|
match(Set dst (MaxReductionV dsrc vsrc));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_maxD $dst, $dsrc, $vsrc" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fmaxp($dst$$FloatRegister, $vsrc$$FloatRegister, __ D);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_fmaxv($dst$$FloatRegister, __ D, ptrue, $vsrc$$FloatRegister);
|
|
}
|
|
__ fmaxd($dst$$FloatRegister, $dst$$FloatRegister, $dsrc$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction max - predicated
|
|
|
|
instruct reduce_maxI_masked(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, pRegGov pg,
|
|
vRegD tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 &&
|
|
(Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_BYTE ||
|
|
Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_SHORT ||
|
|
Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_INT));
|
|
match(Set dst (MaxReductionV (Binary isrc vsrc) pg));
|
|
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
|
|
format %{ "reduce_maxI_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
$pg$$PRegister, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_maxL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg,
|
|
vRegD tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_LONG);
|
|
match(Set dst (MaxReductionV (Binary isrc vsrc) pg));
|
|
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
|
|
format %{ "reduce_maxL_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
$pg$$PRegister, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_maxF_masked(vRegF dst, vRegF fsrc, vReg vsrc, pRegGov pg) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_FLOAT);
|
|
match(Set dst (MaxReductionV (Binary fsrc vsrc) pg));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_maxF_masked $dst, $fsrc, $pg, $vsrc" %}
|
|
ins_encode %{
|
|
__ sve_fmaxv($dst$$FloatRegister, __ S, $pg$$PRegister, $vsrc$$FloatRegister);
|
|
__ fmaxs($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_maxD_masked(vRegD dst, vRegD dsrc, vReg vsrc, pRegGov pg) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_DOUBLE);
|
|
match(Set dst (MaxReductionV (Binary dsrc vsrc) pg));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_maxD_masked $dst, $dsrc, $pg, $vsrc" %}
|
|
ins_encode %{
|
|
__ sve_fmaxv($dst$$FloatRegister, __ D, $pg$$PRegister, $vsrc$$FloatRegister);
|
|
__ fmaxd($dst$$FloatRegister, $dst$$FloatRegister, $dsrc$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector reduction min -------------------------
|
|
|
|
// reduction minI
|
|
|
|
instruct reduce_minI_neon(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
|
|
vReg tmp, rFlagsReg cr) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) &&
|
|
(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
|
|
Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
|
|
Matcher::vector_element_basic_type(n->in(2)) == T_INT));
|
|
match(Set dst (MinReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
|
|
format %{ "reduce_minI_neon $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
__ neon_reduce_minmax_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
length_in_bytes, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_minI_sve(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
|
|
vRegD tmp, rFlagsReg cr) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) &&
|
|
(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
|
|
Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
|
|
Matcher::vector_element_basic_type(n->in(2)) == T_INT));
|
|
match(Set dst (MinReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
|
|
format %{ "reduce_minI_sve $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
ptrue, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction minL
|
|
|
|
instruct reduce_minL_neon(iRegLNoSp dst, iRegL isrc, vReg vsrc, rFlagsReg cr) %{
|
|
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
|
|
match(Set dst (MinReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst, KILL cr);
|
|
format %{ "reduce_minL_neon $dst, $isrc, $vsrc\t# 2L. KILL cr" %}
|
|
ins_encode %{
|
|
__ neon_reduce_minmax_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
/* vector_length_in_bytes */ 16, fnoreg);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_minL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc,
|
|
vRegD tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
|
|
match(Set dst (MinReductionV isrc vsrc));
|
|
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
|
|
format %{ "reduce_minL_sve $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
ptrue, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction minF
|
|
|
|
instruct reduce_minF(vRegF dst, vRegF fsrc, vReg vsrc) %{
|
|
predicate(Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT);
|
|
match(Set dst (MinReductionV fsrc vsrc));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_minF $dst, $fsrc, $vsrc" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
if (length_in_bytes == 8) {
|
|
__ fminp($dst$$FloatRegister, $vsrc$$FloatRegister, __ S);
|
|
} else {
|
|
__ fminv($dst$$FloatRegister, __ T4S, $vsrc$$FloatRegister);
|
|
}
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_fminv($dst$$FloatRegister, __ S, ptrue, $vsrc$$FloatRegister);
|
|
}
|
|
__ fmins($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction minD
|
|
|
|
instruct reduce_minD(vRegD dst, vRegD dsrc, vReg vsrc) %{
|
|
predicate(Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE);
|
|
match(Set dst (MinReductionV dsrc vsrc));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_minD $dst, $dsrc, $vsrc" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ fminp($dst$$FloatRegister, $vsrc$$FloatRegister, __ D);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_fminv($dst$$FloatRegister, __ D, ptrue, $vsrc$$FloatRegister);
|
|
}
|
|
__ fmind($dst$$FloatRegister, $dst$$FloatRegister, $dsrc$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// reduction min - predicated
|
|
|
|
instruct reduce_minI_masked(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, pRegGov pg,
|
|
vRegD tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 &&
|
|
(Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_BYTE ||
|
|
Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_SHORT ||
|
|
Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_INT));
|
|
match(Set dst (MinReductionV (Binary isrc vsrc) pg));
|
|
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
|
|
format %{ "reduce_minI_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
$pg$$PRegister, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_minL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg,
|
|
vRegD tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_LONG);
|
|
match(Set dst (MinReductionV (Binary isrc vsrc) pg));
|
|
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
|
|
format %{ "reduce_minL_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
|
|
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
|
|
$isrc$$Register, $vsrc$$FloatRegister,
|
|
$pg$$PRegister, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_minF_masked(vRegF dst, vRegF fsrc, vReg vsrc, pRegGov pg) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_FLOAT);
|
|
match(Set dst (MinReductionV (Binary fsrc vsrc) pg));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_minF_masked $dst, $fsrc, $pg, $vsrc" %}
|
|
ins_encode %{
|
|
__ sve_fminv($dst$$FloatRegister, __ S, $pg$$PRegister, $vsrc$$FloatRegister);
|
|
__ fmins($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reduce_minD_masked(vRegD dst, vRegD dsrc, vReg vsrc, pRegGov pg) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_DOUBLE);
|
|
match(Set dst (MinReductionV (Binary dsrc vsrc) pg));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "reduce_minD_masked $dst, $dsrc, $pg, $vsrc" %}
|
|
ins_encode %{
|
|
__ sve_fminv($dst$$FloatRegister, __ D, $pg$$PRegister, $vsrc$$FloatRegister);
|
|
__ fmind($dst$$FloatRegister, $dst$$FloatRegister, $dsrc$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector reinterpret ---------------------------
|
|
|
|
instruct reinterpret_same_size(vReg dst_src) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
|
|
match(Set dst_src (VectorReinterpret dst_src));
|
|
ins_cost(0);
|
|
format %{ "reinterpret_same_size $dst_src\t# do nothing" %}
|
|
ins_encode(/* empty encoding */);
|
|
ins_pipe(pipe_class_empty);
|
|
%}
|
|
|
|
instruct reinterpret_resize_le128b(vReg dst, vReg src) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) != Matcher::vector_length_in_bytes(n->in(1)) &&
|
|
Matcher::vector_length_in_bytes(n) <= 16 &&
|
|
Matcher::vector_length_in_bytes(n->in(1)) <= 16);
|
|
match(Set dst (VectorReinterpret src));
|
|
format %{ "reinterpret_resize_le128b $dst, $src\t# vector <= 128 bits." %}
|
|
ins_encode %{
|
|
uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
|
|
uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
|
|
// The higher bits in "dst" register must be cleared to zero.
|
|
if ((length_in_bytes_src == 4 && length_in_bytes_dst == 8) ||
|
|
(length_in_bytes_src == 8 && length_in_bytes_dst == 4)) {
|
|
// Reinterpret between 32 bits and 64 bits
|
|
__ dup($dst$$FloatRegister, __ S, $src$$FloatRegister);
|
|
} else if ((length_in_bytes_src == 4 && length_in_bytes_dst == 16) ||
|
|
(length_in_bytes_src == 16 && length_in_bytes_dst == 4)) {
|
|
// Reinterpret between 32 bits and 128 bits
|
|
__ dup($dst$$FloatRegister, __ S, $src$$FloatRegister);
|
|
} else if ((length_in_bytes_src == 8 && length_in_bytes_dst == 16) ||
|
|
(length_in_bytes_src == 16 && length_in_bytes_dst == 8)) {
|
|
// Reinterpret between 64 bits and 128 bits
|
|
__ orr($dst$$FloatRegister, __ T8B, $src$$FloatRegister, $src$$FloatRegister);
|
|
} else {
|
|
assert(false, "invalid vector length");
|
|
ShouldNotReachHere();
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct reinterpret_resize_gt128b(vReg dst, vReg src, pReg ptmp, rFlagsReg cr) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) != Matcher::vector_length_in_bytes(n->in(1)) &&
|
|
(Matcher::vector_length_in_bytes(n) > 16 ||
|
|
Matcher::vector_length_in_bytes(n->in(1)) > 16));
|
|
match(Set dst (VectorReinterpret src));
|
|
effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
|
|
format %{ "reinterpret_resize_gt128b $dst, $src\t# vector > 128 bits. KILL $ptmp, cr" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
|
|
uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
|
|
uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
|
|
length_in_bytes_src : length_in_bytes_dst;
|
|
assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
|
|
"invalid vector length");
|
|
__ sve_gen_mask_imm($ptmp$$PRegister, T_BYTE, length_in_bytes_resize);
|
|
__ sve_dup($dst$$FloatRegister, __ B, 0);
|
|
__ sve_sel($dst$$FloatRegister, __ B, $ptmp$$PRegister,
|
|
$src$$FloatRegister, $dst$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ---------------------------- Vector zero extend --------------------------------
|
|
|
|
instruct vzeroExtBtoX(vReg dst, vReg src) %{
|
|
match(Set dst (VectorUCastB2X src));
|
|
format %{ "vzeroExtBtoX $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
assert(bt == T_SHORT || bt == T_INT || bt == T_LONG, "must be");
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
// 4B to 4S/4I, 8B to 8S
|
|
__ neon_vector_extend($dst$$FloatRegister, bt, length_in_bytes,
|
|
$src$$FloatRegister, T_BYTE, /* is_unsigned */ true);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_vector_extend($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$src$$FloatRegister, __ B, /* is_unsigned */ true);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vzeroExtStoX(vReg dst, vReg src) %{
|
|
match(Set dst (VectorUCastS2X src));
|
|
format %{ "vzeroExtStoX $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
assert(bt == T_INT || bt == T_LONG, "must be");
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
// 2S to 2I/2L, 4S to 4I
|
|
__ neon_vector_extend($dst$$FloatRegister, bt, length_in_bytes,
|
|
$src$$FloatRegister, T_SHORT, /* is_unsigned */ true);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_vector_extend($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$src$$FloatRegister, __ H, /* is_unsigned */ true);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vzeroExtItoX(vReg dst, vReg src) %{
|
|
match(Set dst (VectorUCastI2X src));
|
|
format %{ "vzeroExtItoX $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
assert(bt == T_LONG, "must be");
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
// 2I to 2L
|
|
__ neon_vector_extend($dst$$FloatRegister, bt, length_in_bytes,
|
|
$src$$FloatRegister, T_INT, /* is_unsigned */ true);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_vector_extend($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$src$$FloatRegister, __ S, /* is_unsigned */ true);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector cast ----------------------------------
|
|
|
|
// VectorCastB2X
|
|
|
|
instruct vcvtBtoX(vReg dst, vReg src) %{
|
|
match(Set dst (VectorCastB2X src));
|
|
format %{ "vcvtBtoX $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
// 4B to 4S/4I/4F, 8B to 8S
|
|
__ neon_vector_extend($dst$$FloatRegister, bt == T_FLOAT ? T_INT : bt,
|
|
length_in_bytes, $src$$FloatRegister, T_BYTE);
|
|
if (bt == T_FLOAT) {
|
|
__ scvtfv(__ T4S, $dst$$FloatRegister, $dst$$FloatRegister);
|
|
}
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_vector_extend($dst$$FloatRegister, size, $src$$FloatRegister, __ B);
|
|
if (is_floating_point_type(bt)) {
|
|
__ sve_scvtf($dst$$FloatRegister, size, ptrue, $dst$$FloatRegister, size);
|
|
}
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// VectorCastS2X
|
|
|
|
instruct vcvtStoB_neon(vReg dst, vReg src) %{
|
|
predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
|
|
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
|
|
match(Set dst (VectorCastS2X src));
|
|
format %{ "vcvtStoB_neon $dst, $src" %}
|
|
ins_encode %{
|
|
// 4S/8S to 4B/8B
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
|
|
__ neon_vector_narrow($dst$$FloatRegister, T_BYTE,
|
|
$src$$FloatRegister, T_SHORT, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcvtStoB_sve(vReg dst, vReg src, vReg tmp) %{
|
|
predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
|
|
!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
|
|
match(Set dst (VectorCastS2X src));
|
|
effect(TEMP tmp);
|
|
format %{ "vcvtStoB_sve $dst, $src\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_vector_narrow($dst$$FloatRegister, __ B,
|
|
$src$$FloatRegister, __ H, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcvtStoX_extend(vReg dst, vReg src) %{
|
|
predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) >= 4);
|
|
match(Set dst (VectorCastS2X src));
|
|
format %{ "vcvtStoX_extend $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
if (is_floating_point_type(bt)) {
|
|
// 2S to 2F/2D, 4S to 4F
|
|
__ neon_vector_extend($dst$$FloatRegister, bt == T_FLOAT ? T_INT : T_LONG,
|
|
length_in_bytes, $src$$FloatRegister, T_SHORT);
|
|
__ scvtfv(get_arrangement(this), $dst$$FloatRegister, $dst$$FloatRegister);
|
|
} else {
|
|
// 2S to 2I/2L, 4S to 4I
|
|
__ neon_vector_extend($dst$$FloatRegister, bt, length_in_bytes,
|
|
$src$$FloatRegister, T_SHORT);
|
|
}
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_vector_extend($dst$$FloatRegister, size, $src$$FloatRegister, __ H);
|
|
if (is_floating_point_type(bt)) {
|
|
__ sve_scvtf($dst$$FloatRegister, size, ptrue, $dst$$FloatRegister, size);
|
|
}
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// VectorCastI2X
|
|
|
|
instruct vcvtItoX_narrow_neon(vReg dst, vReg src) %{
|
|
predicate((Matcher::vector_element_basic_type(n) == T_BYTE ||
|
|
Matcher::vector_element_basic_type(n) == T_SHORT) &&
|
|
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
|
|
match(Set dst (VectorCastI2X src));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "vcvtItoX_narrow_neon $dst, $src" %}
|
|
ins_encode %{
|
|
// 2I to 2S, 4I to 4B/4S
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
|
|
__ neon_vector_narrow($dst$$FloatRegister, bt,
|
|
$src$$FloatRegister, T_INT, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcvtItoX_narrow_sve(vReg dst, vReg src, vReg tmp) %{
|
|
predicate((Matcher::vector_element_basic_type(n) == T_BYTE ||
|
|
Matcher::vector_element_basic_type(n) == T_SHORT) &&
|
|
!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
|
|
match(Set dst (VectorCastI2X src));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "vcvtItoX_narrow_sve $dst, $src\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_vector_narrow($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$src$$FloatRegister, __ S, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcvtItoX(vReg dst, vReg src) %{
|
|
predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) >= 4);
|
|
match(Set dst (VectorCastI2X src));
|
|
format %{ "vcvtItoX $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (bt == T_FLOAT) {
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
// 2I/4I to 2F/4F
|
|
__ scvtfv(get_arrangement(this), $dst$$FloatRegister, $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_scvtf($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ S);
|
|
}
|
|
} else {
|
|
assert(type2aelembytes(bt) == 8, "unsupported type");
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
// 2I to 2L/2D
|
|
__ neon_vector_extend($dst$$FloatRegister, T_LONG, length_in_bytes,
|
|
$src$$FloatRegister, T_INT);
|
|
if (bt == T_DOUBLE) {
|
|
__ scvtfv(__ T2D, $dst$$FloatRegister, $dst$$FloatRegister);
|
|
}
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_vector_extend($dst$$FloatRegister, __ D, $src$$FloatRegister, __ S);
|
|
if (bt == T_DOUBLE) {
|
|
__ sve_scvtf($dst$$FloatRegister, __ D, ptrue, $dst$$FloatRegister, __ D);
|
|
}
|
|
}
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// VectorCastL2X
|
|
|
|
instruct vcvtLtoX_narrow_neon(vReg dst, vReg src) %{
|
|
predicate((Matcher::vector_element_basic_type(n) == T_INT ||
|
|
Matcher::vector_element_basic_type(n) == T_SHORT) &&
|
|
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
|
|
match(Set dst (VectorCastL2X src));
|
|
format %{ "vcvtLtoX_narrow_neon $dst, $src" %}
|
|
ins_encode %{
|
|
// 2L to 2S/2I
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
|
|
__ neon_vector_narrow($dst$$FloatRegister, bt,
|
|
$src$$FloatRegister, T_LONG, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcvtLtoX_narrow_sve(vReg dst, vReg src, vReg tmp) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))) &&
|
|
!is_floating_point_type(Matcher::vector_element_basic_type(n)) &&
|
|
type2aelembytes(Matcher::vector_element_basic_type(n)) <= 4);
|
|
match(Set dst (VectorCastL2X src));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "vcvtLtoX_narrow_sve $dst, $src\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_vector_narrow($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$src$$FloatRegister, __ D, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcvtLtoF_neon(vReg dst, vReg src, vRegF tmp) %{
|
|
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
|
|
match(Set dst (VectorCastL2X src));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "vcvtLtoF_neon $dst, $src\t# 2L to 2F. KILL $tmp" %}
|
|
ins_encode %{
|
|
// 2L to 2F
|
|
__ umov(rscratch1, $src$$FloatRegister, __ D, 0);
|
|
__ scvtfs($dst$$FloatRegister, rscratch1);
|
|
__ umov(rscratch1, $src$$FloatRegister, __ D, 1);
|
|
__ scvtfs($tmp$$FloatRegister, rscratch1);
|
|
__ ins($dst$$FloatRegister, __ S, $tmp$$FloatRegister, 1, 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcvtLtoF_sve(vReg dst, vReg src, vReg tmp) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
|
|
match(Set dst (VectorCastL2X src));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "vcvtLtoF_sve $dst, $src\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
__ sve_scvtf($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ D);
|
|
__ sve_vector_narrow($dst$$FloatRegister, __ S,
|
|
$dst$$FloatRegister, __ D, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcvtLtoD(vReg dst, vReg src) %{
|
|
predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
|
|
match(Set dst (VectorCastL2X src));
|
|
format %{ "vcvtLtoD $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
// 2L to 2D
|
|
__ scvtfv(__ T2D, $dst$$FloatRegister, $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_scvtf($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister, __ D);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// VectorCastF2X
|
|
|
|
instruct vcvtFtoX_narrow_neon(vReg dst, vReg src) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))) &&
|
|
(Matcher::vector_element_basic_type(n) == T_BYTE ||
|
|
Matcher::vector_element_basic_type(n) == T_SHORT));
|
|
match(Set dst (VectorCastF2X src));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "vcvtFtoX_narrow_neon $dst, $src" %}
|
|
ins_encode %{
|
|
// 2F to 2S, 4F to 4B/4S
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
|
|
__ fcvtzs($dst$$FloatRegister, length_in_bytes == 16 ? __ T4S : __ T2S,
|
|
$src$$FloatRegister);
|
|
__ neon_vector_narrow($dst$$FloatRegister, bt,
|
|
$dst$$FloatRegister, T_INT, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcvtFtoX_narrow_sve(vReg dst, vReg src, vReg tmp) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))) &&
|
|
(Matcher::vector_element_basic_type(n) == T_BYTE ||
|
|
Matcher::vector_element_basic_type(n) == T_SHORT));
|
|
match(Set dst (VectorCastF2X src));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "vcvtFtoX_narrow_sve $dst, $src\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_fcvtzs($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ S);
|
|
__ sve_vector_narrow($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$dst$$FloatRegister, __ S, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcvtFtoX(vReg dst, vReg src) %{
|
|
predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) >= 4);
|
|
match(Set dst (VectorCastF2X src));
|
|
format %{ "vcvtFtoX $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (bt == T_INT) {
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
// 2F/4F to 2I/4I
|
|
__ fcvtzs($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fcvtzs($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ S);
|
|
}
|
|
} else if (bt == T_LONG) {
|
|
if (UseSVE == 0) {
|
|
// 2F to 2L
|
|
__ fcvtl($dst$$FloatRegister, __ T2D, $src$$FloatRegister, __ T2S);
|
|
__ fcvtzs($dst$$FloatRegister, __ T2D, $dst$$FloatRegister);
|
|
} else {
|
|
__ sve_vector_extend($dst$$FloatRegister, __ D, $src$$FloatRegister, __ S);
|
|
__ sve_fcvtzs($dst$$FloatRegister, __ D, ptrue, $dst$$FloatRegister, __ S);
|
|
}
|
|
} else {
|
|
assert(bt == T_DOUBLE, "unsupported type");
|
|
if (length_in_bytes == 16) {
|
|
// 2F to 2D
|
|
__ fcvtl($dst$$FloatRegister, __ T2D, $src$$FloatRegister, __ T2S);
|
|
} else {
|
|
assert(UseSVE > 0 && length_in_bytes > 16, "must be");
|
|
__ sve_vector_extend($dst$$FloatRegister, __ D, $src$$FloatRegister, __ S);
|
|
__ sve_fcvt($dst$$FloatRegister, __ D, ptrue, $dst$$FloatRegister, __ S);
|
|
}
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// VectorCastD2X
|
|
|
|
instruct vcvtDtoI_neon(vReg dst, vReg src) %{
|
|
predicate(UseSVE == 0 &&
|
|
(Matcher::vector_element_basic_type(n) == T_INT ||
|
|
Matcher::vector_element_basic_type(n) == T_SHORT));
|
|
match(Set dst (VectorCastD2X src));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "vcvtDtoI_neon $dst, $src\t# 2D to 2S/2I" %}
|
|
ins_encode %{
|
|
// 2D to 2S/2I
|
|
__ ins($dst$$FloatRegister, __ D, $src$$FloatRegister, 0, 1);
|
|
// We can't use fcvtzs(vector, integer) instruction here because we need
|
|
// saturation arithmetic. See JDK-8276151.
|
|
__ fcvtzdw(rscratch1, $src$$FloatRegister);
|
|
__ fcvtzdw(rscratch2, $dst$$FloatRegister);
|
|
__ fmovs($dst$$FloatRegister, rscratch1);
|
|
__ mov($dst$$FloatRegister, __ S, 1, rscratch2);
|
|
if (Matcher::vector_element_basic_type(this) == T_SHORT) {
|
|
__ neon_vector_narrow($dst$$FloatRegister, T_SHORT,
|
|
$dst$$FloatRegister, T_INT, 8);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcvtDtoI_sve(vReg dst, vReg src, vReg tmp) %{
|
|
predicate(UseSVE > 0 &&
|
|
(Matcher::vector_element_basic_type(n) == T_BYTE ||
|
|
Matcher::vector_element_basic_type(n) == T_SHORT ||
|
|
Matcher::vector_element_basic_type(n) == T_INT));
|
|
match(Set dst (VectorCastD2X src));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "vcvtDtoI_sve $dst, $src\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_fcvtzs($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ D);
|
|
__ sve_vector_narrow($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$dst$$FloatRegister, __ D, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcvtDtoL(vReg dst, vReg src) %{
|
|
predicate(Matcher::vector_element_basic_type(n) == T_LONG);
|
|
match(Set dst (VectorCastD2X src));
|
|
format %{ "vcvtDtoL $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
// 2D to 2L
|
|
__ fcvtzs($dst$$FloatRegister, __ T2D, $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fcvtzs($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister, __ D);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcvtDtoF_64b(vReg dst, vReg src) %{
|
|
predicate(Matcher::vector_element_basic_type(n) == T_FLOAT &&
|
|
Matcher::vector_length_in_bytes(n) == 8);
|
|
match(Set dst (VectorCastD2X src));
|
|
format %{ "vcvtDtoF_64b $dst, $src\t# 2D to 2F" %}
|
|
ins_encode %{
|
|
// 2D to 2F
|
|
__ fcvtn($dst$$FloatRegister, __ T2S, $src$$FloatRegister, __ T2D);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcvtDtoF_gt64b(vReg dst, vReg src, vReg tmp) %{
|
|
predicate(Matcher::vector_element_basic_type(n) == T_FLOAT &&
|
|
Matcher::vector_length_in_bytes(n) > 8);
|
|
match(Set dst (VectorCastD2X src));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "vcvtDtoF_gt64b $dst, $src\t# vector > 64 bits. KILL $tmp" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fcvt($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ D);
|
|
__ sve_vector_narrow($dst$$FloatRegister, __ S,
|
|
$dst$$FloatRegister, __ D, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// VectorCastHF2F
|
|
|
|
instruct vcvtHFtoF(vReg dst, vReg src) %{
|
|
match(Set dst (VectorCastHF2F src));
|
|
format %{ "vcvtHFtoF $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
// 2HF to 2F, 4HF to 4F
|
|
__ fcvtl($dst$$FloatRegister, __ T4S, $src$$FloatRegister, __ T4H);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_vector_extend($dst$$FloatRegister, __ S, $src$$FloatRegister, __ H);
|
|
__ sve_fcvt($dst$$FloatRegister, __ S, ptrue, $dst$$FloatRegister, __ H);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// VectorCastF2HF
|
|
|
|
instruct vcvtFtoHF_neon(vReg dst, vReg src) %{
|
|
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
|
|
match(Set dst (VectorCastF2HF src));
|
|
format %{ "vcvtFtoHF_neon $dst, $src\t# 2F/4F to 2HF/4HF" %}
|
|
ins_encode %{
|
|
// 2F to 2HF, 4F to 4HF
|
|
__ fcvtn($dst$$FloatRegister, __ T4H, $src$$FloatRegister, __ T4S);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcvtFtoHF_sve(vReg dst, vReg src, vReg tmp) %{
|
|
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
|
|
match(Set dst (VectorCastF2HF src));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "vcvtFtoHF_sve $dst, $src\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_fcvt($dst$$FloatRegister, __ H, ptrue, $src$$FloatRegister, __ S);
|
|
__ sve_vector_narrow($dst$$FloatRegister, __ H,
|
|
$dst$$FloatRegister, __ S, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Replicate ------------------------------------
|
|
|
|
// replicate from reg
|
|
|
|
instruct replicateI(vReg dst, iRegIorL2I src) %{
|
|
match(Set dst (Replicate src));
|
|
format %{ "replicateI $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ dup($dst$$FloatRegister, get_arrangement(this), $src$$Register);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_dup($dst$$FloatRegister, get_reg_variant(this), $src$$Register);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct replicateL(vReg dst, iRegL src) %{
|
|
match(Set dst (Replicate src));
|
|
format %{ "replicateL $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ dup($dst$$FloatRegister, get_arrangement(this), $src$$Register);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_dup($dst$$FloatRegister, get_reg_variant(this), $src$$Register);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct replicateF(vReg dst, vRegF src) %{
|
|
match(Set dst (Replicate src));
|
|
format %{ "replicateF $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ dup($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_cpy($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct replicateD(vReg dst, vRegD src) %{
|
|
match(Set dst (Replicate src));
|
|
format %{ "replicateD $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ dup($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_cpy($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Replicate a half-precision float value held in a floating point register
|
|
instruct replicateHF(vReg dst, vRegF src) %{
|
|
predicate(Matcher::vector_element_basic_type(n) == T_SHORT);
|
|
match(Set dst (Replicate src));
|
|
format %{ "replicateHF $dst, $src\t# replicate half-precision float" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ dup($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else { // length_in_bytes must be > 16 and SVE should be enabled
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_cpy($dst$$FloatRegister, __ H, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// replicate from imm
|
|
|
|
instruct replicateI_imm_le128b(vReg dst, immI con) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) <= 16 &&
|
|
Matcher::is_non_long_integral_vector(n));
|
|
match(Set dst (Replicate con));
|
|
format %{ "replicateI_imm_le128b $dst, $con\t# vector <= 128 bits" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
int imm = (int)$con$$constant;
|
|
if (type2aelembytes(bt) == 1) {
|
|
// Refine imm for B
|
|
imm = imm & 0xff;
|
|
} else if (type2aelembytes(bt) == 2) {
|
|
// Refine imm for S
|
|
imm = imm & 0xffff;
|
|
}
|
|
__ mov($dst$$FloatRegister, get_arrangement(this), imm);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct replicateB_imm8_gt128b(vReg dst, immI8 con) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) > 16 &&
|
|
Matcher::vector_element_basic_type(n) == T_BYTE);
|
|
match(Set dst (Replicate con));
|
|
format %{ "replicateB_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_dup($dst$$FloatRegister, __ B, (int)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct replicateI_imm8_gt128b(vReg dst, immIDupV con) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) > 16 &&
|
|
(Matcher::vector_element_basic_type(n) == T_SHORT ||
|
|
Matcher::vector_element_basic_type(n) == T_INT));
|
|
match(Set dst (Replicate con));
|
|
format %{ "replicateI_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_dup($dst$$FloatRegister, get_reg_variant(this), (int)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct replicateL_imm_128b(vReg dst, immL con) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) == 16);
|
|
match(Set dst (Replicate con));
|
|
format %{ "replicateL_imm_128b $dst, $con\t# vector > 128 bits" %}
|
|
ins_encode %{
|
|
__ mov($dst$$FloatRegister, __ T2D, (uint64_t)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct replicateL_imm8_gt128b(vReg dst, immLDupV con) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) > 16);
|
|
match(Set dst (Replicate con));
|
|
format %{ "replicateL_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_dup($dst$$FloatRegister, __ D, (int)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Replicate an immediate 16-bit half precision float value
|
|
instruct replicateHF_imm_le128b(vReg dst, immH con) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) <= 16);
|
|
match(Set dst (Replicate con));
|
|
format %{ "replicateHF_imm_le128b $dst, $con\t# vector <= 128 bits" %}
|
|
ins_encode %{
|
|
int imm = (int)($con$$constant) & 0xffff;
|
|
__ mov($dst$$FloatRegister, get_arrangement(this), imm);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Replicate a 16-bit half precision float which is within the limits
|
|
// for the operand - immHDupV
|
|
instruct replicateHF_imm8_gt128b(vReg dst, immHDupV con) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) > 16);
|
|
match(Set dst (Replicate con));
|
|
format %{ "replicateHF_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_dup($dst$$FloatRegister, __ H, (int)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector insert --------------------------------
|
|
|
|
// BYTE, SHORT, INT
|
|
|
|
instruct insertI_le128b(vReg dst, vReg src, iRegIorL2I val, immI idx) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) <= 16 &&
|
|
(Matcher::vector_element_basic_type(n) == T_BYTE ||
|
|
Matcher::vector_element_basic_type(n) == T_SHORT ||
|
|
Matcher::vector_element_basic_type(n) == T_INT));
|
|
match(Set dst (VectorInsert (Binary src val) idx));
|
|
format %{ "insertI_le128b $dst, $src, $val, $idx\t# vector <= 128 bits" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if ($dst$$FloatRegister != $src$$FloatRegister) {
|
|
__ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src$$FloatRegister, $src$$FloatRegister);
|
|
}
|
|
__ mov($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
(int)($idx$$constant), $val$$Register);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct insertI_index_lt32(vReg dst, vReg src, iRegIorL2I val, immI idx,
|
|
vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
|
|
predicate(n->in(2)->get_int() < 32 &&
|
|
Matcher::vector_length_in_bytes(n) > 16 &&
|
|
(Matcher::vector_element_basic_type(n) == T_BYTE ||
|
|
Matcher::vector_element_basic_type(n) == T_SHORT ||
|
|
Matcher::vector_element_basic_type(n) == T_INT));
|
|
match(Set dst (VectorInsert (Binary src val) idx));
|
|
effect(TEMP tmp, TEMP pgtmp, KILL cr);
|
|
format %{ "insertI_index_lt32 $dst, $src, $val, $idx\t# vector > 128 bits, index < 31. KILL $tmp, $pgtmp, cr" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $src);
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_index($tmp$$FloatRegister, size, -16, 1);
|
|
__ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, size, ptrue,
|
|
$tmp$$FloatRegister, (int)($idx$$constant) - 16);
|
|
if ($dst$$FloatRegister != $src$$FloatRegister) {
|
|
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
|
|
}
|
|
__ sve_cpy($dst$$FloatRegister, size, $pgtmp$$PRegister, $val$$Register);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct insertI_index_ge32(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1,
|
|
vReg tmp2, pRegGov pgtmp, rFlagsReg cr) %{
|
|
predicate(n->in(2)->get_int() >= 32 &&
|
|
(Matcher::vector_element_basic_type(n) == T_BYTE ||
|
|
Matcher::vector_element_basic_type(n) == T_SHORT ||
|
|
Matcher::vector_element_basic_type(n) == T_INT));
|
|
match(Set dst (VectorInsert (Binary src val) idx));
|
|
effect(TEMP tmp1, TEMP tmp2, TEMP pgtmp, KILL cr);
|
|
format %{ "insertI_index_ge32 $dst, $src, $val, $idx\t# index >= 32. KILL $tmp1, $tmp2, $pgtmp, cr" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $src);
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_index($tmp1$$FloatRegister, size, 0, 1);
|
|
__ sve_dup($tmp2$$FloatRegister, size, (int)($idx$$constant));
|
|
__ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, size, ptrue,
|
|
$tmp1$$FloatRegister, $tmp2$$FloatRegister);
|
|
if ($dst$$FloatRegister != $src$$FloatRegister) {
|
|
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
|
|
}
|
|
__ sve_cpy($dst$$FloatRegister, size, $pgtmp$$PRegister, $val$$Register);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// LONG
|
|
|
|
instruct insertL_128b(vReg dst, vReg src, iRegL val, immI idx) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) == 16 &&
|
|
Matcher::vector_element_basic_type(n) == T_LONG);
|
|
match(Set dst (VectorInsert (Binary src val) idx));
|
|
format %{ "insertL_128b $dst, $src, $val, $idx\t# 2L" %}
|
|
ins_encode %{
|
|
if ($dst$$FloatRegister != $src$$FloatRegister) {
|
|
__ orr($dst$$FloatRegister, __ T16B, $src$$FloatRegister, $src$$FloatRegister);
|
|
}
|
|
__ mov($dst$$FloatRegister, __ D, (int)($idx$$constant), $val$$Register);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct insertL_gt128b(vReg dst, vReg src, iRegL val, immI idx,
|
|
vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) > 16 &&
|
|
Matcher::vector_element_basic_type(n) == T_LONG);
|
|
match(Set dst (VectorInsert (Binary src val) idx));
|
|
effect(TEMP tmp, TEMP pgtmp, KILL cr);
|
|
format %{ "insertL_gt128b $dst, $src, $val, $idx\t# vector > 128 bits. KILL $tmp, $pgtmp, cr" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_index($tmp$$FloatRegister, __ D, -16, 1);
|
|
__ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, __ D, ptrue,
|
|
$tmp$$FloatRegister, (int)($idx$$constant) - 16);
|
|
if ($dst$$FloatRegister != $src$$FloatRegister) {
|
|
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
|
|
}
|
|
__ sve_cpy($dst$$FloatRegister, __ D, $pgtmp$$PRegister, $val$$Register);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// FLOAT
|
|
|
|
instruct insertF_le128b(vReg dst, vReg src, vRegF val, immI idx) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) <= 16 &&
|
|
Matcher::vector_element_basic_type(n) == T_FLOAT);
|
|
match(Set dst (VectorInsert (Binary src val) idx));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "insertF_le128b $dst, $src, $val, $idx\t# vector <= 128 bits" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if ($dst$$FloatRegister != $src$$FloatRegister) {
|
|
__ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src$$FloatRegister, $src$$FloatRegister);
|
|
}
|
|
__ ins($dst$$FloatRegister, __ S, $val$$FloatRegister, (int)($idx$$constant), 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct insertF_index_lt32(vReg dst, vReg src, vRegF val, immI idx,
|
|
pRegGov pgtmp, rFlagsReg cr) %{
|
|
predicate(n->in(2)->get_int() < 32 &&
|
|
Matcher::vector_length_in_bytes(n) > 16 &&
|
|
Matcher::vector_element_basic_type(n) == T_FLOAT);
|
|
match(Set dst (VectorInsert (Binary src val) idx));
|
|
effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
|
|
format %{ "insertF_index_lt32 $dst, $src, $val, $idx\t# vector > 128 bits, index < 32. KILL $pgtmp, cr" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_index($dst$$FloatRegister, __ S, -16, 1);
|
|
__ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, __ S, ptrue,
|
|
$dst$$FloatRegister, (int)($idx$$constant) - 16);
|
|
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
|
|
__ sve_cpy($dst$$FloatRegister, __ S, $pgtmp$$PRegister, $val$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct insertF_index_ge32(vReg dst, vReg src, vRegF val, immI idx, vReg tmp,
|
|
pRegGov pgtmp, rFlagsReg cr) %{
|
|
predicate(n->in(2)->get_int() >= 32 &&
|
|
Matcher::vector_element_basic_type(n) == T_FLOAT);
|
|
match(Set dst (VectorInsert (Binary src val) idx));
|
|
effect(TEMP_DEF dst, TEMP tmp, TEMP pgtmp, KILL cr);
|
|
format %{ "insertF_index_ge32 $dst, $src, $val, $idx\t# index >= 32. KILL $tmp, $pgtmp, cr" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_index($tmp$$FloatRegister, __ S, 0, 1);
|
|
__ sve_dup($dst$$FloatRegister, __ S, (int)($idx$$constant));
|
|
__ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, __ S, ptrue,
|
|
$tmp$$FloatRegister, $dst$$FloatRegister);
|
|
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
|
|
__ sve_cpy($dst$$FloatRegister, __ S, $pgtmp$$PRegister, $val$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// DOUBLE
|
|
|
|
instruct insertD_128b(vReg dst, vReg src, vRegD val, immI idx) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) == 16 &&
|
|
Matcher::vector_element_basic_type(n) == T_DOUBLE);
|
|
match(Set dst (VectorInsert (Binary src val) idx));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "insertD_128b $dst, $src, $val, $idx\t# 2D" %}
|
|
ins_encode %{
|
|
if ($dst$$FloatRegister != $src$$FloatRegister) {
|
|
__ orr($dst$$FloatRegister, __ T16B, $src$$FloatRegister, $src$$FloatRegister);
|
|
}
|
|
__ ins($dst$$FloatRegister, __ D, $val$$FloatRegister, (int)($idx$$constant), 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct insertD_gt128b(vReg dst, vReg src, vRegD val, immI idx,
|
|
pRegGov pgtmp, rFlagsReg cr) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) > 16 &&
|
|
Matcher::vector_element_basic_type(n) == T_DOUBLE);
|
|
match(Set dst (VectorInsert (Binary src val) idx));
|
|
effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
|
|
format %{ "insertD_gt128b $dst, $src, $val, $idx\t# vector > 128 bits. KILL $pgtmp, cr" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_index($dst$$FloatRegister, __ D, -16, 1);
|
|
__ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, __ D, ptrue,
|
|
$dst$$FloatRegister, (int)($idx$$constant) - 16);
|
|
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
|
|
__ sve_cpy($dst$$FloatRegister, __ D, $pgtmp$$PRegister, $val$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Extract --------------------------------------
|
|
|
|
// BOOLEAN
|
|
|
|
instruct extractUB_ireg(iRegINoSp dst, vReg src, iRegI idx, vReg tmp) %{
|
|
match(Set dst (ExtractUB src idx));
|
|
effect(TEMP tmp);
|
|
format %{ "extractUB_ireg $dst, $src, $idx\t# variable index. KILL $tmp" %}
|
|
ins_encode %{
|
|
// Input "src" is a vector of boolean represented as
|
|
// bytes with 0x00/0x01 as element values.
|
|
// "idx" is expected to be in range.
|
|
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
|
|
__ mov($tmp$$FloatRegister, __ B, 0, $idx$$Register);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ tbl($tmp$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src$$FloatRegister, 1, $tmp$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_tbl($tmp$$FloatRegister, __ B, $src$$FloatRegister, $tmp$$FloatRegister);
|
|
}
|
|
__ smov($dst$$Register, $tmp$$FloatRegister, __ B, 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct extractUB_index_lt16(iRegINoSp dst, vReg src, immI idx) %{
|
|
predicate(n->in(2)->get_int() < 16);
|
|
match(Set dst (ExtractUB src idx));
|
|
format %{ "extractUB_index_lt16 $dst, $src, $idx\t# index < 16" %}
|
|
ins_encode %{
|
|
__ smov($dst$$Register, $src$$FloatRegister, __ B, (int)($idx$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct extractUB_index_ge16(iRegINoSp dst, vReg src, immI idx, vReg tmp) %{
|
|
predicate(n->in(2)->get_int() >= 16);
|
|
match(Set dst (ExtractUB src idx));
|
|
effect(TEMP tmp);
|
|
format %{ "extractUB_index_ge16 $dst, $src, $idx\t# index >=16. KILL $tmp" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_extract_integral($dst$$Register, T_BYTE, $src$$FloatRegister,
|
|
(int)($idx$$constant), $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// BYTE
|
|
|
|
instruct extractB_index_lt16(iRegINoSp dst, vReg src, immI idx) %{
|
|
predicate(n->in(2)->get_int() < 16);
|
|
match(Set dst (ExtractB src idx));
|
|
format %{ "extractB_index_lt16 $dst, $src, $idx\t# index < 16" %}
|
|
ins_encode %{
|
|
__ smov($dst$$Register, $src$$FloatRegister, __ B, (int)($idx$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct extractB_index_ge16(iRegINoSp dst, vReg src, immI idx, vReg tmp) %{
|
|
predicate(n->in(2)->get_int() >= 16);
|
|
match(Set dst (ExtractB src idx));
|
|
effect(TEMP tmp);
|
|
format %{ "extractB_index_ge16 $dst, $src, $idx\t# index >=16. KILL $tmp" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_extract_integral($dst$$Register, T_BYTE, $src$$FloatRegister,
|
|
(int)($idx$$constant), $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// SHORT
|
|
|
|
instruct extractS_index_lt8(iRegINoSp dst, vReg src, immI idx) %{
|
|
predicate(n->in(2)->get_int() < 8);
|
|
match(Set dst (ExtractS src idx));
|
|
format %{ "extractS_index_lt8 $dst, $src, $idx\t# index < 8" %}
|
|
ins_encode %{
|
|
__ smov($dst$$Register, $src$$FloatRegister, __ H, (int)($idx$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct extractS_index_ge8(iRegINoSp dst, vReg src, immI idx, vReg tmp) %{
|
|
predicate(n->in(2)->get_int() >= 8);
|
|
match(Set dst (ExtractS src idx));
|
|
effect(TEMP tmp);
|
|
format %{ "extractS_index_ge8 $dst, $src, $idx\t# index >=8. KILL $tmp" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_extract_integral($dst$$Register, T_SHORT, $src$$FloatRegister,
|
|
(int)($idx$$constant), $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// INT
|
|
|
|
instruct extractI_index_lt4(iRegINoSp dst, vReg src, immI idx) %{
|
|
predicate(n->in(2)->get_int() < 4);
|
|
match(Set dst (ExtractI src idx));
|
|
format %{ "extractI_index_lt4 $dst, $src, $idx\t# index < 4" %}
|
|
ins_encode %{
|
|
__ umov($dst$$Register, $src$$FloatRegister, __ S, (int)($idx$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct extractI_index_ge4(iRegINoSp dst, vReg src, immI idx, vReg tmp) %{
|
|
predicate(n->in(2)->get_int() >= 4);
|
|
match(Set dst (ExtractI src idx));
|
|
effect(TEMP tmp);
|
|
format %{ "extractI_index_ge4 $dst, $src, $idx\t# index >=4. KILL $tmp" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_extract_integral($dst$$Register, T_INT, $src$$FloatRegister,
|
|
(int)($idx$$constant), $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// LONG
|
|
|
|
instruct extractL_index_lt2(iRegLNoSp dst, vReg src, immI idx) %{
|
|
predicate(n->in(2)->get_int() < 2);
|
|
match(Set dst (ExtractL src idx));
|
|
format %{ "extractL_index_lt2 $dst, $src, $idx\t# index < 2" %}
|
|
ins_encode %{
|
|
__ umov($dst$$Register, $src$$FloatRegister, __ D, (int)($idx$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct extractL_index_ge2(iRegLNoSp dst, vReg src, immI idx, vReg tmp) %{
|
|
predicate(n->in(2)->get_int() >= 2);
|
|
match(Set dst (ExtractL src idx));
|
|
effect(TEMP tmp);
|
|
format %{ "extractL_index_ge2 $dst, $src, $idx\t# index >=2. KILL $tmp" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_extract_integral($dst$$Register, T_LONG, $src$$FloatRegister,
|
|
(int)($idx$$constant), $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// FLOAT
|
|
|
|
instruct extractF(vRegF dst, vReg src, immI idx) %{
|
|
match(Set dst (ExtractF src idx));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "extractF $dst, $src, $idx" %}
|
|
ins_encode %{
|
|
int index = (int)$idx$$constant;
|
|
if (index == 0) {
|
|
__ fmovs($dst$$FloatRegister, $src$$FloatRegister);
|
|
} else if (index < 4) {
|
|
__ ins($dst$$FloatRegister, __ S, $src$$FloatRegister, 0, index);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
|
|
__ sve_ext($dst$$FloatRegister, $dst$$FloatRegister, index << 2);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// DOUBLE
|
|
|
|
instruct extractD(vRegD dst, vReg src, immI idx) %{
|
|
match(Set dst (ExtractD src idx));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "extractD $dst, $src, $idx" %}
|
|
ins_encode %{
|
|
int index = (int)$idx$$constant;
|
|
if (index == 0) {
|
|
__ fmovd($dst$$FloatRegister, $src$$FloatRegister);
|
|
} else if (index < 2) {
|
|
__ ins($dst$$FloatRegister, __ D, $src$$FloatRegister, 0, index);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
|
|
__ sve_ext($dst$$FloatRegister, $dst$$FloatRegister, index << 3);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector mask load/store -----------------------
|
|
|
|
// vector load mask
|
|
|
|
instruct vloadmask_neon(vReg dst, vReg src) %{
|
|
predicate(UseSVE == 0 &&
|
|
(Matcher::vector_length_in_bytes(n) == 8 ||
|
|
Matcher::vector_length_in_bytes(n) == 16));
|
|
match(Set dst (VectorLoadMask src ));
|
|
format %{ "vloadmask_neon $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
if (bt == T_BYTE) {
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
__ negr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src$$FloatRegister);
|
|
} else {
|
|
__ uxtl($dst$$FloatRegister, __ T8H, $src$$FloatRegister, __ T8B);
|
|
if (type2aelembytes(bt) >= 4) {
|
|
__ uxtl($dst$$FloatRegister, __ T4S, $dst$$FloatRegister, __ T4H);
|
|
}
|
|
if (type2aelembytes(bt) == 8) {
|
|
__ uxtl($dst$$FloatRegister, __ T2D, $dst$$FloatRegister, __ T2S);
|
|
}
|
|
__ negr($dst$$FloatRegister, get_arrangement(this), $dst$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vloadmaskB_sve(pReg dst, vReg src, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
|
|
match(Set dst (VectorLoadMask src));
|
|
effect(KILL cr);
|
|
format %{ "vloadmaskB_sve $dst, $src\t# KILL cr" %}
|
|
ins_encode %{
|
|
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ B,
|
|
ptrue, $src$$FloatRegister, 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vloadmask_extend_sve(pReg dst, vReg src, vReg tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) != T_BYTE);
|
|
match(Set dst (VectorLoadMask src));
|
|
effect(TEMP tmp, KILL cr);
|
|
format %{ "vloadmask_extend_sve $dst, $src\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_vector_extend($tmp$$FloatRegister, size, $src$$FloatRegister, __ B);
|
|
__ sve_cmp(Assembler::NE, $dst$$PRegister, size, ptrue, $tmp$$FloatRegister, 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vloadmaskB_masked(pReg dst, vReg src, pRegGov pg, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
|
|
match(Set dst (VectorLoadMask src pg));
|
|
effect(KILL cr);
|
|
format %{ "vloadmaskB_masked $dst, $pg, $src\t# KILL cr" %}
|
|
ins_encode %{
|
|
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ B,
|
|
$pg$$PRegister, $src$$FloatRegister, 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vloadmask_extend_masked(pReg dst, vReg src, pRegGov pg, vReg tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) != T_BYTE);
|
|
match(Set dst (VectorLoadMask src pg));
|
|
effect(TEMP tmp, KILL cr);
|
|
format %{ "vloadmask_extend_masked $dst, $pg, $src\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_vector_extend($tmp$$FloatRegister, size, $src$$FloatRegister, __ B);
|
|
__ sve_cmp(Assembler::NE, $dst$$PRegister, size,
|
|
$pg$$PRegister, $tmp$$FloatRegister, 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector store mask - neon
|
|
|
|
instruct vstoremaskB_neon(vReg dst, vReg src, immI_1 size) %{
|
|
predicate(UseSVE == 0);
|
|
match(Set dst (VectorStoreMask src size));
|
|
format %{ "vstoremaskB_neon $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
|
|
__ negr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vstoremask_narrow_neon(vReg dst, vReg src, immI_gt_1 size) %{
|
|
predicate(UseSVE == 0);
|
|
match(Set dst (VectorStoreMask src size));
|
|
format %{ "vstoremask_narrow_neon $dst, $src" %}
|
|
ins_encode %{
|
|
int esize = (int)$size$$constant;
|
|
if (esize == 2) {
|
|
__ xtn($dst$$FloatRegister, __ T8B, $src$$FloatRegister, __ T8H);
|
|
} else if (esize == 4) {
|
|
__ xtn($dst$$FloatRegister, __ T4H, $src$$FloatRegister, __ T4S);
|
|
__ xtn($dst$$FloatRegister, __ T8B, $dst$$FloatRegister, __ T8H);
|
|
} else {
|
|
assert(esize == 8, "must be");
|
|
__ xtn($dst$$FloatRegister, __ T2S, $src$$FloatRegister, __ T2D);
|
|
__ xtn($dst$$FloatRegister, __ T4H, $dst$$FloatRegister, __ T4S);
|
|
__ xtn($dst$$FloatRegister, __ T8B, $dst$$FloatRegister, __ T8H);
|
|
}
|
|
__ negr($dst$$FloatRegister, __ T8B, $dst$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector store mask - sve
|
|
|
|
instruct vstoremaskB_sve(vReg dst, pReg src, immI_1 size) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (VectorStoreMask src size));
|
|
format %{ "vstoremaskB_sve $dst, $src" %}
|
|
ins_encode %{
|
|
__ sve_cpy($dst$$FloatRegister, __ B, $src$$PRegister, 1, false);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vstoremask_narrow_sve(vReg dst, pReg src, immI_gt_1 size, vReg tmp) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (VectorStoreMask src size));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "vstoremask_narrow_sve $dst, $src\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
|
|
__ sve_cpy($dst$$FloatRegister, size, $src$$PRegister, 1, false);
|
|
__ sve_vector_narrow($dst$$FloatRegister, __ B,
|
|
$dst$$FloatRegister, size, $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Combined rules for vector mask load when the vector element type is not T_BYTE
|
|
|
|
// VectorLoadMask+LoadVector, and the VectorLoadMask is unpredicated.
|
|
instruct vloadmask_loadV(pReg dst, indirect mem, vReg tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 &&
|
|
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
|
|
match(Set dst (VectorLoadMask (LoadVector mem)));
|
|
effect(TEMP tmp, KILL cr);
|
|
format %{ "vloadmask_loadV $dst, $mem\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
// Load mask values which are boolean type, and extend them to the
|
|
// defined vector element type. Convert the vector to predicate.
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
|
|
ptrue, T_BOOLEAN, bt, $mem->opcode(),
|
|
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
|
|
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $tmp$$FloatRegister, 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// VectorLoadMask+LoadVector, and the VectorLoadMask is predicated.
|
|
instruct vloadmask_loadV_masked(pReg dst, indirect mem, pRegGov pg,
|
|
vReg tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 &&
|
|
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
|
|
match(Set dst (VectorLoadMask (LoadVector mem) pg));
|
|
effect(TEMP tmp, KILL cr);
|
|
format %{ "vloadmask_loadV_masked $dst, $pg, $mem\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
// Load valid mask values which are boolean type, and extend them to the
|
|
// defined vector element type. Convert the vector to predicate.
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
|
|
$pg$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
|
|
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
|
|
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $tmp$$FloatRegister, 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// VectorLoadMask+LoadVectorMasked, and the VectorLoadMask is unpredicated.
|
|
instruct vloadmask_loadVMasked(pReg dst, vmemA mem, pRegGov pg, vReg tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 &&
|
|
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
|
|
match(Set dst (VectorLoadMask (LoadVectorMasked mem pg)));
|
|
effect(TEMP tmp, KILL cr);
|
|
format %{ "vloadmask_loadVMasked $dst, $mem\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
// Load mask values which are boolean type, and extend them to the
|
|
// defined vector element type. Convert the vector to predicate.
|
|
//
|
|
// Note that we cannot use "pg" here, since it is the predicate used
|
|
// for the vector load with boolean type. But the predicate used in
|
|
// the extending "sve_ld1b" is based on the final extended vector type,
|
|
// which is the full-sized predicate (ptrue) used in VectorLoadMask.
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
|
|
ptrue, T_BOOLEAN, bt, $mem->opcode(),
|
|
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
|
|
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $tmp$$FloatRegister, 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// VectorLoadMask+LoadVectorMasked, and the VectorLoadMask is predicated.
|
|
instruct vloadmask_loadVMasked_masked(pReg dst, vmemA mem, pRegGov pg1, pRegGov pg2,
|
|
vReg tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 &&
|
|
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
|
|
match(Set dst (VectorLoadMask (LoadVectorMasked mem pg1) pg2));
|
|
effect(TEMP tmp, KILL cr);
|
|
format %{ "vloadmask_loadVMasked_masked $dst, $pg2, $mem\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
// Load valid mask values which are boolean type, and extend them to the
|
|
// defined vector element type. Convert the vector to predicate.
|
|
//
|
|
// Note that we cannot use "pg1" here, since it is the predicate used
|
|
// for the vector load with boolean type. But the predicate used in
|
|
// the extending "sve_ld1b" is based on the final extended vector type,
|
|
// which is the "pg2" used in VectorLoadMask.
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
|
|
$pg2$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
|
|
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
|
|
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
|
|
$pg2$$PRegister, $tmp$$FloatRegister, 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Combined rules for vector mask store when the vector element type is not T_BYTE
|
|
|
|
// StoreVector+VectorStoreMask, and the vector size of "src" is equal to the MaxVectorSize.
|
|
instruct storeV_vstoremask(indirect mem, pReg src, immI_gt_1 esize, vReg tmp) %{
|
|
predicate(UseSVE > 0 &&
|
|
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
|
|
match(Set mem (StoreVector mem (VectorStoreMask src esize)));
|
|
effect(TEMP tmp);
|
|
format %{ "storeV_vstoremask $mem, $src\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
// Convert the valid src predicate to vector, and store the vector elements
|
|
// as boolean values.
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $src);
|
|
assert(type2aelembytes(bt) == (int)$esize$$constant, "unsupported type");
|
|
Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
|
|
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
|
|
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
|
|
ptrue, T_BOOLEAN, bt, $mem->opcode(),
|
|
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// StoreVector+VectorStoreMask, and the vector size of "src" is less than the MaxVectorSize.
|
|
instruct storeV_vstoremask_masked(indirect mem, pReg src, immI_gt_1 esize,
|
|
vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 &&
|
|
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
|
|
match(Set mem (StoreVector mem (VectorStoreMask src esize)));
|
|
effect(TEMP tmp, TEMP pgtmp, KILL cr);
|
|
format %{ "storeV_vstoremask_masked $mem, $src\t# KILL $tmp, $pgtmp, cr" %}
|
|
ins_encode %{
|
|
// Convert the valid src predicate to vector, and store the vector elements
|
|
// as boolean values.
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $src);
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
|
|
__ sve_gen_mask_imm($pgtmp$$PRegister, bt, Matcher::vector_length(this, $src));
|
|
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
|
|
$pgtmp$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
|
|
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// StoreVectorMasked+VectorStoreMask, and the vector size of "src" is equal to the MaxVectorSize.
|
|
instruct storeVMasked_vstoremask(vmemA mem, pReg src, pRegGov pg, immI_gt_1 esize, vReg tmp) %{
|
|
predicate(UseSVE > 0 &&
|
|
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
|
|
match(Set mem (StoreVectorMasked mem (Binary (VectorStoreMask src esize) pg)));
|
|
effect(TEMP tmp);
|
|
format %{ "storeVMasked_vstoremask $mem, $src\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
// Convert the valid src predicate to vector, and store the vector elements
|
|
// as boolean values.
|
|
//
|
|
// Note that we cannot use "pg" here, since it is the predicate used
|
|
// for the vector store with boolean type. But the predicate used in
|
|
// the narrowing "sve_st1b" is based on the "src" vector type, which
|
|
// is the full-sized predicate (ptrue) here.
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $src);
|
|
assert(type2aelembytes(bt) == (int)$esize$$constant, "unsupported type.");
|
|
Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
|
|
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
|
|
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
|
|
ptrue, T_BOOLEAN, bt, $mem->opcode(),
|
|
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// StoreVectorMasked+VectorStoreMask, and the vector size of "src" is less than the MaxVectorSize.
|
|
instruct storeVMasked_vstoremask_masked(vmemA mem, pReg src, pRegGov pg, immI_gt_1 esize,
|
|
vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 &&
|
|
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
|
|
match(Set mem (StoreVectorMasked mem (Binary (VectorStoreMask src esize) pg)));
|
|
effect(TEMP tmp, TEMP pgtmp, KILL cr);
|
|
format %{ "storeVMasked_vstoremask_masked $mem, $src\t# KILL $tmp, $pgtmp, cr" %}
|
|
ins_encode %{
|
|
// Convert the valid src predicate to vector, and store the vector elements
|
|
// as boolean values.
|
|
//
|
|
// Note that we cannot use "pg" here, since it is the predicate used for the
|
|
// vector store with boolean type. But the predicate used in the narrowing
|
|
// "sve_st1b" is based on the "src" vector type, which needed to be generated
|
|
// here.
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $src);
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
|
|
__ sve_gen_mask_imm($pgtmp$$PRegister, bt, Matcher::vector_length(this, $src));
|
|
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
|
|
$pgtmp$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
|
|
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector mask basic OPs ------------------------
|
|
|
|
// vector mask logical ops: and/or/xor/and_not
|
|
|
|
instruct vmask_and(pReg pd, pReg pn, pReg pm) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set pd (AndVMask pn pm));
|
|
format %{ "vmask_and $pd, $pn, $pm" %}
|
|
ins_encode %{
|
|
__ sve_and($pd$$PRegister, ptrue, $pn$$PRegister, $pm$$PRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmask_or(pReg pd, pReg pn, pReg pm) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set pd (OrVMask pn pm));
|
|
format %{ "vmask_or $pd, $pn, $pm" %}
|
|
ins_encode %{
|
|
__ sve_orr($pd$$PRegister, ptrue, $pn$$PRegister, $pm$$PRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmask_xor(pReg pd, pReg pn, pReg pm) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set pd (XorVMask pn pm));
|
|
format %{ "vmask_xor $pd, $pn, $pm" %}
|
|
ins_encode %{
|
|
__ sve_eor($pd$$PRegister, ptrue, $pn$$PRegister, $pm$$PRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmask_and_notI(pReg pd, pReg pn, pReg pm, immI_M1 m1) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
|
|
format %{ "vmask_and_notI $pd, $pn, $pm" %}
|
|
ins_encode %{
|
|
__ sve_bic($pd$$PRegister, ptrue, $pn$$PRegister, $pm$$PRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmask_and_notL(pReg pd, pReg pn, pReg pm, immL_M1 m1) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
|
|
format %{ "vmask_and_notL $pd, $pn, $pm" %}
|
|
ins_encode %{
|
|
__ sve_bic($pd$$PRegister, ptrue, $pn$$PRegister, $pm$$PRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector mask compare
|
|
|
|
instruct vmaskcmp_neon(vReg dst, vReg src1, vReg src2, immI cond) %{
|
|
predicate(UseSVE == 0 &&
|
|
(Matcher::vector_length_in_bytes(n) == 8 ||
|
|
Matcher::vector_length_in_bytes(n) == 16));
|
|
match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
|
|
format %{ "vmaskcmp_neon $dst, $src1, $src2, $cond" %}
|
|
ins_encode %{
|
|
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
__ neon_compare($dst$$FloatRegister, bt, $src1$$FloatRegister,
|
|
$src2$$FloatRegister, condition, /* isQ */ length_in_bytes == 16);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskcmp_zeroI_neon(vReg dst, vReg src, immI0 zero, immI_cmp_cond cond) %{
|
|
predicate(UseSVE == 0);
|
|
match(Set dst (VectorMaskCmp (Binary src (Replicate zero)) cond));
|
|
format %{ "vmaskcmp_zeroI_neon $dst, $src, #0, $cond" %}
|
|
ins_encode %{
|
|
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
__ neon_compare_zero($dst$$FloatRegister, Matcher::vector_element_basic_type(this),
|
|
$src$$FloatRegister,
|
|
condition, /* isQ */ length_in_bytes == 16);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskcmp_zeroL_neon(vReg dst, vReg src, immL0 zero, immI_cmp_cond cond) %{
|
|
predicate(UseSVE == 0);
|
|
match(Set dst (VectorMaskCmp (Binary src (Replicate zero)) cond));
|
|
format %{ "vmaskcmp_zeroL_neon $dst, $src, #0, $cond" %}
|
|
ins_encode %{
|
|
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
__ neon_compare_zero($dst$$FloatRegister, T_LONG,
|
|
$src$$FloatRegister,
|
|
condition, /* isQ */ length_in_bytes == 16);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskcmp_zeroF_neon(vReg dst, vReg src, immF0 zero, immI_cmp_cond cond) %{
|
|
predicate(UseSVE == 0);
|
|
match(Set dst (VectorMaskCmp (Binary src (Replicate zero)) cond));
|
|
format %{ "vmaskcmp_zeroF_neon $dst, $src, #0, $cond" %}
|
|
ins_encode %{
|
|
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
__ neon_compare_zero($dst$$FloatRegister, T_FLOAT,
|
|
$src$$FloatRegister,
|
|
condition, /* isQ */ length_in_bytes == 16);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskcmp_zeroD_neon(vReg dst, vReg src, immD0 zero, immI_cmp_cond cond) %{
|
|
predicate(UseSVE == 0);
|
|
match(Set dst (VectorMaskCmp (Binary src (Replicate zero)) cond));
|
|
format %{ "vmaskcmp_zeroD_neon $dst, $src, #0, $cond" %}
|
|
ins_encode %{
|
|
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
__ neon_compare_zero($dst$$FloatRegister, T_DOUBLE,
|
|
$src$$FloatRegister,
|
|
condition, /* isQ */ length_in_bytes == 16);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskcmp_sve(pReg dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
|
|
effect(KILL cr);
|
|
format %{ "vmaskcmp_sve $dst, $src1, $src2, $cond\t# KILL cr" %}
|
|
ins_encode %{
|
|
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_compare($dst$$PRegister, bt, ptrue, $src1$$FloatRegister,
|
|
$src2$$FloatRegister, condition);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskcmp_immI_sve(pReg dst, vReg src, immI5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (VectorMaskCmp (Binary src (Replicate imm)) cond));
|
|
effect(KILL cr);
|
|
format %{ "vmaskcmp_immI_sve $dst, $src, $imm, $cond\t# KILL cr" %}
|
|
ins_encode %{
|
|
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_cmp(condition, $dst$$PRegister, get_reg_variant(in(operand_index($src))),
|
|
ptrue, $src$$FloatRegister, (int)$imm$$constant);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskcmpU_immI_sve(pReg dst, vReg src, immIU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (VectorMaskCmp (Binary src (Replicate imm)) cond));
|
|
effect(KILL cr);
|
|
format %{ "vmaskcmpU_immI_sve $dst, $src, $imm, $cond\t# KILL cr" %}
|
|
ins_encode %{
|
|
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_cmp(condition, $dst$$PRegister, get_reg_variant(in(operand_index($src))),
|
|
ptrue, $src$$FloatRegister, (int)$imm$$constant);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskcmp_immL_sve(pReg dst, vReg src, immL5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (VectorMaskCmp (Binary src (Replicate imm)) cond));
|
|
effect(KILL cr);
|
|
format %{ "vmaskcmp_immL_sve $dst, $src, $imm, $cond\t# KILL cr" %}
|
|
ins_encode %{
|
|
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_cmp(condition, $dst$$PRegister, get_reg_variant(in(operand_index($src))),
|
|
ptrue, $src$$FloatRegister, (int)$imm$$constant);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskcmpU_immL_sve(pReg dst, vReg src, immLU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (VectorMaskCmp (Binary src (Replicate imm)) cond));
|
|
effect(KILL cr);
|
|
format %{ "vmaskcmpU_immL_sve $dst, $src, $imm, $cond\t# KILL cr" %}
|
|
ins_encode %{
|
|
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_cmp(condition, $dst$$PRegister, get_reg_variant(in(operand_index($src))),
|
|
ptrue, $src$$FloatRegister, (int)$imm$$constant);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskcmp_masked(pReg dst, vReg src1, vReg src2, immI cond,
|
|
pRegGov pg, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
|
|
effect(KILL cr);
|
|
format %{ "vmaskcmp_masked $dst, $pg, $src1, $src2, $cond\t# KILL cr" %}
|
|
ins_encode %{
|
|
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_compare($dst$$PRegister, bt, $pg$$PRegister, $src1$$FloatRegister,
|
|
$src2$$FloatRegister, condition);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector mask cast
|
|
|
|
instruct vmaskcast_same_esize_neon(vReg dst_src) %{
|
|
predicate(UseSVE == 0 &&
|
|
Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)) &&
|
|
(Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16));
|
|
match(Set dst_src (VectorMaskCast dst_src));
|
|
ins_cost(0);
|
|
format %{ "vmaskcast_same_esize_neon $dst_src\t# do nothing" %}
|
|
ins_encode(/* empty encoding */);
|
|
ins_pipe(pipe_class_empty);
|
|
%}
|
|
|
|
instruct vmaskcast_extend_neon(vReg dst, vReg src) %{
|
|
predicate(UseSVE == 0 &&
|
|
Matcher::vector_length_in_bytes(n) > Matcher::vector_length_in_bytes(n->in(1)));
|
|
match(Set dst (VectorMaskCast src));
|
|
format %{ "vmaskcast_extend_neon $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType dst_bt = Matcher::vector_element_basic_type(this);
|
|
if (is_floating_point_type(dst_bt)) {
|
|
dst_bt = (dst_bt == T_FLOAT) ? T_INT : T_LONG;
|
|
}
|
|
uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
|
|
BasicType src_bt = Matcher::vector_element_basic_type(this, $src);
|
|
if (is_floating_point_type(src_bt)) {
|
|
src_bt = (src_bt == T_FLOAT) ? T_INT : T_LONG;
|
|
}
|
|
__ neon_vector_extend($dst$$FloatRegister, dst_bt, length_in_bytes_dst,
|
|
$src$$FloatRegister, src_bt);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskcast_narrow_neon(vReg dst, vReg src) %{
|
|
predicate(UseSVE == 0 &&
|
|
Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)));
|
|
match(Set dst (VectorMaskCast src));
|
|
format %{ "vmaskcast_narrow_neon $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType dst_bt = Matcher::vector_element_basic_type(this);
|
|
if (is_floating_point_type(dst_bt)) {
|
|
dst_bt = (dst_bt == T_FLOAT) ? T_INT : T_LONG;
|
|
}
|
|
BasicType src_bt = Matcher::vector_element_basic_type(this, $src);
|
|
if (is_floating_point_type(src_bt)) {
|
|
src_bt = (src_bt == T_FLOAT) ? T_INT : T_LONG;
|
|
}
|
|
uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
|
|
__ neon_vector_narrow($dst$$FloatRegister, dst_bt,
|
|
$src$$FloatRegister, src_bt, length_in_bytes_src);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskcast_same_esize_sve(pReg dst_src) %{
|
|
predicate(UseSVE > 0 &&
|
|
Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
|
|
match(Set dst_src (VectorMaskCast dst_src));
|
|
ins_cost(0);
|
|
format %{ "vmaskcast_same_esize_sve $dst_src\t# do nothing" %}
|
|
ins_encode(/* empty encoding */);
|
|
ins_pipe(pipe_class_empty);
|
|
%}
|
|
|
|
instruct vmaskcast_extend_sve(pReg dst, pReg src) %{
|
|
predicate(UseSVE > 0 &&
|
|
Matcher::vector_length_in_bytes(n) > Matcher::vector_length_in_bytes(n->in(1)));
|
|
match(Set dst (VectorMaskCast src));
|
|
format %{ "vmaskcast_extend_sve $dst, $src" %}
|
|
ins_encode %{
|
|
uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
|
|
uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
|
|
assert(length_in_bytes_dst == 2 * length_in_bytes_src ||
|
|
length_in_bytes_dst == 4 * length_in_bytes_src ||
|
|
length_in_bytes_dst == 8 * length_in_bytes_src, "invalid vector length");
|
|
__ sve_vmaskcast_extend($dst$$PRegister, $src$$PRegister,
|
|
length_in_bytes_dst, length_in_bytes_src);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskcast_narrow_sve(pReg dst, pReg src, pReg ptmp) %{
|
|
predicate(UseSVE > 0 &&
|
|
Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)));
|
|
match(Set dst (VectorMaskCast src));
|
|
effect(TEMP_DEF dst, TEMP ptmp);
|
|
format %{ "vmaskcast_narrow_sve $dst, $src\t# KILL $ptmp" %}
|
|
ins_encode %{
|
|
uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
|
|
uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
|
|
assert(length_in_bytes_dst * 2 == length_in_bytes_src ||
|
|
length_in_bytes_dst * 4 == length_in_bytes_src ||
|
|
length_in_bytes_dst * 8 == length_in_bytes_src, "invalid vector length");
|
|
__ sve_vmaskcast_narrow($dst$$PRegister, $src$$PRegister, $ptmp$$PRegister,
|
|
length_in_bytes_dst, length_in_bytes_src);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector mask reinterpret
|
|
|
|
instruct vmask_reinterpret_same_esize(pReg dst_src) %{
|
|
predicate(UseSVE > 0 &&
|
|
Matcher::vector_length(n) == Matcher::vector_length(n->in(1)) &&
|
|
Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
|
|
match(Set dst_src (VectorReinterpret dst_src));
|
|
ins_cost(0);
|
|
format %{ "vmask_reinterpret_same_esize $dst_src\t# do nothing" %}
|
|
ins_encode(/* empty encoding */);
|
|
ins_pipe(pipe_class_empty);
|
|
%}
|
|
|
|
instruct vmask_reinterpret_diff_esize(pReg dst, pReg src, vReg tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0 &&
|
|
Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
|
|
Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
|
|
match(Set dst (VectorReinterpret src));
|
|
effect(TEMP tmp, KILL cr);
|
|
format %{ "vmask_reinterpret_diff_esize $dst, $src\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
|
|
Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
|
|
BasicType to_bt = Matcher::vector_element_basic_type(this);
|
|
Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
|
|
__ sve_cpy($tmp$$FloatRegister, from_size, $src$$PRegister, -1, false);
|
|
__ sve_cmp(Assembler::EQ, $dst$$PRegister, to_size, ptrue, $tmp$$FloatRegister, -1);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector mask reductions -----------------------
|
|
|
|
// true count
|
|
|
|
instruct vmask_truecount_neon(iRegINoSp dst, vReg src, vReg tmp) %{
|
|
predicate(UseSVE == 0);
|
|
match(Set dst (VectorMaskTrueCount src));
|
|
effect(TEMP tmp);
|
|
format %{ "vmask_truecount_neon $dst, $src\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
// Input "src" is a vector of boolean represented as bytes with
|
|
// 0x00/0x01 as element values.
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $src);
|
|
assert(bt == T_BOOLEAN, "unsupported type");
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
|
|
__ addv($tmp$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src$$FloatRegister);
|
|
__ umov($dst$$Register, $tmp$$FloatRegister, __ B, 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmask_truecount_sve(iRegINoSp dst, pReg src) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (VectorMaskTrueCount src));
|
|
format %{ "vmask_truecount_sve $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $src);
|
|
__ sve_cntp($dst$$Register, __ elemType_to_regVariant(bt),
|
|
ptrue, $src$$PRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Combined rule for VectorMaskTrueCount (VectorStoreMask) when the vector element type is not T_BYTE.
|
|
|
|
instruct vstoremask_truecount_neon(iRegINoSp dst, vReg src, immI_gt_1 size, vReg vtmp) %{
|
|
match(Set dst (VectorMaskTrueCount (VectorStoreMask src size)));
|
|
effect(TEMP vtmp);
|
|
format %{ "vstoremask_truecount_neon $dst, $src\t# KILL $vtmp" %}
|
|
ins_encode %{
|
|
// Input "src" is a vector mask represented as lanes with
|
|
// 0/-1 as element values.
|
|
uint esize = (uint)$size$$constant;
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
|
|
Assembler::SIMD_Arrangement arrangement = Assembler::esize2arrangement(esize,
|
|
/* isQ */ length_in_bytes == 16);
|
|
if (arrangement == __ T2D || arrangement == __ T2S) {
|
|
__ addpv($vtmp$$FloatRegister, arrangement, $src$$FloatRegister, $src$$FloatRegister);
|
|
} else {
|
|
__ addv($vtmp$$FloatRegister, arrangement, $src$$FloatRegister);
|
|
}
|
|
__ smov($dst$$Register, $vtmp$$FloatRegister, __ B, 0);
|
|
__ neg($dst$$Register, $dst$$Register);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// first true
|
|
|
|
instruct vmask_firsttrue_neon(iRegINoSp dst, vReg src) %{
|
|
predicate(UseSVE == 0);
|
|
match(Set dst (VectorMaskFirstTrue src));
|
|
format %{ "vmask_firsttrue_neon $dst, $src" %}
|
|
ins_encode %{
|
|
// Returns the index of the first active lane of the
|
|
// vector mask, or VLENGTH if no lane is active.
|
|
//
|
|
// Input "src" is a vector of boolean represented as
|
|
// bytes with 0x00/0x01 as element values.
|
|
//
|
|
// Computed by reversing the bits and counting the leading
|
|
// zero bytes.
|
|
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $src);
|
|
assert(bt == T_BOOLEAN, "unsupported type");
|
|
uint vlength = Matcher::vector_length(this, $src);
|
|
if (vlength <= 8) {
|
|
__ fmovd($dst$$Register, $src$$FloatRegister);
|
|
if (vlength == 2 || vlength == 4) {
|
|
// Special handling for 2B or 4B cases:
|
|
// Vector mask is moved to a 64-bit general register, but only the low 16/32 bits are
|
|
// significant for 2B/4B cases. We initialize the 16th/32nd bit as bit 1, so as to generate
|
|
// the expected result (i.e. VLENGTH) for the case that all lanes are zero.
|
|
__ orr($dst$$Register, $dst$$Register, vlength == 2 ? 0x10000 : 0x100000000);
|
|
}
|
|
__ rbit($dst$$Register, $dst$$Register);
|
|
__ clz($dst$$Register, $dst$$Register);
|
|
__ lsrw($dst$$Register, $dst$$Register, 3);
|
|
} else {
|
|
assert(vlength == 16, "must be");
|
|
Label FIRST_TRUE_INDEX;
|
|
|
|
// Try to compute the result from lower 64 bits.
|
|
__ fmovd($dst$$Register, $src$$FloatRegister);
|
|
__ movw(rscratch1, zr);
|
|
__ cbnz($dst$$Register, FIRST_TRUE_INDEX);
|
|
|
|
// Compute the result from the higher 64 bits.
|
|
__ fmovhid($dst$$Register, $src$$FloatRegister);
|
|
__ movw(rscratch1, 8);
|
|
|
|
// Reverse the bits and count the leading zero bytes.
|
|
__ bind(FIRST_TRUE_INDEX);
|
|
__ rbit($dst$$Register, $dst$$Register);
|
|
__ clz($dst$$Register, $dst$$Register);
|
|
__ addw($dst$$Register, rscratch1, $dst$$Register, Assembler::LSR, 3);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// Return the index of the first mask lane that is set, or vector length if none of
|
|
// them are set.
|
|
|
|
instruct vmask_firsttrue_sve(iRegINoSp dst, pReg src, pReg ptmp) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (VectorMaskFirstTrue src));
|
|
effect(TEMP ptmp);
|
|
format %{ "vmask_firsttrue_sve $dst, $src\t# KILL $ptmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $src);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_brkb($ptmp$$PRegister, ptrue, $src$$PRegister, false);
|
|
__ sve_cntp($dst$$Register, __ elemType_to_regVariant(bt), ptrue, $ptmp$$PRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmask_firsttrue_masked(iRegINoSp dst, pReg src, pReg pg, pReg ptmp) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (VectorMaskFirstTrue src pg));
|
|
effect(TEMP ptmp);
|
|
format %{ "vmask_firsttrue_masked $dst, $pg, $src\t# KILL $ptmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $src);
|
|
__ sve_brkb($ptmp$$PRegister, $pg$$PRegister, $src$$PRegister, false);
|
|
__ sve_cntp($dst$$Register, __ elemType_to_regVariant(bt), ptrue, $ptmp$$PRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// last true
|
|
|
|
instruct vmask_lasttrue_neon(iRegINoSp dst, vReg src) %{
|
|
predicate(UseSVE == 0);
|
|
match(Set dst (VectorMaskLastTrue src));
|
|
format %{ "vmask_lasttrue_neon $dst, $src" %}
|
|
ins_encode %{
|
|
// Returns the index of the last active lane of the
|
|
// vector mask, or -1 if no lane is active.
|
|
//
|
|
// Input "src" is a vector of boolean represented as
|
|
// bytes with 0x00/0x01 as element values.
|
|
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $src);
|
|
assert(bt == T_BOOLEAN, "unsupported type");
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
|
|
if (length_in_bytes <= 8) {
|
|
// Computed by counting the leading zero bytes and
|
|
// subtracting it by 7 (VLENGTH - 1).
|
|
__ fmovd($dst$$Register, $src$$FloatRegister);
|
|
__ clz($dst$$Register, $dst$$Register);
|
|
__ movw(rscratch1, 7);
|
|
__ subw($dst$$Register, rscratch1, $dst$$Register, Assembler::LSR, 3);
|
|
} else {
|
|
assert(length_in_bytes == 16, "must be");
|
|
Label LAST_TRUE_INDEX;
|
|
|
|
// Try to compute the result from higher 64 bits.
|
|
__ fmovhid($dst$$Register, $src$$FloatRegister);
|
|
__ movw(rscratch1, 16 - 1);
|
|
__ cbnz($dst$$Register, LAST_TRUE_INDEX);
|
|
|
|
// Compute the result from the lower 64 bits.
|
|
__ fmovd($dst$$Register, $src$$FloatRegister);
|
|
__ movw(rscratch1, 8 - 1);
|
|
|
|
// Count the leading zero bytes and subtract it by 15 (VLENGTH - 1).
|
|
__ bind(LAST_TRUE_INDEX);
|
|
__ clz($dst$$Register, $dst$$Register);
|
|
__ subw($dst$$Register, rscratch1, $dst$$Register, Assembler::LSR, 3);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmask_lasttrue_sve(iRegINoSp dst, pReg src, pReg ptmp) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (VectorMaskLastTrue src));
|
|
effect(TEMP ptmp);
|
|
format %{ "vmask_lasttrue_sve $dst, $src\t# KILL $ptmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this, $src);
|
|
__ sve_vmask_lasttrue($dst$$Register, bt, $src$$PRegister, $ptmp$$PRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// tolong
|
|
|
|
instruct vmask_tolong_neon(iRegLNoSp dst, vReg src) %{
|
|
predicate(UseSVE == 0);
|
|
match(Set dst (VectorMaskToLong src));
|
|
format %{ "vmask_tolong_neon $dst, $src" %}
|
|
ins_encode %{
|
|
// Input "src" is a vector of boolean represented as
|
|
// bytes with 0x00/0x01 as element values.
|
|
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
|
|
if (length_in_bytes <= 8) {
|
|
__ fmovd($dst$$Register, $src$$FloatRegister);
|
|
__ bytemask_compress($dst$$Register);
|
|
} else {
|
|
assert(length_in_bytes == 16, "must be");
|
|
__ umov($dst$$Register, $src$$FloatRegister, __ D, 0);
|
|
__ umov(rscratch1, $src$$FloatRegister, __ D, 1);
|
|
__ bytemask_compress($dst$$Register);
|
|
__ bytemask_compress(rscratch1);
|
|
__ orr($dst$$Register, $dst$$Register, rscratch1, Assembler::LSL, 8);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmask_tolong_sve(iRegLNoSp dst, vReg src, vReg tmp) %{
|
|
predicate(UseSVE > 0 && !VM_Version::supports_svebitperm());
|
|
match(Set dst (VectorMaskToLong src));
|
|
effect(TEMP tmp);
|
|
format %{ "vmask_tolong_sve $dst, $src\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
// Input "src" is a vector of boolean represented as
|
|
// bytes with 0x00/0x01 as element values.
|
|
__ sve_vmask_tolong($dst$$Register, $src$$FloatRegister,
|
|
$tmp$$FloatRegister, Matcher::vector_length(this, $src));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmask_tolong_sve2(iRegLNoSp dst, vReg src, vReg tmp1, vReg tmp2) %{
|
|
predicate(VM_Version::supports_svebitperm());
|
|
match(Set dst (VectorMaskToLong src));
|
|
effect(TEMP tmp1, TEMP tmp2);
|
|
format %{ "vmask_tolong_sve2 $dst, $src\t# KILL $tmp1, $tmp2" %}
|
|
ins_encode %{
|
|
// Input "src" is a vector of boolean represented as
|
|
// bytes with 0x00/0x01 as element values.
|
|
__ sve2_vmask_tolong($dst$$Register, $src$$FloatRegister,
|
|
$tmp1$$FloatRegister, $tmp2$$FloatRegister,
|
|
Matcher::vector_length(this, $src));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// fromlong
|
|
|
|
instruct vmask_fromlong(vReg dst, iRegL src, vReg tmp) %{
|
|
match(Set dst (VectorLongToMask src));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "vmask_fromlong $dst, $src\t# vector (sve2). KILL $tmp" %}
|
|
ins_encode %{
|
|
__ sve_vmask_fromlong($dst$$FloatRegister, $src$$Register,
|
|
$tmp$$FloatRegister, Matcher::vector_length(this));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector mask generation -----------------------
|
|
|
|
// maskAll
|
|
|
|
instruct vmaskAll_immI(pReg dst, immI src, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (MaskAll src));
|
|
effect(KILL cr);
|
|
format %{ "vmaskAll_immI $dst, $src\t# KILL cr" %}
|
|
ins_encode %{
|
|
int con = (int)$src$$constant;
|
|
if (con == 0) {
|
|
__ sve_pfalse($dst$$PRegister);
|
|
} else {
|
|
assert(con == -1, "invalid constant value for mask");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_gen_mask_imm($dst$$PRegister, bt, Matcher::vector_length(this));
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskAllI(pReg dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (MaskAll src));
|
|
effect(TEMP tmp, KILL cr);
|
|
format %{ "vmaskAllI $dst, $src\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_dup($tmp$$FloatRegister, size, $src$$Register);
|
|
__ sve_cmp(Assembler::NE, $dst$$PRegister, size, ptrue, $tmp$$FloatRegister, 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskAllI_masked(pReg dst, iRegIorL2I src, pRegGov pg, vReg tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (MaskAll src pg));
|
|
effect(TEMP tmp, KILL cr);
|
|
format %{ "vmaskAllI_masked $dst, $pg, $src\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_dup($tmp$$FloatRegister, size, $src$$Register);
|
|
__ sve_cmp(Assembler::NE, $dst$$PRegister, size,
|
|
$pg$$PRegister, $tmp$$FloatRegister, 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskAll_immL(pReg dst, immL src, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (MaskAll src));
|
|
effect(KILL cr);
|
|
format %{ "vmaskAll_immL $dst, $src\t# KILL cr" %}
|
|
ins_encode %{
|
|
long con = (long)$src$$constant;
|
|
if (con == 0) {
|
|
__ sve_pfalse($dst$$PRegister);
|
|
} else {
|
|
assert(con == -1, "invalid constant value for mask");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_gen_mask_imm($dst$$PRegister, bt, Matcher::vector_length(this));
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskAllL(pReg dst, iRegL src, vReg tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (MaskAll src));
|
|
effect(TEMP tmp, KILL cr);
|
|
format %{ "vmaskAllL $dst, $src\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_dup($tmp$$FloatRegister, size, $src$$Register);
|
|
__ sve_cmp(Assembler::NE, $dst$$PRegister, size, ptrue, $tmp$$FloatRegister, 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmaskAllL_masked(pReg dst, iRegL src, pRegGov pg, vReg tmp, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (MaskAll src pg));
|
|
effect(TEMP tmp, KILL cr);
|
|
format %{ "vmaskAllL_masked $dst, $pg, $src\t# KILL $tmp, cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_dup($tmp$$FloatRegister, size, $src$$Register);
|
|
__ sve_cmp(Assembler::NE, $dst$$PRegister, size,
|
|
$pg$$PRegister, $tmp$$FloatRegister, 0);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vetcor mask generation
|
|
|
|
instruct vmask_gen_I(pReg pd, iRegIorL2I src, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set pd (VectorMaskGen (ConvI2L src)));
|
|
effect(KILL cr);
|
|
format %{ "vmask_gen_I $pd, $src\t# KILL cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_whileltw($pd$$PRegister, __ elemType_to_regVariant(bt), zr, $src$$Register);
|
|
%}
|
|
ins_pipe(pipe_class_default);
|
|
%}
|
|
|
|
instruct vmask_gen_L(pReg pd, iRegL src, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set pd (VectorMaskGen src));
|
|
effect(KILL cr);
|
|
format %{ "vmask_gen_L $pd, $src\t# KILL cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_whilelt($pd$$PRegister, __ elemType_to_regVariant(bt), zr, $src$$Register);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmask_gen_imm(pReg pd, immL con, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set pd (VectorMaskGen con));
|
|
effect(KILL cr);
|
|
format %{ "vmask_gen_imm $pd, $con\t# KILL cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_gen_mask_imm($pd$$PRegister, bt, (uint)($con$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vmask_gen_sub(pReg pd, iRegL src1, iRegL src2, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set pd (VectorMaskGen (SubL src1 src2)));
|
|
effect(KILL cr);
|
|
format %{ "vmask_gen_sub $pd, $src2, $src1\t# KILL cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_whilelt($pd$$PRegister, __ elemType_to_regVariant(bt), $src2$$Register, $src1$$Register);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Popcount vector ------------------------------
|
|
|
|
// vector popcount - INT
|
|
|
|
instruct vpopcountI(vReg dst, vReg src) %{
|
|
match(Set dst (PopCountVI src));
|
|
format %{ "vpopcountI $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (bt == T_BYTE) {
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ cnt($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_cnt($dst$$FloatRegister, __ B, ptrue, $src$$FloatRegister);
|
|
}
|
|
} else {
|
|
assert(bt == T_SHORT || bt == T_INT, "unsupported");
|
|
if (UseSVE == 0) {
|
|
assert(length_in_bytes <= 16, "unsupported");
|
|
bool isQ = length_in_bytes == 16;
|
|
__ cnt($dst$$FloatRegister, isQ ? __ T16B : __ T8B, $src$$FloatRegister);
|
|
__ uaddlp($dst$$FloatRegister, isQ ? __ T16B : __ T8B, $dst$$FloatRegister);
|
|
if (bt == T_INT) {
|
|
__ uaddlp($dst$$FloatRegister, isQ ? __ T8H : __ T4H, $dst$$FloatRegister);
|
|
}
|
|
} else {
|
|
__ sve_cnt($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src$$FloatRegister);
|
|
}
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector popcount - LONG
|
|
|
|
instruct vpopcountL(vReg dst, vReg src) %{
|
|
match(Set dst (PopCountVL src));
|
|
format %{ "vpopcountL $dst, $src" %}
|
|
ins_encode %{
|
|
if (UseSVE == 0) {
|
|
__ cnt($dst$$FloatRegister, __ T16B, $src$$FloatRegister);
|
|
__ uaddlp($dst$$FloatRegister, __ T16B, $dst$$FloatRegister);
|
|
__ uaddlp($dst$$FloatRegister, __ T8H, $dst$$FloatRegister);
|
|
__ uaddlp($dst$$FloatRegister, __ T4S, $dst$$FloatRegister);
|
|
} else {
|
|
__ sve_cnt($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// vector popcount - predicated
|
|
|
|
instruct vpopcountI_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (PopCountVI dst_src pg));
|
|
format %{ "vpopcountI_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_cnt($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vpopcountL_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (PopCountVL dst_src pg));
|
|
format %{ "vpopcountL_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
__ sve_cnt($dst_src$$FloatRegister, __ D,
|
|
$pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector blend ---------------------------------
|
|
|
|
instruct vblend_neon(vReg dst, vReg src1, vReg src2) %{
|
|
predicate(UseSVE == 0);
|
|
match(Set dst (VectorBlend (Binary src1 src2) dst));
|
|
format %{ "vblend_neon $dst, $src1, $src2" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes <= 16, "must be");
|
|
__ bsl($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src2$$FloatRegister, $src1$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vblend_sve(vReg dst, vReg src1, vReg src2, pReg pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (VectorBlend (Binary src1 src2) pg));
|
|
format %{ "vblend_sve $dst, $pg, $src1, $src2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_sel($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $src2$$FloatRegister, $src1$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector round ---------------------------------
|
|
|
|
// vector Math.round
|
|
|
|
instruct vround_le128b(vReg dst, vReg src, vReg tmp1, vReg tmp2,
|
|
vReg tmp3, rFlagsReg cr) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) <= 16);
|
|
match(Set dst (RoundVF src));
|
|
match(Set dst (RoundVD src));
|
|
effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
|
|
format %{ "vround_le128b $dst, $src\t# vector <= 128 bits. KILL $tmp1, $tmp2, $tmp3, cr" %}
|
|
ins_encode %{
|
|
__ vector_round_neon($dst$$FloatRegister, $src$$FloatRegister,
|
|
$tmp1$$FloatRegister, $tmp2$$FloatRegister,
|
|
$tmp3$$FloatRegister, get_arrangement(this));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vround_gt128b(vReg dst, vReg src, vReg tmp1, vReg tmp2,
|
|
pRegGov pgtmp, rFlagsReg cr) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) > 16);
|
|
match(Set dst (RoundVF src));
|
|
match(Set dst (RoundVD src));
|
|
effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP pgtmp, KILL cr);
|
|
format %{ "vround_gt128b $dst, $src\t# vector > 128 bits. KILL $tmp1, $tmp2, $pgtmp, cr" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ vector_round_sve($dst$$FloatRegister, $src$$FloatRegister,
|
|
$tmp1$$FloatRegister, $tmp2$$FloatRegister,
|
|
$pgtmp$$PRegister, __ elemType_to_regVariant(bt));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ RoundDouble ----------------------------------
|
|
|
|
// vector Math.rint, floor, ceil
|
|
|
|
instruct vroundD(vReg dst, vReg src, immI rmode) %{
|
|
predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
|
|
match(Set dst (RoundDoubleModeV src rmode));
|
|
format %{ "vroundD $dst, $src, $rmode" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
switch ($rmode$$constant) {
|
|
case RoundDoubleModeNode::rmode_rint:
|
|
__ frintn($dst$$FloatRegister, __ T2D, $src$$FloatRegister);
|
|
break;
|
|
case RoundDoubleModeNode::rmode_floor:
|
|
__ frintm($dst$$FloatRegister, __ T2D, $src$$FloatRegister);
|
|
break;
|
|
case RoundDoubleModeNode::rmode_ceil:
|
|
__ frintp($dst$$FloatRegister, __ T2D, $src$$FloatRegister);
|
|
break;
|
|
default:
|
|
assert(false, "unsupported");
|
|
ShouldNotReachHere();
|
|
}
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
switch ($rmode$$constant) {
|
|
case RoundDoubleModeNode::rmode_rint:
|
|
__ sve_frintn($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
|
|
break;
|
|
case RoundDoubleModeNode::rmode_floor:
|
|
__ sve_frintm($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
|
|
break;
|
|
case RoundDoubleModeNode::rmode_ceil:
|
|
__ sve_frintp($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
|
|
break;
|
|
default:
|
|
assert(false, "unsupported");
|
|
ShouldNotReachHere();
|
|
}
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ VectorTest -----------------------------------
|
|
|
|
// anytrue
|
|
|
|
instruct vtest_anytrue_neon(rFlagsReg cr, vReg src1, vReg src2, vReg tmp) %{
|
|
predicate(UseSVE == 0 &&
|
|
static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
|
|
match(Set cr (VectorTest src1 src2));
|
|
effect(TEMP tmp);
|
|
format %{ "vtest_anytrue_neon $src1\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
// No need to use src2.
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src1);
|
|
assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
|
|
__ addv($tmp$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B, $src1$$FloatRegister);
|
|
__ umov(rscratch1, $tmp$$FloatRegister, __ B, 0);
|
|
__ cmpw(rscratch1, zr);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vtest_anytrue_sve(rFlagsReg cr, pReg src1, pReg src2) %{
|
|
predicate(UseSVE > 0 &&
|
|
static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
|
|
match(Set cr (VectorTest src1 src2));
|
|
format %{ "vtest_anytrue_sve $src1" %}
|
|
ins_encode %{
|
|
// "src2" is not used for sve.
|
|
__ sve_ptest(ptrue, $src1$$PRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// alltrue
|
|
|
|
instruct vtest_alltrue_neon(rFlagsReg cr, vReg src1, vReg src2, vReg tmp) %{
|
|
predicate(UseSVE == 0 &&
|
|
static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
|
|
match(Set cr (VectorTest src1 src2));
|
|
effect(TEMP tmp);
|
|
format %{ "vtest_alltrue_neon $src1\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
// No need to use src2.
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src1);
|
|
assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
|
|
__ uminv($tmp$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B, $src1$$FloatRegister);
|
|
__ umov(rscratch1, $tmp$$FloatRegister, __ B, 0);
|
|
__ cmpw(rscratch1, 0xff);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vtest_alltrue_sve(rFlagsReg cr, pReg src1, pReg src2, pReg ptmp) %{
|
|
predicate(UseSVE > 0 &&
|
|
static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
|
|
match(Set cr (VectorTest src1 src2));
|
|
effect(TEMP ptmp);
|
|
format %{ "vtest_alltrue_sve $src1, $src2\t# KILL $ptmp" %}
|
|
ins_encode %{
|
|
__ sve_eors($ptmp$$PRegister, ptrue, $src1$$PRegister, $src2$$PRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector rearrange -----------------------------
|
|
|
|
instruct rearrange_HSD_neon(vReg dst, vReg src, vReg shuffle, vReg tmp) %{
|
|
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) != T_BYTE);
|
|
match(Set dst (VectorRearrange src shuffle));
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
format %{ "rearrange_HSD_neon $dst, $src, $shuffle\t# vector (4H/8H/2S/4S/2D). KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
|
|
__ neon_rearrange_hsd($dst$$FloatRegister, $src$$FloatRegister,
|
|
$shuffle$$FloatRegister, $tmp$$FloatRegister,
|
|
bt, length_in_bytes == 16);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct rearrange(vReg dst, vReg src, vReg shuffle) %{
|
|
predicate(UseSVE > 0 || Matcher::vector_element_basic_type(n) == T_BYTE);
|
|
match(Set dst (VectorRearrange src shuffle));
|
|
format %{ "rearrange $dst, $src, $shuffle" %}
|
|
ins_encode %{
|
|
BasicType bt_dst = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (bt_dst == T_BYTE && VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ tbl($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src$$FloatRegister, 1, $shuffle$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt_src = Matcher::vector_element_basic_type(this, $src);
|
|
__ sve_tbl($dst$$FloatRegister, __ elemType_to_regVariant(bt_src),
|
|
$src$$FloatRegister, $shuffle$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector Load Gather ---------------------------
|
|
|
|
instruct gather_loadS(vReg dst, indirect mem, vReg idx) %{
|
|
predicate(UseSVE > 0 &&
|
|
type2aelembytes(Matcher::vector_element_basic_type(n)) == 4);
|
|
match(Set dst (LoadVectorGather mem idx));
|
|
format %{ "gather_loadS $dst, $mem, $idx\t# vector (sve)" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_ld1w_gather($dst$$FloatRegister, ptrue,
|
|
as_Register($mem$$base), $idx$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct gather_loadD(vReg dst, indirect mem, vReg idx, vReg tmp) %{
|
|
predicate(UseSVE > 0 &&
|
|
type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
|
|
match(Set dst (LoadVectorGather mem idx));
|
|
effect(TEMP tmp);
|
|
format %{ "gather_loadD $dst, $mem, $idx\t# vector (sve). KILL $tmp" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_uunpklo($tmp$$FloatRegister, __ D, $idx$$FloatRegister);
|
|
__ sve_ld1d_gather($dst$$FloatRegister, ptrue, as_Register($mem$$base),
|
|
$tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct gather_loadS_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
|
|
predicate(UseSVE > 0 &&
|
|
type2aelembytes(Matcher::vector_element_basic_type(n)) == 4);
|
|
match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
|
|
format %{ "gather_loadS_masked $dst, $pg, $mem, $idx" %}
|
|
ins_encode %{
|
|
__ sve_ld1w_gather($dst$$FloatRegister, $pg$$PRegister,
|
|
as_Register($mem$$base), $idx$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct gather_loadD_masked(vReg dst, indirect mem, vReg idx, pRegGov pg, vReg tmp) %{
|
|
predicate(UseSVE > 0 &&
|
|
type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
|
|
match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
|
|
effect(TEMP tmp);
|
|
format %{ "gather_loadD_masked $dst, $pg, $mem, $idx\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
__ sve_uunpklo($tmp$$FloatRegister, __ D, $idx$$FloatRegister);
|
|
__ sve_ld1d_gather($dst$$FloatRegister, $pg$$PRegister,
|
|
as_Register($mem$$base), $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector Store Scatter -------------------------
|
|
|
|
instruct scatter_storeS(indirect mem, vReg src, vReg idx) %{
|
|
predicate(UseSVE > 0 &&
|
|
type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4);
|
|
match(Set mem (StoreVectorScatter mem (Binary src idx)));
|
|
format %{ "scatter_storeS $mem, $idx, $src\t# vector (sve)" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_st1w_scatter($src$$FloatRegister, ptrue,
|
|
as_Register($mem$$base), $idx$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct scatter_storeD(indirect mem, vReg src, vReg idx, vReg tmp) %{
|
|
predicate(UseSVE > 0 &&
|
|
type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
|
|
match(Set mem (StoreVectorScatter mem (Binary src idx)));
|
|
effect(TEMP tmp);
|
|
format %{ "scatter_storeD $mem, $idx, $src\t# vector (sve). KILL $tmp" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
|
|
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
|
|
__ sve_uunpklo($tmp$$FloatRegister, __ D, $idx$$FloatRegister);
|
|
__ sve_st1d_scatter($src$$FloatRegister, ptrue,
|
|
as_Register($mem$$base), $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct scatter_storeS_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
|
|
predicate(UseSVE > 0 &&
|
|
type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4);
|
|
match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
|
|
format %{ "scatter_storeS_masked $mem, $pg, $idx, $src" %}
|
|
ins_encode %{
|
|
__ sve_st1w_scatter($src$$FloatRegister, $pg$$PRegister,
|
|
as_Register($mem$$base), $idx$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct scatter_storeD_masked(indirect mem, vReg src, vReg idx, pRegGov pg, vReg tmp) %{
|
|
predicate(UseSVE > 0 &&
|
|
type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
|
|
match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
|
|
effect(TEMP tmp);
|
|
format %{ "scatter_storeD_masked $mem, $pg, $idx, $src\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
__ sve_uunpklo($tmp$$FloatRegister, __ D, $idx$$FloatRegister);
|
|
__ sve_st1d_scatter($src$$FloatRegister, $pg$$PRegister,
|
|
as_Register($mem$$base), $tmp$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ CountLeadingZerosV ---------------------------
|
|
|
|
instruct vcountLeadingZeros(vReg dst, vReg src) %{
|
|
match(Set dst (CountLeadingZerosV src));
|
|
format %{ "vcountLeadingZeros $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (bt == T_LONG) {
|
|
if (UseSVE == 0) {
|
|
__ umov(rscratch1, $src$$FloatRegister, __ D, 0);
|
|
__ clz(rscratch1, rscratch1);
|
|
__ mov($dst$$FloatRegister, __ D, 0, rscratch1);
|
|
__ umov(rscratch1, $src$$FloatRegister, __ D, 1);
|
|
__ clz(rscratch1, rscratch1);
|
|
__ mov($dst$$FloatRegister, __ D, 1, rscratch1);
|
|
} else {
|
|
__ sve_clz($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
|
|
}
|
|
} else {
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ clz($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_clz($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src$$FloatRegister);
|
|
}
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// The dst and src should use the same register to make sure the
|
|
// inactive lanes in dst save the same elements as src.
|
|
|
|
instruct vcountLeadingZeros_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (CountLeadingZerosV dst_src pg));
|
|
format %{ "vcountLeadingZeros_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_clz($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ CountTrailingZerosV --------------------------
|
|
|
|
instruct vcountTrailingZeros(vReg dst, vReg src) %{
|
|
match(Set dst (CountTrailingZerosV src));
|
|
format %{ "vcountTrailingZeros $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (bt == T_BYTE) {
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ rbit($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src$$FloatRegister);
|
|
__ clz($dst$$FloatRegister, get_arrangement(this), $dst$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_rbit($dst$$FloatRegister, __ B, ptrue, $src$$FloatRegister);
|
|
__ sve_clz($dst$$FloatRegister, __ B, ptrue, $dst$$FloatRegister);
|
|
}
|
|
} else {
|
|
assert(bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported type");
|
|
if (UseSVE == 0) {
|
|
assert(length_in_bytes <= 16, "unsupported");
|
|
__ neon_reverse_bits($dst$$FloatRegister, $src$$FloatRegister,
|
|
bt, /* isQ */ length_in_bytes == 16);
|
|
if (bt != T_LONG) {
|
|
__ clz($dst$$FloatRegister, get_arrangement(this), $dst$$FloatRegister);
|
|
} else {
|
|
__ umov(rscratch1, $dst$$FloatRegister, __ D, 0);
|
|
__ clz(rscratch1, rscratch1);
|
|
__ mov($dst$$FloatRegister, __ D, 0, rscratch1);
|
|
__ umov(rscratch1, $dst$$FloatRegister, __ D, 1);
|
|
__ clz(rscratch1, rscratch1);
|
|
__ mov($dst$$FloatRegister, __ D, 1, rscratch1);
|
|
}
|
|
} else {
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_rbit($dst$$FloatRegister, size, ptrue, $src$$FloatRegister);
|
|
__ sve_clz($dst$$FloatRegister, size, ptrue, $dst$$FloatRegister);
|
|
}
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// The dst and src should use the same register to make sure the
|
|
// inactive lanes in dst save the same elements as src.
|
|
instruct vcountTrailingZeros_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (CountTrailingZerosV dst_src pg));
|
|
format %{ "vcountTrailingZeros_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_rbit($dst_src$$FloatRegister, size,
|
|
$pg$$PRegister, $dst_src$$FloatRegister);
|
|
__ sve_clz($dst_src$$FloatRegister, size,
|
|
$pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ ReverseV -------------------------------------
|
|
|
|
instruct vreverse(vReg dst, vReg src) %{
|
|
match(Set dst (ReverseV src));
|
|
format %{ "vreverse $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (bt == T_BYTE) {
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
__ rbit($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src$$FloatRegister);
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
__ sve_rbit($dst$$FloatRegister, __ B, ptrue, $src$$FloatRegister);
|
|
}
|
|
} else {
|
|
assert(bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported type");
|
|
if (UseSVE == 0) {
|
|
assert(length_in_bytes <= 16, "unsupported");
|
|
__ neon_reverse_bits($dst$$FloatRegister, $src$$FloatRegister,
|
|
bt, /* isQ */ length_in_bytes == 16);
|
|
} else {
|
|
__ sve_rbit($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src$$FloatRegister);
|
|
}
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// The dst and src should use the same register to make sure the
|
|
// inactive lanes in dst save the same elements as src.
|
|
|
|
instruct vreverse_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (ReverseV dst_src pg));
|
|
format %{ "vreverse_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_rbit($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $dst_src$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ ReverseBytesV --------------------------------
|
|
|
|
instruct vreverseBytes(vReg dst, vReg src) %{
|
|
match(Set dst (ReverseBytesV src));
|
|
format %{ "vreverseBytes $dst, $src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
|
|
assert(length_in_bytes <= 16, "unsupported");
|
|
if (bt == T_BYTE) {
|
|
if ($dst$$FloatRegister != $src$$FloatRegister) {
|
|
__ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
|
|
$src$$FloatRegister, $src$$FloatRegister);
|
|
}
|
|
} else {
|
|
__ neon_reverse_bytes($dst$$FloatRegister, $src$$FloatRegister,
|
|
bt, /* isQ */ length_in_bytes == 16);
|
|
}
|
|
} else {
|
|
assert(UseSVE > 0, "must be sve");
|
|
if (bt == T_BYTE) {
|
|
if ($dst$$FloatRegister != $src$$FloatRegister) {
|
|
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
|
|
}
|
|
} else {
|
|
__ sve_revb($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
ptrue, $src$$FloatRegister);
|
|
}
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// The dst and src should use the same register to make sure the
|
|
// inactive lanes in dst save the same elements as src.
|
|
instruct vreverseBytes_masked(vReg dst_src, pRegGov pg) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst_src (ReverseBytesV dst_src pg));
|
|
format %{ "vreverseBytes_masked $dst_src, $pg, $dst_src" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
if (bt == T_BYTE) {
|
|
// do nothing
|
|
} else {
|
|
__ sve_revb($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$pg$$PRegister, $dst_src$$FloatRegister);
|
|
}
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Populate Index to a Vector -------------------
|
|
|
|
instruct populateindex(vReg dst, iRegIorL2I src1, immI src2) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (PopulateIndex src1 src2));
|
|
format %{ "populateindex $dst, $src1, $src2\t # populate index (sve)" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_index($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$src1$$Register, (int)($src2$$constant));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Compress/Expand Operations -------------------
|
|
|
|
instruct mcompress(pReg dst, pReg pg, rFlagsReg cr) %{
|
|
predicate(UseSVE > 0);
|
|
match(Set dst (CompressM pg));
|
|
effect(KILL cr);
|
|
format %{ "mcompress $dst, $pg\t# KILL cr" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_cntp(rscratch1, size, ptrue, $pg$$PRegister);
|
|
__ sve_whilelt(as_PRegister($dst$$reg), size, zr, rscratch1);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcompress(vReg dst, vReg src, pRegGov pg) %{
|
|
predicate(UseSVE > 0 &&
|
|
!is_subword_type(Matcher::vector_element_basic_type(n)));
|
|
match(Set dst (CompressV src pg));
|
|
format %{ "vcompress $dst, $src, $pg" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ sve_compact($dst$$FloatRegister, __ elemType_to_regVariant(bt),
|
|
$src$$FloatRegister, $pg$$PRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcompressB(vReg dst, vReg src, pReg pg, vReg tmp1, vReg tmp2,
|
|
vReg tmp3, pReg ptmp, pRegGov pgtmp) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
|
|
effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP ptmp, TEMP pgtmp);
|
|
match(Set dst (CompressV src pg));
|
|
format %{ "vcompressB $dst, $src, $pg\t# KILL $tmp1, $tmp2, $tmp3, $ptmp, $pgtmp" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
__ sve_compress_byte($dst$$FloatRegister, $src$$FloatRegister, $pg$$PRegister,
|
|
$tmp1$$FloatRegister, $tmp2$$FloatRegister, $tmp3$$FloatRegister,
|
|
$ptmp$$PRegister, $pgtmp$$PRegister, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vcompressS(vReg dst, vReg src, pReg pg, vReg tmp1, vReg tmp2, pRegGov pgtmp) %{
|
|
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_SHORT);
|
|
effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP pgtmp);
|
|
match(Set dst (CompressV src pg));
|
|
format %{ "vcompressS $dst, $src, $pg\t# KILL $tmp1, $tmp2, $pgtmp" %}
|
|
ins_encode %{
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
__ sve_dup($tmp1$$FloatRegister, __ H, 0);
|
|
__ sve_compress_short($dst$$FloatRegister, $src$$FloatRegister, $pg$$PRegister,
|
|
$tmp1$$FloatRegister, $tmp2$$FloatRegister, $pgtmp$$PRegister,
|
|
length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vexpand_neon(vReg dst, vReg src, vReg mask, vReg tmp1, vReg tmp2) %{
|
|
predicate(UseSVE == 0);
|
|
match(Set dst (ExpandV src mask));
|
|
effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
|
|
format %{ "vexpand_neon $dst, $src, $mask\t# KILL $tmp1, $tmp2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
int length_in_bytes = (int) Matcher::vector_length_in_bytes(this);
|
|
__ vector_expand_neon($dst$$FloatRegister, $src$$FloatRegister, $mask$$FloatRegister,
|
|
$tmp1$$FloatRegister, $tmp2$$FloatRegister, bt, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vexpand_sve(vReg dst, vReg src, pRegGov pg, vReg tmp1, vReg tmp2) %{
|
|
predicate(UseSVE == 1 || (UseSVE == 2 && type2aelembytes(Matcher::vector_element_basic_type(n)) < 4));
|
|
match(Set dst (ExpandV src pg));
|
|
effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
|
|
format %{ "vexpand_sve $dst, $src, $pg\t# KILL $tmp1, $tmp2" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
int length_in_bytes = (int) Matcher::vector_length_in_bytes(this);
|
|
__ vector_expand_sve($dst$$FloatRegister, $src$$FloatRegister, $pg$$PRegister,
|
|
$tmp1$$FloatRegister, $tmp2$$FloatRegister, bt, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vexpand_sve2_SD(vReg dst, vReg src, pRegGov pg) %{
|
|
predicate(UseSVE == 2 && type2aelembytes(Matcher::vector_element_basic_type(n)) >= 4);
|
|
match(Set dst (ExpandV src pg));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "vexpand_sve2_SD $dst, $src, $pg" %}
|
|
ins_encode %{
|
|
// Example input: src = 1 2 3 4 5 6 7 8
|
|
// pg = 1 0 0 1 1 0 1 1
|
|
// Expected result: dst = 4 0 0 5 6 0 7 8
|
|
|
|
// The basic idea is to use TBL which can shuffle the elements in the given
|
|
// vector flexibly. HISTCNT + SUB is used to generate the second source input
|
|
// for TBL whose value is used to select the indexed element from src vector.
|
|
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
// dst = 0 0 0 0 0 0 0 0
|
|
__ sve_dup($dst$$FloatRegister, size, 0);
|
|
// dst = 5 0 0 4 3 0 2 1
|
|
__ sve_histcnt($dst$$FloatRegister, size, $pg$$PRegister,
|
|
$dst$$FloatRegister, $dst$$FloatRegister);
|
|
// dst = 4 -1 -1 3 2 -1 1 0
|
|
__ sve_sub($dst$$FloatRegister, size, 1);
|
|
// dst = 4 0 0 5 6 0 7 8
|
|
__ sve_tbl($dst$$FloatRegister, size, $src$$FloatRegister, $dst$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------ Vector signum --------------------------------
|
|
|
|
// Vector Math.signum
|
|
|
|
instruct vsignum_le128b(vReg dst, vReg src, vReg zero, vReg one) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) <= 16);
|
|
match(Set dst (SignumVF src (Binary zero one)));
|
|
match(Set dst (SignumVD src (Binary zero one)));
|
|
effect(TEMP_DEF dst);
|
|
format %{ "vsignum_le128b $dst, $src\t# vector <= 128 bits" %}
|
|
ins_encode %{
|
|
__ vector_signum_neon($dst$$FloatRegister, $src$$FloatRegister, $zero$$FloatRegister,
|
|
$one$$FloatRegister, get_arrangement(this));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vsignum_gt128b(vReg dst, vReg src, vReg zero, vReg one, vReg tmp, pRegGov pgtmp) %{
|
|
predicate(Matcher::vector_length_in_bytes(n) > 16);
|
|
match(Set dst (SignumVF src (Binary zero one)));
|
|
match(Set dst (SignumVD src (Binary zero one)));
|
|
effect(TEMP_DEF dst, TEMP tmp, TEMP pgtmp);
|
|
format %{ "vsignum_gt128b $dst, $src\t# vector > 128 bits. KILL $tmp, $pgtmp" %}
|
|
ins_encode %{
|
|
assert(UseSVE > 0, "must be sve");
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
__ vector_signum_sve($dst$$FloatRegister, $src$$FloatRegister, $zero$$FloatRegister,
|
|
$one$$FloatRegister, $tmp$$FloatRegister, $pgtmp$$PRegister,
|
|
__ elemType_to_regVariant(bt));
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ---------------------------------- CompressBitsV --------------------------------
|
|
|
|
instruct vcompressBits(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (CompressBitsV src1 src2));
|
|
format %{ "vcompressBits $dst, $src1, $src2\t# vector (sve)" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_bext($dst$$FloatRegister, size,
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ----------------------------------- ExpandBitsV ---------------------------------
|
|
|
|
instruct vexpandBits(vReg dst, vReg src1, vReg src2) %{
|
|
match(Set dst (ExpandBitsV src1 src2));
|
|
format %{ "vexpandBits $dst, $src1, $src2\t# vector (sve)" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
|
|
__ sve_bdep($dst$$FloatRegister, size,
|
|
$src1$$FloatRegister, $src2$$FloatRegister);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
// ------------------------------------- SelectFromTwoVector ------------------------------------
|
|
// The Neon and SVE2 tbl instruction for two vector lookup requires both the source vectors to be
|
|
// consecutive. The match rules for SelectFromTwoVector reserve two consecutive vector registers
|
|
// for src1 and src2.
|
|
// Four combinations of vector registers for vselect_from_two_vectors are chosen at random
|
|
// (two from volatile and two from non-volatile set) which gives more freedom to the register
|
|
// allocator to choose the best pair of source registers at that point.
|
|
|
|
instruct vselect_from_two_vectors_10_11(vReg dst, vReg_V10 src1, vReg_V11 src2,
|
|
vReg index, vReg tmp) %{
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
match(Set dst (SelectFromTwoVector (Binary index src1) src2));
|
|
format %{ "vselect_from_two_vectors_10_11 $dst, $src1, $src2, $index\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
__ select_from_two_vectors($dst$$FloatRegister, $src1$$FloatRegister,
|
|
$src2$$FloatRegister, $index$$FloatRegister,
|
|
$tmp$$FloatRegister, bt, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vselect_from_two_vectors_12_13(vReg dst, vReg_V12 src1, vReg_V13 src2,
|
|
vReg index, vReg tmp) %{
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
match(Set dst (SelectFromTwoVector (Binary index src1) src2));
|
|
format %{ "vselect_from_two_vectors_12_13 $dst, $src1, $src2, $index\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
__ select_from_two_vectors($dst$$FloatRegister, $src1$$FloatRegister,
|
|
$src2$$FloatRegister, $index$$FloatRegister,
|
|
$tmp$$FloatRegister, bt, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vselect_from_two_vectors_17_18(vReg dst, vReg_V17 src1, vReg_V18 src2,
|
|
vReg index, vReg tmp) %{
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
match(Set dst (SelectFromTwoVector (Binary index src1) src2));
|
|
format %{ "vselect_from_two_vectors_17_18 $dst, $src1, $src2, $index\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
__ select_from_two_vectors($dst$$FloatRegister, $src1$$FloatRegister,
|
|
$src2$$FloatRegister, $index$$FloatRegister,
|
|
$tmp$$FloatRegister, bt, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|
|
|
|
instruct vselect_from_two_vectors_23_24(vReg dst, vReg_V23 src1, vReg_V24 src2,
|
|
vReg index, vReg tmp) %{
|
|
effect(TEMP_DEF dst, TEMP tmp);
|
|
match(Set dst (SelectFromTwoVector (Binary index src1) src2));
|
|
format %{ "vselect_from_two_vectors_23_24 $dst, $src1, $src2, $index\t# KILL $tmp" %}
|
|
ins_encode %{
|
|
BasicType bt = Matcher::vector_element_basic_type(this);
|
|
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
|
|
__ select_from_two_vectors($dst$$FloatRegister, $src1$$FloatRegister,
|
|
$src2$$FloatRegister, $index$$FloatRegister,
|
|
$tmp$$FloatRegister, bt, length_in_bytes);
|
|
%}
|
|
ins_pipe(pipe_slow);
|
|
%}
|