infernap12/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2026-07-27 03:13:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

8181616: FMA Vectorization on x86

Browse Source 
Reviewed-by: kvn
This commit is contained in:
Vivek Deshpande 2017-06-07 13:09:46 -07:00
parent c04bd11452
commit 4fd6f92702
14 changed files with 293 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
hotspot/src/cpu/x86/vm/assembler_x86.cpp
View File

@ -5092,6 +5092,42 @@ void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector
emit_operand(dst, src);
}
void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
assert(VM_Version::supports_fma(), "");
InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int8((unsigned char)0xB8);
emit_int8((unsigned char)(0xC0 | encode));
}
void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
assert(VM_Version::supports_fma(), "");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int8((unsigned char)0xB8);
emit_int8((unsigned char)(0xC0 | encode));
}
void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
assert(VM_Version::supports_fma(), "");
InstructionMark im(this);
InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int8((unsigned char)0xB8);
emit_operand(dst, src2);
}
void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
assert(VM_Version::supports_fma(), "");
InstructionMark im(this);
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int8((unsigned char)0xB8);
emit_operand(dst, src2);
}
void Assembler::divpd(XMMRegister dst, XMMRegister src) {
NOT_LP64(assert(VM_Version::supports_sse2(), ""));
InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);

5
hotspot/src/cpu/x86/vm/assembler_x86.hpp
View File

@ -1906,6 +1906,11 @@ private:
void vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
void vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
void vfmadd231pd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vfmadd231ps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vfmadd231pd(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
void vfmadd231ps(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
// Divide Packed Floating-Point Values
void divpd(XMMRegister dst, XMMRegister src);
void divps(XMMRegister dst, XMMRegister src);

29
hotspot/src/cpu/x86/vm/macroAssembler_x86.cpp
View File

@ -3165,8 +3165,37 @@ void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegi
}
}
// dst = c = a * b + c
void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
Assembler::vfmadd231pd(c, a, b, vector_len);
if (dst != c) {
vmovdqu(dst, c);
}
}
// dst = c = a * b + c
void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
Assembler::vfmadd231ps(c, a, b, vector_len);
if (dst != c) {
vmovdqu(dst, c);
}
}
// dst = c = a * b + c
void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
Assembler::vfmadd231pd(c, a, b, vector_len);
if (dst != c) {
vmovdqu(dst, c);
}
}
// dst = c = a * b + c
void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
Assembler::vfmadd231ps(c, a, b, vector_len);
if (dst != c) {
vmovdqu(dst, c);
}
}
void MacroAssembler::incrementl(AddressLiteral dst) {
if (reachable(dst)) {

5
hotspot/src/cpu/x86/vm/macroAssembler_x86.hpp
View File

@ -456,6 +456,11 @@ class MacroAssembler: public Assembler {
void fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c);
void fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c);
void vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len);
void vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len);
void vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len);
void vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len);
// same as fcmp2int, but using SSE2
void cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less);

2
hotspot/src/cpu/x86/vm/vm_version_x86.cpp
View File

@ -812,7 +812,7 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseGHASHIntrinsics, false);
}
if (supports_fma() && UseSSE >= 2) {
if (supports_fma()) {
if (FLAG_IS_DEFAULT(UseFMA)) {
UseFMA = true;
}

2
hotspot/src/cpu/x86/vm/vm_version_x86.hpp
View File

@ -732,7 +732,7 @@ public:
static bool supports_avx256only() { return (supports_avx2() && !supports_evex()); }
static bool supports_avxonly() { return ((supports_avx2() || supports_avx()) && !supports_evex()); }
static bool supports_sha() { return (_features & CPU_SHA) != 0; }
static bool supports_fma() { return (_features & CPU_FMA) != 0; }
static bool supports_fma() { return (_features & CPU_FMA) != 0 && supports_avx(); }
static bool supports_vzeroupper() { return (_features & CPU_VZEROUPPER) != 0; }
// Intel features

158
hotspot/src/cpu/x86/vm/x86.ad
View File

@ -10520,3 +10520,161 @@ instruct vxor64B_mem(vecZ dst, vecZ src, memory mem) %{
ins_pipe( pipe_slow );
%}
// --------------------------------- FMA --------------------------------------
// a * b + c
instruct vfma2D_reg(vecX a, vecX b, vecX c) %{
predicate(UseFMA && n->as_Vector()->length() == 2);
match(Set c (FmaVD c (Binary a b)));
format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packed2D" %}
ins_cost(150);
ins_encode %{
int vector_len = 0;
__ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vector_len);
%}
ins_pipe( pipe_slow );
%}
// a * b + c
instruct vfma2D_mem(vecX a, memory b, vecX c) %{
predicate(UseFMA && n->as_Vector()->length() == 2);
match(Set c (FmaVD c (Binary a (LoadVector b))));
format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packed2D" %}
ins_cost(150);
ins_encode %{
int vector_len = 0;
__ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vector_len);
%}
ins_pipe( pipe_slow );
%}
// a * b + c
instruct vfma4D_reg(vecY a, vecY b, vecY c) %{
predicate(UseFMA && n->as_Vector()->length() == 4);
match(Set c (FmaVD c (Binary a b)));
format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packed4D" %}
ins_cost(150);
ins_encode %{
int vector_len = 1;
__ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vector_len);
%}
ins_pipe( pipe_slow );
%}
// a * b + c
instruct vfma4D_mem(vecY a, memory b, vecY c) %{
predicate(UseFMA && n->as_Vector()->length() == 4);
match(Set c (FmaVD c (Binary a (LoadVector b))));
format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packed4D" %}
ins_cost(150);
ins_encode %{
int vector_len = 1;
__ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vector_len);
%}
ins_pipe( pipe_slow );
%}
// a * b + c
instruct vfma8D_reg(vecZ a, vecZ b, vecZ c) %{
predicate(UseFMA && n->as_Vector()->length() == 8);
match(Set c (FmaVD c (Binary a b)));
format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packed8D" %}
ins_cost(150);
ins_encode %{
int vector_len = 2;
__ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vector_len);
%}
ins_pipe( pipe_slow );
%}
// a * b + c
instruct vfma8D_mem(vecZ a, memory b, vecZ c) %{
predicate(UseFMA && n->as_Vector()->length() == 8);
match(Set c (FmaVD c (Binary a (LoadVector b))));
format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packed8D" %}
ins_cost(150);
ins_encode %{
int vector_len = 2;
__ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vector_len);
%}
ins_pipe( pipe_slow );
%}
// a * b + c
instruct vfma4F_reg(vecX a, vecX b, vecX c) %{
predicate(UseFMA && n->as_Vector()->length() == 4);
match(Set c (FmaVF c (Binary a b)));
format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packed4F" %}
ins_cost(150);
ins_encode %{
int vector_len = 0;
__ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vector_len);
%}
ins_pipe( pipe_slow );
%}
// a * b + c
instruct vfma4F_mem(vecX a, memory b, vecX c) %{
predicate(UseFMA && n->as_Vector()->length() == 4);
match(Set c (FmaVF c (Binary a (LoadVector b))));
format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packed4F" %}
ins_cost(150);
ins_encode %{
int vector_len = 0;
__ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vector_len);
%}
ins_pipe( pipe_slow );
%}
// a * b + c
instruct vfma8F_reg(vecY a, vecY b, vecY c) %{
predicate(UseFMA && n->as_Vector()->length() == 8);
match(Set c (FmaVF c (Binary a b)));
format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packed8F" %}
ins_cost(150);
ins_encode %{
int vector_len = 1;
__ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vector_len);
%}
ins_pipe( pipe_slow );
%}
// a * b + c
instruct vfma8F_mem(vecY a, memory b, vecY c) %{
predicate(UseFMA && n->as_Vector()->length() == 8);
match(Set c (FmaVF c (Binary a (LoadVector b))));
format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packed8F" %}
ins_cost(150);
ins_encode %{
int vector_len = 1;
__ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vector_len);
%}
ins_pipe( pipe_slow );
%}
// a * b + c
instruct vfma16F_reg(vecZ a, vecZ b, vecZ c) %{
predicate(UseFMA && n->as_Vector()->length() == 16);
match(Set c (FmaVF c (Binary a b)));
format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packed16F" %}
ins_cost(150);
ins_encode %{
int vector_len = 2;
__ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vector_len);
%}
ins_pipe( pipe_slow );
%}
// a * b + c
instruct vfma16F_mem(vecZ a, memory b, vecZ c) %{
predicate(UseFMA && n->as_Vector()->length() == 16);
match(Set c (FmaVF c (Binary a (LoadVector b))));
format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packed16F" %}
ins_cost(150);
ins_encode %{
int vector_len = 2;
__ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vector_len);
%}
ins_pipe( pipe_slow );
%}

1
hotspot/src/share/vm/adlc/formssel.cpp
View File

@ -4179,6 +4179,7 @@ bool MatchRule::is_vector() const {
"URShiftVB","URShiftVS","URShiftVI","URShiftVL",
"ReplicateB","ReplicateS","ReplicateI","ReplicateL","ReplicateF","ReplicateD",
"LoadVector","StoreVector",
"FmaVD", "FmaVF",
// Next are not supported currently.
"PackB","PackS","PackI","PackL","PackF","PackD","Pack2L","Pack2D",
"ExtractB","ExtractUB","ExtractC","ExtractS","ExtractI","ExtractL","ExtractF","ExtractD"

2
hotspot/src/share/vm/opto/classes.hpp
View File

@ -310,6 +310,8 @@ macro(MulVF)
macro(MulReductionVF)
macro(MulVD)
macro(MulReductionVD)
macro(FmaVD)
macro(FmaVF)
macro(DivVF)
macro(DivVD)
macro(AbsVF)

7
hotspot/src/share/vm/opto/matcher.cpp
View File

@ -977,7 +977,6 @@ Node *Matcher::xform( Node *n, int max_stack ) {
// Use one stack to keep both: child's node/state and parent's node/index
MStack mstack(max_stack * 2 * 2); // usually: C->live_nodes() * 2 * 2
mstack.push(n, Visit, NULL, -1); // set NULL as parent to indicate root
while (mstack.is_nonempty()) {
C->check_node_count(NodeLimitFudgeFactor, "too many nodes matching instructions");
if (C->failing()) return NULL;
@ -2122,6 +2121,8 @@ void Matcher::find_shared( Node *n ) {
case Op_EncodeISOArray:
case Op_FmaD:
case Op_FmaF:
case Op_FmaVD:
case Op_FmaVF:
set_shared(n); // Force result into register (it will be anyways)
break;
case Op_ConP: { // Convert pointers above the centerline to NUL
@ -2311,7 +2312,9 @@ void Matcher::find_shared( Node *n ) {
break;
}
case Op_FmaD:
case Op_FmaF: {
case Op_FmaF:
case Op_FmaVD:
case Op_FmaVF: {
// Restructure into a binary tree for Matching.
Node* pair = new BinaryNode(n->in(1), n->in(2));
n->set_req(2, pair);

7
hotspot/src/share/vm/opto/superword.cpp
View File

@ -2324,6 +2324,13 @@ void SuperWord::output() {
const TypeVect* vt = TypeVect::make(bt, vlen);
vn = new CMoveVDNode(cc, src1, src2, vt);
NOT_PRODUCT(if(is_trace_cmov()) {tty->print("SWPointer::output: created new CMove node %d: ", vn->_idx); vn->dump();})
} else if (opc == Op_FmaD || opc == Op_FmaF) {
// Promote operands to vector
Node* in1 = vector_opd(p, 1);
Node* in2 = vector_opd(p, 2);
Node* in3 = vector_opd(p, 3);
vn = VectorNode::make(opc, in1, in2, in3, vlen, velt_basic_type(n));
vlen_in_bytes = vn->as_Vector()->length_in_bytes();
} else {
if (do_reserve_copy()) {
NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: ShouldNotReachHere, exiting SuperWord");})

24
hotspot/src/share/vm/opto/vectornode.cpp
View File

@ -86,6 +86,12 @@ int VectorNode::opcode(int sopc, BasicType bt) {
case Op_MulD:
assert(bt == T_DOUBLE, "must be");
return Op_MulVD;
case Op_FmaD:
assert(bt == T_DOUBLE, "must be");
return Op_FmaVD;
case Op_FmaF:
assert(bt == T_FLOAT, "must be");
return Op_FmaVF;
case Op_CMoveD:
assert(bt == T_DOUBLE, "must be");
return Op_CMoveVD;
@ -259,6 +265,11 @@ void VectorNode::vector_operands(Node* n, uint* start, uint* end) {
*start = 2;
*end = n->req();
break;
case Op_FmaD:
case Op_FmaF:
*start = 1;
*end = 4; // 3 vector operands
break;
default:
*start = 1;
*end = n->req(); // default is all operands
@ -328,6 +339,19 @@ VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType b
}
VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) {
const TypeVect* vt = TypeVect::make(bt, vlen);
int vopc = VectorNode::opcode(opc, bt);
// This method should not be called for unimplemented vectors.
guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
switch (vopc) {
case Op_FmaVD: return new FmaVDNode(n1, n2, n3, vt);
case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt);
}
fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
return NULL;
}
// Scalar promotion
VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t) {
BasicType bt = opd_t->array_element_basic_type();

17
hotspot/src/share/vm/opto/vectornode.hpp
View File

@ -62,6 +62,7 @@ class VectorNode : public TypeNode {
static VectorNode* scalar2vector(Node* s, uint vlen, const Type* opd_t);
static VectorNode* shift_count(Node* shift, Node* cnt, uint vlen, BasicType bt);
static VectorNode* make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt);
static VectorNode* make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt);
static int opcode(int opc, BasicType bt);
static bool implemented(int opc, uint vlen, BasicType bt);
@ -260,6 +261,22 @@ public:
virtual int Opcode() const;
};
//------------------------------FmaVDNode--------------------------------------
// Vector multiply double
class FmaVDNode : public VectorNode {
public:
FmaVDNode(Node* in1, Node* in2, Node* in3, const TypeVect* vt) : VectorNode(in1, in2, in3, vt) {}
virtual int Opcode() const;
};
//------------------------------FmaVFNode--------------------------------------
// Vector multiply float
class FmaVFNode : public VectorNode {
public:
FmaVFNode(Node* in1, Node* in2, Node* in3, const TypeVect* vt) : VectorNode(in1, in2, in3, vt) {}
virtual int Opcode() const;
};
//------------------------------CMoveVDNode--------------------------------------
// Vector multiply double
class CMoveVDNode : public VectorNode {

2
hotspot/src/share/vm/runtime/vmStructs.cpp
View File

@ -2057,6 +2057,8 @@ typedef RehashableHashtable<Symbol*, mtSymbol> RehashableSymbolHashtable;
declare_c2_type(MulVFNode, VectorNode) \
declare_c2_type(MulReductionVFNode, ReductionNode) \
declare_c2_type(MulVDNode, VectorNode) \
declare_c2_type(FmaVDNode, VectorNode) \
declare_c2_type(FmaVFNode, VectorNode) \
declare_c2_type(CMoveVDNode, VectorNode) \
declare_c2_type(MulReductionVDNode, ReductionNode) \
declare_c2_type(DivVFNode, VectorNode) \