infernap12/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2026-03-14 18:03:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

8366357: C2 SuperWord: refactor VTransformNode::apply with VTransformApplyState

Browse Source 
Reviewed-by: chagedorn, kvn, mhaessig
This commit is contained in:
Emanuel Peter 2025-09-01 06:56:48 +00:00
parent a6e2a329a0
commit dbac620b99
3 changed files with 105 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/hotspot/share/opto/superword.cpp
View File

@ -2108,19 +2108,14 @@ void VTransformGraph::apply_memops_reordering_with_schedule() const {
void VTransformGraph::apply_vectorization_for_each_vtnode(uint& max_vector_length, uint& max_vector_width) const {
ResourceMark rm;
// We keep track of the resulting Nodes from every "VTransformNode::apply" call.
// Since "apply" is called on defs before uses, this allows us to find the
// generated def (input) nodes when we are generating the use nodes in "apply".
int length = _vtnodes.length();
GrowableArray<Node*> vtnode_idx_to_transformed_node(length, length, nullptr);
VTransformApplyState apply_state(_vloop_analyzer, _vtnodes.length());
for (int i = 0; i < _schedule.length(); i++) {
VTransformNode* vtn = _schedule.at(i);
VTransformApplyResult result = vtn->apply(_vloop_analyzer,
vtnode_idx_to_transformed_node);
VTransformApplyResult result = vtn->apply(apply_state);
NOT_PRODUCT( if (_trace._verbose) { result.trace(vtn); } )
vtnode_idx_to_transformed_node.at_put(vtn->_idx, result.node());
apply_state.set_transformed_node(vtn, result.node());
max_vector_length = MAX2(max_vector_length, result.vector_length());
max_vector_width = MAX2(max_vector_width, result.vector_width());
}
@ -3074,7 +3069,7 @@ void VTransform::adjust_pre_loop_limit_to_align_main_loop_vectors() {
const bool is_sub = iv_scale * iv_stride > 0;
// 1.1: con
Node* xbic = igvn().intcon(is_sub ? -con : con);
Node* xbic = phase()->intcon(is_sub ? -con : con);
TRACE_ALIGN_VECTOR_NODE(xbic);
// 1.2: invar = SUM(invar_summands)
@ -3091,7 +3086,7 @@ void VTransform::adjust_pre_loop_limit_to_align_main_loop_vectors() {
phase()->register_new_node(invar_variable, pre_ctrl);
TRACE_ALIGN_VECTOR_NODE(invar_variable);
}
Node* invar_scale_con = igvn().intcon(invar_scale);
Node* invar_scale_con = phase()->intcon(invar_scale);
TRACE_ALIGN_VECTOR_NODE(invar_scale_con);
Node* invar_summand = new MulINode(invar_variable, invar_scale_con);
phase()->register_new_node(invar_summand, pre_ctrl);
@ -3143,7 +3138,7 @@ void VTransform::adjust_pre_loop_limit_to_align_main_loop_vectors() {
// 2: Compute (14):
// XBIC = xbic / abs(iv_scale)
// The division is executed as shift
Node* log2_abs_iv_scale = igvn().intcon(exact_log2(abs(iv_scale)));
Node* log2_abs_iv_scale = phase()->intcon(exact_log2(abs(iv_scale)));
Node* XBIC = new URShiftINode(xbic, log2_abs_iv_scale);
phase()->register_new_node(XBIC, pre_ctrl);
TRACE_ALIGN_VECTOR_NODE(log2_abs_iv_scale);
@ -3168,7 +3163,7 @@ void VTransform::adjust_pre_loop_limit_to_align_main_loop_vectors() {
// = XBIC_OP_old_limit AND (AW - 1)
// Since AW is a power of 2, the modulo operation can be replaced with
// a bitmask operation.
Node* mask_AW = igvn().intcon(AW-1);
Node* mask_AW = phase()->intcon(AW-1);
Node* adjust_pre_iter = new AndINode(XBIC_OP_old_limit, mask_AW);
phase()->register_new_node(adjust_pre_iter, pre_ctrl);
TRACE_ALIGN_VECTOR_NODE(mask_AW);

123
src/hotspot/share/opto/vtransform.cpp
View File

@ -203,13 +203,13 @@ void VTransform::add_speculative_alignment_check(Node* node, juint alignment) {
TRACE_SPECULATIVE_ALIGNMENT_CHECK(node);
}
Node* mask_alignment = igvn().intcon(alignment-1);
Node* mask_alignment = phase()->intcon(alignment-1);
Node* base_alignment = new AndINode(node, mask_alignment);
phase()->register_new_node(base_alignment, ctrl);
TRACE_SPECULATIVE_ALIGNMENT_CHECK(mask_alignment);
TRACE_SPECULATIVE_ALIGNMENT_CHECK(base_alignment);
Node* zero = igvn().intcon(0);
Node* zero = phase()->intcon(0);
Node* cmp_alignment = CmpNode::make(base_alignment, zero, T_INT, false);
BoolNode* bol_alignment = new BoolNode(cmp_alignment, BoolTest::eq);
phase()->register_new_node(cmp_alignment, ctrl);
@ -697,69 +697,68 @@ bool VTransformGraph::has_store_to_load_forwarding_failure(const VLoopAnalyzer&
return false;
}
Node* VTransformNode::find_transformed_input(int i, const GrowableArray<Node*>& vnode_idx_to_transformed_node) const {
Node* n = vnode_idx_to_transformed_node.at(in_req(i)->_idx);
assert(n != nullptr, "must find input IR node");
void VTransformApplyState::set_transformed_node(VTransformNode* vtn, Node* n) {
assert(_vtnode_idx_to_transformed_node.at(vtn->_idx) == nullptr, "only set once");
_vtnode_idx_to_transformed_node.at_put(vtn->_idx, n);
}
Node* VTransformApplyState::transformed_node(const VTransformNode* vtn) const {
Node* n = _vtnode_idx_to_transformed_node.at(vtn->_idx);
assert(n != nullptr, "must find IR node for vtnode");
return n;
}
VTransformApplyResult VTransformScalarNode::apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const {
VTransformApplyResult VTransformScalarNode::apply(VTransformApplyState& apply_state) const {
// This was just wrapped. Now we simply unwap without touching the inputs.
return VTransformApplyResult::make_scalar(_node);
}
VTransformApplyResult VTransformReplicateNode::apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const {
Node* val = find_transformed_input(1, vnode_idx_to_transformed_node);
VTransformApplyResult VTransformReplicateNode::apply(VTransformApplyState& apply_state) const {
Node* val = apply_state.transformed_node(in_req(1));
VectorNode* vn = VectorNode::scalar2vector(val, _vlen, _element_type);
register_new_node_from_vectorization(vloop_analyzer, vn, val);
register_new_node_from_vectorization(apply_state, vn, val);
return VTransformApplyResult::make_vector(vn, _vlen, vn->length_in_bytes());
}
VTransformApplyResult VTransformConvI2LNode::apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const {
Node* val = find_transformed_input(1, vnode_idx_to_transformed_node);
VTransformApplyResult VTransformConvI2LNode::apply(VTransformApplyState& apply_state) const {
Node* val = apply_state.transformed_node(in_req(1));
Node* n = new ConvI2LNode(val);
register_new_node_from_vectorization(vloop_analyzer, n, val);
register_new_node_from_vectorization(apply_state, n, val);
return VTransformApplyResult::make_scalar(n);
}
VTransformApplyResult VTransformShiftCountNode::apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const {
PhaseIdealLoop* phase = vloop_analyzer.vloop().phase();
Node* shift_count_in = find_transformed_input(1, vnode_idx_to_transformed_node);
VTransformApplyResult VTransformShiftCountNode::apply(VTransformApplyState& apply_state) const {
PhaseIdealLoop* phase = apply_state.phase();
Node* shift_count_in = apply_state.transformed_node(in_req(1));
assert(shift_count_in->bottom_type()->isa_int(), "int type only for shift count");
// The shift_count_in would be automatically truncated to the lowest _mask
// bits in a scalar shift operation. But vector shift does not truncate, so
// we must apply the mask now.
Node* shift_count_masked = new AndINode(shift_count_in, phase->igvn().intcon(_mask));
register_new_node_from_vectorization(vloop_analyzer, shift_count_masked, shift_count_in);
Node* shift_count_masked = new AndINode(shift_count_in, phase->intcon(_mask));
register_new_node_from_vectorization(apply_state, shift_count_masked, shift_count_in);
// Now that masked value is "boadcast" (some platforms only set the lowest element).
VectorNode* vn = VectorNode::shift_count(_shift_opcode, shift_count_masked, _vlen, _element_bt);
register_new_node_from_vectorization(vloop_analyzer, vn, shift_count_in);
register_new_node_from_vectorization(apply_state, vn, shift_count_in);
return VTransformApplyResult::make_vector(vn, _vlen, vn->length_in_bytes());
}
VTransformApplyResult VTransformPopulateIndexNode::apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const {
PhaseIdealLoop* phase = vloop_analyzer.vloop().phase();
Node* val = find_transformed_input(1, vnode_idx_to_transformed_node);
VTransformApplyResult VTransformPopulateIndexNode::apply(VTransformApplyState& apply_state) const {
PhaseIdealLoop* phase = apply_state.phase();
Node* val = apply_state.transformed_node(in_req(1));
assert(val->is_Phi(), "expected to be iv");
assert(VectorNode::is_populate_index_supported(_element_bt), "should support");
const TypeVect* vt = TypeVect::make(_element_bt, _vlen);
VectorNode* vn = new PopulateIndexNode(val, phase->igvn().intcon(1), vt);
register_new_node_from_vectorization(vloop_analyzer, vn, val);
VectorNode* vn = new PopulateIndexNode(val, phase->intcon(1), vt);
register_new_node_from_vectorization(apply_state, vn, val);
return VTransformApplyResult::make_vector(vn, _vlen, vn->length_in_bytes());
}
VTransformApplyResult VTransformElementWiseVectorNode::apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const {
VTransformApplyResult VTransformElementWiseVectorNode::apply(VTransformApplyState& apply_state) const {
Node* first = nodes().at(0);
uint vlen = nodes().length();
int opc = first->Opcode();
BasicType bt = vloop_analyzer.types().velt_basic_type(first);
BasicType bt = apply_state.vloop_analyzer().types().velt_basic_type(first);
if (first->is_Cmp()) {
// Cmp + Bool -> VectorMaskCmp
@ -769,9 +768,9 @@ VTransformApplyResult VTransformElementWiseVectorNode::apply(const VLoopAnalyzer
assert(2 <= req() && req() <= 4, "Must have 1-3 inputs");
VectorNode* vn = nullptr;
Node* in1 = find_transformed_input(1, vnode_idx_to_transformed_node);
Node* in2 = (req() >= 3) ? find_transformed_input(2, vnode_idx_to_transformed_node) : nullptr;
Node* in3 = (req() >= 4) ? find_transformed_input(3, vnode_idx_to_transformed_node) : nullptr;
Node* in1 = apply_state.transformed_node(in_req(1));
Node* in2 = (req() >= 3) ? apply_state.transformed_node(in_req(2)) : nullptr;
Node* in3 = (req() >= 4) ? apply_state.transformed_node(in_req(3)) : nullptr;
if (first->is_CMove()) {
assert(req() == 4, "three inputs expected: mask, blend1, blend2");
@ -791,7 +790,7 @@ VTransformApplyResult VTransformElementWiseVectorNode::apply(const VLoopAnalyzer
// The scalar operation was a long -> int operation.
// However, the vector operation is long -> long.
VectorNode* long_vn = VectorNode::make(opc, in1, nullptr, vlen, T_LONG);
register_new_node_from_vectorization(vloop_analyzer, long_vn, first);
register_new_node_from_vectorization(apply_state, long_vn, first);
// Cast long -> int, to mimic the scalar long -> int operation.
vn = VectorCastNode::make(Op_VectorCastL2X, long_vn, T_INT, vlen);
} else if (req() == 3 ||
@ -809,50 +808,47 @@ VTransformApplyResult VTransformElementWiseVectorNode::apply(const VLoopAnalyzer
vn = VectorNode::make(opc, in1, in2, in3, vlen, bt); // ternary
}
register_new_node_from_vectorization_and_replace_scalar_nodes(vloop_analyzer, vn);
register_new_node_from_vectorization_and_replace_scalar_nodes(apply_state, vn);
return VTransformApplyResult::make_vector(vn, vlen, vn->length_in_bytes());
}
VTransformApplyResult VTransformBoolVectorNode::apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const {
VTransformApplyResult VTransformBoolVectorNode::apply(VTransformApplyState& apply_state) const {
BoolNode* first = nodes().at(0)->as_Bool();
uint vlen = nodes().length();
BasicType bt = vloop_analyzer.types().velt_basic_type(first);
BasicType bt = apply_state.vloop_analyzer().types().velt_basic_type(first);
// Cmp + Bool -> VectorMaskCmp
VTransformElementWiseVectorNode* vtn_cmp = in_req(1)->isa_ElementWiseVector();
assert(vtn_cmp != nullptr && vtn_cmp->nodes().at(0)->is_Cmp(),
"bool vtn expects cmp vtn as input");
Node* cmp_in1 = vtn_cmp->find_transformed_input(1, vnode_idx_to_transformed_node);
Node* cmp_in2 = vtn_cmp->find_transformed_input(2, vnode_idx_to_transformed_node);
Node* cmp_in1 = apply_state.transformed_node(vtn_cmp->in_req(1));
Node* cmp_in2 = apply_state.transformed_node(vtn_cmp->in_req(2));
BoolTest::mask mask = test()._mask;
PhaseIdealLoop* phase = vloop_analyzer.vloop().phase();
ConINode* mask_node = phase->igvn().intcon((int)mask);
PhaseIdealLoop* phase = apply_state.phase();
ConINode* mask_node = phase->intcon((int)mask);
const TypeVect* vt = TypeVect::make(bt, vlen);
VectorNode* vn = new VectorMaskCmpNode(mask, cmp_in1, cmp_in2, mask_node, vt);
register_new_node_from_vectorization_and_replace_scalar_nodes(vloop_analyzer, vn);
register_new_node_from_vectorization_and_replace_scalar_nodes(apply_state, vn);
return VTransformApplyResult::make_vector(vn, vlen, vn->vect_type()->length_in_bytes());
}
VTransformApplyResult VTransformReductionVectorNode::apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const {
VTransformApplyResult VTransformReductionVectorNode::apply(VTransformApplyState& apply_state) const {
Node* first = nodes().at(0);
uint vlen = nodes().length();
int opc = first->Opcode();
BasicType bt = first->bottom_type()->basic_type();
Node* init = find_transformed_input(1, vnode_idx_to_transformed_node);
Node* vec = find_transformed_input(2, vnode_idx_to_transformed_node);
Node* init = apply_state.transformed_node(in_req(1));
Node* vec = apply_state.transformed_node(in_req(2));
ReductionNode* vn = ReductionNode::make(opc, nullptr, init, vec, bt);
register_new_node_from_vectorization_and_replace_scalar_nodes(vloop_analyzer, vn);
register_new_node_from_vectorization_and_replace_scalar_nodes(apply_state, vn);
return VTransformApplyResult::make_vector(vn, vlen, vn->vect_type()->length_in_bytes());
}
VTransformApplyResult VTransformLoadVectorNode::apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const {
VTransformApplyResult VTransformLoadVectorNode::apply(VTransformApplyState& apply_state) const {
LoadNode* first = nodes().at(0)->as_Load();
uint vlen = nodes().length();
Node* ctrl = first->in(MemNode::Control);
@ -860,14 +856,14 @@ VTransformApplyResult VTransformLoadVectorNode::apply(const VLoopAnalyzer& vloop
Node* adr = first->in(MemNode::Address);
int opc = first->Opcode();
const TypePtr* adr_type = first->adr_type();
BasicType bt = vloop_analyzer.types().velt_basic_type(first);
BasicType bt = apply_state.vloop_analyzer().types().velt_basic_type(first);
// Set the memory dependency of the LoadVector as early as possible.
// Walk up the memory chain, and ignore any StoreVector that provably
// does not have any memory dependency.
const VPointer& load_p = vpointer(vloop_analyzer);
const VPointer& load_p = vpointer(apply_state.vloop_analyzer());
while (mem->is_StoreVector()) {
VPointer store_p(mem->as_Mem(), vloop_analyzer.vloop());
VPointer store_p(mem->as_Mem(), apply_state.vloop());
if (store_p.never_overlaps_with(load_p)) {
mem = mem->in(MemNode::Memory);
} else {
@ -878,12 +874,11 @@ VTransformApplyResult VTransformLoadVectorNode::apply(const VLoopAnalyzer& vloop
LoadVectorNode* vn = LoadVectorNode::make(opc, ctrl, mem, adr, adr_type, vlen, bt,
control_dependency());
DEBUG_ONLY( if (VerifyAlignVector) { vn->set_must_verify_alignment(); } )
register_new_node_from_vectorization_and_replace_scalar_nodes(vloop_analyzer, vn);
register_new_node_from_vectorization_and_replace_scalar_nodes(apply_state, vn);
return VTransformApplyResult::make_vector(vn, vlen, vn->memory_size());
}
VTransformApplyResult VTransformStoreVectorNode::apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const {
VTransformApplyResult VTransformStoreVectorNode::apply(VTransformApplyState& apply_state) const {
StoreNode* first = nodes().at(0)->as_Store();
uint vlen = nodes().length();
Node* ctrl = first->in(MemNode::Control);
@ -892,18 +887,18 @@ VTransformApplyResult VTransformStoreVectorNode::apply(const VLoopAnalyzer& vloo
int opc = first->Opcode();
const TypePtr* adr_type = first->adr_type();
Node* value = find_transformed_input(MemNode::ValueIn, vnode_idx_to_transformed_node);
Node* value = apply_state.transformed_node(in_req(MemNode::ValueIn));
StoreVectorNode* vn = StoreVectorNode::make(opc, ctrl, mem, adr, adr_type, value, vlen);
DEBUG_ONLY( if (VerifyAlignVector) { vn->set_must_verify_alignment(); } )
register_new_node_from_vectorization_and_replace_scalar_nodes(vloop_analyzer, vn);
register_new_node_from_vectorization_and_replace_scalar_nodes(apply_state, vn);
return VTransformApplyResult::make_vector(vn, vlen, vn->memory_size());
}
void VTransformVectorNode::register_new_node_from_vectorization_and_replace_scalar_nodes(const VLoopAnalyzer& vloop_analyzer, Node* vn) const {
PhaseIdealLoop* phase = vloop_analyzer.vloop().phase();
void VTransformVectorNode::register_new_node_from_vectorization_and_replace_scalar_nodes(VTransformApplyState& apply_state, Node* vn) const {
PhaseIdealLoop* phase = apply_state.phase();
Node* first = nodes().at(0);
register_new_node_from_vectorization(vloop_analyzer, vn, first);
register_new_node_from_vectorization(apply_state, vn, first);
for (int i = 0; i < _nodes.length(); i++) {
Node* n = _nodes.at(i);
@ -911,8 +906,8 @@ void VTransformVectorNode::register_new_node_from_vectorization_and_replace_scal
}
}
void VTransformNode::register_new_node_from_vectorization(const VLoopAnalyzer& vloop_analyzer, Node* vn, Node* old_node) const {
PhaseIdealLoop* phase = vloop_analyzer.vloop().phase();
void VTransformNode::register_new_node_from_vectorization(VTransformApplyState& apply_state, Node* vn, Node* old_node) const {
PhaseIdealLoop* phase = apply_state.phase();
phase->register_new_node_with_ctrl_of(vn, old_node);
phase->igvn()._worklist.push(vn);
VectorNode::trace_new_vector(vn, "AutoVectorization");

63
src/hotspot/share/opto/vtransform.hpp
View File

@ -260,6 +260,32 @@ private:
void apply_vectorization() const;
};
// Keeps track of the state during "VTransform::apply"
// -> keep track of the already transformed nodes
class VTransformApplyState : public StackObj {
private:
const VLoopAnalyzer& _vloop_analyzer;
// We keep track of the resulting Nodes from every "VTransformNode::apply" call.
// Since "apply" is called on defs before uses, this allows us to find the
// generated def (input) nodes when we are generating the use nodes in "apply".
GrowableArray<Node*> _vtnode_idx_to_transformed_node;
public:
VTransformApplyState(const VLoopAnalyzer& vloop_analyzer, int num_vtnodes) :
_vloop_analyzer(vloop_analyzer),
_vtnode_idx_to_transformed_node(num_vtnodes, num_vtnodes, nullptr)
{
}
const VLoop& vloop() const { return _vloop_analyzer.vloop(); }
PhaseIdealLoop* phase() const { return vloop().phase(); }
const VLoopAnalyzer& vloop_analyzer() const { return _vloop_analyzer; }
void set_transformed_node(VTransformNode* vtn, Node* n);
Node* transformed_node(const VTransformNode* vtn) const;
};
// The vtnodes (VTransformNode) resemble the C2 IR Nodes, and model a part of the
// VTransform. Many such vtnodes make up the VTransformGraph. The vtnodes represent
// the resulting scalar and vector nodes as closely as possible.
@ -410,12 +436,11 @@ public:
virtual bool is_load_or_store_in_loop() const { return false; }
virtual const VPointer& vpointer(const VLoopAnalyzer& vloop_analyzer) const { ShouldNotReachHere(); }
virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const = 0;
virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const = 0;
Node* find_transformed_input(int i, const GrowableArray<Node*>& vnode_idx_to_transformed_node) const;
void register_new_node_from_vectorization(const VLoopAnalyzer& vloop_analyzer, Node* vn, Node* old_node) const;
void register_new_node_from_vectorization(VTransformApplyState& apply_state, Node* vn, Node* old_node) const;
NOT_PRODUCT(virtual const char* name() const = 0;)
NOT_PRODUCT(void print() const;)
@ -435,8 +460,7 @@ public:
virtual bool is_load_in_loop() const override { return _node->is_Load(); }
virtual bool is_load_or_store_in_loop() const override { return _node->is_Load() || _node->is_Store(); }
virtual const VPointer& vpointer(const VLoopAnalyzer& vloop_analyzer) const override { return vloop_analyzer.vpointers().vpointer(node()->as_Mem()); }
virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const override;
virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override;
NOT_PRODUCT(virtual const char* name() const override { return "Scalar"; };)
NOT_PRODUCT(virtual void print_spec() const override;)
};
@ -462,8 +486,7 @@ private:
public:
VTransformReplicateNode(VTransform& vtransform, int vlen, BasicType element_type) :
VTransformNode(vtransform, 2), _vlen(vlen), _element_type(element_type) {}
virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const override;
virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override;
NOT_PRODUCT(virtual const char* name() const override { return "Replicate"; };)
NOT_PRODUCT(virtual void print_spec() const override;)
};
@ -472,8 +495,7 @@ public:
class VTransformConvI2LNode : public VTransformNode {
public:
VTransformConvI2LNode(VTransform& vtransform) : VTransformNode(vtransform, 2) {}
virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const override;
virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override;
NOT_PRODUCT(virtual const char* name() const override { return "ConvI2L"; };)
};
@ -487,8 +509,7 @@ private:
public:
VTransformShiftCountNode(VTransform& vtransform, int vlen, BasicType element_bt, juint mask, int shift_opcode) :
VTransformNode(vtransform, 2), _vlen(vlen), _element_bt(element_bt), _mask(mask), _shift_opcode(shift_opcode) {}
virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const override;
virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override;
NOT_PRODUCT(virtual const char* name() const override { return "ShiftCount"; };)
NOT_PRODUCT(virtual void print_spec() const override;)
};
@ -501,8 +522,7 @@ private:
public:
VTransformPopulateIndexNode(VTransform& vtransform, int vlen, const BasicType element_bt) :
VTransformNode(vtransform, 2), _vlen(vlen), _element_bt(element_bt) {}
virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const override;
virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override;
NOT_PRODUCT(virtual const char* name() const override { return "PopulateIndex"; };)
NOT_PRODUCT(virtual void print_spec() const override;)
};
@ -523,7 +543,7 @@ public:
const GrowableArray<Node*>& nodes() const { return _nodes; }
virtual VTransformVectorNode* isa_Vector() override { return this; }
void register_new_node_from_vectorization_and_replace_scalar_nodes(const VLoopAnalyzer& vloop_analyzer, Node* vn) const;
void register_new_node_from_vectorization_and_replace_scalar_nodes(VTransformApplyState& apply_state, Node* vn) const;
NOT_PRODUCT(virtual void print_spec() const override;)
};
@ -533,8 +553,7 @@ public:
VTransformElementWiseVectorNode(VTransform& vtransform, uint req, uint number_of_nodes) :
VTransformVectorNode(vtransform, req, number_of_nodes) {}
virtual VTransformElementWiseVectorNode* isa_ElementWiseVector() override { return this; }
virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const override;
virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override;
NOT_PRODUCT(virtual const char* name() const override { return "ElementWiseVector"; };)
};
@ -554,8 +573,7 @@ public:
VTransformElementWiseVectorNode(vtransform, 2, number_of_nodes), _test(test) {}
VTransformBoolTest test() const { return _test; }
virtual VTransformBoolVectorNode* isa_BoolVector() override { return this; }
virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const override;
virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override;
NOT_PRODUCT(virtual const char* name() const override { return "BoolVector"; };)
};
@ -565,8 +583,7 @@ public:
VTransformReductionVectorNode(VTransform& vtransform, uint number_of_nodes) :
VTransformVectorNode(vtransform, 3, number_of_nodes) {}
virtual VTransformReductionVectorNode* isa_ReductionVector() override { return this; }
virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const override;
virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override;
NOT_PRODUCT(virtual const char* name() const override { return "ReductionVector"; };)
};
@ -592,8 +609,7 @@ public:
LoadNode::ControlDependency control_dependency() const;
virtual VTransformLoadVectorNode* isa_LoadVector() override { return this; }
virtual bool is_load_in_loop() const override { return true; }
virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const override;
virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override;
NOT_PRODUCT(virtual const char* name() const override { return "LoadVector"; };)
};
@ -604,8 +620,7 @@ public:
VTransformMemVectorNode(vtransform, 4, number_of_nodes, vpointer) {}
virtual VTransformStoreVectorNode* isa_StoreVector() override { return this; }
virtual bool is_load_in_loop() const override { return false; }
virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer,
const GrowableArray<Node*>& vnode_idx_to_transformed_node) const override;
virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override;
NOT_PRODUCT(virtual const char* name() const override { return "StoreVector"; };)
};