From dbac620b996713087f0d1b1189e543e51a0bb09f Mon Sep 17 00:00:00 2001 From: Emanuel Peter Date: Mon, 1 Sep 2025 06:56:48 +0000 Subject: [PATCH] 8366357: C2 SuperWord: refactor VTransformNode::apply with VTransformApplyState Reviewed-by: chagedorn, kvn, mhaessig --- src/hotspot/share/opto/superword.cpp | 19 ++-- src/hotspot/share/opto/vtransform.cpp | 123 ++++++++++++-------------- src/hotspot/share/opto/vtransform.hpp | 63 ++++++++----- 3 files changed, 105 insertions(+), 100 deletions(-) diff --git a/src/hotspot/share/opto/superword.cpp b/src/hotspot/share/opto/superword.cpp index cb92febf803..7b47992e77e 100644 --- a/src/hotspot/share/opto/superword.cpp +++ b/src/hotspot/share/opto/superword.cpp @@ -2108,19 +2108,14 @@ void VTransformGraph::apply_memops_reordering_with_schedule() const { void VTransformGraph::apply_vectorization_for_each_vtnode(uint& max_vector_length, uint& max_vector_width) const { ResourceMark rm; - // We keep track of the resulting Nodes from every "VTransformNode::apply" call. - // Since "apply" is called on defs before uses, this allows us to find the - // generated def (input) nodes when we are generating the use nodes in "apply". - int length = _vtnodes.length(); - GrowableArray vtnode_idx_to_transformed_node(length, length, nullptr); + VTransformApplyState apply_state(_vloop_analyzer, _vtnodes.length()); for (int i = 0; i < _schedule.length(); i++) { VTransformNode* vtn = _schedule.at(i); - VTransformApplyResult result = vtn->apply(_vloop_analyzer, - vtnode_idx_to_transformed_node); + VTransformApplyResult result = vtn->apply(apply_state); NOT_PRODUCT( if (_trace._verbose) { result.trace(vtn); } ) - vtnode_idx_to_transformed_node.at_put(vtn->_idx, result.node()); + apply_state.set_transformed_node(vtn, result.node()); max_vector_length = MAX2(max_vector_length, result.vector_length()); max_vector_width = MAX2(max_vector_width, result.vector_width()); } @@ -3074,7 +3069,7 @@ void VTransform::adjust_pre_loop_limit_to_align_main_loop_vectors() { const bool is_sub = iv_scale * iv_stride > 0; // 1.1: con - Node* xbic = igvn().intcon(is_sub ? -con : con); + Node* xbic = phase()->intcon(is_sub ? -con : con); TRACE_ALIGN_VECTOR_NODE(xbic); // 1.2: invar = SUM(invar_summands) @@ -3091,7 +3086,7 @@ void VTransform::adjust_pre_loop_limit_to_align_main_loop_vectors() { phase()->register_new_node(invar_variable, pre_ctrl); TRACE_ALIGN_VECTOR_NODE(invar_variable); } - Node* invar_scale_con = igvn().intcon(invar_scale); + Node* invar_scale_con = phase()->intcon(invar_scale); TRACE_ALIGN_VECTOR_NODE(invar_scale_con); Node* invar_summand = new MulINode(invar_variable, invar_scale_con); phase()->register_new_node(invar_summand, pre_ctrl); @@ -3143,7 +3138,7 @@ void VTransform::adjust_pre_loop_limit_to_align_main_loop_vectors() { // 2: Compute (14): // XBIC = xbic / abs(iv_scale) // The division is executed as shift - Node* log2_abs_iv_scale = igvn().intcon(exact_log2(abs(iv_scale))); + Node* log2_abs_iv_scale = phase()->intcon(exact_log2(abs(iv_scale))); Node* XBIC = new URShiftINode(xbic, log2_abs_iv_scale); phase()->register_new_node(XBIC, pre_ctrl); TRACE_ALIGN_VECTOR_NODE(log2_abs_iv_scale); @@ -3168,7 +3163,7 @@ void VTransform::adjust_pre_loop_limit_to_align_main_loop_vectors() { // = XBIC_OP_old_limit AND (AW - 1) // Since AW is a power of 2, the modulo operation can be replaced with // a bitmask operation. - Node* mask_AW = igvn().intcon(AW-1); + Node* mask_AW = phase()->intcon(AW-1); Node* adjust_pre_iter = new AndINode(XBIC_OP_old_limit, mask_AW); phase()->register_new_node(adjust_pre_iter, pre_ctrl); TRACE_ALIGN_VECTOR_NODE(mask_AW); diff --git a/src/hotspot/share/opto/vtransform.cpp b/src/hotspot/share/opto/vtransform.cpp index 1675b8d9fdb..f8efe333941 100644 --- a/src/hotspot/share/opto/vtransform.cpp +++ b/src/hotspot/share/opto/vtransform.cpp @@ -203,13 +203,13 @@ void VTransform::add_speculative_alignment_check(Node* node, juint alignment) { TRACE_SPECULATIVE_ALIGNMENT_CHECK(node); } - Node* mask_alignment = igvn().intcon(alignment-1); + Node* mask_alignment = phase()->intcon(alignment-1); Node* base_alignment = new AndINode(node, mask_alignment); phase()->register_new_node(base_alignment, ctrl); TRACE_SPECULATIVE_ALIGNMENT_CHECK(mask_alignment); TRACE_SPECULATIVE_ALIGNMENT_CHECK(base_alignment); - Node* zero = igvn().intcon(0); + Node* zero = phase()->intcon(0); Node* cmp_alignment = CmpNode::make(base_alignment, zero, T_INT, false); BoolNode* bol_alignment = new BoolNode(cmp_alignment, BoolTest::eq); phase()->register_new_node(cmp_alignment, ctrl); @@ -697,69 +697,68 @@ bool VTransformGraph::has_store_to_load_forwarding_failure(const VLoopAnalyzer& return false; } -Node* VTransformNode::find_transformed_input(int i, const GrowableArray& vnode_idx_to_transformed_node) const { - Node* n = vnode_idx_to_transformed_node.at(in_req(i)->_idx); - assert(n != nullptr, "must find input IR node"); +void VTransformApplyState::set_transformed_node(VTransformNode* vtn, Node* n) { + assert(_vtnode_idx_to_transformed_node.at(vtn->_idx) == nullptr, "only set once"); + _vtnode_idx_to_transformed_node.at_put(vtn->_idx, n); +} + +Node* VTransformApplyState::transformed_node(const VTransformNode* vtn) const { + Node* n = _vtnode_idx_to_transformed_node.at(vtn->_idx); + assert(n != nullptr, "must find IR node for vtnode"); return n; } -VTransformApplyResult VTransformScalarNode::apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const { +VTransformApplyResult VTransformScalarNode::apply(VTransformApplyState& apply_state) const { // This was just wrapped. Now we simply unwap without touching the inputs. return VTransformApplyResult::make_scalar(_node); } -VTransformApplyResult VTransformReplicateNode::apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const { - Node* val = find_transformed_input(1, vnode_idx_to_transformed_node); +VTransformApplyResult VTransformReplicateNode::apply(VTransformApplyState& apply_state) const { + Node* val = apply_state.transformed_node(in_req(1)); VectorNode* vn = VectorNode::scalar2vector(val, _vlen, _element_type); - register_new_node_from_vectorization(vloop_analyzer, vn, val); + register_new_node_from_vectorization(apply_state, vn, val); return VTransformApplyResult::make_vector(vn, _vlen, vn->length_in_bytes()); } -VTransformApplyResult VTransformConvI2LNode::apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const { - Node* val = find_transformed_input(1, vnode_idx_to_transformed_node); +VTransformApplyResult VTransformConvI2LNode::apply(VTransformApplyState& apply_state) const { + Node* val = apply_state.transformed_node(in_req(1)); Node* n = new ConvI2LNode(val); - register_new_node_from_vectorization(vloop_analyzer, n, val); + register_new_node_from_vectorization(apply_state, n, val); return VTransformApplyResult::make_scalar(n); } -VTransformApplyResult VTransformShiftCountNode::apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const { - PhaseIdealLoop* phase = vloop_analyzer.vloop().phase(); - Node* shift_count_in = find_transformed_input(1, vnode_idx_to_transformed_node); +VTransformApplyResult VTransformShiftCountNode::apply(VTransformApplyState& apply_state) const { + PhaseIdealLoop* phase = apply_state.phase(); + Node* shift_count_in = apply_state.transformed_node(in_req(1)); assert(shift_count_in->bottom_type()->isa_int(), "int type only for shift count"); // The shift_count_in would be automatically truncated to the lowest _mask // bits in a scalar shift operation. But vector shift does not truncate, so // we must apply the mask now. - Node* shift_count_masked = new AndINode(shift_count_in, phase->igvn().intcon(_mask)); - register_new_node_from_vectorization(vloop_analyzer, shift_count_masked, shift_count_in); + Node* shift_count_masked = new AndINode(shift_count_in, phase->intcon(_mask)); + register_new_node_from_vectorization(apply_state, shift_count_masked, shift_count_in); // Now that masked value is "boadcast" (some platforms only set the lowest element). VectorNode* vn = VectorNode::shift_count(_shift_opcode, shift_count_masked, _vlen, _element_bt); - register_new_node_from_vectorization(vloop_analyzer, vn, shift_count_in); + register_new_node_from_vectorization(apply_state, vn, shift_count_in); return VTransformApplyResult::make_vector(vn, _vlen, vn->length_in_bytes()); } -VTransformApplyResult VTransformPopulateIndexNode::apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const { - PhaseIdealLoop* phase = vloop_analyzer.vloop().phase(); - Node* val = find_transformed_input(1, vnode_idx_to_transformed_node); +VTransformApplyResult VTransformPopulateIndexNode::apply(VTransformApplyState& apply_state) const { + PhaseIdealLoop* phase = apply_state.phase(); + Node* val = apply_state.transformed_node(in_req(1)); assert(val->is_Phi(), "expected to be iv"); assert(VectorNode::is_populate_index_supported(_element_bt), "should support"); const TypeVect* vt = TypeVect::make(_element_bt, _vlen); - VectorNode* vn = new PopulateIndexNode(val, phase->igvn().intcon(1), vt); - register_new_node_from_vectorization(vloop_analyzer, vn, val); + VectorNode* vn = new PopulateIndexNode(val, phase->intcon(1), vt); + register_new_node_from_vectorization(apply_state, vn, val); return VTransformApplyResult::make_vector(vn, _vlen, vn->length_in_bytes()); } -VTransformApplyResult VTransformElementWiseVectorNode::apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const { +VTransformApplyResult VTransformElementWiseVectorNode::apply(VTransformApplyState& apply_state) const { Node* first = nodes().at(0); uint vlen = nodes().length(); int opc = first->Opcode(); - BasicType bt = vloop_analyzer.types().velt_basic_type(first); + BasicType bt = apply_state.vloop_analyzer().types().velt_basic_type(first); if (first->is_Cmp()) { // Cmp + Bool -> VectorMaskCmp @@ -769,9 +768,9 @@ VTransformApplyResult VTransformElementWiseVectorNode::apply(const VLoopAnalyzer assert(2 <= req() && req() <= 4, "Must have 1-3 inputs"); VectorNode* vn = nullptr; - Node* in1 = find_transformed_input(1, vnode_idx_to_transformed_node); - Node* in2 = (req() >= 3) ? find_transformed_input(2, vnode_idx_to_transformed_node) : nullptr; - Node* in3 = (req() >= 4) ? find_transformed_input(3, vnode_idx_to_transformed_node) : nullptr; + Node* in1 = apply_state.transformed_node(in_req(1)); + Node* in2 = (req() >= 3) ? apply_state.transformed_node(in_req(2)) : nullptr; + Node* in3 = (req() >= 4) ? apply_state.transformed_node(in_req(3)) : nullptr; if (first->is_CMove()) { assert(req() == 4, "three inputs expected: mask, blend1, blend2"); @@ -791,7 +790,7 @@ VTransformApplyResult VTransformElementWiseVectorNode::apply(const VLoopAnalyzer // The scalar operation was a long -> int operation. // However, the vector operation is long -> long. VectorNode* long_vn = VectorNode::make(opc, in1, nullptr, vlen, T_LONG); - register_new_node_from_vectorization(vloop_analyzer, long_vn, first); + register_new_node_from_vectorization(apply_state, long_vn, first); // Cast long -> int, to mimic the scalar long -> int operation. vn = VectorCastNode::make(Op_VectorCastL2X, long_vn, T_INT, vlen); } else if (req() == 3 || @@ -809,50 +808,47 @@ VTransformApplyResult VTransformElementWiseVectorNode::apply(const VLoopAnalyzer vn = VectorNode::make(opc, in1, in2, in3, vlen, bt); // ternary } - register_new_node_from_vectorization_and_replace_scalar_nodes(vloop_analyzer, vn); + register_new_node_from_vectorization_and_replace_scalar_nodes(apply_state, vn); return VTransformApplyResult::make_vector(vn, vlen, vn->length_in_bytes()); } -VTransformApplyResult VTransformBoolVectorNode::apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const { +VTransformApplyResult VTransformBoolVectorNode::apply(VTransformApplyState& apply_state) const { BoolNode* first = nodes().at(0)->as_Bool(); uint vlen = nodes().length(); - BasicType bt = vloop_analyzer.types().velt_basic_type(first); + BasicType bt = apply_state.vloop_analyzer().types().velt_basic_type(first); // Cmp + Bool -> VectorMaskCmp VTransformElementWiseVectorNode* vtn_cmp = in_req(1)->isa_ElementWiseVector(); assert(vtn_cmp != nullptr && vtn_cmp->nodes().at(0)->is_Cmp(), "bool vtn expects cmp vtn as input"); - Node* cmp_in1 = vtn_cmp->find_transformed_input(1, vnode_idx_to_transformed_node); - Node* cmp_in2 = vtn_cmp->find_transformed_input(2, vnode_idx_to_transformed_node); + Node* cmp_in1 = apply_state.transformed_node(vtn_cmp->in_req(1)); + Node* cmp_in2 = apply_state.transformed_node(vtn_cmp->in_req(2)); BoolTest::mask mask = test()._mask; - PhaseIdealLoop* phase = vloop_analyzer.vloop().phase(); - ConINode* mask_node = phase->igvn().intcon((int)mask); + PhaseIdealLoop* phase = apply_state.phase(); + ConINode* mask_node = phase->intcon((int)mask); const TypeVect* vt = TypeVect::make(bt, vlen); VectorNode* vn = new VectorMaskCmpNode(mask, cmp_in1, cmp_in2, mask_node, vt); - register_new_node_from_vectorization_and_replace_scalar_nodes(vloop_analyzer, vn); + register_new_node_from_vectorization_and_replace_scalar_nodes(apply_state, vn); return VTransformApplyResult::make_vector(vn, vlen, vn->vect_type()->length_in_bytes()); } -VTransformApplyResult VTransformReductionVectorNode::apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const { +VTransformApplyResult VTransformReductionVectorNode::apply(VTransformApplyState& apply_state) const { Node* first = nodes().at(0); uint vlen = nodes().length(); int opc = first->Opcode(); BasicType bt = first->bottom_type()->basic_type(); - Node* init = find_transformed_input(1, vnode_idx_to_transformed_node); - Node* vec = find_transformed_input(2, vnode_idx_to_transformed_node); + Node* init = apply_state.transformed_node(in_req(1)); + Node* vec = apply_state.transformed_node(in_req(2)); ReductionNode* vn = ReductionNode::make(opc, nullptr, init, vec, bt); - register_new_node_from_vectorization_and_replace_scalar_nodes(vloop_analyzer, vn); + register_new_node_from_vectorization_and_replace_scalar_nodes(apply_state, vn); return VTransformApplyResult::make_vector(vn, vlen, vn->vect_type()->length_in_bytes()); } -VTransformApplyResult VTransformLoadVectorNode::apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const { +VTransformApplyResult VTransformLoadVectorNode::apply(VTransformApplyState& apply_state) const { LoadNode* first = nodes().at(0)->as_Load(); uint vlen = nodes().length(); Node* ctrl = first->in(MemNode::Control); @@ -860,14 +856,14 @@ VTransformApplyResult VTransformLoadVectorNode::apply(const VLoopAnalyzer& vloop Node* adr = first->in(MemNode::Address); int opc = first->Opcode(); const TypePtr* adr_type = first->adr_type(); - BasicType bt = vloop_analyzer.types().velt_basic_type(first); + BasicType bt = apply_state.vloop_analyzer().types().velt_basic_type(first); // Set the memory dependency of the LoadVector as early as possible. // Walk up the memory chain, and ignore any StoreVector that provably // does not have any memory dependency. - const VPointer& load_p = vpointer(vloop_analyzer); + const VPointer& load_p = vpointer(apply_state.vloop_analyzer()); while (mem->is_StoreVector()) { - VPointer store_p(mem->as_Mem(), vloop_analyzer.vloop()); + VPointer store_p(mem->as_Mem(), apply_state.vloop()); if (store_p.never_overlaps_with(load_p)) { mem = mem->in(MemNode::Memory); } else { @@ -878,12 +874,11 @@ VTransformApplyResult VTransformLoadVectorNode::apply(const VLoopAnalyzer& vloop LoadVectorNode* vn = LoadVectorNode::make(opc, ctrl, mem, adr, adr_type, vlen, bt, control_dependency()); DEBUG_ONLY( if (VerifyAlignVector) { vn->set_must_verify_alignment(); } ) - register_new_node_from_vectorization_and_replace_scalar_nodes(vloop_analyzer, vn); + register_new_node_from_vectorization_and_replace_scalar_nodes(apply_state, vn); return VTransformApplyResult::make_vector(vn, vlen, vn->memory_size()); } -VTransformApplyResult VTransformStoreVectorNode::apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const { +VTransformApplyResult VTransformStoreVectorNode::apply(VTransformApplyState& apply_state) const { StoreNode* first = nodes().at(0)->as_Store(); uint vlen = nodes().length(); Node* ctrl = first->in(MemNode::Control); @@ -892,18 +887,18 @@ VTransformApplyResult VTransformStoreVectorNode::apply(const VLoopAnalyzer& vloo int opc = first->Opcode(); const TypePtr* adr_type = first->adr_type(); - Node* value = find_transformed_input(MemNode::ValueIn, vnode_idx_to_transformed_node); + Node* value = apply_state.transformed_node(in_req(MemNode::ValueIn)); StoreVectorNode* vn = StoreVectorNode::make(opc, ctrl, mem, adr, adr_type, value, vlen); DEBUG_ONLY( if (VerifyAlignVector) { vn->set_must_verify_alignment(); } ) - register_new_node_from_vectorization_and_replace_scalar_nodes(vloop_analyzer, vn); + register_new_node_from_vectorization_and_replace_scalar_nodes(apply_state, vn); return VTransformApplyResult::make_vector(vn, vlen, vn->memory_size()); } -void VTransformVectorNode::register_new_node_from_vectorization_and_replace_scalar_nodes(const VLoopAnalyzer& vloop_analyzer, Node* vn) const { - PhaseIdealLoop* phase = vloop_analyzer.vloop().phase(); +void VTransformVectorNode::register_new_node_from_vectorization_and_replace_scalar_nodes(VTransformApplyState& apply_state, Node* vn) const { + PhaseIdealLoop* phase = apply_state.phase(); Node* first = nodes().at(0); - register_new_node_from_vectorization(vloop_analyzer, vn, first); + register_new_node_from_vectorization(apply_state, vn, first); for (int i = 0; i < _nodes.length(); i++) { Node* n = _nodes.at(i); @@ -911,8 +906,8 @@ void VTransformVectorNode::register_new_node_from_vectorization_and_replace_scal } } -void VTransformNode::register_new_node_from_vectorization(const VLoopAnalyzer& vloop_analyzer, Node* vn, Node* old_node) const { - PhaseIdealLoop* phase = vloop_analyzer.vloop().phase(); +void VTransformNode::register_new_node_from_vectorization(VTransformApplyState& apply_state, Node* vn, Node* old_node) const { + PhaseIdealLoop* phase = apply_state.phase(); phase->register_new_node_with_ctrl_of(vn, old_node); phase->igvn()._worklist.push(vn); VectorNode::trace_new_vector(vn, "AutoVectorization"); diff --git a/src/hotspot/share/opto/vtransform.hpp b/src/hotspot/share/opto/vtransform.hpp index 397d712366b..fb76a3b89d6 100644 --- a/src/hotspot/share/opto/vtransform.hpp +++ b/src/hotspot/share/opto/vtransform.hpp @@ -260,6 +260,32 @@ private: void apply_vectorization() const; }; +// Keeps track of the state during "VTransform::apply" +// -> keep track of the already transformed nodes +class VTransformApplyState : public StackObj { +private: + const VLoopAnalyzer& _vloop_analyzer; + + // We keep track of the resulting Nodes from every "VTransformNode::apply" call. + // Since "apply" is called on defs before uses, this allows us to find the + // generated def (input) nodes when we are generating the use nodes in "apply". + GrowableArray _vtnode_idx_to_transformed_node; + +public: + VTransformApplyState(const VLoopAnalyzer& vloop_analyzer, int num_vtnodes) : + _vloop_analyzer(vloop_analyzer), + _vtnode_idx_to_transformed_node(num_vtnodes, num_vtnodes, nullptr) + { + } + + const VLoop& vloop() const { return _vloop_analyzer.vloop(); } + PhaseIdealLoop* phase() const { return vloop().phase(); } + const VLoopAnalyzer& vloop_analyzer() const { return _vloop_analyzer; } + + void set_transformed_node(VTransformNode* vtn, Node* n); + Node* transformed_node(const VTransformNode* vtn) const; +}; + // The vtnodes (VTransformNode) resemble the C2 IR Nodes, and model a part of the // VTransform. Many such vtnodes make up the VTransformGraph. The vtnodes represent // the resulting scalar and vector nodes as closely as possible. @@ -410,12 +436,11 @@ public: virtual bool is_load_or_store_in_loop() const { return false; } virtual const VPointer& vpointer(const VLoopAnalyzer& vloop_analyzer) const { ShouldNotReachHere(); } - virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const = 0; + virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const = 0; Node* find_transformed_input(int i, const GrowableArray& vnode_idx_to_transformed_node) const; - void register_new_node_from_vectorization(const VLoopAnalyzer& vloop_analyzer, Node* vn, Node* old_node) const; + void register_new_node_from_vectorization(VTransformApplyState& apply_state, Node* vn, Node* old_node) const; NOT_PRODUCT(virtual const char* name() const = 0;) NOT_PRODUCT(void print() const;) @@ -435,8 +460,7 @@ public: virtual bool is_load_in_loop() const override { return _node->is_Load(); } virtual bool is_load_or_store_in_loop() const override { return _node->is_Load() || _node->is_Store(); } virtual const VPointer& vpointer(const VLoopAnalyzer& vloop_analyzer) const override { return vloop_analyzer.vpointers().vpointer(node()->as_Mem()); } - virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const override; + virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override; NOT_PRODUCT(virtual const char* name() const override { return "Scalar"; };) NOT_PRODUCT(virtual void print_spec() const override;) }; @@ -462,8 +486,7 @@ private: public: VTransformReplicateNode(VTransform& vtransform, int vlen, BasicType element_type) : VTransformNode(vtransform, 2), _vlen(vlen), _element_type(element_type) {} - virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const override; + virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override; NOT_PRODUCT(virtual const char* name() const override { return "Replicate"; };) NOT_PRODUCT(virtual void print_spec() const override;) }; @@ -472,8 +495,7 @@ public: class VTransformConvI2LNode : public VTransformNode { public: VTransformConvI2LNode(VTransform& vtransform) : VTransformNode(vtransform, 2) {} - virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const override; + virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override; NOT_PRODUCT(virtual const char* name() const override { return "ConvI2L"; };) }; @@ -487,8 +509,7 @@ private: public: VTransformShiftCountNode(VTransform& vtransform, int vlen, BasicType element_bt, juint mask, int shift_opcode) : VTransformNode(vtransform, 2), _vlen(vlen), _element_bt(element_bt), _mask(mask), _shift_opcode(shift_opcode) {} - virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const override; + virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override; NOT_PRODUCT(virtual const char* name() const override { return "ShiftCount"; };) NOT_PRODUCT(virtual void print_spec() const override;) }; @@ -501,8 +522,7 @@ private: public: VTransformPopulateIndexNode(VTransform& vtransform, int vlen, const BasicType element_bt) : VTransformNode(vtransform, 2), _vlen(vlen), _element_bt(element_bt) {} - virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const override; + virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override; NOT_PRODUCT(virtual const char* name() const override { return "PopulateIndex"; };) NOT_PRODUCT(virtual void print_spec() const override;) }; @@ -523,7 +543,7 @@ public: const GrowableArray& nodes() const { return _nodes; } virtual VTransformVectorNode* isa_Vector() override { return this; } - void register_new_node_from_vectorization_and_replace_scalar_nodes(const VLoopAnalyzer& vloop_analyzer, Node* vn) const; + void register_new_node_from_vectorization_and_replace_scalar_nodes(VTransformApplyState& apply_state, Node* vn) const; NOT_PRODUCT(virtual void print_spec() const override;) }; @@ -533,8 +553,7 @@ public: VTransformElementWiseVectorNode(VTransform& vtransform, uint req, uint number_of_nodes) : VTransformVectorNode(vtransform, req, number_of_nodes) {} virtual VTransformElementWiseVectorNode* isa_ElementWiseVector() override { return this; } - virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const override; + virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override; NOT_PRODUCT(virtual const char* name() const override { return "ElementWiseVector"; };) }; @@ -554,8 +573,7 @@ public: VTransformElementWiseVectorNode(vtransform, 2, number_of_nodes), _test(test) {} VTransformBoolTest test() const { return _test; } virtual VTransformBoolVectorNode* isa_BoolVector() override { return this; } - virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const override; + virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override; NOT_PRODUCT(virtual const char* name() const override { return "BoolVector"; };) }; @@ -565,8 +583,7 @@ public: VTransformReductionVectorNode(VTransform& vtransform, uint number_of_nodes) : VTransformVectorNode(vtransform, 3, number_of_nodes) {} virtual VTransformReductionVectorNode* isa_ReductionVector() override { return this; } - virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const override; + virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override; NOT_PRODUCT(virtual const char* name() const override { return "ReductionVector"; };) }; @@ -592,8 +609,7 @@ public: LoadNode::ControlDependency control_dependency() const; virtual VTransformLoadVectorNode* isa_LoadVector() override { return this; } virtual bool is_load_in_loop() const override { return true; } - virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const override; + virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override; NOT_PRODUCT(virtual const char* name() const override { return "LoadVector"; };) }; @@ -604,8 +620,7 @@ public: VTransformMemVectorNode(vtransform, 4, number_of_nodes, vpointer) {} virtual VTransformStoreVectorNode* isa_StoreVector() override { return this; } virtual bool is_load_in_loop() const override { return false; } - virtual VTransformApplyResult apply(const VLoopAnalyzer& vloop_analyzer, - const GrowableArray& vnode_idx_to_transformed_node) const override; + virtual VTransformApplyResult apply(VTransformApplyState& apply_state) const override; NOT_PRODUCT(virtual const char* name() const override { return "StoreVector"; };) };