diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index 99742a598e8..6e537054643 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -1309,19 +1309,6 @@ Node *PhaseIdealLoop::clone_up_backedge_goo(Node *back_ctrl, Node *preheader_ctr return n; } -Node* PhaseIdealLoop::cast_incr_before_loop(Node* incr, Node* ctrl, Node* loop) { - Node* castii = new CastIINode(ctrl, incr, TypeInt::INT, ConstraintCastNode::UnconditionalDependency); - register_new_node(castii, ctrl); - for (DUIterator_Fast imax, i = incr->fast_outs(imax); i < imax; i++) { - Node* n = incr->fast_out(i); - if (n->is_Phi() && n->in(0) == loop) { - int nrep = n->replace_edge(incr, castii, &_igvn); - return castii; - } - } - return nullptr; -} - #ifdef ASSERT void PhaseIdealLoop::ensure_zero_trip_guard_proj(Node* node, bool is_main_loop) { assert(node->is_IfProj(), "must be the zero trip guard If node"); @@ -1680,14 +1667,11 @@ void PhaseIdealLoop::insert_pre_post_loops(IdealLoopTree *loop, Node_List &old_n // variable value and the induction variable Phi to preserve correct // dependencies. - // CastII for the main loop: - Node* castii = cast_incr_before_loop(pre_incr, min_taken, main_head); - assert(castii != nullptr, "no castII inserted"); assert(post_head->in(1)->is_IfProj(), "must be zero-trip guard If node projection of the post loop"); - copy_assertion_predicates_to_main_loop(pre_head, castii, stride, outer_loop, outer_main_head, dd_main_head, + copy_assertion_predicates_to_main_loop(pre_head, pre_incr, stride, outer_loop, outer_main_head, dd_main_head, idx_before_pre_post, idx_after_post_before_pre, min_taken, post_head->in(1), old_new); - copy_assertion_predicates_to_post_loop(outer_main_head, post_head, post_incr, stride); + copy_assertion_predicates_to_post_loop(outer_main_head, post_head, stride); // Step B4: Shorten the pre-loop to run only 1 iteration (for now). // RCE and alignment may change this later. @@ -1812,7 +1796,7 @@ void PhaseIdealLoop::insert_vector_post_loop(IdealLoopTree *loop, Node_List &old // In this case we throw away the result as we are not using it to connect anything else. CountedLoopNode *post_head = nullptr; insert_post_loop(loop, old_new, main_head, main_end, incr, limit, post_head); - copy_assertion_predicates_to_post_loop(main_head->skip_strip_mined(), post_head, incr, main_head->stride()); + copy_assertion_predicates_to_post_loop(main_head->skip_strip_mined(), post_head, main_head->stride()); // It's difficult to be precise about the trip-counts // for post loops. They are usually very short, @@ -1915,10 +1899,6 @@ Node *PhaseIdealLoop::insert_post_loop(IdealLoopTree* loop, Node_List& old_new, } } - // CastII for the new post loop: - incr = cast_incr_before_loop(zer_opaq->in(1), zer_taken, post_head); - assert(incr != nullptr, "no castII inserted"); - return new_main_exit; } @@ -1934,12 +1914,6 @@ bool IdealLoopTree::is_invariant(Node* n) const { // to the new stride. void PhaseIdealLoop::update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, const int stride_con) { - if (init->is_CastII()) { - // skip over the cast added by PhaseIdealLoop::cast_incr_before_loop() when pre/post/main loops are created because - // it can get in the way of type propagation - assert(init->as_CastII()->carry_dependency() && loop_head->skip_assertion_predicates_with_halt() == init->in(0), "casted iv phi from pre loop expected"); - init = init->in(1); - } Node* entry = ctrl; Node* prev_proj = ctrl; LoopNode* outer_loop_head = loop_head->skip_strip_mined(); @@ -1988,7 +1962,9 @@ void PhaseIdealLoop::update_main_loop_assertion_predicates(Node* ctrl, CountedLo // Go over the Assertion Predicates of the main loop and make a copy for the post loop with its initial iv value and // stride as inputs. void PhaseIdealLoop::copy_assertion_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head, - Node* init, Node* stride) { + Node* stride) { + Node* opaq = post_loop_head->is_canonical_loop_entry(); + Node* init = opaq->in(1); Node* post_loop_entry = post_loop_head->in(LoopNode::EntryControl); Node* main_loop_entry = main_loop_head->in(LoopNode::EntryControl); IdealLoopTree* post_loop = get_loop(post_loop_head); diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 1217caa41eb..369f446adbb 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -938,8 +938,6 @@ private: return ctrl; } - Node* cast_incr_before_loop(Node* incr, Node* ctrl, Node* loop); - #ifdef ASSERT void ensure_zero_trip_guard_proj(Node* node, bool is_main_loop); #endif @@ -960,7 +958,7 @@ private: static bool assertion_predicate_has_loop_opaque_node(IfNode* iff); static void get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false); void update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con); - void copy_assertion_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head, Node* init, + void copy_assertion_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head, Node* stride); void initialize_assertion_predicates_for_peeled_loop(const PredicateBlock* predicate_block, LoopNode* outer_loop_head, int dd_outer_loop_head, Node* init, Node* stride, @@ -1534,6 +1532,8 @@ public: // Attempt to use a conditional move instead of a phi/branch Node *conditional_move( Node *n ); + bool split_thru_phi_could_prevent_vectorization(Node* n, Node* n_blk); + // Check for aggressive application of 'split-if' optimization, // using basic block level info. void split_if_with_blocks ( VectorSet &visited, Node_Stack &nstack); diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index b838b15032f..7f42d2d4beb 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -1089,6 +1089,25 @@ void PhaseIdealLoop::try_move_store_after_loop(Node* n) { } } +// Split some nodes that take a counted loop phi as input at a counted +// loop can cause vectorization of some expressions to fail +bool PhaseIdealLoop::split_thru_phi_could_prevent_vectorization(Node* n, Node* n_blk) { + if (!n_blk->is_CountedLoop()) { + return false; + } + + int opcode = n->Opcode(); + + if (opcode != Op_AndI && + opcode != Op_MulI && + opcode != Op_RotateRight && + opcode != Op_RShiftI) { + return false; + } + + return n->in(1) == n_blk->as_BaseCountedLoop()->phi(); +} + //------------------------------split_if_with_blocks_pre----------------------- // Do the real work in a non-recursive function. Data nodes want to be // cloned in the pre-order so they can feed each other nicely. @@ -1175,6 +1194,10 @@ Node *PhaseIdealLoop::split_if_with_blocks_pre( Node *n ) { return n; } + if (split_thru_phi_could_prevent_vectorization(n, n_blk)) { + return n; + } + // Check for having no control input; not pinned. Allow // dominating control. if (n->in(0)) {