diff --git a/src/hotspot/share/opto/cfgnode.hpp b/src/hotspot/share/opto/cfgnode.hpp index 869ac68bd9e..de37f4cdd65 100644 --- a/src/hotspot/share/opto/cfgnode.hpp +++ b/src/hotspot/share/opto/cfgnode.hpp @@ -493,6 +493,11 @@ class ParsePredicateNode : public IfNode { _useless = false; } + // Return the uncommon trap If projection of this Parse Predicate. + ParsePredicateUncommonProj* uncommon_proj() const { + return proj_out(0)->as_IfFalse(); + } + Node* uncommon_trap() const; Node* Ideal(PhaseGVN* phase, bool can_reshape) { diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp index 278f636d9c7..9fcfbaf5e64 100644 --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -99,40 +99,38 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred, // We will create a region to guard the uct call if there is no one there. // The continuation projection (if_cont) of the new_iff is returned which // is an IfTrue projection. This code is also used to clone predicates to cloned loops. -IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, - Deoptimization::DeoptReason reason, +IfTrueNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_success_proj, + Node* new_entry, const Deoptimization::DeoptReason reason, const int opcode, const bool rewire_uncommon_proj_phi_inputs) { - assert(parse_predicate_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!"); - ParsePredicateNode* parse_predicate = parse_predicate_proj->in(0)->as_ParsePredicate(); - - ProjNode* uncommon_proj = parse_predicate->proj_out(false); - Node* uct_region = uncommon_proj->unique_ctrl_out(); - assert(uct_region->is_Region() || uct_region->is_Call(), "must be a region or call uct"); + assert(parse_predicate_success_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!"); + ParsePredicateNode* parse_predicate = parse_predicate_success_proj->in(0)->as_ParsePredicate(); + ParsePredicateUncommonProj* uncommon_proj = parse_predicate->uncommon_proj(); + Node* uncommon_trap = parse_predicate->uncommon_trap(); uint proj_index = 1; // region's edge corresponding to uncommon_proj - if (!uct_region->is_Region()) { // create a region to guard the call - assert(uct_region->is_Call(), "must be call uct"); - CallNode* call = uct_region->as_Call(); + if (!uncommon_trap->is_Region()) { // create a region to guard the call + assert(uncommon_trap->is_Call(), "must be call uct"); + CallNode* call = uncommon_trap->as_Call(); IdealLoopTree* loop = get_loop(call); - uct_region = new RegionNode(1); + uncommon_trap = new RegionNode(1); Node* uncommon_proj_orig = uncommon_proj; - uncommon_proj = uncommon_proj->clone()->as_Proj(); + uncommon_proj = uncommon_proj->clone()->as_IfFalse(); register_control(uncommon_proj, loop, parse_predicate); - uct_region->add_req(uncommon_proj); - register_control(uct_region, loop, uncommon_proj); - _igvn.replace_input_of(call, 0, uct_region); + uncommon_trap->add_req(uncommon_proj); + register_control(uncommon_trap, loop, uncommon_proj); + _igvn.replace_input_of(call, 0, uncommon_trap); // When called from beautify_loops() idom is not constructed yet. if (_idom != nullptr) { - set_idom(call, uct_region, dom_depth(uct_region)); + set_idom(call, uncommon_trap, dom_depth(uncommon_trap)); } // Move nodes pinned on the projection or whose control is set to // the projection to the region. - lazy_replace(uncommon_proj_orig, uct_region); + lazy_replace(uncommon_proj_orig, uncommon_trap); } else { // Find region's edge corresponding to uncommon_proj - for (; proj_index < uct_region->req(); proj_index++) - if (uct_region->in(proj_index) == uncommon_proj) break; - assert(proj_index < uct_region->req(), "sanity"); + for (; proj_index < uncommon_trap->req(); proj_index++) + if (uncommon_trap->in(proj_index) == uncommon_proj) break; + assert(proj_index < uncommon_trap->req(), "sanity"); } Node* entry = parse_predicate->in(0); @@ -157,22 +155,23 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessPro fatal("no other If variant here"); } register_control(new_iff, lp, entry); - IfProjNode* if_cont = new IfTrueNode(new_iff); - IfProjNode* if_uct = new IfFalseNode(new_iff); + + IfTrueNode* if_cont = new IfTrueNode(new_iff); + IfFalseNode* if_uct = new IfFalseNode(new_iff); register_control(if_cont, lp, new_iff); - register_control(if_uct, get_loop(uct_region), new_iff); + register_control(if_uct, get_loop(uncommon_trap), new_iff); - _igvn.add_input_to(uct_region, if_uct); + _igvn.add_input_to(uncommon_trap, if_uct); // If rgn has phis add new edges which has the same // value as on original uncommon_proj pass. - assert(uct_region->in(uct_region->req() - 1) == if_uct, "new edge should be last"); + assert(uncommon_trap->in(uncommon_trap->req() - 1) == if_uct, "new edge should be last"); bool has_phi = false; - for (DUIterator_Fast imax, i = uct_region->fast_outs(imax); i < imax; i++) { - Node* use = uct_region->fast_out(i); + for (DUIterator_Fast imax, i = uncommon_trap->fast_outs(imax); i < imax; i++) { + Node* use = uncommon_trap->fast_out(i); if (use->is_Phi() && use->outcnt() > 0) { - assert(use->in(0) == uct_region, ""); + assert(use->in(0) == uncommon_trap, ""); _igvn.rehash_node_delayed(use); Node* phi_input = use->in(proj_index); @@ -193,7 +192,7 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessPro has_phi = true; } } - assert(!has_phi || uct_region->req() > 3, "no phis when region is created"); + assert(!has_phi || uncommon_trap->req() > 3, "no phis when region is created"); if (new_entry == nullptr) { // Attach if_cont to iff @@ -205,12 +204,12 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessPro // When called from beautify_loops() idom is not constructed yet. if (_idom != nullptr) { - Node* ridom = idom(uct_region); + Node* ridom = idom(uncommon_trap); Node* nrdom = dom_lca_internal(ridom, new_iff); - set_idom(uct_region, nrdom, dom_depth(uct_region)); + set_idom(uncommon_trap, nrdom, dom_depth(uncommon_trap)); } - return if_cont->as_IfProj(); + return if_cont; } // Update ctrl and control inputs of all data nodes starting from 'node' to 'new_ctrl' which have 'old_ctrl' as @@ -757,8 +756,8 @@ bool IdealLoopTree::is_range_check_if(IfProjNode* if_success_proj, PhaseIdealLoo // max(scale*i + offset) = scale*(limit-stride) + offset // (2) stride*scale < 0 // max(scale*i + offset) = scale*init + offset -BoolNode* PhaseIdealLoop::rc_predicate(IdealLoopTree* loop, Node* ctrl, int scale, Node* offset, Node* init, - Node* limit, jint stride, Node* range, bool upper, bool& overflow) { +BoolNode* PhaseIdealLoop::rc_predicate(Node* ctrl, const int scale, Node* offset, Node* init, Node* limit, + const jint stride, Node* range, const bool upper, bool& overflow) { jint con_limit = (limit != nullptr && limit->is_Con()) ? limit->get_int() : 0; jint con_init = init->is_Con() ? init->get_int() : 0; jint con_offset = offset->is_Con() ? offset->get_int() : 0; @@ -1165,18 +1164,19 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod reason, iff->Opcode()); Node* ctrl = new_predicate_proj->in(0)->as_If()->in(0); - BoolNode* new_predicate_bol = invar.clone(bol, ctrl)->as_Bool(); + BoolNode* hoisted_check_predicate_bool = invar.clone(bol, ctrl)->as_Bool(); // Negate test if necessary (Parse Predicates always have IfTrue as success projection and IfFalse as uncommon trap) bool negated = false; if (if_success_proj->is_IfFalse()) { - new_predicate_bol = new BoolNode(new_predicate_bol->in(1), new_predicate_bol->_test.negate()); - register_new_node(new_predicate_bol, ctrl); + hoisted_check_predicate_bool = new BoolNode(hoisted_check_predicate_bool->in(1), + hoisted_check_predicate_bool->_test.negate()); + register_new_node(hoisted_check_predicate_bool, ctrl); negated = true; } IfNode* new_predicate_iff = new_predicate_proj->in(0)->as_If(); _igvn.hash_delete(new_predicate_iff); - new_predicate_iff->set_req(1, new_predicate_bol); + new_predicate_iff->set_req(1, hoisted_check_predicate_bool); C->print_method(PHASE_AFTER_LOOP_PREDICATION_IC, 4, new_predicate_proj->in(0)); @@ -1220,7 +1220,8 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod // Perform cloning to keep Invariance state correct since the // late schedule will place invariant things in the loop. - Node* ctrl = parse_predicate_proj->in(0)->as_If()->in(0); + ParsePredicateNode* parse_predicate = parse_predicate_proj->in(0)->as_ParsePredicate(); + Node* ctrl = parse_predicate->in(0); rng = invar.clone(rng, ctrl); if (offset && offset != zero) { assert(invar.is_invariant(offset), "offset must be loop invariant"); @@ -1229,25 +1230,28 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod // If predicate expressions may overflow in the integer range, longs are used. bool overflow = false; // Test the lower bound - BoolNode* lower_bound_bol = rc_predicate(loop, ctrl, scale, offset, init, limit, stride, rng, false, overflow); + BoolNode* lower_bound_bol = rc_predicate(ctrl, scale, offset, init, limit, stride, rng, false, overflow); const int if_opcode = iff->Opcode(); IfProjNode* lower_bound_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : if_opcode); IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If(); _igvn.hash_delete(lower_bound_iff); lower_bound_iff->set_req(1, lower_bound_bol); - if (TraceLoopPredicate) tty->print_cr("lower bound check if: %d", lower_bound_iff->_idx); + if (TraceLoopPredicate) { + tty->print_cr("lower bound check if: %d", lower_bound_iff->_idx); + } // Test the upper bound - BoolNode* upper_bound_bol = rc_predicate(loop, lower_bound_proj, scale, offset, init, limit, stride, rng, true, - overflow); + BoolNode* upper_bound_bol = rc_predicate(lower_bound_proj, scale, offset, init, limit, stride, rng, true, overflow); IfProjNode* upper_bound_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : if_opcode); assert(upper_bound_proj->in(0)->as_If()->in(0) == lower_bound_proj, "should dominate"); IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If(); _igvn.hash_delete(upper_bound_iff); upper_bound_iff->set_req(1, upper_bound_bol); - if (TraceLoopPredicate) tty->print_cr("upper bound check if: %d", upper_bound_iff->_idx); + if (TraceLoopPredicate) { + tty->print_cr("upper bound check if: %d", upper_bound_iff->_idx); + } // Fall through into rest of the cleanup code which will move any dependent nodes to the skeleton predicates of the // upper bound test. We always need to create skeleton predicates in order to properly remove dead loops when later @@ -1294,7 +1298,7 @@ IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealL Node* opaque_init = new OpaqueLoopInitNode(C, init); register_new_node(opaque_init, upper_bound_proj); bool negate = (if_proj->_con != parse_predicate_proj->_con); - BoolNode* bol = rc_predicate(loop, upper_bound_proj, scale, offset, opaque_init, limit, stride, rng, + BoolNode* bol = rc_predicate(upper_bound_proj, scale, offset, opaque_init, limit, stride, rng, (stride > 0) != (scale > 0), overflow); Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); // This will go away once loop opts are over C->add_template_assertion_predicate_opaq(opaque_bol); @@ -1317,7 +1321,7 @@ IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealL max_value = new CastIINode(max_value, loop->_head->as_CountedLoop()->phi()->bottom_type()); register_new_node(max_value, parse_predicate_proj); - bol = rc_predicate(loop, new_proj, scale, offset, max_value, limit, stride, rng, (stride > 0) != (scale > 0), + bol = rc_predicate(new_proj, scale, offset, max_value, limit, stride, rng, (stride > 0) != (scale > 0), overflow); opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); C->add_template_assertion_predicate_opaq(opaque_bol); @@ -1340,10 +1344,6 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree* loop) { return false; } - if (head->is_OuterStripMinedLoop()) { - return false; - } - CountedLoopNode *cl = nullptr; if (head->is_valid_counted_loop(T_INT)) { cl = head->as_CountedLoop(); @@ -1514,19 +1514,20 @@ bool PhaseIdealLoop::can_create_loop_predicates(const PredicateBlock* profiled_l //------------------------------loop_predication-------------------------------- // driver routine for loop predication optimization -bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) { +bool IdealLoopTree::loop_predication(PhaseIdealLoop* phase) { bool hoisted = false; // Recursively promote predicates if (_child) { hoisted = _child->loop_predication( phase); } - // self + // Self if (can_apply_loop_predication()) { hoisted |= phase->loop_predication_impl(this); } - if (_next) { //sibling + // Sibling + if (_next) { hoisted |= _next->loop_predication( phase); } @@ -1534,5 +1535,9 @@ bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) { } bool IdealLoopTree::can_apply_loop_predication() { - return _head->is_Loop() && !_irreducible && !tail()->is_top(); + return !_head->is_Root() && + _head->is_Loop() && + !_head->is_OuterStripMinedLoop() && + !_irreducible && + !tail()->is_top(); } diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index 24861cf2725..f2022d35cf6 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -2772,7 +2772,7 @@ Node* PhaseIdealLoop::add_range_check_elimination_assertion_predicate(IdealLoopT Node* offset, Node* limit, jint stride_con, Node* value) { bool overflow = false; - BoolNode* bol = rc_predicate(loop, ctrl, scale_con, offset, value, nullptr, stride_con, + BoolNode* bol = rc_predicate(ctrl, scale_con, offset, value, nullptr, stride_con, limit, (stride_con > 0) != (scale_con > 0), overflow); Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); register_new_node(opaque_bol, ctrl); diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index fcd0e1a4df2..fac7cf32f60 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4320,7 +4320,7 @@ void PhaseIdealLoop::mark_loop_associated_parse_predicates_useful() { } void PhaseIdealLoop::mark_useful_parse_predicates_for_loop(IdealLoopTree* loop) { - Node* entry = loop->_head->in(LoopNode::EntryControl); + Node* entry = loop->_head->as_Loop()->skip_strip_mined()->in(LoopNode::EntryControl); const Predicates predicates(entry); ParsePredicateIterator iterator(predicates); while (iterator.has_next()) { @@ -4359,7 +4359,7 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates(Unique_Node_Li void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(IdealLoopTree* loop, Unique_Node_List &useful_predicates) { - Node* entry = loop->_head->in(LoopNode::EntryControl); + Node* entry = loop->_head->as_Loop()->skip_strip_mined()->in(LoopNode::EntryControl); const Predicates predicates(entry); if (UseProfiledLoopPredicate) { const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block(); diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 75276aae36d..f7ba4578a51 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1337,7 +1337,7 @@ public: bool* p_short_scale, int depth); // Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted - IfProjNode* create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, + IfTrueNode* create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason, int opcode, bool rewire_uncommon_proj_phi_inputs = false); @@ -1360,7 +1360,7 @@ public: } // Construct a range check for a predicate if - BoolNode* rc_predicate(IdealLoopTree* loop, Node* ctrl, int scale, Node* offset, Node* init, Node* limit, + BoolNode* rc_predicate(Node* ctrl, int scale, Node* offset, Node* init, Node* limit, jint stride, Node* range, bool upper, bool& overflow); // Implementation of the loop predication to promote checks outside the loop