diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp index 71068f76043..21e6a068de1 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp @@ -734,7 +734,7 @@ Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { Node* u = n->fast_out(j); if (u->is_CFG()) { - if (!wq.member(u) && !u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) { + if (!wq.member(u) && !u->as_Proj()->is_uncommon_trap_proj()) { return NodeSentinel; } } @@ -743,7 +743,7 @@ Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one } } else if (c->is_Proj()) { if (c->is_IfProj()) { - if (c->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) != nullptr) { + if (c->as_Proj()->is_uncommon_trap_if_pattern() != nullptr) { // continue; } else { if (!allow_one_proj) { @@ -1138,7 +1138,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { if (u_t->meet(TypePtr::NULL_PTR) != u_t && u->in(0)->Opcode() == Op_IfTrue && - u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && + u->in(0)->as_Proj()->is_uncommon_trap_if_pattern() && u->in(0)->in(0)->is_If() && u->in(0)->in(0)->in(1)->Opcode() == Op_Bool && u->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && diff --git a/src/hotspot/share/opto/cfgnode.hpp b/src/hotspot/share/opto/cfgnode.hpp index 9fc68f56309..e0a640aae94 100644 --- a/src/hotspot/share/opto/cfgnode.hpp +++ b/src/hotspot/share/opto/cfgnode.hpp @@ -463,8 +463,9 @@ public: // More information about predicates can be found in loopPredicate.cpp. class ParsePredicateNode : public IfNode { Deoptimization::DeoptReason _deopt_reason; + bool _useless; // If the associated loop dies, this parse predicate becomes useless and can be cleaned up by Value(). public: - ParsePredicateNode(Node* control, Node* bol, Deoptimization::DeoptReason deopt_reason); + ParsePredicateNode(Node* control, Deoptimization::DeoptReason deopt_reason, PhaseGVN* gvn); virtual int Opcode() const; virtual uint size_of() const { return sizeof(*this); } @@ -472,8 +473,25 @@ class ParsePredicateNode : public IfNode { return _deopt_reason; } + bool is_useless() const { + return _useless; + } + + void mark_useless() { + _useless = true; + } + + void mark_useful() { + _useless = false; + } + Node* uncommon_trap() const; + Node* Ideal(PhaseGVN* phase, bool can_reshape) { + return nullptr; // Don't optimize + } + + const Type* Value(PhaseGVN* phase) const; NOT_PRODUCT(void dump_spec(outputStream* st) const;) }; diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index d9628459cf8..b715936f237 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -360,10 +360,11 @@ void Compile::remove_useless_late_inlines(GrowableArray* inlines assert(found <= 1, "not unique"); } -void Compile::remove_useless_nodes(GrowableArray& node_list, Unique_Node_List& useful) { +template::value)> +void Compile::remove_useless_nodes(GrowableArray& node_list, Unique_Node_List& useful) { for (int i = node_list.length() - 1; i >= 0; i--) { - Node* n = node_list.at(i); - if (!useful.member(n)) { + N* node = node_list.at(i); + if (!useful.member(node)) { node_list.delete_at(i); // replaces i-th with last element which is known to be useful (already processed) } } @@ -389,6 +390,9 @@ void Compile::remove_useless_node(Node* dead) { if (dead->Opcode() == Op_Opaque4) { remove_template_assertion_predicate_opaq(dead); } + if (dead->is_ParsePredicate()) { + remove_parse_predicate(dead->as_ParsePredicate()); + } if (dead->for_post_loop_opts_igvn()) { remove_from_post_loop_opts_igvn(dead); } @@ -436,7 +440,7 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis } remove_useless_nodes(_macro_nodes, useful); // remove useless macro nodes - remove_useless_nodes(_parse_predicate_opaqs, useful); // remove useless Parse Predicate opaque nodes + remove_useless_nodes(_parse_predicates, useful); // remove useless Parse Predicate nodes remove_useless_nodes(_template_assertion_predicate_opaqs, useful); // remove useless Assertion Predicate opaque nodes remove_useless_nodes(_expensive_nodes, useful); // remove useless expensive nodes remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass @@ -631,7 +635,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, _failure_reason(nullptr), _intrinsics (comp_arena(), 0, 0, nullptr), _macro_nodes (comp_arena(), 8, 0, nullptr), - _parse_predicate_opaqs (comp_arena(), 8, 0, nullptr), + _parse_predicates (comp_arena(), 8, 0, nullptr), _template_assertion_predicate_opaqs (comp_arena(), 8, 0, nullptr), _expensive_nodes (comp_arena(), 8, 0, nullptr), _for_post_loop_igvn(comp_arena(), 8, 0, nullptr), @@ -1807,18 +1811,18 @@ bool Compile::can_alias(const TypePtr* adr_type, int alias_idx) { return adr_idx == alias_idx; } -// Remove the opaque nodes that protect the Parse Predicates so that all unused -// checks and uncommon_traps will be eliminated from the ideal graph. -void Compile::cleanup_parse_predicates(PhaseIterGVN& igvn) const { +// Mark all ParsePredicateNodes as useless. They will later be removed from the graph in IGVN together with their +// uncommon traps if no Runtime Predicates were created from the Parse Predicates. +void Compile::mark_parse_predicate_nodes_useless(PhaseIterGVN& igvn) { if (parse_predicate_count() == 0) { return; } - for (int i = parse_predicate_count(); i > 0; i--) { - Node* n = parse_predicate_opaque1_node(i - 1); - assert(n->Opcode() == Op_Opaque1, "must be"); - igvn.replace_node(n, n->in(1)); + for (int i = 0; i < parse_predicate_count(); i++) { + ParsePredicateNode* parse_predicate = _parse_predicates.at(i); + parse_predicate->mark_useless(); + igvn._worklist.push(parse_predicate); } - assert(parse_predicate_count() == 0, "should be clean!"); + _parse_predicates.clear(); } void Compile::record_for_post_loop_opts_igvn(Node* n) { @@ -1851,6 +1855,7 @@ void Compile::process_for_post_loop_opts_igvn(PhaseIterGVN& igvn) { } igvn.optimize(); assert(_for_post_loop_igvn.length() == 0, "no more delayed nodes allowed"); + assert(C->parse_predicate_count() == 0, "all parse predicates should have been removed now"); // Sometimes IGVN sets major progress (e.g., when processing loop nodes). if (C->major_progress()) { diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index a8d7f78942c..1eb021feebf 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -69,6 +69,7 @@ class Node_Notes; class NodeHash; class NodeCloneInfo; class OptoReg; +class ParsePredicateNode; class PhaseCFG; class PhaseGVN; class PhaseIterGVN; @@ -357,7 +358,7 @@ class Compile : public Phase { const char* _failure_reason; // for record_failure/failing pattern GrowableArray _intrinsics; // List of intrinsics. GrowableArray _macro_nodes; // List of nodes which need to be expanded before matching. - GrowableArray _parse_predicate_opaqs; // List of Opaque1 nodes for the Parse Predicates. + GrowableArray _parse_predicates; // List of Parse Predicates. GrowableArray _template_assertion_predicate_opaqs; // List of Opaque4 nodes for Template Assertion Predicates. GrowableArray _expensive_nodes; // List of nodes that are expensive to compute and that we'd better not let the GVN freely common GrowableArray _for_post_loop_igvn; // List of nodes for IGVN after loop opts are over @@ -703,13 +704,13 @@ private: #endif int macro_count() const { return _macro_nodes.length(); } - int parse_predicate_count() const { return _parse_predicate_opaqs.length(); } + int parse_predicate_count() const { return _parse_predicates.length(); } int template_assertion_predicate_count() const { return _template_assertion_predicate_opaqs.length(); } int expensive_count() const { return _expensive_nodes.length(); } int coarsened_count() const { return _coarsened_locks.length(); } Node* macro_node(int idx) const { return _macro_nodes.at(idx); } - Node* parse_predicate_opaque1_node(int idx) const { return _parse_predicate_opaqs.at(idx); } + ParsePredicateNode* parse_predicate(int idx) const { return _parse_predicates.at(idx); } Node* template_assertion_predicate_opaq_node(int idx) const { return _template_assertion_predicate_opaqs.at(idx); @@ -728,10 +729,6 @@ private: // this function may be called twice for a node so we can only remove it // if it's still existing. _macro_nodes.remove_if_existing(n); - // remove from _parse_predicate_opaqs list also if it is there - if (parse_predicate_count() > 0) { - _parse_predicate_opaqs.remove_if_existing(n); - } // Remove from coarsened locks list if present if (coarsened_count() > 0) { remove_coarsened_lock(n); @@ -741,16 +738,24 @@ private: void remove_expensive_node(Node* n) { _expensive_nodes.remove_if_existing(n); } - void add_parse_predicate_opaq(Node* n) { - assert(!_parse_predicate_opaqs.contains(n), "duplicate entry in Parse Predicate opaque1 list"); - assert(_macro_nodes.contains(n), "should have already been in macro list"); - _parse_predicate_opaqs.append(n); + + void add_parse_predicate(ParsePredicateNode* n) { + assert(!_parse_predicates.contains(n), "duplicate entry in Parse Predicate list"); + _parse_predicates.append(n); } + + void remove_parse_predicate(ParsePredicateNode* n) { + if (parse_predicate_count() > 0) { + _parse_predicates.remove_if_existing(n); + } + } + void add_template_assertion_predicate_opaq(Node* n) { assert(!_template_assertion_predicate_opaqs.contains(n), "duplicate entry in template assertion predicate opaque4 list"); _template_assertion_predicate_opaqs.append(n); } + void remove_template_assertion_predicate_opaq(Node* n) { if (template_assertion_predicate_count() > 0) { _template_assertion_predicate_opaqs.remove_if_existing(n); @@ -775,12 +780,7 @@ private: void sort_macro_nodes(); - // Remove the opaque nodes that protect the Parse Predicates so that the unused checks and - // uncommon traps will be eliminated from the graph. - void cleanup_parse_predicates(PhaseIterGVN &igvn) const; - bool is_predicate_opaq(Node* n) const { - return _parse_predicate_opaqs.contains(n); - } + void mark_parse_predicate_nodes_useless(PhaseIterGVN& igvn); // Are there candidate expensive nodes for optimization? bool should_optimize_expensive_nodes(PhaseIterGVN &igvn); @@ -1021,7 +1021,8 @@ private: _vector_reboxing_late_inlines.push(cg); } - void remove_useless_nodes (GrowableArray& node_list, Unique_Node_List &useful); + template::value)> + void remove_useless_nodes(GrowableArray& node_list, Unique_Node_List& useful); void remove_useless_late_inlines(GrowableArray* inlines, Unique_Node_List &useful); void remove_useless_late_inlines(GrowableArray* inlines, Node* dead); diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index d9ea82c03c1..c7a2698de12 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -3996,11 +3996,7 @@ void GraphKit::add_parse_predicate(Deoptimization::DeoptReason reason, const int return; } - Node* cont = _gvn.intcon(1); - Node* opaq = _gvn.transform(new Opaque1Node(C, cont)); - C->add_parse_predicate_opaq(opaq); - Node* bol = _gvn.transform(new Conv2BNode(opaq)); - ParsePredicateNode* parse_predicate = new ParsePredicateNode(control(), bol, reason); + ParsePredicateNode* parse_predicate = new ParsePredicateNode(control(), reason, &_gvn); _gvn.set_type(parse_predicate, parse_predicate->Value(&_gvn)); Node* if_false = _gvn.transform(new IfFalseNode(parse_predicate)); { diff --git a/src/hotspot/share/opto/ifnode.cpp b/src/hotspot/share/opto/ifnode.cpp index e10b7f9c5ad..b1192825379 100644 --- a/src/hotspot/share/opto/ifnode.cpp +++ b/src/hotspot/share/opto/ifnode.cpp @@ -800,7 +800,7 @@ bool IfNode::is_dominator_unc(CallStaticJavaNode* dom_unc, CallStaticJavaNode* u // Return projection that leads to an uncommon trap if any ProjNode* IfNode::uncommon_trap_proj(CallStaticJavaNode*& call) const { for (int i = 0; i < 2; i++) { - call = proj_out(i)->is_uncommon_trap_proj(Deoptimization::Reason_none); + call = proj_out(i)->is_uncommon_trap_proj(); if (call != nullptr) { return proj_out(i); } @@ -811,7 +811,7 @@ ProjNode* IfNode::uncommon_trap_proj(CallStaticJavaNode*& call) const { // Do this If and the dominating If both branch out to an uncommon trap bool IfNode::has_only_uncommon_traps(ProjNode* proj, ProjNode*& success, ProjNode*& fail, PhaseIterGVN* igvn) { ProjNode* otherproj = proj->other_if_proj(); - CallStaticJavaNode* dom_unc = otherproj->is_uncommon_trap_proj(Deoptimization::Reason_none); + CallStaticJavaNode* dom_unc = otherproj->is_uncommon_trap_proj(); if (otherproj->outcnt() == 1 && dom_unc != nullptr) { // We need to re-execute the folded Ifs after deoptimization from the merged traps @@ -1076,8 +1076,8 @@ Node* IfNode::merge_uncommon_traps(ProjNode* proj, ProjNode* success, ProjNode* ProjNode* otherproj = proj->other_if_proj(); - CallStaticJavaNode* unc = success->is_uncommon_trap_proj(Deoptimization::Reason_none); - CallStaticJavaNode* dom_unc = otherproj->is_uncommon_trap_proj(Deoptimization::Reason_none); + CallStaticJavaNode* unc = success->is_uncommon_trap_proj(); + CallStaticJavaNode* dom_unc = otherproj->is_uncommon_trap_proj(); if (unc != dom_unc) { Node* r = new RegionNode(3); @@ -1241,13 +1241,13 @@ bool IfNode::is_side_effect_free_test(ProjNode* proj, PhaseIterGVN* igvn) { if (proj == nullptr) { return false; } - CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); + CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern(); if (unc != nullptr && proj->outcnt() <= 2) { if (proj->outcnt() == 1 || // Allow simple null check from LoadRange (is_cmp_with_loadrange(proj) && is_null_check(proj, igvn))) { - CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); - CallStaticJavaNode* dom_unc = proj->in(0)->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); + CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern(); + CallStaticJavaNode* dom_unc = proj->in(0)->in(0)->as_Proj()->is_uncommon_trap_if_pattern(); assert(dom_unc != nullptr, "is_uncommon_trap_if_pattern returned null"); // reroute_side_effect_free_unc changes the state of this @@ -1278,9 +1278,9 @@ bool IfNode::is_side_effect_free_test(ProjNode* proj, PhaseIterGVN* igvn) { // where the first CmpI would have prevented it from executing: on a // trap, we need to restart execution at the state of the first CmpI void IfNode::reroute_side_effect_free_unc(ProjNode* proj, ProjNode* dom_proj, PhaseIterGVN* igvn) { - CallStaticJavaNode* dom_unc = dom_proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); + CallStaticJavaNode* dom_unc = dom_proj->is_uncommon_trap_if_pattern(); ProjNode* otherproj = proj->other_if_proj(); - CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); + CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern(); Node* call_proj = dom_unc->unique_ctrl_out(); Node* halt = call_proj->unique_ctrl_out(); @@ -1975,11 +1975,13 @@ Node* RangeCheckNode::Ideal(PhaseGVN *phase, bool can_reshape) { return dominated_by(prev_dom, igvn); } -ParsePredicateNode::ParsePredicateNode(Node* control, Node* bol, Deoptimization::DeoptReason deopt_reason) - : IfNode(control, bol, PROB_MAX, COUNT_UNKNOWN), - _deopt_reason(deopt_reason) { +ParsePredicateNode::ParsePredicateNode(Node* control, Deoptimization::DeoptReason deopt_reason, PhaseGVN* gvn) + : IfNode(control, gvn->intcon(1), PROB_MAX, COUNT_UNKNOWN), + _deopt_reason(deopt_reason), + _useless(false) { init_class_id(Class_ParsePredicate); - assert(bol->Opcode() == Op_Conv2B && bol->in(1) != nullptr && bol->in(1)->is_Opaque1(), "wrong boolean input"); + gvn->C->add_parse_predicate(this); + gvn->C->record_for_post_loop_opts_igvn(this); #ifdef ASSERT switch (deopt_reason) { case Deoptimization::Reason_predicate: @@ -1999,6 +2001,18 @@ Node* ParsePredicateNode::uncommon_trap() const { return uct_region_or_call; } +// Fold this node away once it becomes useless or at latest in post loop opts IGVN. +const Type* ParsePredicateNode::Value(PhaseGVN* phase) const { + if (phase->type(in(0)) == Type::TOP) { + return Type::TOP; + } + if (_useless || phase->C->post_loop_opts_phase()) { + return TypeTuple::IFTRUE; + } else { + return bottom_type(); + } +} + #ifndef PRODUCT void ParsePredicateNode::dump_spec(outputStream* st) const { st->print(" #"); diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp index 6c655e6f137..d569bfcfab5 100644 --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -99,43 +99,43 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred, // We will create a region to guard the uct call if there is no one there. // The continuation projection (if_cont) of the new_iff is returned which // is an IfTrue projection. This code is also used to clone predicates to cloned loops. -IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(IfProjNode* cont_proj, Node* new_entry, +IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason, const int opcode, const bool rewire_uncommon_proj_phi_inputs) { - assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!"); - IfNode* iff = cont_proj->in(0)->as_If(); + assert(parse_predicate_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!"); + ParsePredicateNode* parse_predicate = parse_predicate_proj->in(0)->as_ParsePredicate(); - ProjNode *uncommon_proj = iff->proj_out(1 - cont_proj->_con); - Node *rgn = uncommon_proj->unique_ctrl_out(); - assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct"); + ProjNode* uncommon_proj = parse_predicate->proj_out(false); + Node* uct_region = uncommon_proj->unique_ctrl_out(); + assert(uct_region->is_Region() || uct_region->is_Call(), "must be a region or call uct"); uint proj_index = 1; // region's edge corresponding to uncommon_proj - if (!rgn->is_Region()) { // create a region to guard the call - assert(rgn->is_Call(), "must be call uct"); - CallNode* call = rgn->as_Call(); + if (!uct_region->is_Region()) { // create a region to guard the call + assert(uct_region->is_Call(), "must be call uct"); + CallNode* call = uct_region->as_Call(); IdealLoopTree* loop = get_loop(call); - rgn = new RegionNode(1); + uct_region = new RegionNode(1); Node* uncommon_proj_orig = uncommon_proj; uncommon_proj = uncommon_proj->clone()->as_Proj(); - register_control(uncommon_proj, loop, iff); - rgn->add_req(uncommon_proj); - register_control(rgn, loop, uncommon_proj); - _igvn.replace_input_of(call, 0, rgn); + register_control(uncommon_proj, loop, parse_predicate); + uct_region->add_req(uncommon_proj); + register_control(uct_region, loop, uncommon_proj); + _igvn.replace_input_of(call, 0, uct_region); // When called from beautify_loops() idom is not constructed yet. if (_idom != nullptr) { - set_idom(call, rgn, dom_depth(rgn)); + set_idom(call, uct_region, dom_depth(uct_region)); } // Move nodes pinned on the projection or whose control is set to // the projection to the region. - lazy_replace(uncommon_proj_orig, rgn); + lazy_replace(uncommon_proj_orig, uct_region); } else { // Find region's edge corresponding to uncommon_proj - for (; proj_index < rgn->req(); proj_index++) - if (rgn->in(proj_index) == uncommon_proj) break; - assert(proj_index < rgn->req(), "sanity"); + for (; proj_index < uct_region->req(); proj_index++) + if (uct_region->in(proj_index) == uncommon_proj) break; + assert(proj_index < uct_region->req(), "sanity"); } - Node* entry = iff->in(0); + Node* entry = parse_predicate->in(0); if (new_entry != nullptr) { // Cloning the predicate to new location. entry = new_entry; @@ -145,13 +145,13 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(IfProjNode* cont_proj, N IfNode* new_iff = nullptr; switch (opcode) { case Op_If: - new_iff = new IfNode(entry, iff->in(1), iff->_prob, iff->_fcnt); + new_iff = new IfNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt); break; case Op_RangeCheck: - new_iff = new RangeCheckNode(entry, iff->in(1), iff->_prob, iff->_fcnt); + new_iff = new RangeCheckNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt); break; case Op_ParsePredicate: - new_iff = new ParsePredicateNode(entry, iff->in(1), reason); + new_iff = new ParsePredicateNode(entry, reason, &_igvn); break; default: fatal("no other If variant here"); @@ -160,23 +160,19 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(IfProjNode* cont_proj, N IfProjNode* if_cont = new IfTrueNode(new_iff); IfProjNode* if_uct = new IfFalseNode(new_iff); - if (cont_proj->is_IfFalse()) { - // Swap - IfProjNode* tmp = if_uct; if_uct = if_cont; if_cont = tmp; - } register_control(if_cont, lp, new_iff); - register_control(if_uct, get_loop(rgn), new_iff); + register_control(if_uct, get_loop(uct_region), new_iff); - _igvn.add_input_to(rgn, if_uct); + _igvn.add_input_to(uct_region, if_uct); // If rgn has phis add new edges which has the same // value as on original uncommon_proj pass. - assert(rgn->in(rgn->req() -1) == if_uct, "new edge should be last"); + assert(uct_region->in(uct_region->req() - 1) == if_uct, "new edge should be last"); bool has_phi = false; - for (DUIterator_Fast imax, i = rgn->fast_outs(imax); i < imax; i++) { - Node* use = rgn->fast_out(i); + for (DUIterator_Fast imax, i = uct_region->fast_outs(imax); i < imax; i++) { + Node* use = uct_region->fast_out(i); if (use->is_Phi() && use->outcnt() > 0) { - assert(use->in(0) == rgn, ""); + assert(use->in(0) == uct_region, ""); _igvn.rehash_node_delayed(use); Node* phi_input = use->in(proj_index); @@ -197,21 +193,21 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(IfProjNode* cont_proj, N has_phi = true; } } - assert(!has_phi || rgn->req() > 3, "no phis when region is created"); + assert(!has_phi || uct_region->req() > 3, "no phis when region is created"); if (new_entry == nullptr) { // Attach if_cont to iff - _igvn.replace_input_of(iff, 0, if_cont); + _igvn.replace_input_of(parse_predicate, 0, if_cont); if (_idom != nullptr) { - set_idom(iff, if_cont, dom_depth(iff)); + set_idom(parse_predicate, if_cont, dom_depth(parse_predicate)); } } // When called from beautify_loops() idom is not constructed yet. if (_idom != nullptr) { - Node* ridom = idom(rgn); + Node* ridom = idom(uct_region); Node* nrdom = dom_lca_internal(ridom, new_iff); - set_idom(rgn, nrdom, dom_depth(rgn)); + set_idom(uct_region, nrdom, dom_depth(uct_region)); } return if_cont->as_IfProj(); @@ -303,24 +299,14 @@ void PhaseIdealLoop::rewire_inputs_of_clones_to_clones(Node* new_ctrl, Node* clo } } -IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* predicate_proj, +IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason, const bool slow_loop) { - IfProjNode* new_predicate_proj = create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_ParsePredicate, + IfProjNode* new_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, new_entry, reason, Op_ParsePredicate, slow_loop); - IfNode* iff = new_predicate_proj->in(0)->as_If(); - Node* ctrl = iff->in(0); - - // Match original condition since predicate's projections could be swapped. - assert(predicate_proj->in(0)->in(1)->in(1)->Opcode()==Op_Opaque1, "must be"); - Node* opq = new Opaque1Node(C, predicate_proj->in(0)->in(1)->in(1)->in(1)); - C->add_parse_predicate_opaq(opq); - Node* bol = new Conv2BNode(opq); - register_new_node(opq, ctrl); - register_new_node(bol, ctrl); - _igvn.hash_delete(iff); - iff->set_req(1, bol); + assert(new_predicate_proj->is_IfTrue(), "the success projection of a Parse Predicate is a true projection"); + ParsePredicateNode* parse_predicate = new_predicate_proj->in(0)->as_ParsePredicate(); return new_predicate_proj; } @@ -329,9 +315,11 @@ IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredic // cloned predicates. void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, Deoptimization::DeoptReason reason, - IfProjNode* old_predicate_proj, IfProjNode* iffast_pred, - IfProjNode* ifslow_pred) { - assert(iffast_pred->in(0)->is_If() && ifslow_pred->in(0)->is_If(), "sanity check"); + IfProjNode* old_predicate_proj, + ParsePredicateSuccessProj* fast_loop_parse_predicate_proj, + ParsePredicateSuccessProj* slow_loop_parse_predicate_proj) { + assert(fast_loop_parse_predicate_proj->in(0)->is_ParsePredicate() && + slow_loop_parse_predicate_proj->in(0)->is_ParsePredicate(), "sanity check"); // Only need to clone range check predicates as those can be changed and duplicated by inserting pre/main/post loops // and doing loop unrolling. Push the original predicates on a list to later process them in reverse order to keep the // original predicate order. @@ -350,9 +338,9 @@ void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree assert(predicate->is_Proj() && predicate->as_Proj()->is_IfProj(), "predicate must be a projection of an if node"); IfProjNode* predicate_proj = predicate->as_IfProj(); - IfProjNode* fast_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, iffast_pred); + IfProjNode* fast_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, fast_loop_parse_predicate_proj); assert(assertion_predicate_has_loop_opaque_node(fast_proj->in(0)->as_If()), "must find Assertion Predicate for fast loop"); - IfProjNode* slow_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, ifslow_pred); + IfProjNode* slow_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, slow_loop_parse_predicate_proj); assert(assertion_predicate_has_loop_opaque_node(slow_proj->in(0)->as_If()), "must find Assertion Predicate for slow loop"); // Update control dependent data nodes. @@ -378,14 +366,13 @@ void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree // Put all Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque' // is set, then the Opaque4 nodes of the Assertion Predicates are put on the list instead of the projections. void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque) { - IfNode* iff = predicate->in(0)->as_If(); - ProjNode* uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con); + ParsePredicateNode* parse_predicate = predicate->in(0)->as_ParsePredicate(); + ProjNode* uncommon_proj = parse_predicate->proj_out(1 - predicate->as_Proj()->_con); Node* rgn = uncommon_proj->unique_ctrl_out(); assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct"); - assert(iff->in(1)->in(1)->Opcode() == Op_Opaque1, "unexpected predicate shape"); - predicate = iff->in(0); + predicate = parse_predicate->in(0); while (predicate != nullptr && predicate->is_Proj() && predicate->in(0)->is_If()) { - iff = predicate->in(0)->as_If(); + IfNode* iff = predicate->in(0)->as_If(); uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con); if (uncommon_proj->unique_ctrl_out() != rgn) { break; @@ -408,12 +395,12 @@ void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List& // predicate again). IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate, Deoptimization::DeoptReason reason, - IfProjNode* output_proj) { - Node* bol = create_bool_from_template_assertion_predicate(iff, nullptr, nullptr, output_proj); - IfProjNode* if_proj = create_new_if_for_predicate(output_proj, nullptr, reason, iff->Opcode(), false); + ParsePredicateSuccessProj* parse_predicate_proj) { + Node* bol = create_bool_from_template_assertion_predicate(iff, nullptr, nullptr, parse_predicate_proj); + IfProjNode* if_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, iff->Opcode(), false); _igvn.replace_input_of(if_proj->in(0), 1, bol); - _igvn.replace_input_of(output_proj->in(0), 0, if_proj); - set_idom(output_proj->in(0), if_proj, dom_depth(if_proj)); + _igvn.replace_input_of(parse_predicate_proj->in(0), 0, if_proj); + set_idom(parse_predicate_proj->in(0), if_proj, dom_depth(if_proj)); return if_proj; } @@ -446,9 +433,13 @@ void PhaseIdealLoop::clone_loop_predication_predicates_to_unswitched_loop(IdealL IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred) { if (predicate_block->has_parse_predicate()) { + // We currently only clone Assertion Predicates if there are Parse Predicates. This is not entirely correct and will + // be changed with the complete fix for Assertion Predicates. clone_parse_predicate_to_unswitched_loops(predicate_block, reason, iffast_pred, ifslow_pred); + assert(iffast_pred->in(0)->is_ParsePredicate() && ifslow_pred->in(0)->is_ParsePredicate(), + "must be success projections of the cloned Parse Predicates"); clone_assertion_predicates_to_unswitched_loop(loop, old_new, reason, predicate_block->parse_predicate_success_proj(), - iffast_pred, ifslow_pred); + iffast_pred->as_IfTrue(), ifslow_pred->as_IfTrue()); } } @@ -1156,7 +1147,7 @@ void PhaseIdealLoop::loop_predication_follow_branches(Node *n, IdealLoopTree *lo stack.push(in, 1); break; } else if (in->is_IfProj() && - in->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && + in->as_Proj()->is_uncommon_trap_if_pattern() && (in->in(0)->Opcode() == Op_If || in->in(0)->Opcode() == Op_RangeCheck)) { if (pf.to(in) * loop_trip_cnt >= 1) { @@ -1305,19 +1296,20 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod // them by making a copy of them when splitting a loop into sub loops. The Assertion Predicates ensure that dead sub // loops are removed properly. IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj, - IfProjNode* predicate_proj, IfProjNode* upper_bound_proj, - int scale, Node* offset, Node* init, Node* limit, jint stride, + ParsePredicateSuccessProj* parse_predicate_proj, + IfProjNode* upper_bound_proj, const int scale, Node* offset, + Node* init, Node* limit, const jint stride, Node* rng, bool& overflow, Deoptimization::DeoptReason reason) { // First predicate for the initial value on first loop iteration Node* opaque_init = new OpaqueLoopInitNode(C, init); register_new_node(opaque_init, upper_bound_proj); - bool negate = (if_proj->_con != predicate_proj->_con); + bool negate = (if_proj->_con != parse_predicate_proj->_con); BoolNode* bol = rc_predicate(loop, upper_bound_proj, scale, offset, opaque_init, limit, stride, rng, (stride > 0) != (scale > 0), overflow); Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); // This will go away once loop opts are over C->add_template_assertion_predicate_opaq(opaque_bol); register_new_node(opaque_bol, upper_bound_proj); - IfProjNode* new_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode()); + IfProjNode* new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode()); _igvn.replace_input_of(new_proj->in(0), 1, opaque_bol); assert(opaque_init->outcnt() > 0, "should be used"); @@ -1333,14 +1325,14 @@ IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealL register_new_node(max_value, new_proj); // init + (current stride - initial stride) is within the loop so narrow its type by leveraging the type of the iv Phi max_value = new CastIINode(max_value, loop->_head->as_CountedLoop()->phi()->bottom_type()); - register_new_node(max_value, predicate_proj); + register_new_node(max_value, parse_predicate_proj); bol = rc_predicate(loop, new_proj, scale, offset, max_value, limit, stride, rng, (stride > 0) != (scale > 0), overflow); opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); C->add_template_assertion_predicate_opaq(opaque_bol); register_new_node(opaque_bol, new_proj); - new_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode()); + new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode()); _igvn.replace_input_of(new_proj->in(0), 1, opaque_bol); assert(max_value->outcnt() > 0, "should be used"); assert(assertion_predicate_has_loop_opaque_node(new_proj->in(0)->as_If()), "unexpected"); @@ -1350,13 +1342,7 @@ IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealL // Insert Hoisted Check Predicates for null checks and range checks and additional Template Assertion Predicates for // range checks. -bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) { - if (!UseLoopPredicate) return false; - - if (!loop->_head->is_Loop()) { - // Could be a simple region when irreducible loops are present. - return false; - } +bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree* loop) { LoopNode* head = loop->_head->as_Loop(); if (head->unique_ctrl_out()->is_NeverBranch()) { @@ -1436,7 +1422,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) { IfProjNode* if_proj = n->as_IfProj(); IfNode* iff = if_proj->in(0)->as_If(); - CallStaticJavaNode* call = if_proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); + CallStaticJavaNode* call = if_proj->is_uncommon_trap_if_pattern(); if (call == nullptr) { if (loop->is_loop_exit(iff)) { // stop processing the remaining projs in the list because the execution of them @@ -1474,7 +1460,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) { while (if_proj_list.size() > 0) { Node* if_proj = if_proj_list.pop(); float f = pf.to(if_proj); - if (if_proj->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && + if (if_proj->as_Proj()->is_uncommon_trap_if_pattern() && f * loop_trip_cnt >= 1) { ParsePredicateSuccessProj* profiled_loop_parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj(); @@ -1546,7 +1532,7 @@ bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) { } // self - if (!_irreducible && !tail()->is_top()) { + if (can_apply_loop_predication()) { hoisted |= phase->loop_predication_impl(this); } @@ -1556,3 +1542,7 @@ bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) { return hoisted; } + +bool IdealLoopTree::can_apply_loop_predication() { + return _head->is_Loop() && !_irreducible && !tail()->is_top(); +} diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 5107c6e9692..7bd92f33af7 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -177,7 +177,7 @@ Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) { // expensive nodes will notice the loop and skip over it to try to // move the node further up. if (ctl->is_CountedLoop() && ctl->in(1) != nullptr && ctl->in(1)->in(0) != nullptr && ctl->in(1)->in(0)->is_If()) { - if (!ctl->in(1)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) { + if (!ctl->in(1)->as_Proj()->is_uncommon_trap_if_pattern()) { break; } next = idom(ctl->in(1)->in(0)); @@ -191,7 +191,7 @@ Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) { } else if (parent_ctl->is_CountedLoopEnd() && parent_ctl->as_CountedLoopEnd()->loopnode() != nullptr) { next = parent_ctl->as_CountedLoopEnd()->loopnode()->init_control(); } else if (parent_ctl->is_If()) { - if (!ctl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) { + if (!ctl->as_Proj()->is_uncommon_trap_if_pattern()) { break; } assert(idom(ctl) == parent_ctl, "strange"); @@ -350,11 +350,6 @@ void PhaseIdealLoop::insert_loop_limit_check_predicate(ParsePredicateSuccessProj Deoptimization::Reason_loop_limit_check, Op_If); Node* iff = new_predicate_proj->in(0); - assert(iff->Opcode() == Op_If, "bad graph shape"); - Node* conv = iff->in(1); - assert(conv->Opcode() == Op_Conv2B, "bad graph shape"); - Node* opaq = conv->in(1); - assert(opaq->Opcode() == Op_Opaque1, "bad graph shape"); cmp_limit = _igvn.register_new_node_with_optimizer(cmp_limit); bol = _igvn.register_new_node_with_optimizer(bol); set_subtree_ctrl(bol, false); @@ -564,19 +559,12 @@ Node* PhaseIdealLoop::loop_nest_replace_iv(Node* iv_to_replace, Node* inner_iv, void PhaseIdealLoop::add_parse_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop, SafePointNode* sfpt) { if (!C->too_many_traps(reason)) { - Node* cont = _igvn.intcon(1); - Node* opaq = new Opaque1Node(C, cont); - _igvn.register_new_node_with_optimizer(opaq); - Node* bol = new Conv2BNode(opaq); - _igvn.register_new_node_with_optimizer(bol); - set_subtree_ctrl(bol, false); - ParsePredicateNode* iff = new ParsePredicateNode(inner_head->in(LoopNode::EntryControl), bol, reason); - register_control(iff, loop, inner_head->in(LoopNode::EntryControl)); - Node* if_false = new IfFalseNode(iff); - register_control(if_false, _ltree_root, iff); - Node* if_true = new IfTrueNode(iff); - register_control(if_true, loop, iff); - C->add_parse_predicate_opaq(opaq); + ParsePredicateNode* parse_predicate = new ParsePredicateNode(inner_head->in(LoopNode::EntryControl), reason, &_igvn); + register_control(parse_predicate, loop, inner_head->in(LoopNode::EntryControl)); + Node* if_false = new IfFalseNode(parse_predicate); + register_control(if_false, _ltree_root, parse_predicate); + Node* if_true = new IfTrueNode(parse_predicate); + register_control(if_true, loop, parse_predicate); int trap_request = Deoptimization::make_trap_request(reason, Deoptimization::Action_maybe_recompile); address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point(); @@ -1089,7 +1077,7 @@ int PhaseIdealLoop::extract_long_range_checks(const IdealLoopTree* loop, jlong s Node* c = loop->_body.at(i); if (c->is_IfProj() && c->in(0)->is_RangeCheck()) { IfProjNode* if_proj = c->as_IfProj(); - CallStaticJavaNode* call = if_proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); + CallStaticJavaNode* call = if_proj->is_uncommon_trap_if_pattern(); if (call != nullptr) { Node* range = nullptr; Node* offset = nullptr; @@ -4052,78 +4040,107 @@ void PhaseIdealLoop::log_loop_tree() { } } -//---------------------collect_potentially_useful_predicates----------------------- -// Helper function to collect potentially useful predicates to prevent them from -// being eliminated by PhaseIdealLoop::eliminate_useless_predicates -void PhaseIdealLoop::collect_potentially_useful_predicates(IdealLoopTree* loop, Unique_Node_List &useful_predicates) { - if (loop->_child) { // child - collect_potentially_useful_predicates(loop->_child, useful_predicates); +// Eliminate all Parse and Template Assertion Predicates that are not associated with a loop anymore. The eliminated +// predicates will be removed during the next round of IGVN. +void PhaseIdealLoop::eliminate_useless_predicates() { + if (C->parse_predicate_count() == 0 && C->template_assertion_predicate_count() == 0) { + return; // No predicates left. } - // self (only loops that we can apply loop predication may use their predicates) - if (loop->_head->is_Loop() && - !loop->_irreducible && - !loop->tail()->is_top()) { - LoopNode* lpn = loop->_head->as_Loop(); - Node* entry = lpn->in(LoopNode::EntryControl); - const Predicates predicates(entry); - const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); - if (loop_limit_check_predicate_block->has_parse_predicate()) { // right pattern that can be used by loop predication - IfProjNode* parse_predicate_proj = loop_limit_check_predicate_block->parse_predicate_success_proj(); - assert(parse_predicate_proj->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1, "must be"); - useful_predicates.push(parse_predicate_proj->in(0)->in(1)->in(1)); // good one - } - if (UseProfiledLoopPredicate) { - const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block(); - if (profiled_loop_predicate_block->has_parse_predicate()) { // right pattern that can be used by loop predication - IfProjNode* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj(); - useful_predicates.push(parse_predicate_proj->in(0)->in(1)->in(1)); // good one - get_assertion_predicates(parse_predicate_proj, useful_predicates, true); - } - } + eliminate_useless_parse_predicates(); + eliminate_useless_template_assertion_predicates(); +} - if (UseLoopPredicate) { - const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block(); - if (loop_predicate_block->has_parse_predicate()) { // right pattern that can be used by loop predication - IfProjNode* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj(); - useful_predicates.push(parse_predicate_proj->in(0)->in(1)->in(1)); // good one - get_assertion_predicates(parse_predicate_proj, useful_predicates, true); - } - } +// Eliminate all Parse Predicates that do not belong to a loop anymore by marking them useless. These will be removed +// during the next round of IGVN. +void PhaseIdealLoop::eliminate_useless_parse_predicates() { + mark_all_parse_predicates_useless(); + if (C->has_loops()) { + mark_loop_associated_parse_predicates_useful(); } + add_useless_parse_predicates_to_igvn_worklist(); +} - if (loop->_next) { // sibling - collect_potentially_useful_predicates(loop->_next, useful_predicates); +void PhaseIdealLoop::mark_all_parse_predicates_useless() const { + for (int i = 0; i < C->parse_predicate_count(); i++) { + C->parse_predicate(i)->mark_useless(); } } -//------------------------eliminate_useless_predicates----------------------------- -// Eliminate all inserted predicates if they could not be used by loop predication. -// Note: it will also eliminates loop limits check predicate since it also uses -// Opaque1 node (see Parse::add_predicate()). -void PhaseIdealLoop::eliminate_useless_predicates() { - if (C->parse_predicate_count() == 0 && C->template_assertion_predicate_count() == 0) { - return; // no predicate left +void PhaseIdealLoop::mark_loop_associated_parse_predicates_useful() { + for (LoopTreeIterator iterator(_ltree_root); !iterator.done(); iterator.next()) { + IdealLoopTree* loop = iterator.current(); + if (loop->can_apply_loop_predication()) { + mark_useful_parse_predicates_for_loop(loop); + } } +} - Unique_Node_List useful_predicates; // to store useful predicates +void PhaseIdealLoop::mark_useful_parse_predicates_for_loop(IdealLoopTree* loop) { + Node* entry = loop->_head->in(LoopNode::EntryControl); + const Predicates predicates(entry); + ParsePredicateIterator iterator(predicates); + while (iterator.has_next()) { + iterator.next()->mark_useful(); + } +} + +void PhaseIdealLoop::add_useless_parse_predicates_to_igvn_worklist() { + for (int i = 0; i < C->parse_predicate_count(); i++) { + ParsePredicateNode* parse_predicate_node = C->parse_predicate(i); + if (parse_predicate_node->is_useless()) { + _igvn._worklist.push(parse_predicate_node); + } + } +} + + +// Eliminate all Template Assertion Predicates that do not belong to their originally associated loop anymore by +// replacing the Opaque4 node of the If node with true. These nodes will be removed during the next round of IGVN. +void PhaseIdealLoop::eliminate_useless_template_assertion_predicates() { + Unique_Node_List useful_predicates; if (C->has_loops()) { - collect_potentially_useful_predicates(_ltree_root->_child, useful_predicates); + collect_useful_template_assertion_predicates(useful_predicates); + } + eliminate_useless_template_assertion_predicates(useful_predicates); +} + +void PhaseIdealLoop::collect_useful_template_assertion_predicates(Unique_Node_List& useful_predicates) { + for (LoopTreeIterator iterator(_ltree_root); !iterator.done(); iterator.next()) { + IdealLoopTree* loop = iterator.current(); + if (loop->can_apply_loop_predication()) { + collect_useful_template_assertion_predicates_for_loop(loop, useful_predicates); + } + } +} + +void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(IdealLoopTree* loop, + Unique_Node_List &useful_predicates) { + Node* entry = loop->_head->in(LoopNode::EntryControl); + const Predicates predicates(entry); + if (UseProfiledLoopPredicate) { + const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block(); + if (profiled_loop_predicate_block->has_parse_predicate()) { + IfProjNode* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj(); + get_assertion_predicates(parse_predicate_proj, useful_predicates, true); + } } - for (int i = C->parse_predicate_count(); i > 0; i--) { - Node* n = C->parse_predicate_opaque1_node(i - 1); - assert(n->Opcode() == Op_Opaque1, "must be"); - if (!useful_predicates.member(n)) { // not in the useful list - _igvn.replace_node(n, n->in(1)); - } + if (UseLoopPredicate) { + const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block(); + if (loop_predicate_block->has_parse_predicate()) { + IfProjNode* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj(); + get_assertion_predicates(parse_predicate_proj, useful_predicates, true); + } } +} - for (int i = C->template_assertion_predicate_count(); i > 0; i--) { - Node* n = C->template_assertion_predicate_opaq_node(i - 1); - assert(n->Opcode() == Op_Opaque4, "must be"); - if (!useful_predicates.member(n)) { // not in the useful list - _igvn.replace_node(n, n->in(2)); +void PhaseIdealLoop::eliminate_useless_template_assertion_predicates(Unique_Node_List& useful_predicates) { + for (int i = 0; i < C->template_assertion_predicate_count(); i++) { + Node* opaque4 = C->template_assertion_predicate_opaq_node(i); + assert(opaque4->Opcode() == Op_Opaque4, "must be"); + if (!useful_predicates.member(opaque4)) { // not in the useful list + _igvn.replace_node(opaque4, opaque4->in(2)); } } } @@ -4561,7 +4578,7 @@ void PhaseIdealLoop::build_and_optimize() { } // Perform loop predication before iteration splitting - if (C->has_loops() && !C->major_progress() && (C->parse_predicate_count() > 0)) { + if (UseLoopPredicate && C->has_loops() && !C->major_progress() && (C->parse_predicate_count() > 0)) { _ltree_root->_child->loop_predication(this); } @@ -4614,7 +4631,8 @@ void PhaseIdealLoop::build_and_optimize() { // until no more loop optimizations could be done. // After that switch predicates off and do more loop optimizations. if (!C->major_progress() && (C->parse_predicate_count() > 0)) { - C->cleanup_parse_predicates(_igvn); + C->mark_parse_predicate_nodes_useless(_igvn); + assert(C->parse_predicate_count() == 0, "should be zero now"); if (TraceLoopOpts) { tty->print_cr("PredicatesOff"); } @@ -6096,7 +6114,7 @@ void PhaseIdealLoop::build_loop_late_post_work(Node *n, bool pinned) { if (!new_ctrl->is_Proj()) { break; } - CallStaticJavaNode* call = new_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); + CallStaticJavaNode* call = new_ctrl->as_Proj()->is_uncommon_trap_if_pattern(); if (call == nullptr) { break; } diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index b238de71a95..bb12e7b97d4 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -646,6 +646,7 @@ public: // Perform optimization to use the loop predicates for null checks and range checks. // Applies to any loop level (not just the innermost one) bool loop_predication( PhaseIdealLoop *phase); + bool can_apply_loop_predication(); // Perform iteration-splitting on inner loops. Split iterations to // avoid range checks or one-shot null checks. Returns false if the @@ -1330,8 +1331,9 @@ public: bool* p_short_scale, int depth); // Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted - IfProjNode* create_new_if_for_predicate(IfProjNode* cont_proj, Node* new_entry, Deoptimization::DeoptReason reason, - int opcode, bool rewire_uncommon_proj_phi_inputs = false); + IfProjNode* create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, + Deoptimization::DeoptReason reason, int opcode, + bool rewire_uncommon_proj_phi_inputs = false); private: // Helper functions for create_new_if_for_predicate() @@ -1362,15 +1364,27 @@ public: void loop_predication_follow_branches(Node *c, IdealLoopTree *loop, float loop_trip_cnt, PathFrequency& pf, Node_Stack& stack, VectorSet& seen, Node_List& if_proj_list); - IfProjNode* add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj, IfProjNode* predicate_proj, + IfProjNode* add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj, + ParsePredicateSuccessProj* parse_predicate_proj, IfProjNode* upper_bound_proj, int scale, Node* offset, Node* init, Node* limit, jint stride, Node* rng, bool& overflow, Deoptimization::DeoptReason reason); Node* add_range_check_elimination_assertion_predicate(IdealLoopTree* loop, Node* predicate_proj, int scale_con, Node* offset, Node* limit, jint stride_con, Node* value); // Helper function to collect predicate for eliminating the useless ones - void collect_potentially_useful_predicates(IdealLoopTree *loop, Unique_Node_List &predicate_opaque1); void eliminate_useless_predicates(); + + void eliminate_useless_parse_predicates(); + void mark_all_parse_predicates_useless() const; + void mark_loop_associated_parse_predicates_useful(); + static void mark_useful_parse_predicates_for_loop(IdealLoopTree* loop); + void add_useless_parse_predicates_to_igvn_worklist(); + + void eliminate_useless_template_assertion_predicates(); + void collect_useful_template_assertion_predicates(Unique_Node_List& useful_predicates); + static void collect_useful_template_assertion_predicates_for_loop(IdealLoopTree* loop, Unique_Node_List& useful_predicates); + void eliminate_useless_template_assertion_predicates(Unique_Node_List& useful_predicates); + void eliminate_useless_zero_trip_guard(); bool has_control_dependencies_from_predicates(LoopNode* head) const; @@ -1621,14 +1635,15 @@ private: IfProjNode*& ifslow_pred); void clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, Deoptimization::DeoptReason reason, IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred); - IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* predicate_proj, Node* new_entry, + IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason, bool slow_loop); void clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, Deoptimization::DeoptReason reason, IfProjNode* old_predicate_proj, - IfProjNode* iffast_pred, IfProjNode* ifslow_pred); + ParsePredicateSuccessProj* fast_loop_parse_predicate_proj, + ParsePredicateSuccessProj* slow_loop_parse_predicate_proj); IfProjNode* clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate, Deoptimization::DeoptReason reason, - IfProjNode* output_proj); + ParsePredicateSuccessProj* parse_predicate_proj); static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN; bool _created_loop_node; diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index 80a90d7933d..a899f0b370f 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -2068,7 +2068,7 @@ void PhaseIdealLoop::clone_loop_handle_data_uses(Node* old, Node_List &old_new, // make sure the Bool/Cmp input is cloned down to avoid a Phi between // the AllocateArray node and its ValidLengthTest input that could cause // split if to break. - if (use->is_If() || use->is_CMove() || C->is_predicate_opaq(use) || use->Opcode() == Op_Opaque4 || + if (use->is_If() || use->is_CMove() || use->Opcode() == Op_Opaque4 || (use->Opcode() == Op_AllocateArray && use->in(AllocateNode::ValidLengthTest) == old)) { // Since this code is highly unlikely, we lazily build the worklist // of such Nodes to go split. diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp index 6b950f33e71..37d3f756bf7 100644 --- a/src/hotspot/share/opto/macro.cpp +++ b/src/hotspot/share/opto/macro.cpp @@ -537,7 +537,7 @@ Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, Node *sfpt_ctl, BasicType } else if (mem->is_ArrayCopy()) { Node* ctl = mem->in(0); Node* m = mem->in(TypeFunc::Memory); - if (sfpt_ctl->is_Proj() && sfpt_ctl->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) { + if (sfpt_ctl->is_Proj() && sfpt_ctl->as_Proj()->is_uncommon_trap_proj()) { // pin the loads in the uncommon trap path ctl = sfpt_ctl; m = sfpt_mem; diff --git a/src/hotspot/share/opto/multnode.cpp b/src/hotspot/share/opto/multnode.cpp index 814f20ca224..904c9470b6d 100644 --- a/src/hotspot/share/opto/multnode.cpp +++ b/src/hotspot/share/opto/multnode.cpp @@ -184,9 +184,9 @@ uint ProjNode::ideal_reg() const { //-------------------------------is_uncommon_trap_proj---------------------------- // Return uncommon trap call node if proj is for "proj->[region->..]call_uct" // null otherwise -CallStaticJavaNode* ProjNode::is_uncommon_trap_proj(Deoptimization::DeoptReason reason) { - int path_limit = 10; - Node* out = this; +CallStaticJavaNode* ProjNode::is_uncommon_trap_proj(Deoptimization::DeoptReason reason) const { + const int path_limit = 10; + const Node* out = this; for (int ct = 0; ct < path_limit; ct++) { out = out->unique_ctrl_out_or_null(); if (out == nullptr) @@ -213,31 +213,14 @@ CallStaticJavaNode* ProjNode::is_uncommon_trap_proj(Deoptimization::DeoptReason // | // V // other_proj->[region->..]call_uct" -// null otherwise -// "must_reason_predicate" means the uct reason must be Reason_predicate -CallStaticJavaNode* ProjNode::is_uncommon_trap_if_pattern(Deoptimization::DeoptReason reason) { - Node *in0 = in(0); - if (!in0->is_If()) return nullptr; - // Variation of a dead If node. - if (in0->outcnt() < 2) return nullptr; - IfNode* iff = in0->as_If(); - - // we need "If(Conv2B(Opaque1(...)))" pattern for reason_predicate - if (reason != Deoptimization::Reason_none) { - if (iff->in(1)->Opcode() != Op_Conv2B || - iff->in(1)->in(1)->Opcode() != Op_Opaque1) { - return nullptr; - } +// or null otherwise. +CallStaticJavaNode* ProjNode::is_uncommon_trap_if_pattern(Deoptimization::DeoptReason reason) const { + Node* iff = in(0); + if (!iff->is_If() || iff->outcnt() < 2) { + // Not a projection of an If or variation of a dead If node. + return nullptr; } - - ProjNode* other_proj = iff->proj_out(1-_con); - CallStaticJavaNode* call = other_proj->is_uncommon_trap_proj(reason); - if (call != nullptr) { - assert(reason == Deoptimization::Reason_none || - Compile::current()->is_predicate_opaq(iff->in(1)->in(1)), "should be on the list"); - return call; - } - return nullptr; + return other_if_proj()->is_uncommon_trap_proj(reason); } ProjNode* ProjNode::other_if_proj() const { diff --git a/src/hotspot/share/opto/multnode.hpp b/src/hotspot/share/opto/multnode.hpp index 09552508aa3..25dad70a50a 100644 --- a/src/hotspot/share/opto/multnode.hpp +++ b/src/hotspot/share/opto/multnode.hpp @@ -93,13 +93,13 @@ public: // Return uncommon trap call node if proj is for "proj->[region->..]call_uct" // null otherwise - CallStaticJavaNode* is_uncommon_trap_proj(Deoptimization::DeoptReason reason); + CallStaticJavaNode* is_uncommon_trap_proj(Deoptimization::DeoptReason reason = Deoptimization::Reason_none) const; // Return uncommon trap call node for "if(test)-> proj -> ... // | // V // other_proj->[region->..]call_uct" // null otherwise - CallStaticJavaNode* is_uncommon_trap_if_pattern(Deoptimization::DeoptReason reason); + CallStaticJavaNode* is_uncommon_trap_if_pattern(Deoptimization::DeoptReason reason = Deoptimization::Reason_none) const; // Return other proj node when this is a If proj node ProjNode* other_if_proj() const; diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp index 527b0ba8938..44c37768cdc 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -508,6 +508,10 @@ Node *Node::clone() const { // If it is applicable, it will happen anyway when the cloned node is registered with IGVN. n->remove_flag(Node::NodeFlags::Flag_for_post_loop_opts_igvn); } + if (n->is_ParsePredicate()) { + C->add_parse_predicate(n->as_ParsePredicate()); + } + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); bs->register_potential_barrier_node(n); @@ -609,6 +613,9 @@ void Node::destruct(PhaseValues* phase) { if (Opcode() == Op_Opaque4) { compile->remove_template_assertion_predicate_opaq(this); } + if (is_ParsePredicate()) { + compile->remove_parse_predicate(as_ParsePredicate()); + } if (for_post_loop_opts_igvn()) { compile->remove_from_post_loop_opts_igvn(this); } diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index 45ec2f3faa0..63e627070a1 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -69,7 +69,7 @@ ParsePredicateNode* ParsePredicate::init_parse_predicate(Node* parse_predicate_p } Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* if_proj) { - CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); + CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern(); if (uct_call == nullptr) { return Deoptimization::Reason_none; } @@ -84,6 +84,30 @@ bool RuntimePredicate::is_success_proj(Node* node, Deoptimization::DeoptReason d } } +ParsePredicateIterator::ParsePredicateIterator(const Predicates& predicates) : _current_index(0) { + const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); + if (loop_limit_check_predicate_block->has_parse_predicate()) { + _parse_predicates.push(loop_limit_check_predicate_block->parse_predicate()); + } + if (UseProfiledLoopPredicate) { + const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block(); + if (profiled_loop_predicate_block->has_parse_predicate()) { + _parse_predicates.push(profiled_loop_predicate_block->parse_predicate()); + } + } + if (UseLoopPredicate) { + const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block(); + if (loop_predicate_block->has_parse_predicate()) { + _parse_predicates.push(loop_predicate_block->parse_predicate()); + } + } +} + +ParsePredicateNode* ParsePredicateIterator::next() { + assert(has_next(), "always check has_next() first"); + return _parse_predicates.at(_current_index++); +} + // Walk over all Regular Predicates of this block (if any) and return the first node not belonging to the block // anymore (i.e. entry to the first Regular Predicate in this block if any or `regular_predicate_proj` otherwise). Node* PredicateBlock::skip_regular_predicates(Node* regular_predicate_proj, Deoptimization::DeoptReason deopt_reason) { diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index 0652cea2992..0f5c8fe3264 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -355,4 +355,19 @@ class Predicates : public StackObj { return _entry != _loop_entry; } }; + +// This class iterates over the Parse Predicates of a loop. +class ParsePredicateIterator : public StackObj { + GrowableArray _parse_predicates; + int _current_index; + + public: + ParsePredicateIterator(const Predicates& predicates); + + bool has_next() const { + return _current_index < _parse_predicates.length(); + } + + ParsePredicateNode* next(); +}; #endif // SHARE_OPTO_PREDICATES_HPP