diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp index 0ff08712624..a1d542440c9 100644 --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -100,8 +100,7 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred, // is an IfTrue projection. This code is also used to clone predicates to cloned loops. IfTrueNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_success_proj, Node* new_entry, const Deoptimization::DeoptReason reason, - const int opcode, const bool rewire_uncommon_proj_phi_inputs, - AssertionPredicateType assertion_predicate_type) { + const int opcode, const bool rewire_uncommon_proj_phi_inputs) { assert(parse_predicate_success_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!"); ParsePredicateNode* parse_predicate = parse_predicate_success_proj->in(0)->as_ParsePredicate(); ParsePredicateUncommonProj* uncommon_proj = parse_predicate->uncommon_proj(); @@ -143,12 +142,10 @@ IfTrueNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessPro IfNode* new_iff = nullptr; switch (opcode) { case Op_If: - new_iff = new IfNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt - NOT_PRODUCT(COMMA assertion_predicate_type)); + new_iff = new IfNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt); break; case Op_RangeCheck: - new_iff = new RangeCheckNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt - NOT_PRODUCT(COMMA assertion_predicate_type)); + new_iff = new RangeCheckNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt); break; case Op_ParsePredicate: new_iff = new ParsePredicateNode(entry, reason, &_igvn); @@ -272,150 +269,15 @@ void PhaseIdealLoop::fix_cloned_data_node_controls(const ProjNode* old_uncommon_ orig_to_clone.iterate_all(orig_clone_action); } -IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, - Node* new_entry, Deoptimization::DeoptReason reason, - const bool slow_loop) { - - IfProjNode* new_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, new_entry, reason, Op_ParsePredicate, - slow_loop); - assert(new_predicate_proj->is_IfTrue(), "the success projection of a Parse Predicate is a true projection"); - ParsePredicateNode* parse_predicate = new_predicate_proj->in(0)->as_ParsePredicate(); - return new_predicate_proj; -} - -// Clones Template Assertion Predicates to both unswitched loops starting at 'old_predicate_proj' by following its -// control inputs. It also rewires the control edges of data nodes with dependencies in the loop from the old predicates -// to the new cloned predicates. -void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, - ParsePredicateSuccessProj* old_parse_predicate_proj, - ParsePredicateNode* true_path_loop_parse_predicate, - ParsePredicateNode* false_path_loop_parse_predicate) { - // Push the original Template Assertion Predicates on a list to later process them in reverse order to keep the - // original predicate order. - Unique_Node_List list; - get_template_assertion_predicates(old_parse_predicate_proj, list); - - Node_List to_process; - for (int i = list.size() - 1; i >= 0; i--) { - IfTrueNode* template_assertion_predicate_success_proj = list.at(i)->as_IfTrue(); - assert(template_assertion_predicate_success_proj->in(0)->is_If(), "must be If node"); - - IfTrueNode* true_path_loop_proj = - clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj, - true_path_loop_parse_predicate); - IfTrueNode* false_path_loop_proj = - clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj, - false_path_loop_parse_predicate); - - // Update control dependent data nodes. - for (DUIterator j = template_assertion_predicate_success_proj->outs(); - template_assertion_predicate_success_proj->has_out(j); - j++) { - Node* true_path_loop_node = template_assertion_predicate_success_proj->out(j); - if (loop->is_member(get_loop(ctrl_or_self(true_path_loop_node)))) { - assert(true_path_loop_node->in(0) == template_assertion_predicate_success_proj, "only control edge"); - Node* false_path_loop_node = old_new[true_path_loop_node->_idx]; - assert(false_path_loop_node->in(0) == template_assertion_predicate_success_proj, "only control edge"); - _igvn.replace_input_of(true_path_loop_node, 0, true_path_loop_proj); - to_process.push(false_path_loop_node); - --j; - } - } - // Have to delay updates to the false path loop so uses of predicate are not modified while we iterate on them. - while (to_process.size() > 0) { - Node* slow_node = to_process.pop(); - _igvn.replace_input_of(slow_node, 0, false_path_loop_proj); - } - } -} - -// Put all Template Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque' -// is set, then the OpaqueTemplateAssertionPredicateNode nodes of the Assertion Predicates are put on the list instead -// of the projections. -void PhaseIdealLoop::get_template_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list, - const bool get_opaque) { +// Put all OpaqueTemplateAssertionPredicate nodes on a list, starting at 'predicate' and going up in the tree. +void PhaseIdealLoop::get_opaque_template_assertion_predicate_nodes(ParsePredicateSuccessProj* parse_predicate_proj, + Unique_Node_List& list) { Deoptimization::DeoptReason deopt_reason = parse_predicate_proj->in(0)->as_ParsePredicate()->deopt_reason(); PredicateBlockIterator predicate_iterator(parse_predicate_proj, deopt_reason); - TemplateAssertionPredicateCollector template_assertion_predicate_collector(list, get_opaque); - predicate_iterator.for_each(template_assertion_predicate_collector); + OpaqueTemplateAssertionPredicateCollector opaque_template_assertion_predicate_collector(list); + predicate_iterator.for_each(opaque_template_assertion_predicate_collector); } -// Clone an Assertion Predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon -// traps are kept for the predicate (a Halt node is used later when creating pre/main/post loops and copying this cloned -// predicate again). -IfTrueNode* -PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj, - ParsePredicateNode* unswitched_loop_parse_predicate) { - TemplateAssertionPredicate template_assertion_predicate(template_assertion_predicate_success_proj); - IfTrueNode* template_success_proj = template_assertion_predicate.clone(unswitched_loop_parse_predicate->in(0), this); - _igvn.replace_input_of(unswitched_loop_parse_predicate, 0, template_success_proj); - set_idom(unswitched_loop_parse_predicate, template_success_proj, dom_depth(template_success_proj)); - return template_success_proj; -} - -// Clone the old Parse Predicates and Assertion Predicates before the unswitch If to the unswitched loops after the -// unswitch If. -void PhaseIdealLoop::clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, - IfProjNode*& true_path_loop_entry, - IfProjNode*& false_path_loop_entry) { - LoopNode* head = loop->_head->as_Loop(); - Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl); - - const Predicates predicates(entry); - clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.loop_predicate_block(), - Deoptimization::Reason_predicate, true_path_loop_entry, false_path_loop_entry); - clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.profiled_loop_predicate_block(), - Deoptimization::Reason_profile_predicate, true_path_loop_entry, false_path_loop_entry); - - const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); - if (loop_limit_check_predicate_block->has_parse_predicate() && !head->is_CountedLoop()) { - // Don't clone the Loop Limit Check Parse Predicate if we already have a counted loop (a Loop Limit Check Predicate - // is only created when converting a LoopNode to a CountedLoopNode). - clone_parse_predicate_to_unswitched_loops(loop_limit_check_predicate_block, Deoptimization::Reason_loop_limit_check, - true_path_loop_entry, false_path_loop_entry); - } -} - -// Clone the Parse Predicate and Template Assertion Predicates of a Loop Predication related Predicate Block. -void PhaseIdealLoop::clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, - const PredicateBlock* predicate_block, - Deoptimization::DeoptReason reason, - IfProjNode*& true_path_loop_entry, - IfProjNode*& false_path_loop_entry) { - if (predicate_block->has_parse_predicate()) { - // We currently only clone Assertion Predicates if there are Parse Predicates. This is not entirely correct and will - // be changed with the complete fix for Assertion Predicates. - clone_parse_predicate_to_unswitched_loops(predicate_block, reason, true_path_loop_entry, false_path_loop_entry); - assert(true_path_loop_entry->in(0)->is_ParsePredicate() && false_path_loop_entry->in(0)->is_ParsePredicate(), - "must be success projections of the cloned Parse Predicates"); - clone_assertion_predicates_to_unswitched_loop(loop, old_new, predicate_block->parse_predicate_success_proj(), - true_path_loop_entry->in(0)->as_ParsePredicate(), - false_path_loop_entry->in(0)->as_ParsePredicate()); - } -} - -void PhaseIdealLoop::clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, - Deoptimization::DeoptReason reason, - IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred) { - assert(predicate_block->has_parse_predicate(), "must have parse predicate"); - ParsePredicateSuccessProj* parse_predicate_proj = predicate_block->parse_predicate_success_proj(); - iffast_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, iffast_pred, reason, false); - check_cloned_parse_predicate_for_unswitching(iffast_pred, true); - - ifslow_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, ifslow_pred, reason, true); - check_cloned_parse_predicate_for_unswitching(ifslow_pred, false); -} - -#ifndef PRODUCT -void PhaseIdealLoop::check_cloned_parse_predicate_for_unswitching(const Node* new_entry, const bool is_fast_loop) { - assert(new_entry != nullptr, "IfTrue or IfFalse after clone predicate"); - if (TraceLoopPredicate) { - tty->print("Parse Predicate cloned to %s loop: ", is_fast_loop ? "fast" : "slow"); - new_entry->in(0)->dump(); - } -} -#endif - //------------------------------Invariance----------------------------------- // Helper class for loop_predication_impl to compute invariance on the fly and // clone invariants. diff --git a/src/hotspot/share/opto/loopUnswitch.cpp b/src/hotspot/share/opto/loopUnswitch.cpp index a5a90b7e481..39e34c7dde3 100644 --- a/src/hotspot/share/opto/loopUnswitch.cpp +++ b/src/hotspot/share/opto/loopUnswitch.cpp @@ -236,15 +236,47 @@ class OriginalLoop : public StackObj { _phase(loop->_phase) {} NONCOPYABLE(OriginalLoop); - private: - void fix_loop_entries(IfProjNode* true_path_loop_entry, IfProjNode* false_path_loop_entry) { - _phase->replace_loop_entry(_loop_head, true_path_loop_entry); - LoopNode* false_path_loop_strip_mined_head = old_to_new(_loop_head)->as_Loop(); - _phase->replace_loop_entry(false_path_loop_strip_mined_head, false_path_loop_entry); + // Unswitch the original loop on the invariant loop selector by creating a true-path-loop and a false-path-loop. + // Remove the unswitch candidate If from both unswitched loop versions which are now covered by the loop selector If. + void unswitch(const UnswitchedLoopSelector& unswitched_loop_selector) { + const uint first_false_path_loop_node_index = _phase->C->unique(); + clone_loop(unswitched_loop_selector); + + move_parse_and_template_assertion_predicates_to_unswitched_loops(unswitched_loop_selector, + first_false_path_loop_node_index); + DEBUG_ONLY(verify_unswitched_loop_versions(_loop->_head->as_Loop(), unswitched_loop_selector);) + + _phase->recompute_dom_depth(); + remove_unswitch_candidate_from_loops(unswitched_loop_selector); } - Node* old_to_new(const Node* old) const { - return _old_new[old->_idx]; + private: + void clone_loop(const UnswitchedLoopSelector& unswitched_loop_selector) { + _phase->clone_loop(_loop, _old_new, _phase->dom_depth(_loop_head), + PhaseIdealLoop::CloneIncludesStripMined, unswitched_loop_selector.selector()); + fix_loop_entries(unswitched_loop_selector); + } + + void fix_loop_entries(const UnswitchedLoopSelector& unswitched_loop_selector) { + _phase->replace_loop_entry(_loop_head, unswitched_loop_selector.true_path_loop_proj()); + LoopNode* false_path_loop_strip_mined_head = old_to_new(_loop_head)->as_Loop(); + _phase->replace_loop_entry(false_path_loop_strip_mined_head, + unswitched_loop_selector.false_path_loop_proj()); + } + + // Moves the Parse And Template Assertion Predicates to the true and false path loop. They are inserted between the + // loop heads and the loop selector If projections. The old Parse and Template Assertion Predicates before + // the unswitched loop selector are killed. + void move_parse_and_template_assertion_predicates_to_unswitched_loops( + const UnswitchedLoopSelector& unswitched_loop_selector, const uint first_false_path_loop_node_index) const { + const NodeInOriginalLoopBody node_in_true_path_loop_body(first_false_path_loop_node_index, _old_new); + const NodeInClonedLoopBody node_in_false_path_loop_body(first_false_path_loop_node_index); + CloneUnswitchedLoopPredicatesVisitor + clone_unswitched_loop_predicates_visitor(_loop_head, old_to_new(_loop_head)->as_Loop(), node_in_true_path_loop_body, + node_in_false_path_loop_body, _phase); + Node* source_loop_entry = unswitched_loop_selector.selector()->in(0); + PredicateIterator predicate_iterator(source_loop_entry); + predicate_iterator.for_each(clone_unswitched_loop_predicates_visitor); } #ifdef ASSERT @@ -263,6 +295,10 @@ class OriginalLoop : public StackObj { } #endif // ASSERT + Node* old_to_new(const Node* old) const { + return _old_new[old->_idx]; + } + // Remove the unswitch candidate If nodes in both unswitched loop versions which are now dominated by the loop selector // If node. Keep the true-path-path in the true-path-loop and the false-path-path in the false-path-loop by setting // the bool input accordingly. The unswitch candidate If nodes are folded in the next IGVN round. @@ -276,28 +312,6 @@ class OriginalLoop : public StackObj { _phase->dominated_by(unswitched_loop_selector.false_path_loop_proj(), unswitching_candidate_clone); } - public: - // Unswitch the original loop on the invariant loop selector by creating a true-path-loop and a false-path-loop. - // Remove the unswitch candidate If from both unswitched loop versions which are now covered by the loop selector If. - void unswitch(const UnswitchedLoopSelector& unswitched_loop_selector) { - _phase->clone_loop(_loop, _old_new, _phase->dom_depth(_loop_head), - PhaseIdealLoop::CloneIncludesStripMined, unswitched_loop_selector.selector()); - - // At this point, the selector If projections are the corresponding loop entries. - // clone_parse_and_assertion_predicates_to_unswitched_loop() could clone additional predicates after the selector - // If projections. The loop entries are updated accordingly. - IfProjNode* true_path_loop_entry = unswitched_loop_selector.true_path_loop_proj(); - IfProjNode* false_path_loop_entry = unswitched_loop_selector.false_path_loop_proj(); - _phase->clone_parse_and_assertion_predicates_to_unswitched_loop(_loop, _old_new, - true_path_loop_entry, false_path_loop_entry); - - fix_loop_entries(true_path_loop_entry, false_path_loop_entry); - - DEBUG_ONLY(verify_unswitched_loop_versions(_loop->_head->as_Loop(), unswitched_loop_selector);) - - _phase->recompute_dom_depth(); - remove_unswitch_candidate_from_loops(unswitched_loop_selector); - } }; // See comments below file header for more information about Loop Unswitching. diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 72870613cb0..79874f01876 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4458,7 +4458,7 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block(); if (profiled_loop_predicate_block->has_parse_predicate()) { ParsePredicateSuccessProj* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj(); - get_template_assertion_predicates(parse_predicate_proj, useful_predicates, true); + get_opaque_template_assertion_predicate_nodes(parse_predicate_proj, useful_predicates); } } @@ -4466,7 +4466,7 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block(); if (loop_predicate_block->has_parse_predicate()) { ParsePredicateSuccessProj* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj(); - get_template_assertion_predicates(parse_predicate_proj, useful_predicates, true); + get_opaque_template_assertion_predicate_nodes(parse_predicate_proj, useful_predicates); } } } diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index acafea3fce8..829c3cb8a18 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -942,7 +942,8 @@ private: static void ensure_zero_trip_guard_proj(Node* node, bool is_main_loop); #endif private: - static void get_template_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list, bool get_opaque = false); + static void get_opaque_template_assertion_predicate_nodes(ParsePredicateSuccessProj* parse_predicate_proj, + Unique_Node_List& list); void update_main_loop_assertion_predicates(CountedLoopNode* main_loop_head); void initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head, CountedLoopNode* remaining_loop_head, @@ -1350,9 +1351,8 @@ public: // Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted IfTrueNode* create_new_if_for_predicate( - ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason, int opcode, - bool rewire_uncommon_proj_phi_inputs = false, - AssertionPredicateType assertion_predicate_type = AssertionPredicateType::None); + ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason, int opcode, + bool rewire_uncommon_proj_phi_inputs = false); private: // Helper functions for create_new_if_for_predicate() @@ -1661,28 +1661,7 @@ private: _nodes_required = UINT_MAX; } - public: - // Clone Parse Predicates to slow and fast loop when unswitching a loop - void clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, - IfProjNode*& true_path_loop_entry, - IfProjNode*& false_path_loop_entry); private: - void clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, - const PredicateBlock* predicate_block, - Deoptimization::DeoptReason reason, - IfProjNode*& true_path_loop_entry, - IfProjNode*& false_path_loop_entry); - void clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, Deoptimization::DeoptReason reason, - IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred); - IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, - Deoptimization::DeoptReason reason, bool slow_loop); - void clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, - ParsePredicateSuccessProj* old_parse_predicate_proj, - ParsePredicateNode* true_path_loop_parse_predicate, - ParsePredicateNode* false_path_loop_parse_predicate); - IfTrueNode* clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj, - ParsePredicateNode* unswitched_loop_parse_predicate); - static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN; bool _created_loop_node; DEBUG_ONLY(void dump_idoms(Node* early, Node* wrong_lca);) diff --git a/src/hotspot/share/opto/opaquenode.cpp b/src/hotspot/share/opto/opaquenode.cpp index 8a8eea51db6..94cd84822a1 100644 --- a/src/hotspot/share/opto/opaquenode.cpp +++ b/src/hotspot/share/opto/opaquenode.cpp @@ -23,6 +23,7 @@ */ #include "precompiled.hpp" +#include "opto/connode.hpp" #include "opto/loopnode.hpp" #include "opto/opaquenode.hpp" #include "opto/phaseX.hpp" diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index a9cd1e638c1..fe46cd6e427 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -82,6 +82,25 @@ ParsePredicateNode* ParsePredicate::init_parse_predicate(Node* parse_predicate_p return nullptr; } +ParsePredicateSuccessProj* ParsePredicate::clone_to_unswitched_loop(Node* new_control, const bool is_true_path_loop, + PhaseIdealLoop* phase) const { + ParsePredicateSuccessProj* success_proj = phase->create_new_if_for_predicate(_success_proj, new_control, + _parse_predicate_node->deopt_reason(), + Op_ParsePredicate, is_true_path_loop); + NOT_PRODUCT(trace_cloned_parse_predicate(is_true_path_loop, success_proj)); + return success_proj; +} + +#ifndef PRODUCT +void ParsePredicate::trace_cloned_parse_predicate(const bool is_true_path_loop, + const ParsePredicateSuccessProj* success_proj) { + if (TraceLoopPredicate) { + tty->print("Parse Predicate cloned to %s path loop: ", is_true_path_loop ? "true" : "false"); + success_proj->in(0)->dump(); + } +} +#endif // NOT PRODUCT + Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* if_proj) { CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern(); if (uct_call == nullptr) { @@ -196,6 +215,12 @@ IfTrueNode* TemplateAssertionPredicate::initialize(PhaseIdealLoop* phase, Node* return success_proj; } +// Kills the Template Assertion Predicate by setting the condition to true. Will be folded away in the next IGVN round. +void TemplateAssertionPredicate::kill(PhaseIdealLoop* phase) const { + ConINode* true_con = phase->intcon(1); + phase->igvn().replace_input_of(_if_node, 1, true_con); +} + #ifdef ASSERT // Class to verify Initialized and Template Assertion Predicates by trying to find OpaqueLoop*Nodes. class OpaqueLoopNodesVerifier : public BFSActions { @@ -947,8 +972,8 @@ IfTrueNode* CreateAssertionPredicatesVisitor::clone_template_and_replace_init_in // x // | old target // Template Assertion loop entry -// Predicate 1 old target clone | \ -// | loop entry TAP 2 | cloned Template Assertion +// Predicate 1 old target clone | \ +// | loop entry TAP 2 | cloned Template Assertion // Template Assertion | ======> | Predicate 2 // Predicate 2 target loop | // | target loop #_current_predicate_chain_head @@ -977,6 +1002,73 @@ void CreateAssertionPredicatesVisitor::rewire_to_old_predicate_chain_head( } } +TargetLoopPredicateChain::TargetLoopPredicateChain(LoopNode* loop_head, PhaseIdealLoop* phase) + : DEBUG_ONLY(_old_target_loop_entry(loop_head->in(LoopNode::EntryControl)) COMMA) + DEBUG_ONLY(_node_index_before_cloning(phase->C->unique()) COMMA) + _current_predicate_chain_head(loop_head), + _phase(phase) {} + +// Inserts the provided newly cloned predicate to the head of the target loop predicate chain. +void TargetLoopPredicateChain::insert_predicate(IfTrueNode* predicate_success_proj) { + rewire_to_target_chain_head(predicate_success_proj); + _current_predicate_chain_head = predicate_success_proj->in(0); + assert(predicate_success_proj->_idx >= _node_index_before_cloning, "must be a newly cloned predicate"); + assert(_current_predicate_chain_head->in(0) == _old_target_loop_entry && + _old_target_loop_entry->unique_ctrl_out() == _current_predicate_chain_head , "must be connected now"); +} + +void TargetLoopPredicateChain::rewire_to_target_chain_head(IfTrueNode* template_assertion_predicate_success_proj) const { + if (_current_predicate_chain_head->is_Loop()) { + _phase->replace_loop_entry(_current_predicate_chain_head->as_Loop(), template_assertion_predicate_success_proj); + } else { + _phase->replace_control(_current_predicate_chain_head, template_assertion_predicate_success_proj); + } +} + +ClonePredicateToTargetLoop::ClonePredicateToTargetLoop(LoopNode* target_loop_head, const NodeInLoopBody& node_in_loop_body, + PhaseIdealLoop* phase) + : _old_target_loop_entry(target_loop_head->in(LoopNode::EntryControl)), + _target_loop_predicate_chain(target_loop_head, phase), + _node_in_loop_body(node_in_loop_body), + _phase(phase) {} + + +CloneUnswitchedLoopPredicatesVisitor::CloneUnswitchedLoopPredicatesVisitor( + LoopNode* true_path_loop_head, LoopNode* false_path_loop_head, + const NodeInOriginalLoopBody& node_in_true_path_loop_body, const NodeInClonedLoopBody& node_in_false_path_loop_body, + PhaseIdealLoop* phase) + : _clone_predicate_to_true_path_loop(true_path_loop_head, node_in_true_path_loop_body, phase), + _clone_predicate_to_false_path_loop(false_path_loop_head, node_in_false_path_loop_body, phase), + _phase(phase), + _has_hoisted_check_parse_predicates(false) {} + +// Keep track of whether we are in the correct Predicate Block where Template Assertion Predicates can be found. +// The PredicateIterator will always start at the loop entry and first visits the Loop Limit Check Predicate Block. +void CloneUnswitchedLoopPredicatesVisitor::visit(const ParsePredicate& parse_predicate) { + Deoptimization::DeoptReason deopt_reason = parse_predicate.head()->deopt_reason(); + if (deopt_reason == Deoptimization::Reason_predicate || + deopt_reason == Deoptimization::Reason_profile_predicate) { + _has_hoisted_check_parse_predicates = true; + } + + _clone_predicate_to_true_path_loop.clone_parse_predicate(parse_predicate, true); + _clone_predicate_to_false_path_loop.clone_parse_predicate(parse_predicate, false); + parse_predicate.kill(_phase->igvn()); +} + +// Clone the Template Assertion Predicate, which is currently found before the newly added unswitched loop selector, +// to the true path and false path loop. +void CloneUnswitchedLoopPredicatesVisitor::visit(const TemplateAssertionPredicate& template_assertion_predicate) { + if (!_has_hoisted_check_parse_predicates) { + // Only process if we are in the correct Predicate Block. + return; + } + + _clone_predicate_to_true_path_loop.clone_template_assertion_predicate(template_assertion_predicate); + _clone_predicate_to_false_path_loop.clone_template_assertion_predicate(template_assertion_predicate); + template_assertion_predicate.kill(_phase); +} + // Clone the Template Assertion Predicate and set a new input for the OpaqueLoopStrideNode. void UpdateStrideForAssertionPredicates::visit(const TemplateAssertionPredicate& template_assertion_predicate) { if (!template_assertion_predicate.is_last_value()) { diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index bc8c1a8ebdc..3044b08a9f8 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -26,7 +26,6 @@ #define SHARE_OPTO_PREDICATES_HPP #include "opto/cfgnode.hpp" -#include "opto/connode.hpp" #include "opto/opaquenode.hpp" class IdealLoopTree; @@ -232,7 +231,7 @@ class Predicate : public StackObj { // Generic predicate visitor that does nothing. Subclass this visitor to add customized actions for each predicate. // The visit methods of this visitor are called from the predicate iterator classes which walk the predicate chain. // Use the UnifiedPredicateVisitor if the type of the predicate does not matter. -class PredicateVisitor : StackObj { +class PredicateVisitor : public StackObj { public: virtual void visit(const ParsePredicate& parse_predicate) {} virtual void visit(const RuntimePredicate& runtime_predicate) {} @@ -297,6 +296,8 @@ class ParsePredicate : public Predicate { } static ParsePredicateNode* init_parse_predicate(Node* parse_predicate_proj, Deoptimization::DeoptReason deopt_reason); + NOT_PRODUCT(static void trace_cloned_parse_predicate(bool is_true_path_loop, + const ParsePredicateSuccessProj* success_proj);) public: ParsePredicate(Node* parse_predicate_proj, Deoptimization::DeoptReason deopt_reason) @@ -325,6 +326,15 @@ class ParsePredicate : public Predicate { assert(is_valid(), "must be valid"); return _success_proj; } + + ParsePredicateSuccessProj* clone_to_unswitched_loop(Node* new_control, bool is_true_path_loop, + PhaseIdealLoop* phase) const; + + // Kills this Parse Predicate by marking it useless. Will be folded away in the next IGVN round. + void kill(const PhaseIterGVN& igvn) const { + _parse_predicate_node->mark_useless(); + igvn._worklist.push(_parse_predicate_node); + } }; // Class to represent a Runtime Predicate which always has an associated UCT on the failing path. @@ -399,6 +409,7 @@ class TemplateAssertionPredicate : public Predicate { IfTrueNode* initialize(PhaseIdealLoop* phase, Node* new_control) const; void rewire_loop_data_dependencies(IfTrueNode* target_predicate, const NodeInLoopBody& data_in_loop_body, PhaseIdealLoop* phase) const; + void kill(PhaseIdealLoop* phase) const; static bool is_predicate(Node* node); #ifdef ASSERT @@ -1006,25 +1017,103 @@ class CreateAssertionPredicatesVisitor : public PredicateVisitor { void visit(const TemplateAssertionPredicate& template_assertion_predicate) override; }; -// This visitor collects all Template Assertion Predicates If nodes or the corresponding Opaque nodes, depending on the -// provided 'get_opaque' flag, to the provided list. -class TemplateAssertionPredicateCollector : public PredicateVisitor { - Unique_Node_List& _list; - const bool _get_opaque; +// This class establishes a predicate chain at the target loop by rewiring newly cloned predicates to the current head +// of the predicate chain. +class TargetLoopPredicateChain : public StackObj { + DEBUG_ONLY(const Node* const _old_target_loop_entry;) + DEBUG_ONLY(const node_idx_t _node_index_before_cloning;) + Node* _current_predicate_chain_head; + PhaseIdealLoop* const _phase; + + void rewire_to_target_chain_head(IfTrueNode* template_assertion_predicate_success_proj) const; + +public: + TargetLoopPredicateChain(LoopNode* loop_head, PhaseIdealLoop* phase); + NONCOPYABLE(TargetLoopPredicateChain); + + void insert_predicate(IfTrueNode* predicate_success_proj); +}; + +// This class clones Parse and Template Assertion Predicates to the provided target loop. This also involves rewiring +// of any data pinned to Template Assertion Predicates. The Template Assertion Predicate Expressions are cloned +// without applying any changes to them. +// +// Each time a predicate is cloned, it is inserted at the top of previously cloned predicates. This ensures that the +// target loop predicate chain order of the newly cloned predicates is the same as in the source loop from which the +// predicates were cloned from. +// +// Template Assertion Predicate Example: +// +// x _old_target_loop_entry _old_target_loop_entry +// | | | | +// Template Assertion | Cloned Template 2. rewire data Cloned Template +// Predicate 1. clone | Assertion Predicate and predicate Assertion Predicate +// | \ =======> | ===============> | \ +// | data | | data +// | | | +// source loop head target loop head target loop head +class ClonePredicateToTargetLoop : public StackObj { + Node* const _old_target_loop_entry; // Used as control for each newly cloned predicate. + TargetLoopPredicateChain _target_loop_predicate_chain; + const NodeInLoopBody& _node_in_loop_body; + PhaseIdealLoop* const _phase; + +public: + ClonePredicateToTargetLoop(LoopNode* target_loop_head, const NodeInLoopBody& node_in_loop_body, PhaseIdealLoop* phase); + + // Clones the provided Parse Predicate to the head of the current predicate chain at the target loop. + void clone_parse_predicate(const ParsePredicate& parse_predicate, bool is_true_path_loop) { + ParsePredicateSuccessProj* cloned_parse_predicate_success_proj = + parse_predicate.clone_to_unswitched_loop(_old_target_loop_entry, is_true_path_loop, _phase); + _target_loop_predicate_chain.insert_predicate(cloned_parse_predicate_success_proj); + } + + // Clones the provided Template Assertion Predicate to the head of the current predicate chain at the target loop. + void clone_template_assertion_predicate(const TemplateAssertionPredicate& template_assertion_predicate) { + IfTrueNode* cloned_template_success_proj = template_assertion_predicate.clone(_old_target_loop_entry, + _phase); + template_assertion_predicate.rewire_loop_data_dependencies(cloned_template_success_proj, _node_in_loop_body, _phase); + _target_loop_predicate_chain.insert_predicate(cloned_template_success_proj); + } +}; + +// Visitor to clone Parse and Template Assertion Predicates from a loop to its unswitched true and false path loop. +// The cloned predicates are not updated in any way. Thus, an Initialized Assertion Predicate is also not required to +// be created. Note that the data dependencies from the Template Assertion Predicates are also updated to the newly +// cloned Template Assertion Predicates, depending on whether they belong to the true or false path loop. +class CloneUnswitchedLoopPredicatesVisitor : public PredicateVisitor { + ClonePredicateToTargetLoop _clone_predicate_to_true_path_loop; + ClonePredicateToTargetLoop _clone_predicate_to_false_path_loop; + + PhaseIdealLoop* const _phase; + bool _has_hoisted_check_parse_predicates; public: - TemplateAssertionPredicateCollector(Unique_Node_List& list, const bool get_opaque) - : _list(list), - _get_opaque(get_opaque) {} + CloneUnswitchedLoopPredicatesVisitor(LoopNode* true_path_loop_head, + LoopNode* false_path_loop_head, + const NodeInOriginalLoopBody& node_in_true_path_loop_body, + const NodeInClonedLoopBody& node_in_false_path_loop_body, + PhaseIdealLoop* phase); + NONCOPYABLE(CloneUnswitchedLoopPredicatesVisitor); + + using PredicateVisitor::visit; + + void visit(const ParsePredicate& parse_predicate) override; + void visit(const TemplateAssertionPredicate& template_assertion_predicate) override; +}; + +// This visitor collects all OpaqueTemplateAssertionNodes of Template Assertion Predicates. This is used for cleaning +// up unused Template Assertion Predicates. +class OpaqueTemplateAssertionPredicateCollector : public PredicateVisitor { + Unique_Node_List& _list; + + public: + explicit OpaqueTemplateAssertionPredicateCollector(Unique_Node_List& list) : _list(list) {} using PredicateVisitor::visit; void visit(const TemplateAssertionPredicate& template_assertion_predicate) override { - if (_get_opaque) { - _list.push(template_assertion_predicate.opaque_node()); - } else { - _list.push(template_assertion_predicate.tail()); - } + _list.push(template_assertion_predicate.opaque_node()); } };