mirror of
https://github.com/openjdk/jdk.git
synced 2026-03-15 18:33:41 +00:00
8344035: Replace predicate walking code in Loop Unswitching with a predicate visitor
Reviewed-by: roland, kvn
This commit is contained in:
Christian Hagedorn
parent
b37f123625
commit
ed0b555627
@ -100,8 +100,7 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred,
|
||||
// is an IfTrue projection. This code is also used to clone predicates to cloned loops.
|
||||
IfTrueNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_success_proj,
|
||||
Node* new_entry, const Deoptimization::DeoptReason reason,
|
||||
const int opcode, const bool rewire_uncommon_proj_phi_inputs,
|
||||
AssertionPredicateType assertion_predicate_type) {
|
||||
const int opcode, const bool rewire_uncommon_proj_phi_inputs) {
|
||||
assert(parse_predicate_success_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
|
||||
ParsePredicateNode* parse_predicate = parse_predicate_success_proj->in(0)->as_ParsePredicate();
|
||||
ParsePredicateUncommonProj* uncommon_proj = parse_predicate->uncommon_proj();
|
||||
@ -143,12 +142,10 @@ IfTrueNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessPro
|
||||
IfNode* new_iff = nullptr;
|
||||
switch (opcode) {
|
||||
case Op_If:
|
||||
new_iff = new IfNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt
|
||||
NOT_PRODUCT(COMMA assertion_predicate_type));
|
||||
new_iff = new IfNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt);
|
||||
break;
|
||||
case Op_RangeCheck:
|
||||
new_iff = new RangeCheckNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt
|
||||
NOT_PRODUCT(COMMA assertion_predicate_type));
|
||||
new_iff = new RangeCheckNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt);
|
||||
break;
|
||||
case Op_ParsePredicate:
|
||||
new_iff = new ParsePredicateNode(entry, reason, &_igvn);
|
||||
@ -272,150 +269,15 @@ void PhaseIdealLoop::fix_cloned_data_node_controls(const ProjNode* old_uncommon_
|
||||
orig_to_clone.iterate_all(orig_clone_action);
|
||||
}
|
||||
|
||||
IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj,
|
||||
Node* new_entry, Deoptimization::DeoptReason reason,
|
||||
const bool slow_loop) {
|
||||
|
||||
IfProjNode* new_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, new_entry, reason, Op_ParsePredicate,
|
||||
slow_loop);
|
||||
assert(new_predicate_proj->is_IfTrue(), "the success projection of a Parse Predicate is a true projection");
|
||||
ParsePredicateNode* parse_predicate = new_predicate_proj->in(0)->as_ParsePredicate();
|
||||
return new_predicate_proj;
|
||||
}
|
||||
|
||||
// Clones Template Assertion Predicates to both unswitched loops starting at 'old_predicate_proj' by following its
|
||||
// control inputs. It also rewires the control edges of data nodes with dependencies in the loop from the old predicates
|
||||
// to the new cloned predicates.
|
||||
void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
|
||||
ParsePredicateSuccessProj* old_parse_predicate_proj,
|
||||
ParsePredicateNode* true_path_loop_parse_predicate,
|
||||
ParsePredicateNode* false_path_loop_parse_predicate) {
|
||||
// Push the original Template Assertion Predicates on a list to later process them in reverse order to keep the
|
||||
// original predicate order.
|
||||
Unique_Node_List list;
|
||||
get_template_assertion_predicates(old_parse_predicate_proj, list);
|
||||
|
||||
Node_List to_process;
|
||||
for (int i = list.size() - 1; i >= 0; i--) {
|
||||
IfTrueNode* template_assertion_predicate_success_proj = list.at(i)->as_IfTrue();
|
||||
assert(template_assertion_predicate_success_proj->in(0)->is_If(), "must be If node");
|
||||
|
||||
IfTrueNode* true_path_loop_proj =
|
||||
clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj,
|
||||
true_path_loop_parse_predicate);
|
||||
IfTrueNode* false_path_loop_proj =
|
||||
clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj,
|
||||
false_path_loop_parse_predicate);
|
||||
|
||||
// Update control dependent data nodes.
|
||||
for (DUIterator j = template_assertion_predicate_success_proj->outs();
|
||||
template_assertion_predicate_success_proj->has_out(j);
|
||||
j++) {
|
||||
Node* true_path_loop_node = template_assertion_predicate_success_proj->out(j);
|
||||
if (loop->is_member(get_loop(ctrl_or_self(true_path_loop_node)))) {
|
||||
assert(true_path_loop_node->in(0) == template_assertion_predicate_success_proj, "only control edge");
|
||||
Node* false_path_loop_node = old_new[true_path_loop_node->_idx];
|
||||
assert(false_path_loop_node->in(0) == template_assertion_predicate_success_proj, "only control edge");
|
||||
_igvn.replace_input_of(true_path_loop_node, 0, true_path_loop_proj);
|
||||
to_process.push(false_path_loop_node);
|
||||
--j;
|
||||
}
|
||||
}
|
||||
// Have to delay updates to the false path loop so uses of predicate are not modified while we iterate on them.
|
||||
while (to_process.size() > 0) {
|
||||
Node* slow_node = to_process.pop();
|
||||
_igvn.replace_input_of(slow_node, 0, false_path_loop_proj);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Put all Template Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque'
|
||||
// is set, then the OpaqueTemplateAssertionPredicateNode nodes of the Assertion Predicates are put on the list instead
|
||||
// of the projections.
|
||||
void PhaseIdealLoop::get_template_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list,
|
||||
const bool get_opaque) {
|
||||
// Put all OpaqueTemplateAssertionPredicate nodes on a list, starting at 'predicate' and going up in the tree.
|
||||
void PhaseIdealLoop::get_opaque_template_assertion_predicate_nodes(ParsePredicateSuccessProj* parse_predicate_proj,
|
||||
Unique_Node_List& list) {
|
||||
Deoptimization::DeoptReason deopt_reason = parse_predicate_proj->in(0)->as_ParsePredicate()->deopt_reason();
|
||||
PredicateBlockIterator predicate_iterator(parse_predicate_proj, deopt_reason);
|
||||
TemplateAssertionPredicateCollector template_assertion_predicate_collector(list, get_opaque);
|
||||
predicate_iterator.for_each(template_assertion_predicate_collector);
|
||||
OpaqueTemplateAssertionPredicateCollector opaque_template_assertion_predicate_collector(list);
|
||||
predicate_iterator.for_each(opaque_template_assertion_predicate_collector);
|
||||
}
|
||||
|
||||
// Clone an Assertion Predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon
|
||||
// traps are kept for the predicate (a Halt node is used later when creating pre/main/post loops and copying this cloned
|
||||
// predicate again).
|
||||
IfTrueNode*
|
||||
PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj,
|
||||
ParsePredicateNode* unswitched_loop_parse_predicate) {
|
||||
TemplateAssertionPredicate template_assertion_predicate(template_assertion_predicate_success_proj);
|
||||
IfTrueNode* template_success_proj = template_assertion_predicate.clone(unswitched_loop_parse_predicate->in(0), this);
|
||||
_igvn.replace_input_of(unswitched_loop_parse_predicate, 0, template_success_proj);
|
||||
set_idom(unswitched_loop_parse_predicate, template_success_proj, dom_depth(template_success_proj));
|
||||
return template_success_proj;
|
||||
}
|
||||
|
||||
// Clone the old Parse Predicates and Assertion Predicates before the unswitch If to the unswitched loops after the
|
||||
// unswitch If.
|
||||
void PhaseIdealLoop::clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new,
|
||||
IfProjNode*& true_path_loop_entry,
|
||||
IfProjNode*& false_path_loop_entry) {
|
||||
LoopNode* head = loop->_head->as_Loop();
|
||||
Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
|
||||
|
||||
const Predicates predicates(entry);
|
||||
clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.loop_predicate_block(),
|
||||
Deoptimization::Reason_predicate, true_path_loop_entry, false_path_loop_entry);
|
||||
clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.profiled_loop_predicate_block(),
|
||||
Deoptimization::Reason_profile_predicate, true_path_loop_entry, false_path_loop_entry);
|
||||
|
||||
const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block();
|
||||
if (loop_limit_check_predicate_block->has_parse_predicate() && !head->is_CountedLoop()) {
|
||||
// Don't clone the Loop Limit Check Parse Predicate if we already have a counted loop (a Loop Limit Check Predicate
|
||||
// is only created when converting a LoopNode to a CountedLoopNode).
|
||||
clone_parse_predicate_to_unswitched_loops(loop_limit_check_predicate_block, Deoptimization::Reason_loop_limit_check,
|
||||
true_path_loop_entry, false_path_loop_entry);
|
||||
}
|
||||
}
|
||||
|
||||
// Clone the Parse Predicate and Template Assertion Predicates of a Loop Predication related Predicate Block.
|
||||
void PhaseIdealLoop::clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
|
||||
const PredicateBlock* predicate_block,
|
||||
Deoptimization::DeoptReason reason,
|
||||
IfProjNode*& true_path_loop_entry,
|
||||
IfProjNode*& false_path_loop_entry) {
|
||||
if (predicate_block->has_parse_predicate()) {
|
||||
// We currently only clone Assertion Predicates if there are Parse Predicates. This is not entirely correct and will
|
||||
// be changed with the complete fix for Assertion Predicates.
|
||||
clone_parse_predicate_to_unswitched_loops(predicate_block, reason, true_path_loop_entry, false_path_loop_entry);
|
||||
assert(true_path_loop_entry->in(0)->is_ParsePredicate() && false_path_loop_entry->in(0)->is_ParsePredicate(),
|
||||
"must be success projections of the cloned Parse Predicates");
|
||||
clone_assertion_predicates_to_unswitched_loop(loop, old_new, predicate_block->parse_predicate_success_proj(),
|
||||
true_path_loop_entry->in(0)->as_ParsePredicate(),
|
||||
false_path_loop_entry->in(0)->as_ParsePredicate());
|
||||
}
|
||||
}
|
||||
|
||||
void PhaseIdealLoop::clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block,
|
||||
Deoptimization::DeoptReason reason,
|
||||
IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred) {
|
||||
assert(predicate_block->has_parse_predicate(), "must have parse predicate");
|
||||
ParsePredicateSuccessProj* parse_predicate_proj = predicate_block->parse_predicate_success_proj();
|
||||
iffast_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, iffast_pred, reason, false);
|
||||
check_cloned_parse_predicate_for_unswitching(iffast_pred, true);
|
||||
|
||||
ifslow_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, ifslow_pred, reason, true);
|
||||
check_cloned_parse_predicate_for_unswitching(ifslow_pred, false);
|
||||
}
|
||||
|
||||
#ifndef PRODUCT
|
||||
void PhaseIdealLoop::check_cloned_parse_predicate_for_unswitching(const Node* new_entry, const bool is_fast_loop) {
|
||||
assert(new_entry != nullptr, "IfTrue or IfFalse after clone predicate");
|
||||
if (TraceLoopPredicate) {
|
||||
tty->print("Parse Predicate cloned to %s loop: ", is_fast_loop ? "fast" : "slow");
|
||||
new_entry->in(0)->dump();
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
//------------------------------Invariance-----------------------------------
|
||||
// Helper class for loop_predication_impl to compute invariance on the fly and
|
||||
// clone invariants.
|
||||
|
||||
@ -236,15 +236,47 @@ class OriginalLoop : public StackObj {
|
||||
_phase(loop->_phase) {}
|
||||
NONCOPYABLE(OriginalLoop);
|
||||
|
||||
private:
|
||||
void fix_loop_entries(IfProjNode* true_path_loop_entry, IfProjNode* false_path_loop_entry) {
|
||||
_phase->replace_loop_entry(_loop_head, true_path_loop_entry);
|
||||
LoopNode* false_path_loop_strip_mined_head = old_to_new(_loop_head)->as_Loop();
|
||||
_phase->replace_loop_entry(false_path_loop_strip_mined_head, false_path_loop_entry);
|
||||
// Unswitch the original loop on the invariant loop selector by creating a true-path-loop and a false-path-loop.
|
||||
// Remove the unswitch candidate If from both unswitched loop versions which are now covered by the loop selector If.
|
||||
void unswitch(const UnswitchedLoopSelector& unswitched_loop_selector) {
|
||||
const uint first_false_path_loop_node_index = _phase->C->unique();
|
||||
clone_loop(unswitched_loop_selector);
|
||||
|
||||
move_parse_and_template_assertion_predicates_to_unswitched_loops(unswitched_loop_selector,
|
||||
first_false_path_loop_node_index);
|
||||
DEBUG_ONLY(verify_unswitched_loop_versions(_loop->_head->as_Loop(), unswitched_loop_selector);)
|
||||
|
||||
_phase->recompute_dom_depth();
|
||||
remove_unswitch_candidate_from_loops(unswitched_loop_selector);
|
||||
}
|
||||
|
||||
Node* old_to_new(const Node* old) const {
|
||||
return _old_new[old->_idx];
|
||||
private:
|
||||
void clone_loop(const UnswitchedLoopSelector& unswitched_loop_selector) {
|
||||
_phase->clone_loop(_loop, _old_new, _phase->dom_depth(_loop_head),
|
||||
PhaseIdealLoop::CloneIncludesStripMined, unswitched_loop_selector.selector());
|
||||
fix_loop_entries(unswitched_loop_selector);
|
||||
}
|
||||
|
||||
void fix_loop_entries(const UnswitchedLoopSelector& unswitched_loop_selector) {
|
||||
_phase->replace_loop_entry(_loop_head, unswitched_loop_selector.true_path_loop_proj());
|
||||
LoopNode* false_path_loop_strip_mined_head = old_to_new(_loop_head)->as_Loop();
|
||||
_phase->replace_loop_entry(false_path_loop_strip_mined_head,
|
||||
unswitched_loop_selector.false_path_loop_proj());
|
||||
}
|
||||
|
||||
// Moves the Parse And Template Assertion Predicates to the true and false path loop. They are inserted between the
|
||||
// loop heads and the loop selector If projections. The old Parse and Template Assertion Predicates before
|
||||
// the unswitched loop selector are killed.
|
||||
void move_parse_and_template_assertion_predicates_to_unswitched_loops(
|
||||
const UnswitchedLoopSelector& unswitched_loop_selector, const uint first_false_path_loop_node_index) const {
|
||||
const NodeInOriginalLoopBody node_in_true_path_loop_body(first_false_path_loop_node_index, _old_new);
|
||||
const NodeInClonedLoopBody node_in_false_path_loop_body(first_false_path_loop_node_index);
|
||||
CloneUnswitchedLoopPredicatesVisitor
|
||||
clone_unswitched_loop_predicates_visitor(_loop_head, old_to_new(_loop_head)->as_Loop(), node_in_true_path_loop_body,
|
||||
node_in_false_path_loop_body, _phase);
|
||||
Node* source_loop_entry = unswitched_loop_selector.selector()->in(0);
|
||||
PredicateIterator predicate_iterator(source_loop_entry);
|
||||
predicate_iterator.for_each(clone_unswitched_loop_predicates_visitor);
|
||||
}
|
||||
|
||||
#ifdef ASSERT
|
||||
@ -263,6 +295,10 @@ class OriginalLoop : public StackObj {
|
||||
}
|
||||
#endif // ASSERT
|
||||
|
||||
Node* old_to_new(const Node* old) const {
|
||||
return _old_new[old->_idx];
|
||||
}
|
||||
|
||||
// Remove the unswitch candidate If nodes in both unswitched loop versions which are now dominated by the loop selector
|
||||
// If node. Keep the true-path-path in the true-path-loop and the false-path-path in the false-path-loop by setting
|
||||
// the bool input accordingly. The unswitch candidate If nodes are folded in the next IGVN round.
|
||||
@ -276,28 +312,6 @@ class OriginalLoop : public StackObj {
|
||||
_phase->dominated_by(unswitched_loop_selector.false_path_loop_proj(), unswitching_candidate_clone);
|
||||
}
|
||||
|
||||
public:
|
||||
// Unswitch the original loop on the invariant loop selector by creating a true-path-loop and a false-path-loop.
|
||||
// Remove the unswitch candidate If from both unswitched loop versions which are now covered by the loop selector If.
|
||||
void unswitch(const UnswitchedLoopSelector& unswitched_loop_selector) {
|
||||
_phase->clone_loop(_loop, _old_new, _phase->dom_depth(_loop_head),
|
||||
PhaseIdealLoop::CloneIncludesStripMined, unswitched_loop_selector.selector());
|
||||
|
||||
// At this point, the selector If projections are the corresponding loop entries.
|
||||
// clone_parse_and_assertion_predicates_to_unswitched_loop() could clone additional predicates after the selector
|
||||
// If projections. The loop entries are updated accordingly.
|
||||
IfProjNode* true_path_loop_entry = unswitched_loop_selector.true_path_loop_proj();
|
||||
IfProjNode* false_path_loop_entry = unswitched_loop_selector.false_path_loop_proj();
|
||||
_phase->clone_parse_and_assertion_predicates_to_unswitched_loop(_loop, _old_new,
|
||||
true_path_loop_entry, false_path_loop_entry);
|
||||
|
||||
fix_loop_entries(true_path_loop_entry, false_path_loop_entry);
|
||||
|
||||
DEBUG_ONLY(verify_unswitched_loop_versions(_loop->_head->as_Loop(), unswitched_loop_selector);)
|
||||
|
||||
_phase->recompute_dom_depth();
|
||||
remove_unswitch_candidate_from_loops(unswitched_loop_selector);
|
||||
}
|
||||
};
|
||||
|
||||
// See comments below file header for more information about Loop Unswitching.
|
||||
|
||||
@ -4458,7 +4458,7 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal
|
||||
const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block();
|
||||
if (profiled_loop_predicate_block->has_parse_predicate()) {
|
||||
ParsePredicateSuccessProj* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj();
|
||||
get_template_assertion_predicates(parse_predicate_proj, useful_predicates, true);
|
||||
get_opaque_template_assertion_predicate_nodes(parse_predicate_proj, useful_predicates);
|
||||
}
|
||||
}
|
||||
|
||||
@ -4466,7 +4466,7 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal
|
||||
const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block();
|
||||
if (loop_predicate_block->has_parse_predicate()) {
|
||||
ParsePredicateSuccessProj* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj();
|
||||
get_template_assertion_predicates(parse_predicate_proj, useful_predicates, true);
|
||||
get_opaque_template_assertion_predicate_nodes(parse_predicate_proj, useful_predicates);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -942,7 +942,8 @@ private:
|
||||
static void ensure_zero_trip_guard_proj(Node* node, bool is_main_loop);
|
||||
#endif
|
||||
private:
|
||||
static void get_template_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list, bool get_opaque = false);
|
||||
static void get_opaque_template_assertion_predicate_nodes(ParsePredicateSuccessProj* parse_predicate_proj,
|
||||
Unique_Node_List& list);
|
||||
void update_main_loop_assertion_predicates(CountedLoopNode* main_loop_head);
|
||||
void initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head,
|
||||
CountedLoopNode* remaining_loop_head,
|
||||
@ -1350,9 +1351,8 @@ public:
|
||||
|
||||
// Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted
|
||||
IfTrueNode* create_new_if_for_predicate(
|
||||
ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason, int opcode,
|
||||
bool rewire_uncommon_proj_phi_inputs = false,
|
||||
AssertionPredicateType assertion_predicate_type = AssertionPredicateType::None);
|
||||
ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason, int opcode,
|
||||
bool rewire_uncommon_proj_phi_inputs = false);
|
||||
|
||||
private:
|
||||
// Helper functions for create_new_if_for_predicate()
|
||||
@ -1661,28 +1661,7 @@ private:
|
||||
_nodes_required = UINT_MAX;
|
||||
}
|
||||
|
||||
public:
|
||||
// Clone Parse Predicates to slow and fast loop when unswitching a loop
|
||||
void clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new,
|
||||
IfProjNode*& true_path_loop_entry,
|
||||
IfProjNode*& false_path_loop_entry);
|
||||
private:
|
||||
void clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
|
||||
const PredicateBlock* predicate_block,
|
||||
Deoptimization::DeoptReason reason,
|
||||
IfProjNode*& true_path_loop_entry,
|
||||
IfProjNode*& false_path_loop_entry);
|
||||
void clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, Deoptimization::DeoptReason reason,
|
||||
IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred);
|
||||
IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry,
|
||||
Deoptimization::DeoptReason reason, bool slow_loop);
|
||||
void clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
|
||||
ParsePredicateSuccessProj* old_parse_predicate_proj,
|
||||
ParsePredicateNode* true_path_loop_parse_predicate,
|
||||
ParsePredicateNode* false_path_loop_parse_predicate);
|
||||
IfTrueNode* clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj,
|
||||
ParsePredicateNode* unswitched_loop_parse_predicate);
|
||||
static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN;
|
||||
|
||||
bool _created_loop_node;
|
||||
DEBUG_ONLY(void dump_idoms(Node* early, Node* wrong_lca);)
|
||||
|
||||
@ -23,6 +23,7 @@
|
||||
*/
|
||||
|
||||
#include "precompiled.hpp"
|
||||
#include "opto/connode.hpp"
|
||||
#include "opto/loopnode.hpp"
|
||||
#include "opto/opaquenode.hpp"
|
||||
#include "opto/phaseX.hpp"
|
||||
|
||||
@ -82,6 +82,25 @@ ParsePredicateNode* ParsePredicate::init_parse_predicate(Node* parse_predicate_p
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ParsePredicateSuccessProj* ParsePredicate::clone_to_unswitched_loop(Node* new_control, const bool is_true_path_loop,
|
||||
PhaseIdealLoop* phase) const {
|
||||
ParsePredicateSuccessProj* success_proj = phase->create_new_if_for_predicate(_success_proj, new_control,
|
||||
_parse_predicate_node->deopt_reason(),
|
||||
Op_ParsePredicate, is_true_path_loop);
|
||||
NOT_PRODUCT(trace_cloned_parse_predicate(is_true_path_loop, success_proj));
|
||||
return success_proj;
|
||||
}
|
||||
|
||||
#ifndef PRODUCT
|
||||
void ParsePredicate::trace_cloned_parse_predicate(const bool is_true_path_loop,
|
||||
const ParsePredicateSuccessProj* success_proj) {
|
||||
if (TraceLoopPredicate) {
|
||||
tty->print("Parse Predicate cloned to %s path loop: ", is_true_path_loop ? "true" : "false");
|
||||
success_proj->in(0)->dump();
|
||||
}
|
||||
}
|
||||
#endif // NOT PRODUCT
|
||||
|
||||
Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* if_proj) {
|
||||
CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern();
|
||||
if (uct_call == nullptr) {
|
||||
@ -196,6 +215,12 @@ IfTrueNode* TemplateAssertionPredicate::initialize(PhaseIdealLoop* phase, Node*
|
||||
return success_proj;
|
||||
}
|
||||
|
||||
// Kills the Template Assertion Predicate by setting the condition to true. Will be folded away in the next IGVN round.
|
||||
void TemplateAssertionPredicate::kill(PhaseIdealLoop* phase) const {
|
||||
ConINode* true_con = phase->intcon(1);
|
||||
phase->igvn().replace_input_of(_if_node, 1, true_con);
|
||||
}
|
||||
|
||||
#ifdef ASSERT
|
||||
// Class to verify Initialized and Template Assertion Predicates by trying to find OpaqueLoop*Nodes.
|
||||
class OpaqueLoopNodesVerifier : public BFSActions {
|
||||
@ -947,8 +972,8 @@ IfTrueNode* CreateAssertionPredicatesVisitor::clone_template_and_replace_init_in
|
||||
// x
|
||||
// | old target
|
||||
// Template Assertion loop entry
|
||||
// Predicate 1 old target clone | \
|
||||
// | loop entry TAP 2 | cloned Template Assertion
|
||||
// Predicate 1 old target clone | \
|
||||
// | loop entry TAP 2 | cloned Template Assertion
|
||||
// Template Assertion | ======> | Predicate 2
|
||||
// Predicate 2 target loop |
|
||||
// | target loop #_current_predicate_chain_head
|
||||
@ -977,6 +1002,73 @@ void CreateAssertionPredicatesVisitor::rewire_to_old_predicate_chain_head(
|
||||
}
|
||||
}
|
||||
|
||||
TargetLoopPredicateChain::TargetLoopPredicateChain(LoopNode* loop_head, PhaseIdealLoop* phase)
|
||||
: DEBUG_ONLY(_old_target_loop_entry(loop_head->in(LoopNode::EntryControl)) COMMA)
|
||||
DEBUG_ONLY(_node_index_before_cloning(phase->C->unique()) COMMA)
|
||||
_current_predicate_chain_head(loop_head),
|
||||
_phase(phase) {}
|
||||
|
||||
// Inserts the provided newly cloned predicate to the head of the target loop predicate chain.
|
||||
void TargetLoopPredicateChain::insert_predicate(IfTrueNode* predicate_success_proj) {
|
||||
rewire_to_target_chain_head(predicate_success_proj);
|
||||
_current_predicate_chain_head = predicate_success_proj->in(0);
|
||||
assert(predicate_success_proj->_idx >= _node_index_before_cloning, "must be a newly cloned predicate");
|
||||
assert(_current_predicate_chain_head->in(0) == _old_target_loop_entry &&
|
||||
_old_target_loop_entry->unique_ctrl_out() == _current_predicate_chain_head , "must be connected now");
|
||||
}
|
||||
|
||||
void TargetLoopPredicateChain::rewire_to_target_chain_head(IfTrueNode* template_assertion_predicate_success_proj) const {
|
||||
if (_current_predicate_chain_head->is_Loop()) {
|
||||
_phase->replace_loop_entry(_current_predicate_chain_head->as_Loop(), template_assertion_predicate_success_proj);
|
||||
} else {
|
||||
_phase->replace_control(_current_predicate_chain_head, template_assertion_predicate_success_proj);
|
||||
}
|
||||
}
|
||||
|
||||
ClonePredicateToTargetLoop::ClonePredicateToTargetLoop(LoopNode* target_loop_head, const NodeInLoopBody& node_in_loop_body,
|
||||
PhaseIdealLoop* phase)
|
||||
: _old_target_loop_entry(target_loop_head->in(LoopNode::EntryControl)),
|
||||
_target_loop_predicate_chain(target_loop_head, phase),
|
||||
_node_in_loop_body(node_in_loop_body),
|
||||
_phase(phase) {}
|
||||
|
||||
|
||||
CloneUnswitchedLoopPredicatesVisitor::CloneUnswitchedLoopPredicatesVisitor(
|
||||
LoopNode* true_path_loop_head, LoopNode* false_path_loop_head,
|
||||
const NodeInOriginalLoopBody& node_in_true_path_loop_body, const NodeInClonedLoopBody& node_in_false_path_loop_body,
|
||||
PhaseIdealLoop* phase)
|
||||
: _clone_predicate_to_true_path_loop(true_path_loop_head, node_in_true_path_loop_body, phase),
|
||||
_clone_predicate_to_false_path_loop(false_path_loop_head, node_in_false_path_loop_body, phase),
|
||||
_phase(phase),
|
||||
_has_hoisted_check_parse_predicates(false) {}
|
||||
|
||||
// Keep track of whether we are in the correct Predicate Block where Template Assertion Predicates can be found.
|
||||
// The PredicateIterator will always start at the loop entry and first visits the Loop Limit Check Predicate Block.
|
||||
void CloneUnswitchedLoopPredicatesVisitor::visit(const ParsePredicate& parse_predicate) {
|
||||
Deoptimization::DeoptReason deopt_reason = parse_predicate.head()->deopt_reason();
|
||||
if (deopt_reason == Deoptimization::Reason_predicate ||
|
||||
deopt_reason == Deoptimization::Reason_profile_predicate) {
|
||||
_has_hoisted_check_parse_predicates = true;
|
||||
}
|
||||
|
||||
_clone_predicate_to_true_path_loop.clone_parse_predicate(parse_predicate, true);
|
||||
_clone_predicate_to_false_path_loop.clone_parse_predicate(parse_predicate, false);
|
||||
parse_predicate.kill(_phase->igvn());
|
||||
}
|
||||
|
||||
// Clone the Template Assertion Predicate, which is currently found before the newly added unswitched loop selector,
|
||||
// to the true path and false path loop.
|
||||
void CloneUnswitchedLoopPredicatesVisitor::visit(const TemplateAssertionPredicate& template_assertion_predicate) {
|
||||
if (!_has_hoisted_check_parse_predicates) {
|
||||
// Only process if we are in the correct Predicate Block.
|
||||
return;
|
||||
}
|
||||
|
||||
_clone_predicate_to_true_path_loop.clone_template_assertion_predicate(template_assertion_predicate);
|
||||
_clone_predicate_to_false_path_loop.clone_template_assertion_predicate(template_assertion_predicate);
|
||||
template_assertion_predicate.kill(_phase);
|
||||
}
|
||||
|
||||
// Clone the Template Assertion Predicate and set a new input for the OpaqueLoopStrideNode.
|
||||
void UpdateStrideForAssertionPredicates::visit(const TemplateAssertionPredicate& template_assertion_predicate) {
|
||||
if (!template_assertion_predicate.is_last_value()) {
|
||||
|
||||
@ -26,7 +26,6 @@
|
||||
#define SHARE_OPTO_PREDICATES_HPP
|
||||
|
||||
#include "opto/cfgnode.hpp"
|
||||
#include "opto/connode.hpp"
|
||||
#include "opto/opaquenode.hpp"
|
||||
|
||||
class IdealLoopTree;
|
||||
@ -232,7 +231,7 @@ class Predicate : public StackObj {
|
||||
// Generic predicate visitor that does nothing. Subclass this visitor to add customized actions for each predicate.
|
||||
// The visit methods of this visitor are called from the predicate iterator classes which walk the predicate chain.
|
||||
// Use the UnifiedPredicateVisitor if the type of the predicate does not matter.
|
||||
class PredicateVisitor : StackObj {
|
||||
class PredicateVisitor : public StackObj {
|
||||
public:
|
||||
virtual void visit(const ParsePredicate& parse_predicate) {}
|
||||
virtual void visit(const RuntimePredicate& runtime_predicate) {}
|
||||
@ -297,6 +296,8 @@ class ParsePredicate : public Predicate {
|
||||
}
|
||||
|
||||
static ParsePredicateNode* init_parse_predicate(Node* parse_predicate_proj, Deoptimization::DeoptReason deopt_reason);
|
||||
NOT_PRODUCT(static void trace_cloned_parse_predicate(bool is_true_path_loop,
|
||||
const ParsePredicateSuccessProj* success_proj);)
|
||||
|
||||
public:
|
||||
ParsePredicate(Node* parse_predicate_proj, Deoptimization::DeoptReason deopt_reason)
|
||||
@ -325,6 +326,15 @@ class ParsePredicate : public Predicate {
|
||||
assert(is_valid(), "must be valid");
|
||||
return _success_proj;
|
||||
}
|
||||
|
||||
ParsePredicateSuccessProj* clone_to_unswitched_loop(Node* new_control, bool is_true_path_loop,
|
||||
PhaseIdealLoop* phase) const;
|
||||
|
||||
// Kills this Parse Predicate by marking it useless. Will be folded away in the next IGVN round.
|
||||
void kill(const PhaseIterGVN& igvn) const {
|
||||
_parse_predicate_node->mark_useless();
|
||||
igvn._worklist.push(_parse_predicate_node);
|
||||
}
|
||||
};
|
||||
|
||||
// Class to represent a Runtime Predicate which always has an associated UCT on the failing path.
|
||||
@ -399,6 +409,7 @@ class TemplateAssertionPredicate : public Predicate {
|
||||
IfTrueNode* initialize(PhaseIdealLoop* phase, Node* new_control) const;
|
||||
void rewire_loop_data_dependencies(IfTrueNode* target_predicate, const NodeInLoopBody& data_in_loop_body,
|
||||
PhaseIdealLoop* phase) const;
|
||||
void kill(PhaseIdealLoop* phase) const;
|
||||
static bool is_predicate(Node* node);
|
||||
|
||||
#ifdef ASSERT
|
||||
@ -1006,25 +1017,103 @@ class CreateAssertionPredicatesVisitor : public PredicateVisitor {
|
||||
void visit(const TemplateAssertionPredicate& template_assertion_predicate) override;
|
||||
};
|
||||
|
||||
// This visitor collects all Template Assertion Predicates If nodes or the corresponding Opaque nodes, depending on the
|
||||
// provided 'get_opaque' flag, to the provided list.
|
||||
class TemplateAssertionPredicateCollector : public PredicateVisitor {
|
||||
Unique_Node_List& _list;
|
||||
const bool _get_opaque;
|
||||
// This class establishes a predicate chain at the target loop by rewiring newly cloned predicates to the current head
|
||||
// of the predicate chain.
|
||||
class TargetLoopPredicateChain : public StackObj {
|
||||
DEBUG_ONLY(const Node* const _old_target_loop_entry;)
|
||||
DEBUG_ONLY(const node_idx_t _node_index_before_cloning;)
|
||||
Node* _current_predicate_chain_head;
|
||||
PhaseIdealLoop* const _phase;
|
||||
|
||||
void rewire_to_target_chain_head(IfTrueNode* template_assertion_predicate_success_proj) const;
|
||||
|
||||
public:
|
||||
TargetLoopPredicateChain(LoopNode* loop_head, PhaseIdealLoop* phase);
|
||||
NONCOPYABLE(TargetLoopPredicateChain);
|
||||
|
||||
void insert_predicate(IfTrueNode* predicate_success_proj);
|
||||
};
|
||||
|
||||
// This class clones Parse and Template Assertion Predicates to the provided target loop. This also involves rewiring
|
||||
// of any data pinned to Template Assertion Predicates. The Template Assertion Predicate Expressions are cloned
|
||||
// without applying any changes to them.
|
||||
//
|
||||
// Each time a predicate is cloned, it is inserted at the top of previously cloned predicates. This ensures that the
|
||||
// target loop predicate chain order of the newly cloned predicates is the same as in the source loop from which the
|
||||
// predicates were cloned from.
|
||||
//
|
||||
// Template Assertion Predicate Example:
|
||||
//
|
||||
// x _old_target_loop_entry _old_target_loop_entry
|
||||
// | | | |
|
||||
// Template Assertion | Cloned Template 2. rewire data Cloned Template
|
||||
// Predicate 1. clone | Assertion Predicate and predicate Assertion Predicate
|
||||
// | \ =======> | ===============> | \
|
||||
// | data | | data
|
||||
// | | |
|
||||
// source loop head target loop head target loop head
|
||||
class ClonePredicateToTargetLoop : public StackObj {
|
||||
Node* const _old_target_loop_entry; // Used as control for each newly cloned predicate.
|
||||
TargetLoopPredicateChain _target_loop_predicate_chain;
|
||||
const NodeInLoopBody& _node_in_loop_body;
|
||||
PhaseIdealLoop* const _phase;
|
||||
|
||||
public:
|
||||
ClonePredicateToTargetLoop(LoopNode* target_loop_head, const NodeInLoopBody& node_in_loop_body, PhaseIdealLoop* phase);
|
||||
|
||||
// Clones the provided Parse Predicate to the head of the current predicate chain at the target loop.
|
||||
void clone_parse_predicate(const ParsePredicate& parse_predicate, bool is_true_path_loop) {
|
||||
ParsePredicateSuccessProj* cloned_parse_predicate_success_proj =
|
||||
parse_predicate.clone_to_unswitched_loop(_old_target_loop_entry, is_true_path_loop, _phase);
|
||||
_target_loop_predicate_chain.insert_predicate(cloned_parse_predicate_success_proj);
|
||||
}
|
||||
|
||||
// Clones the provided Template Assertion Predicate to the head of the current predicate chain at the target loop.
|
||||
void clone_template_assertion_predicate(const TemplateAssertionPredicate& template_assertion_predicate) {
|
||||
IfTrueNode* cloned_template_success_proj = template_assertion_predicate.clone(_old_target_loop_entry,
|
||||
_phase);
|
||||
template_assertion_predicate.rewire_loop_data_dependencies(cloned_template_success_proj, _node_in_loop_body, _phase);
|
||||
_target_loop_predicate_chain.insert_predicate(cloned_template_success_proj);
|
||||
}
|
||||
};
|
||||
|
||||
// Visitor to clone Parse and Template Assertion Predicates from a loop to its unswitched true and false path loop.
|
||||
// The cloned predicates are not updated in any way. Thus, an Initialized Assertion Predicate is also not required to
|
||||
// be created. Note that the data dependencies from the Template Assertion Predicates are also updated to the newly
|
||||
// cloned Template Assertion Predicates, depending on whether they belong to the true or false path loop.
|
||||
class CloneUnswitchedLoopPredicatesVisitor : public PredicateVisitor {
|
||||
ClonePredicateToTargetLoop _clone_predicate_to_true_path_loop;
|
||||
ClonePredicateToTargetLoop _clone_predicate_to_false_path_loop;
|
||||
|
||||
PhaseIdealLoop* const _phase;
|
||||
bool _has_hoisted_check_parse_predicates;
|
||||
|
||||
public:
|
||||
TemplateAssertionPredicateCollector(Unique_Node_List& list, const bool get_opaque)
|
||||
: _list(list),
|
||||
_get_opaque(get_opaque) {}
|
||||
CloneUnswitchedLoopPredicatesVisitor(LoopNode* true_path_loop_head,
|
||||
LoopNode* false_path_loop_head,
|
||||
const NodeInOriginalLoopBody& node_in_true_path_loop_body,
|
||||
const NodeInClonedLoopBody& node_in_false_path_loop_body,
|
||||
PhaseIdealLoop* phase);
|
||||
NONCOPYABLE(CloneUnswitchedLoopPredicatesVisitor);
|
||||
|
||||
using PredicateVisitor::visit;
|
||||
|
||||
void visit(const ParsePredicate& parse_predicate) override;
|
||||
void visit(const TemplateAssertionPredicate& template_assertion_predicate) override;
|
||||
};
|
||||
|
||||
// This visitor collects all OpaqueTemplateAssertionNodes of Template Assertion Predicates. This is used for cleaning
|
||||
// up unused Template Assertion Predicates.
|
||||
class OpaqueTemplateAssertionPredicateCollector : public PredicateVisitor {
|
||||
Unique_Node_List& _list;
|
||||
|
||||
public:
|
||||
explicit OpaqueTemplateAssertionPredicateCollector(Unique_Node_List& list) : _list(list) {}
|
||||
|
||||
using PredicateVisitor::visit;
|
||||
|
||||
void visit(const TemplateAssertionPredicate& template_assertion_predicate) override {
|
||||
if (_get_opaque) {
|
||||
_list.push(template_assertion_predicate.opaque_node());
|
||||
} else {
|
||||
_list.push(template_assertion_predicate.tail());
|
||||
}
|
||||
_list.push(template_assertion_predicate.opaque_node());
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user