From be0aec957ae5ccd52edc98394cbc51a8f9c8e89f Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Thu, 26 Mar 2026 16:02:25 -0700 Subject: [PATCH] fixes --- src/hotspot/share/opto/callnode.cpp | 10 ++-- src/hotspot/share/opto/compile.cpp | 2 +- src/hotspot/share/opto/compile.hpp | 3 +- src/hotspot/share/opto/gcm.cpp | 7 +-- src/hotspot/share/opto/loopTransform.cpp | 73 +++++++++++++++++------- src/hotspot/share/opto/loopnode.cpp | 5 +- src/hotspot/share/opto/loopnode.hpp | 4 +- src/hotspot/share/opto/reachability.cpp | 62 ++++++++++++++------ 8 files changed, 112 insertions(+), 54 deletions(-) diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index 48e31789420..6b3ea260db4 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -967,9 +967,7 @@ void CallNode::extract_projections(CallProjections* projs, bool separate_io_proj switch (cpn->_con) { case CatchProjNode::fall_through_index: projs->fallthrough_catchproj = cpn; break; case CatchProjNode::catch_all_index: projs->catchall_catchproj = cpn; break; - default: { - assert(cpn->_con > 1, "not an exception table projection"); // exception table; rethrow case - } + default: break; // exception table; rethrow case } } } @@ -986,8 +984,10 @@ void CallNode::extract_projections(CallProjections* projs, bool separate_io_proj if (e->in(0)->as_CatchProj()->_con == CatchProjNode::catch_all_index) { assert(projs->exobj == nullptr, "only one"); projs->exobj = e; - } else { - assert(e->in(0)->as_CatchProj()->_con > 1, "not an exception table projection"); // exception table for rethrow case + } else { + // exception table for rethrow case + assert(e->in(0)->as_CatchProj()->_con != CatchProjNode::fall_through_index, + "not an exception table projection"); } } } diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index e317116aac0..f737add4e61 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -3984,7 +3984,7 @@ void Compile::final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_R } } - expand_reachability_fences(sfpt); + expand_reachability_edges(sfpt); // Skip next transformation if compressed oops are not used. if ((UseCompressedOops && !Matcher::gen_narrow_oop_implicit_null_checks()) || diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index dadbd813b6a..ff0085d79de 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -1313,7 +1313,8 @@ public: // Definitions of pd methods static void pd_compiler2_init(); - void expand_reachability_fences(Unique_Node_List& safepoints); + // Materialize reachability fences from reachability edges on safepoints. + void expand_reachability_edges(Unique_Node_List& safepoints); // Static parse-time type checking logic for gen_subtype_check: enum SubTypeCheckResult { SSC_always_false, SSC_always_true, SSC_easy_test, SSC_full_test }; diff --git a/src/hotspot/share/opto/gcm.cpp b/src/hotspot/share/opto/gcm.cpp index 2d6402fd6eb..e3d3108a22e 100644 --- a/src/hotspot/share/opto/gcm.cpp +++ b/src/hotspot/share/opto/gcm.cpp @@ -155,10 +155,9 @@ bool PhaseCFG::is_control_proj_or_safepoint(Node* n) const { bool result = n->is_ReachabilityFence() || (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_SafePoint) || (n->is_Proj() && n->as_Proj()->bottom_type() == Type::CONTROL); - assert(!result || - (n->is_ReachabilityFence()) || - (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_SafePoint) || - (n->is_Proj() && n->as_Proj()->_con == 0), "If control projection, it must be projection 0"); + assert(!n->is_Proj() || + n->as_Proj()->bottom_type() != Type::CONTROL || + n->as_Proj()->_con == 0, "If control projection, it must be projection 0"); return result; } diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index 113cf4f3b20..6c2f7a5cffe 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -50,36 +50,65 @@ // Given an IfNode, return the loop-exiting projection or null if both // arms remain in the loop. Node *IdealLoopTree::is_loop_exit(Node *iff) const { - if (iff->outcnt() != 2) return nullptr; // Ignore partially dead tests - PhaseIdealLoop *phase = _phase; + assert(iff->is_If(), "not an If: %s", iff->Name()); + assert(is_member(_phase->get_loop(iff)), "not related"); + + if (iff->outcnt() != 2) { + return nullptr; // Ignore partially dead tests + } // Test is an IfNode, has 2 projections. If BOTH are in the loop // we need loop unswitching instead of peeling. - if (!is_member(phase->get_loop(iff->raw_out(0)))) + if (!is_member(_phase->get_loop(iff->raw_out(0)))) { return iff->raw_out(0); - if (!is_member(phase->get_loop(iff->raw_out(1)))) + } + if (!is_member(_phase->get_loop(iff->raw_out(1)))) { return iff->raw_out(1); + } return nullptr; } //------------------------------unique_loop_exit_or_null---------------------- -// Return the loop-exit projection if it is unique. -IfFalseNode* IdealLoopTree::unique_loop_exit_or_null() { - if (is_loop()) { - if (head()->is_BaseCountedLoop()) { - ProjNode* loop_exit = head()->as_BaseCountedLoop()->loopexit()->proj_out_or_null(0 /* false */); - if (loop_exit != nullptr) { - return loop_exit->as_IfFalse(); - } - } else if (head()->is_OuterStripMinedLoop()) { - return head()->as_OuterStripMinedLoop()->outer_loop_exit(); - } else { - // For now, conservatively report multiple loop exits exist. +// Return the loop-exit projection if loop exit is unique. +IfFalseNode* IdealLoopTree::unique_loop_exit_proj_or_null() { + if (is_loop() && head()->is_BaseCountedLoop()) { + IfNode* loop_end = head()->as_BaseCountedLoop()->loopexit_or_null(); + if (loop_end == nullptr) { + return nullptr; // malformed loop shape } + // Look for other loop exits. + assert(_phase->is_dominator(head(), tail()), "sanity"); + for (Node* ctrl = tail(); ctrl != head(); ctrl = ctrl->in(0)) { + assert(is_member(_phase->get_loop(ctrl)), "sanity"); + if (ctrl->is_If()) { + if (!is_loop_exit(ctrl->as_If())) { + continue; // local branch + } else if (ctrl != loop_end) { + return nullptr; // multiple loop exits + } + } else if (ctrl->is_Region()) { + return nullptr; // give up on control flow merges + } else if (ctrl->is_ReachabilityFence() || ctrl->is_SafePoint() || ctrl->is_MemBar()) { + continue; // skip + } else if (ctrl->is_Proj()) { + if (ctrl->is_IfProj() || ctrl->Opcode() == Op_SCMemProj || ctrl->Opcode() == Op_Proj) { + // skip simple control projections + } else if (ctrl->is_CatchProj() || ctrl->is_JumpProj()) { + return nullptr; // give up on control flow splits + } else { + assert(false, "unknown control projection: %s", ctrl->Name()); + return nullptr; // stop on unknown control node + } + } else { + assert(false, "unknown CFG node: %s", ctrl->Name()); + return nullptr; // stop on unknown control node + } + } + assert(is_loop_exit(loop_end), "not a loop exit?"); + return loop_end->false_proj_or_null(); } - return nullptr; // not found or multiple loop exists exist + return nullptr; // not found or multiple loop exits } - //============================================================================= @@ -3125,9 +3154,13 @@ static CountedLoopNode* locate_pre_from_main(CountedLoopNode* main_loop) { Node* ctrl = main_loop->skip_assertion_predicates_with_halt(); assert(ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, ""); Node* iffm = ctrl->in(0); - assert(iffm->Opcode() == Op_If, ""); + assert(iffm->Opcode() == Op_If, "%s", iffm->Name()); Node* p_f = iffm->in(0); - assert(p_f->Opcode() == Op_IfFalse, ""); + // Skip ReachabilityFences hoisted out of pre-loop. + while (p_f->is_ReachabilityFence()) { + p_f = p_f->in(0); + } + assert(p_f->Opcode() == Op_IfFalse, "%s", p_f->Name()); CountedLoopNode* pre_loop = p_f->in(0)->as_CountedLoopEnd()->loopnode(); assert(pre_loop->is_pre_loop(), "No pre loop found"); return pre_loop; diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 0ddaed31518..0e87b7116c3 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4840,8 +4840,9 @@ void IdealLoopTree::register_reachability_fence(ReachabilityFenceNode* rf) { if (_reachability_fences == nullptr) { _reachability_fences = new Node_List(); } - assert(!_reachability_fences->contains(rf), "already registered"); - _reachability_fences->push(rf); + if (!_reachability_fences->contains(rf)) { + _reachability_fences->push(rf); + } } #ifndef PRODUCT diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 877f56f8d9d..3b006d5f913 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -722,8 +722,8 @@ public: // Check for Node being a loop-breaking test Node *is_loop_exit(Node *iff) const; - // Return unique loop-exit projection or null if there are multiple exits exist. - IfFalseNode* unique_loop_exit_or_null(); + // Return unique loop-exit projection or null if the loop has multiple exits. + IfFalseNode* unique_loop_exit_proj_or_null(); // Remove simplistic dead code from loop body void DCE_loop_body(); diff --git a/src/hotspot/share/opto/reachability.cpp b/src/hotspot/share/opto/reachability.cpp index cee3f337d23..9ed9bfb0161 100644 --- a/src/hotspot/share/opto/reachability.cpp +++ b/src/hotspot/share/opto/reachability.cpp @@ -216,32 +216,56 @@ bool PhaseIdealLoop::optimize_reachability_fences() { // ResourceMark rm; // NB! not safe because insert_rf may trigger _idom reallocation Unique_Node_List redundant_rfs; - GrowableArray> worklist; + typedef Pair LoopExit; + GrowableArray worklist; + for (int i = 0; i < C->reachability_fences_count(); i++) { ReachabilityFenceNode* rf = C->reachability_fence(i); assert(!rf->is_redundant(igvn()), "required"); // Move RFs out of counted loops when possible. IdealLoopTree* lpt = get_loop(rf); Node* referent = rf->referent(); - IfFalseNode* loop_exit = lpt->unique_loop_exit_or_null(); - if (lpt->is_invariant(referent) && loop_exit != nullptr) { - // Switch to the outermost loop. - for (IdealLoopTree* outer_loop = lpt->_parent; - outer_loop->is_invariant(referent) && outer_loop->unique_loop_exit_or_null() != nullptr; - outer_loop = outer_loop->_parent) { - assert(is_member(outer_loop, rf), ""); - loop_exit = outer_loop->unique_loop_exit_or_null(); + if (lpt->is_invariant(referent)) { + IfFalseNode* unique_loop_exit = lpt->unique_loop_exit_proj_or_null(); + if (unique_loop_exit != nullptr) { + // Skip over an outer strip-mined loop. + if (!lpt->is_root()) { + IdealLoopTree* outer_lpt = lpt->_parent; + if (outer_lpt->head()->is_OuterStripMinedLoop()) { + if (outer_lpt->is_invariant(referent)) { + IfNode* outer_loop_end = outer_lpt->head()->as_OuterStripMinedLoop()->outer_loop_end(); + if (outer_loop_end != nullptr && outer_loop_end->false_proj_or_null() != nullptr) { + unique_loop_exit = outer_loop_end->false_proj_or_null(); + } + } else { + // An attempt to insert an RF node inside outer strip-mined loop breaks + // its IR invariants and manifests as assertion failures. + assert(false, "not loop invariant in outer strip-mined loop"); + continue; // skip + } + } + } + + LoopExit p(referent, unique_loop_exit); + worklist.push(p); + redundant_rfs.push(rf); + +#ifndef PRODUCT + if (TraceLoopOpts) { + IdealLoopTree* loop = get_loop(unique_loop_exit->in(0)); + tty->print_cr("ReachabilityFence: N%d: %s N%d/N%d -> loop exit N%d (%s N%d/N%d)", + rf->_idx, lpt->head()->Name(), lpt->head()->_idx, lpt->tail()->_idx, + unique_loop_exit->_idx, + loop->head()->Name(), loop->head()->_idx, loop->tail()->_idx); + } +#endif // !PRODUCT } - assert(loop_exit != nullptr, ""); - Pair p(referent, loop_exit); - worklist.push(p); - redundant_rfs.push(rf); } } - // Populate RFs outside counted loops. + // Populate RFs outside loops. while (worklist.is_nonempty()) { - Pair p = worklist.pop(); + LoopExit p = worklist.pop(); Node* referent = p.first; Node* ctrl_out = p.second; insert_rf(ctrl_out, referent); @@ -437,15 +461,15 @@ static Node* sfpt_ctrl_out(SafePointNode* sfpt) { } } -// Phase 3: expand reachability fences from safepoint info. +// Phase 3: materialize reachability fences from reachability edges on safepoints. // Turn extra safepoint edges into reachability fences immediately following the safepoint. // -// NB! As of now, a special care is needed to properly enumerage reachability edges because -// there are other use cases for non-debug safepoint edges. expand_reachability_fences() runs +// NB! As of now, a special care is needed to properly enumerate reachability edges because +// there are other use cases for non-debug safepoint edges. expand_reachability_edges() runs // after macro expansion where runtime calls during array allocation are annotated with // valid_length_test_input as an extra edges. Until there's a mechanism to distinguish between // different types of non-debug edges, unrelated cases are filtered out explicitly and in ad-hoc manner. -void Compile::expand_reachability_fences(Unique_Node_List& safepoints) { +void Compile::expand_reachability_edges(Unique_Node_List& safepoints) { for (uint i = 0; i < safepoints.size(); i++) { SafePointNode* sfpt = safepoints.at(i)->as_SafePoint();