fixes

src/hotspot/share/opto/callnode.cpp

@@ -967,9 +967,7 @@ void CallNode::extract_projections(CallProjections* projs, bool separate_io_proj
             switch (cpn->_con) {
               case CatchProjNode::fall_through_index: projs->fallthrough_catchproj = cpn; break;
               case CatchProjNode::catch_all_index:    projs->catchall_catchproj    = cpn; break;
-              default: {
-                assert(cpn->_con > 1, "not an exception table projection"); // exception table; rethrow case
-              }
+              default: break; // exception table; rethrow case
             }
           }
         }
@@ -986,8 +984,10 @@ void CallNode::extract_projections(CallProjections* projs, bool separate_io_proj
           if (e->in(0)->as_CatchProj()->_con == CatchProjNode::catch_all_index) {
             assert(projs->exobj == nullptr, "only one");
             projs->exobj = e;
-          } else  {
-            assert(e->in(0)->as_CatchProj()->_con > 1, "not an exception table projection"); // exception table for rethrow case
+          } else {
+            // exception table for rethrow case
+            assert(e->in(0)->as_CatchProj()->_con != CatchProjNode::fall_through_index,
+                   "not an exception table projection");
           }
         }
       }

src/hotspot/share/opto/compile.cpp

@@ -3984,7 +3984,7 @@ void Compile::final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_R
     }
   }
 
-  expand_reachability_fences(sfpt);
+  expand_reachability_edges(sfpt);
 
   // Skip next transformation if compressed oops are not used.
   if ((UseCompressedOops && !Matcher::gen_narrow_oop_implicit_null_checks()) ||

src/hotspot/share/opto/compile.hpp

@@ -1313,7 +1313,8 @@ public:
   // Definitions of pd methods
   static void pd_compiler2_init();
 
-  void expand_reachability_fences(Unique_Node_List& safepoints);
+  // Materialize reachability fences from reachability edges on safepoints.
+  void expand_reachability_edges(Unique_Node_List& safepoints);
 
   // Static parse-time type checking logic for gen_subtype_check:
   enum SubTypeCheckResult { SSC_always_false, SSC_always_true, SSC_easy_test, SSC_full_test };

src/hotspot/share/opto/gcm.cpp

@@ -155,10 +155,9 @@ bool PhaseCFG::is_control_proj_or_safepoint(Node* n) const {
   bool result = n->is_ReachabilityFence() ||
                 (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_SafePoint) ||
                 (n->is_Proj() && n->as_Proj()->bottom_type() == Type::CONTROL);
-  assert(!result ||
-         (n->is_ReachabilityFence()) ||
-         (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_SafePoint) ||
-         (n->is_Proj() && n->as_Proj()->_con == 0), "If control projection, it must be projection 0");
+  assert(!n->is_Proj() ||
+         n->as_Proj()->bottom_type() != Type::CONTROL ||
+         n->as_Proj()->_con == 0, "If control projection, it must be projection 0");
   return result;
 }
 

src/hotspot/share/opto/loopTransform.cpp

@@ -50,36 +50,65 @@
 // Given an IfNode, return the loop-exiting projection or null if both
 // arms remain in the loop.
 Node *IdealLoopTree::is_loop_exit(Node *iff) const {
-  if (iff->outcnt() != 2) return nullptr;  // Ignore partially dead tests
-  PhaseIdealLoop *phase = _phase;
+  assert(iff->is_If(), "not an If: %s", iff->Name());
+  assert(is_member(_phase->get_loop(iff)), "not related");
+
+  if (iff->outcnt() != 2) {
+    return nullptr;  // Ignore partially dead tests
+  }
   // Test is an IfNode, has 2 projections.  If BOTH are in the loop
   // we need loop unswitching instead of peeling.
-  if (!is_member(phase->get_loop(iff->raw_out(0))))
+  if (!is_member(_phase->get_loop(iff->raw_out(0)))) {
     return iff->raw_out(0);
-  if (!is_member(phase->get_loop(iff->raw_out(1))))
+  }
+  if (!is_member(_phase->get_loop(iff->raw_out(1)))) {
     return iff->raw_out(1);
+  }
   return nullptr;
 }
 
 //------------------------------unique_loop_exit_or_null----------------------
-// Return the loop-exit projection if it is unique.
-IfFalseNode* IdealLoopTree::unique_loop_exit_or_null() {
-  if (is_loop()) {
-    if (head()->is_BaseCountedLoop()) {
-      ProjNode* loop_exit = head()->as_BaseCountedLoop()->loopexit()->proj_out_or_null(0 /* false */);
-      if (loop_exit != nullptr) {
-        return loop_exit->as_IfFalse();
-      }
-    } else if (head()->is_OuterStripMinedLoop()) {
-      return head()->as_OuterStripMinedLoop()->outer_loop_exit();
-    } else {
-      // For now, conservatively report multiple loop exits exist.
+// Return the loop-exit projection if loop exit is unique.
+IfFalseNode* IdealLoopTree::unique_loop_exit_proj_or_null() {
+  if (is_loop() && head()->is_BaseCountedLoop()) {
+    IfNode* loop_end = head()->as_BaseCountedLoop()->loopexit_or_null();
+    if (loop_end == nullptr) {
+      return nullptr; // malformed loop shape
     }
+    // Look for other loop exits.
+    assert(_phase->is_dominator(head(), tail()), "sanity");
+    for (Node* ctrl = tail(); ctrl != head(); ctrl = ctrl->in(0)) {
+      assert(is_member(_phase->get_loop(ctrl)), "sanity");
+      if (ctrl->is_If()) {
+        if (!is_loop_exit(ctrl->as_If())) {
+          continue; // local branch
+        } else if (ctrl != loop_end) {
+          return nullptr; // multiple loop exits
+        }
+      } else if (ctrl->is_Region()) {
+        return nullptr; // give up on control flow merges
+      } else if (ctrl->is_ReachabilityFence() || ctrl->is_SafePoint() || ctrl->is_MemBar()) {
+        continue; // skip
+      } else if (ctrl->is_Proj()) {
+        if (ctrl->is_IfProj() || ctrl->Opcode() == Op_SCMemProj || ctrl->Opcode() == Op_Proj) {
+          // skip simple control projections
+        } else if (ctrl->is_CatchProj() || ctrl->is_JumpProj()) {
+          return nullptr; // give up on control flow splits
+        } else {
+          assert(false, "unknown control projection: %s", ctrl->Name());
+          return nullptr; // stop on unknown control node
+        }
+      } else {
+        assert(false, "unknown CFG node: %s", ctrl->Name());
+        return nullptr; // stop on unknown control node
+      }
+    }
+    assert(is_loop_exit(loop_end), "not a loop exit?");
+    return loop_end->false_proj_or_null();
   }
-  return nullptr; // not found or multiple loop exists exist
+  return nullptr; // not found or multiple loop exits
 }
 
-
 //=============================================================================
 
 
@@ -3125,9 +3154,13 @@ static CountedLoopNode* locate_pre_from_main(CountedLoopNode* main_loop) {
   Node* ctrl = main_loop->skip_assertion_predicates_with_halt();
   assert(ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, "");
   Node* iffm = ctrl->in(0);
-  assert(iffm->Opcode() == Op_If, "");
+  assert(iffm->Opcode() == Op_If, "%s", iffm->Name());
   Node* p_f = iffm->in(0);
-  assert(p_f->Opcode() == Op_IfFalse, "");
+  // Skip ReachabilityFences hoisted out of pre-loop.
+  while (p_f->is_ReachabilityFence()) {
+    p_f = p_f->in(0);
+  }
+  assert(p_f->Opcode() == Op_IfFalse, "%s", p_f->Name());
   CountedLoopNode* pre_loop = p_f->in(0)->as_CountedLoopEnd()->loopnode();
   assert(pre_loop->is_pre_loop(), "No pre loop found");
   return pre_loop;

src/hotspot/share/opto/loopnode.cpp

@@ -4840,8 +4840,9 @@ void IdealLoopTree::register_reachability_fence(ReachabilityFenceNode* rf) {
   if (_reachability_fences == nullptr) {
     _reachability_fences = new Node_List();
   }
-  assert(!_reachability_fences->contains(rf), "already registered");
-  _reachability_fences->push(rf);
+  if (!_reachability_fences->contains(rf)) {
+    _reachability_fences->push(rf);
+  }
 }
 
 #ifndef PRODUCT

src/hotspot/share/opto/loopnode.hpp

@@ -722,8 +722,8 @@ public:
   // Check for Node being a loop-breaking test
   Node *is_loop_exit(Node *iff) const;
 
-  // Return unique loop-exit projection or null if there are multiple exits exist.
-  IfFalseNode* unique_loop_exit_or_null();
+  // Return unique loop-exit projection or null if the loop has multiple exits.
+  IfFalseNode* unique_loop_exit_proj_or_null();
 
   // Remove simplistic dead code from loop body
   void DCE_loop_body();

src/hotspot/share/opto/reachability.cpp

@@ -216,32 +216,56 @@ bool PhaseIdealLoop::optimize_reachability_fences() {
 
   // ResourceMark rm; // NB! not safe because insert_rf may trigger _idom reallocation
   Unique_Node_List redundant_rfs;
-  GrowableArray<Pair<Node*,Node*>> worklist;
+  typedef Pair<Node*,Node*> LoopExit;
+  GrowableArray<LoopExit> worklist;
+
   for (int i = 0; i < C->reachability_fences_count(); i++) {
     ReachabilityFenceNode* rf = C->reachability_fence(i);
     assert(!rf->is_redundant(igvn()), "required");
     // Move RFs out of counted loops when possible.
     IdealLoopTree* lpt = get_loop(rf);
     Node* referent = rf->referent();
-    IfFalseNode* loop_exit = lpt->unique_loop_exit_or_null();
-    if (lpt->is_invariant(referent) && loop_exit != nullptr) {
-      // Switch to the outermost loop.
-      for (IdealLoopTree* outer_loop = lpt->_parent;
-           outer_loop->is_invariant(referent) && outer_loop->unique_loop_exit_or_null() != nullptr;
-           outer_loop = outer_loop->_parent) {
-        assert(is_member(outer_loop, rf), "");
-        loop_exit = outer_loop->unique_loop_exit_or_null();
+    if (lpt->is_invariant(referent)) {
+      IfFalseNode* unique_loop_exit = lpt->unique_loop_exit_proj_or_null();
+      if (unique_loop_exit != nullptr) {
+        // Skip over an outer strip-mined loop.
+        if (!lpt->is_root()) {
+          IdealLoopTree* outer_lpt = lpt->_parent;
+          if (outer_lpt->head()->is_OuterStripMinedLoop()) {
+            if (outer_lpt->is_invariant(referent)) {
+              IfNode* outer_loop_end = outer_lpt->head()->as_OuterStripMinedLoop()->outer_loop_end();
+              if (outer_loop_end != nullptr && outer_loop_end->false_proj_or_null() != nullptr) {
+                unique_loop_exit = outer_loop_end->false_proj_or_null();
+              }
+            } else {
+              // An attempt to insert an RF node inside outer strip-mined loop breaks
+              // its IR invariants and manifests as assertion failures.
+              assert(false, "not loop invariant in outer strip-mined loop");
+              continue; // skip
+            }
+          }
+        }
+
+        LoopExit p(referent, unique_loop_exit);
+        worklist.push(p);
+        redundant_rfs.push(rf);
+
+#ifndef PRODUCT
+        if (TraceLoopOpts) {
+          IdealLoopTree* loop = get_loop(unique_loop_exit->in(0));
+          tty->print_cr("ReachabilityFence: N%d: %s N%d/N%d -> loop exit N%d (%s N%d/N%d)",
+                        rf->_idx, lpt->head()->Name(), lpt->head()->_idx, lpt->tail()->_idx,
+                        unique_loop_exit->_idx,
+                        loop->head()->Name(), loop->head()->_idx, loop->tail()->_idx);
+        }
+#endif // !PRODUCT
       }
-      assert(loop_exit != nullptr, "");
-      Pair<Node*,Node*> p(referent, loop_exit);
-      worklist.push(p);
-      redundant_rfs.push(rf);
     }
   }
 
-  // Populate RFs outside counted loops.
+  // Populate RFs outside loops.
   while (worklist.is_nonempty()) {
-    Pair<Node*,Node*> p = worklist.pop();
+    LoopExit p = worklist.pop();
     Node* referent = p.first;
     Node* ctrl_out = p.second;
     insert_rf(ctrl_out, referent);
@@ -437,15 +461,15 @@ static Node* sfpt_ctrl_out(SafePointNode* sfpt) {
   }
 }
 
-// Phase 3: expand reachability fences from safepoint info.
+// Phase 3: materialize reachability fences from reachability edges on safepoints.
 // Turn extra safepoint edges into reachability fences immediately following the safepoint.
 //
-// NB! As of now, a special care is needed to properly enumerage reachability edges because
-// there are other use cases for non-debug safepoint edges. expand_reachability_fences() runs
+// NB! As of now, a special care is needed to properly enumerate reachability edges because
+// there are other use cases for non-debug safepoint edges. expand_reachability_edges() runs
 // after macro expansion where runtime calls during array allocation are annotated with
 // valid_length_test_input as an extra edges. Until there's a mechanism to distinguish between
 // different types of non-debug edges, unrelated cases are filtered out explicitly and in ad-hoc manner.
-void Compile::expand_reachability_fences(Unique_Node_List& safepoints) {
+void Compile::expand_reachability_edges(Unique_Node_List& safepoints) {
   for (uint i = 0; i < safepoints.size(); i++) {
     SafePointNode* sfpt = safepoints.at(i)->as_SafePoint();