8360701: Add bailout when the register allocator interference graph grows unreasonably large

Reviewed-by: mhaessig, thartmann

src/hotspot/share/opto/c2_globals.hpp

@@ -265,6 +265,10 @@
   develop(bool, OptoCoalesce, true,                                         \
           "Use Conservative Copy Coalescing in the Register Allocator")     \
                                                                             \
+  product(uint, IFGEdgesLimit, 10000000, DIAGNOSTIC,                        \
+          "Maximum allowed edges in the interference graphs")               \
+          range(0, max_juint)                                               \
+                                                                            \
   develop(bool, UseUniqueSubclasses, true,                                  \
           "Narrow an abstract reference to the unique concrete subclass")   \
                                                                             \

src/hotspot/share/opto/chaitin.cpp

@@ -429,6 +429,9 @@ void PhaseChaitin::Register_Allocate() {
 
   // Create the interference graph using virtual copies
   build_ifg_virtual();  // Include stack slots this time
+  if (C->failing()) {
+    return;
+  }
 
   // The IFG is/was triangular.  I am 'squaring it up' so Union can run
   // faster.  Union requires a 'for all' operation which is slow on the
@@ -472,6 +475,9 @@ void PhaseChaitin::Register_Allocate() {
   // Build physical interference graph
   uint must_spill = 0;
   must_spill = build_ifg_physical(&live_arena);
+  if (C->failing()) {
+    return;
+  }
   // If we have a guaranteed spill, might as well spill now
   if (must_spill) {
     if(!_lrg_map.max_lrg_id()) {
@@ -513,6 +519,9 @@ void PhaseChaitin::Register_Allocate() {
     C->print_method(PHASE_INITIAL_SPILLING, 4);
 
     build_ifg_physical(&live_arena);
+    if (C->failing()) {
+      return;
+    }
     _ifg->SquareUp();
     _ifg->Compute_Effective_Degree();
     // Only do conservative coalescing if requested
@@ -596,6 +605,9 @@ void PhaseChaitin::Register_Allocate() {
     C->print_method(PHASE_ITERATIVE_SPILLING, 4);
 
     must_spill = build_ifg_physical(&live_arena);
+    if (C->failing()) {
+      return;
+    }
     _ifg->SquareUp();
     _ifg->Compute_Effective_Degree();
 

src/hotspot/share/opto/chaitin.hpp

@@ -246,6 +246,9 @@ class PhaseIFG : public Phase {
   // Live range structure goes here
   LRG *_lrgs;                   // Array of LRG structures
 
+  // Keep track of number of edges to allow bailing out on very large IFGs
+  uint _edges;
+
 public:
   // Largest live-range number
   uint _maxlrg;

src/hotspot/share/opto/ifg.cpp

@@ -43,6 +43,7 @@ PhaseIFG::PhaseIFG( Arena *arena ) : Phase(Interference_Graph), _arena(arena) {
 
 void PhaseIFG::init( uint maxlrg ) {
   _maxlrg = maxlrg;
+  _edges = 0;
   _yanked = new (_arena) VectorSet(_arena);
   _is_square = false;
   // Make uninitialized adjacency lists
@@ -65,7 +66,12 @@ int PhaseIFG::add_edge( uint a, uint b ) {
   // Sort a and b, so that a is bigger
   assert( !_is_square, "only on triangular" );
   if( a < b ) { uint tmp = a; a = b; b = tmp; }
-  return _adjs[a].insert( b );
+  int ret = _adjs[a].insert(b);
+  _edges += ret;
+  if (_edges > IFGEdgesLimit) {
+    C->record_method_not_compilable("out of IFG edges");
+  }
+  return ret;
 }
 
 // Is there an edge between a and b?
@@ -299,6 +305,9 @@ void PhaseChaitin::interfere_with_live(uint lid, IndexSet* liveout) {
       LRG& interfering_lrg = lrgs(interfering_lid);
       if (rm.overlap(interfering_lrg.mask())) {
         _ifg->add_edge(lid, interfering_lid);
+        if (C->failing()) {
+          return;
+        }
       }
       interfering_lid = elements.next();
     }
@@ -348,6 +357,9 @@ void PhaseChaitin::build_ifg_virtual( ) {
 
         // Interfere with everything live
         interfere_with_live(r, liveout);
+        if (C->failing()) {
+          return;
+        }
       }
 
       // Make all inputs live
@@ -391,6 +403,9 @@ void PhaseChaitin::build_ifg_virtual( ) {
           uint kidx = _lrg_map.live_range_id(n->in(k));
           if (kidx != lidx) {
             _ifg->add_edge(r, kidx);
+            if (C->failing()) {
+              return;
+            }
           }
         }
       }
@@ -917,6 +932,9 @@ uint PhaseChaitin::build_ifg_physical( ResourceArea *a ) {
           remove_bound_register_from_interfering_live_ranges(lrg, &liveout, must_spill);
         }
         interfere_with_live(lid, &liveout);
+        if (C->failing()) {
+          return 0;
+        }
       }
 
       // Area remaining in the block