8213384: Move G1/C2 barrier verification into G1BarrierSetC2

Reviewed-by: kvn, roland, eosterlund

src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp

@@ -30,9 +30,11 @@
 #include "gc/g1/g1ThreadLocalData.hpp"
 #include "gc/g1/heapRegion.hpp"
 #include "opto/arraycopynode.hpp"
+#include "opto/compile.hpp"
 #include "opto/graphKit.hpp"
 #include "opto/idealKit.hpp"
 #include "opto/macro.hpp"
+#include "opto/rootnode.hpp"
 #include "opto/type.hpp"
 #include "utilities/macros.hpp"
 
@@ -773,3 +775,65 @@ Node* G1BarrierSetC2::step_over_gc_barrier(Node* c) const {
   }
   return c;
 }
+
+#ifdef ASSERT
+void G1BarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
+  // Verify G1 pre-barriers
+  const int marking_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
+
+  ResourceArea *area = Thread::current()->resource_area();
+  Unique_Node_List visited(area);
+  Node_List worklist(area);
+  // We're going to walk control flow backwards starting from the Root
+  worklist.push(compile->root());
+  while (worklist.size() > 0) {
+    Node* x = worklist.pop();
+    if (x == NULL || x == compile->top()) continue;
+    if (visited.member(x)) {
+      continue;
+    } else {
+      visited.push(x);
+    }
+
+    if (x->is_Region()) {
+      for (uint i = 1; i < x->req(); i++) {
+        worklist.push(x->in(i));
+      }
+    } else {
+      worklist.push(x->in(0));
+      // We are looking for the pattern:
+      //                            /->ThreadLocal
+      // If->Bool->CmpI->LoadB->AddP->ConL(marking_offset)
+      //              \->ConI(0)
+      // We want to verify that the If and the LoadB have the same control
+      // See GraphKit::g1_write_barrier_pre()
+      if (x->is_If()) {
+        IfNode *iff = x->as_If();
+        if (iff->in(1)->is_Bool() && iff->in(1)->in(1)->is_Cmp()) {
+          CmpNode *cmp = iff->in(1)->in(1)->as_Cmp();
+          if (cmp->Opcode() == Op_CmpI && cmp->in(2)->is_Con() && cmp->in(2)->bottom_type()->is_int()->get_con() == 0
+              && cmp->in(1)->is_Load()) {
+            LoadNode* load = cmp->in(1)->as_Load();
+            if (load->Opcode() == Op_LoadB && load->in(2)->is_AddP() && load->in(2)->in(2)->Opcode() == Op_ThreadLocal
+                && load->in(2)->in(3)->is_Con()
+                && load->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == marking_offset) {
+
+              Node* if_ctrl = iff->in(0);
+              Node* load_ctrl = load->in(0);
+
+              if (if_ctrl != load_ctrl) {
+                // Skip possible CProj->NeverBranch in infinite loops
+                if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
+                    && (if_ctrl->in(0)->is_MultiBranch() && if_ctrl->in(0)->Opcode() == Op_NeverBranch)) {
+                  if_ctrl = if_ctrl->in(0)->in(0);
+                }
+              }
+              assert(load_ctrl != NULL && if_ctrl == load_ctrl, "controls must match");
+            }
+          }
+        }
+      }
+    }
+  }
+}
+#endif

src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp

@@ -88,6 +88,10 @@ protected:
   virtual bool is_gc_barrier_node(Node* node) const;
   virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const;
   virtual Node* step_over_gc_barrier(Node* c) const;
+
+#ifdef ASSERT
+  virtual void verify_gc_barriers(Compile* compile, CompilePhase phase) const;
+#endif
 };
 
 #endif // SHARE_GC_SHARED_C2_G1BARRIERSETC2_HPP

src/hotspot/share/gc/shared/c2/barrierSetC2.hpp

@@ -54,6 +54,7 @@ const DecoratorSet C2_TIGHLY_COUPLED_ALLOC   = DECORATOR_LAST << 9;
 // Loads and stores from an arraycopy being optimized
 const DecoratorSet C2_ARRAY_COPY             = DECORATOR_LAST << 10;
 
+class Compile;
 class GraphKit;
 class IdealKit;
 class Node;
@@ -272,7 +273,13 @@ public:
   // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be
   // expanded later, then now is the time to do so.
   virtual bool expand_macro_nodes(PhaseMacroExpand* macro) const { return false; }
-  virtual void verify_gc_barriers(bool post_parse) const {}
+
+  enum CompilePhase {
+    BeforeOptimize, /* post_parse = true */
+    BeforeExpand, /* post_parse = false */
+    BeforeCodeGen
+  };
+  virtual void verify_gc_barriers(Compile* compile, CompilePhase phase) const {}
 };
 
 #endif // SHARE_GC_SHARED_C2_BARRIERSETC2_HPP

src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp

@@ -1464,6 +1464,12 @@ static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) {
   return true;
 }
 
+void ZBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
+  if (phase == BarrierSetC2::BeforeCodeGen) return;
+  bool post_parse = phase == BarrierSetC2::BeforeOptimize;
+  verify_gc_barriers(post_parse);
+}
+
 void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const {
   ZBarrierSetC2State* s = state();
   Compile* C = Compile::current();

src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp

@@ -163,6 +163,10 @@ private:
   void expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const;
   const TypeFunc* load_barrier_Type() const;
 
+#ifdef ASSERT
+  void verify_gc_barriers(bool post_parse) const;
+#endif
+
 protected:
   virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const;
   virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access,
@@ -204,7 +208,7 @@ public:
   static void loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round);
 
 #ifdef ASSERT
-  virtual void verify_gc_barriers(bool post_parse) const;
+  virtual void verify_gc_barriers(Compile* compile, CompilePhase phase) const;
 #endif
 };
 

src/hotspot/share/opto/compile.cpp

@@ -892,7 +892,10 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr
   }
 #endif
 
-  NOT_PRODUCT( verify_barriers(); )
+#ifdef ASSERT
+  BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
+  bs->verify_gc_barriers(this, BarrierSetC2::BeforeCodeGen);
+#endif
 
   // Dump compilation data to replay it.
   if (directive->DumpReplayOption) {
@@ -2193,7 +2196,7 @@ void Compile::Optimize() {
 
 #ifdef ASSERT
   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
-  bs->verify_gc_barriers(true);
+  bs->verify_gc_barriers(this, BarrierSetC2::BeforeOptimize);
 #endif
 
   ResourceMark rm;
@@ -2386,7 +2389,7 @@ void Compile::Optimize() {
 
 #ifdef ASSERT
   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
-  bs->verify_gc_barriers(false);
+  bs->verify_gc_barriers(this, BarrierSetC2::BeforeExpand);
 #endif
 
   {
@@ -3831,74 +3834,6 @@ void Compile::verify_graph_edges(bool no_dead_code) {
     }
   }
 }
-
-// Verify GC barriers consistency
-// Currently supported:
-// - G1 pre-barriers (see GraphKit::g1_write_barrier_pre())
-void Compile::verify_barriers() {
-#if INCLUDE_G1GC
-  if (UseG1GC) {
-    // Verify G1 pre-barriers
-    const int marking_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
-
-    ResourceArea *area = Thread::current()->resource_area();
-    Unique_Node_List visited(area);
-    Node_List worklist(area);
-    // We're going to walk control flow backwards starting from the Root
-    worklist.push(_root);
-    while (worklist.size() > 0) {
-      Node* x = worklist.pop();
-      if (x == NULL || x == top()) continue;
-      if (visited.member(x)) {
-        continue;
-      } else {
-        visited.push(x);
-      }
-
-      if (x->is_Region()) {
-        for (uint i = 1; i < x->req(); i++) {
-          worklist.push(x->in(i));
-        }
-      } else {
-        worklist.push(x->in(0));
-        // We are looking for the pattern:
-        //                            /->ThreadLocal
-        // If->Bool->CmpI->LoadB->AddP->ConL(marking_offset)
-        //              \->ConI(0)
-        // We want to verify that the If and the LoadB have the same control
-        // See GraphKit::g1_write_barrier_pre()
-        if (x->is_If()) {
-          IfNode *iff = x->as_If();
-          if (iff->in(1)->is_Bool() && iff->in(1)->in(1)->is_Cmp()) {
-            CmpNode *cmp = iff->in(1)->in(1)->as_Cmp();
-            if (cmp->Opcode() == Op_CmpI && cmp->in(2)->is_Con() && cmp->in(2)->bottom_type()->is_int()->get_con() == 0
-                && cmp->in(1)->is_Load()) {
-              LoadNode* load = cmp->in(1)->as_Load();
-              if (load->Opcode() == Op_LoadB && load->in(2)->is_AddP() && load->in(2)->in(2)->Opcode() == Op_ThreadLocal
-                  && load->in(2)->in(3)->is_Con()
-                  && load->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == marking_offset) {
-
-                Node* if_ctrl = iff->in(0);
-                Node* load_ctrl = load->in(0);
-
-                if (if_ctrl != load_ctrl) {
-                  // Skip possible CProj->NeverBranch in infinite loops
-                  if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
-                      && (if_ctrl->in(0)->is_MultiBranch() && if_ctrl->in(0)->Opcode() == Op_NeverBranch)) {
-                    if_ctrl = if_ctrl->in(0)->in(0);
-                  }
-                }
-                assert(load_ctrl != NULL && if_ctrl == load_ctrl, "controls must match");
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-#endif
-}
-
 #endif
 
 // The Compile object keeps track of failure reasons separately from the ciEnv.

src/hotspot/share/opto/compile.hpp

@@ -1329,9 +1329,6 @@ class Compile : public Phase {
   // graph is strongly connected from root in both directions.
   void verify_graph_edges(bool no_dead_code = false) PRODUCT_RETURN;
 
-  // Verify GC barrier patterns
-  void verify_barriers() PRODUCT_RETURN;
-
   // End-of-run dumps.
   static void print_statistics() PRODUCT_RETURN;