8037842: Failing to allocate MethodCounters and MDO causes a serious performance drop

Stop allocating compiler profiling metadata when metaspace is full.

Reviewed-by: kvn, anoll

hotspot/src/share/vm/classfile/classLoaderData.cpp

@@ -553,6 +553,7 @@ ClassLoaderData* ClassLoaderDataGraph::_saved_unloading = NULL;
 ClassLoaderData* ClassLoaderDataGraph::_saved_head = NULL;
 
 bool ClassLoaderDataGraph::_should_purge = false;
+bool ClassLoaderDataGraph::_metaspace_oom = false;
 
 // Add a new class loader data node to the list.  Assign the newly created
 // ClassLoaderData into the java/lang/ClassLoader object as a hidden field
@@ -804,12 +805,17 @@ void ClassLoaderDataGraph::purge() {
   ClassLoaderData* list = _unloading;
   _unloading = NULL;
   ClassLoaderData* next = list;
+  bool classes_unloaded = false;
   while (next != NULL) {
     ClassLoaderData* purge_me = next;
     next = purge_me->next();
     delete purge_me;
+    classes_unloaded = true;
+  }
+  if (classes_unloaded) {
+    Metaspace::purge();
+    set_metaspace_oom(false);
   }
-  Metaspace::purge();
 }
 
 void ClassLoaderDataGraph::post_class_unload_events(void) {

hotspot/src/share/vm/classfile/classLoaderData.hpp

@@ -68,6 +68,9 @@ class ClassLoaderDataGraph : public AllStatic {
   static ClassLoaderData* _saved_head;
   static ClassLoaderData* _saved_unloading;
   static bool _should_purge;
+  // OOM has been seen in metaspace allocation. Used to prevent some
+  // allocations until class unloading
+  static bool _metaspace_oom;
 
   static ClassLoaderData* add(Handle class_loader, bool anonymous, TRAPS);
   static void post_class_unload_events(void);
@@ -107,6 +110,9 @@ class ClassLoaderDataGraph : public AllStatic {
     }
   }
 
+  static bool has_metaspace_oom()           { return _metaspace_oom; }
+  static void set_metaspace_oom(bool value) { _metaspace_oom = value; }
+
   static void free_deallocate_lists();
 
   static void dump_on(outputStream * const out) PRODUCT_RETURN;

hotspot/src/share/vm/compiler/compileBroker.cpp

@@ -187,6 +187,14 @@ class CompilationLog : public StringEventLog {
     lm.print("\n");
     log(thread, "%s", (const char*)lm);
   }
+
+  void log_metaspace_failure(const char* reason) {
+    ResourceMark rm;
+    StringLogMessage lm;
+    lm.print("%4d   COMPILE PROFILING SKIPPED: %s", -1, reason);
+    lm.print("\n");
+    log(JavaThread::current(), "%s", (const char*)lm);
+  }
 };
 
 static CompilationLog* _compilation_log = NULL;
@@ -1843,6 +1851,18 @@ void CompileBroker::init_compiler_thread_log() {
     warning("Cannot open log file: %s", file_name);
 }
 
+void CompileBroker::log_metaspace_failure() {
+  const char* message = "some methods may not be compiled because metaspace "
+                        "is out of memory";
+  if (_compilation_log != NULL) {
+    _compilation_log->log_metaspace_failure(message);
+  }
+  if (PrintCompilation) {
+    tty->print_cr("COMPILE PROFILING SKIPPED: %s", message);
+  }
+}
+
+
 // ------------------------------------------------------------------
 // CompileBroker::set_should_block
 //

hotspot/src/share/vm/compiler/compileBroker.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -473,6 +473,9 @@ class CompileBroker: AllStatic {
   static int get_sum_nmethod_code_size() {        return _sum_nmethod_code_size; }
   static long get_peak_compilation_time() {       return _peak_compilation_time; }
   static long get_total_compilation_time() {      return _t_total_compilation.milliseconds(); }
+
+  // Log that compilation profiling is skipped because metaspace is full.
+  static void log_metaspace_failure();
 };
 
 #endif // SHARE_VM_COMPILER_COMPILEBROKER_HPP

hotspot/src/share/vm/oops/method.cpp

@@ -368,6 +368,13 @@ void Method::print_invocation_count() {
 // Build a MethodData* object to hold information about this method
 // collected in the interpreter.
 void Method::build_interpreter_method_data(methodHandle method, TRAPS) {
+  // Do not profile the method if metaspace has hit an OOM previously
+  // allocating profiling data. Callers clear pending exception so don't
+  // add one here.
+  if (ClassLoaderDataGraph::has_metaspace_oom()) {
+    return;
+  }
+
   // Do not profile method if current thread holds the pending list lock,
   // which avoids deadlock for acquiring the MethodData_lock.
   if (InstanceRefKlass::owns_pending_list_lock((JavaThread*)THREAD)) {
@@ -379,7 +386,13 @@ void Method::build_interpreter_method_data(methodHandle method, TRAPS) {
   MutexLocker ml(MethodData_lock, THREAD);
   if (method->method_data() == NULL) {
     ClassLoaderData* loader_data = method->method_holder()->class_loader_data();
-    MethodData* method_data = MethodData::allocate(loader_data, method, CHECK);
+    MethodData* method_data = MethodData::allocate(loader_data, method, THREAD);
+    if (HAS_PENDING_EXCEPTION) {
+      CompileBroker::log_metaspace_failure();
+      ClassLoaderDataGraph::set_metaspace_oom(true);
+      return;   // return the exception (which is cleared)
+    }
+
     method->set_method_data(method_data);
     if (PrintMethodData && (Verbose || WizardMode)) {
       ResourceMark rm(THREAD);
@@ -392,9 +405,19 @@ void Method::build_interpreter_method_data(methodHandle method, TRAPS) {
 }
 
 MethodCounters* Method::build_method_counters(Method* m, TRAPS) {
+  // Do not profile the method if metaspace has hit an OOM previously
+  if (ClassLoaderDataGraph::has_metaspace_oom()) {
+    return NULL;
+  }
+
   methodHandle mh(m);
   ClassLoaderData* loader_data = mh->method_holder()->class_loader_data();
-  MethodCounters* counters = MethodCounters::allocate(loader_data, CHECK_NULL);
+  MethodCounters* counters = MethodCounters::allocate(loader_data, THREAD);
+  if (HAS_PENDING_EXCEPTION) {
+    CompileBroker::log_metaspace_failure();
+    ClassLoaderDataGraph::set_metaspace_oom(true);
+    return NULL;   // return the exception (which is cleared)
+  }
   if (!mh->init_method_counters(counters)) {
     MetadataFactory::free_metadata(loader_data, counters);
   }