8067187: -XX:MaxMetaspaceSize=20m -Xshare:dump caused JVM to crash

Added check for the MaxMetaspaceSize. If it is less than the estimated required size, print an error and exit.

Reviewed-by: jiangli, mseledtsov, stefank

hotspot/src/share/vm/memory/metaspace.cpp

@@ -3170,7 +3170,9 @@ void Metaspace::global_initialize() {
     }
 
     // the min_misc_data_size and min_misc_code_size estimates are based on
-    // MetaspaceShared::generate_vtable_methods()
+    // MetaspaceShared::generate_vtable_methods().
+    // The minimum size only accounts for the vtable methods. Any size less than the
+    // minimum required size would cause vm crash when allocating the vtable methods.
     uint min_misc_data_size = align_size_up(
       MetaspaceShared::num_virtuals * MetaspaceShared::vtbl_list_size * sizeof(void*), max_alignment);
 
@@ -3336,6 +3338,10 @@ void Metaspace::initialize(Mutex* lock, MetaspaceType type) {
   Metachunk* new_chunk = get_initialization_chunk(NonClassType,
                                                   word_size,
                                                   vsm()->medium_chunk_bunch());
+  // For dumping shared archive, report error if allocation has failed.
+  if (DumpSharedSpaces && new_chunk == NULL) {
+    report_insufficient_metaspace(MetaspaceAux::committed_bytes() + word_size * BytesPerWord);
+  }
   assert(!DumpSharedSpaces || new_chunk != NULL, "should have enough space for both chunks");
   if (new_chunk != NULL) {
     // Add to this manager's list of chunks in use and current_chunk().
@@ -3349,6 +3355,11 @@ void Metaspace::initialize(Mutex* lock, MetaspaceType type) {
                                                       class_vsm()->medium_chunk_bunch());
     if (class_chunk != NULL) {
       class_vsm()->add_chunk(class_chunk, true);
+    } else {
+      // For dumping shared archive, report error if allocation has failed.
+      if (DumpSharedSpaces) {
+        report_insufficient_metaspace(MetaspaceAux::committed_bytes() + class_word_size * BytesPerWord);
+      }
     }
   }
 

hotspot/src/share/vm/utilities/debug.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -273,6 +273,14 @@ void report_out_of_shared_space(SharedSpaceType shared_space) {
    exit(2);
 }
 
+void report_insufficient_metaspace(size_t required_size) {
+  warning("\nThe MaxMetaspaceSize of " UINTX_FORMAT " bytes is not large enough.\n"
+          "Either don't specify the -XX:MaxMetaspaceSize=<size>\n"
+          "or increase the size to at least " SIZE_FORMAT ".\n",
+          MaxMetaspaceSize, required_size);
+  exit(2);
+}
+
 void report_java_out_of_memory(const char* message) {
   static jint out_of_memory_reported = 0;
 

hotspot/src/share/vm/utilities/debug.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -251,6 +251,8 @@ enum SharedSpaceType {
 
 void report_out_of_shared_space(SharedSpaceType space_type);
 
+void report_insufficient_metaspace(size_t required_size);
+
 // out of memory reporting
 void report_java_out_of_memory(const char* message);
 

hotspot/test/runtime/SharedArchiveFile/MaxMetaspaceSize.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8067187
+ * @summary Testing CDS dumping with the -XX:MaxMetaspaceSize=<size> option
+ * @library /testlibrary
+ */
+
+import com.oracle.java.testlibrary.*;
+
+public class MaxMetaspaceSize {
+  public static void main(String[] args) throws Exception {
+    ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(
+        "-XX:MaxMetaspaceSize=20m", "-Xshare:dump");
+    OutputAnalyzer output = new OutputAnalyzer(pb.start());
+      output.shouldContain("is not large enough.\nEither don't specify the -XX:MaxMetaspaceSize=<size>\nor increase the size to at least");
+      output.shouldHaveExitValue(2);
+  }
+}