8225035: Thread stack size issue caused by large TLS size

Adjust thread stack size for static TLS on Linux when AdjustStackSizeForTLS is enabled.

Co-authored-by: Florian Weimer <fweimer@redhat.com>
Co-authored-by: Jiangli Zhou <jianglizhou@google.com>
Reviewed-by: dholmes, fweimer, stuefe, rriggs, martin

make/test/JtregNativeHotspot.gmk

@@ -862,12 +862,13 @@ ifeq ($(call isTargetOs, linux), true)
     BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libredefineClasses := -lpthread
     BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exeinvoke := -ljvm -lpthread
     BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exestack-gap := -ljvm -lpthread
+    BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exestack-tls := -ljvm
     BUILD_TEST_exeinvoke_exeinvoke.c_OPTIMIZATION := NONE
     BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exeFPRegs := -ldl
     BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libAsyncGetCallTraceTest := -ldl
 else
   BUILD_HOTSPOT_JTREG_EXCLUDE += libtest-rw.c libtest-rwx.c libTestJNI.c \
-      exeinvoke.c exestack-gap.c libAsyncGetCallTraceTest.cpp
+      exeinvoke.c exestack-gap.c exestack-tls.c libAsyncGetCallTraceTest.cpp
 endif
 
 BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exesigtest := -ljvm

src/hotspot/os/linux/globals_linux.hpp

@@ -64,9 +64,13 @@
                                                                         \
   product(bool, PreferContainerQuotaForCPUCount, true,                  \
           "Calculate the container CPU availability based on the value" \
-          " of quotas (if set), when true. Otherwise, use the CPU"    \
+          " of quotas (if set), when true. Otherwise, use the CPU"      \
           " shares value, provided it is less than quota.")             \
                                                                         \
+  product(bool, AdjustStackSizeForTLS, false,                           \
+          "Increase the thread stack size to include space for glibc "  \
+          "static thread-local storage (TLS) if true")                  \
+                                                                        \
   diagnostic(bool, DumpPrivateMappingsInCore, true,                     \
           "If true, sets bit 2 of /proc/PID/coredump_filter, thus "     \
           "resulting in file-backed private mappings of the process to "\

src/hotspot/os/linux/os_linux.cpp

@@ -801,6 +801,73 @@ static void *thread_native_entry(Thread *thread) {
   return 0;
 }
 
+// On Linux, glibc places static TLS blocks (for __thread variables) on
+// the thread stack. This decreases the stack size actually available
+// to threads.
+//
+// For large static TLS sizes, this may cause threads to malfunction due
+// to insufficient stack space. This is a well-known issue in glibc:
+// http://sourceware.org/bugzilla/show_bug.cgi?id=11787.
+//
+// As a workaround, we call a private but assumed-stable glibc function,
+// __pthread_get_minstack() to obtain the minstack size and derive the
+// static TLS size from it. We then increase the user requested stack
+// size by this TLS size.
+//
+// Due to compatibility concerns, this size adjustment is opt-in and
+// controlled via AdjustStackSizeForTLS.
+typedef size_t (*GetMinStack)(const pthread_attr_t *attr);
+
+GetMinStack _get_minstack_func = NULL;
+
+static void get_minstack_init() {
+  _get_minstack_func =
+        (GetMinStack)dlsym(RTLD_DEFAULT, "__pthread_get_minstack");
+  log_info(os, thread)("Lookup of __pthread_get_minstack %s",
+                       _get_minstack_func == NULL ? "failed" : "succeeded");
+}
+
+// Returns the size of the static TLS area glibc puts on thread stacks.
+// The value is cached on first use, which occurs when the first thread
+// is created during VM initialization.
+static size_t get_static_tls_area_size(const pthread_attr_t *attr) {
+  size_t tls_size = 0;
+  if (_get_minstack_func != NULL) {
+    // Obtain the pthread minstack size by calling __pthread_get_minstack.
+    size_t minstack_size = _get_minstack_func(attr);
+
+    // Remove non-TLS area size included in minstack size returned
+    // by __pthread_get_minstack() to get the static TLS size.
+    // In glibc before 2.27, minstack size includes guard_size.
+    // In glibc 2.27 and later, guard_size is automatically added
+    // to the stack size by pthread_create and is no longer included
+    // in minstack size. In both cases, the guard_size is taken into
+    // account, so there is no need to adjust the result for that.
+    //
+    // Although __pthread_get_minstack() is a private glibc function,
+    // it is expected to have a stable behavior across future glibc
+    // versions while glibc still allocates the static TLS blocks off
+    // the stack. Following is glibc 2.28 __pthread_get_minstack():
+    //
+    // size_t
+    // __pthread_get_minstack (const pthread_attr_t *attr)
+    // {
+    //   return GLRO(dl_pagesize) + __static_tls_size + PTHREAD_STACK_MIN;
+    // }
+    //
+    //
+    // The following 'minstack_size > os::vm_page_size() + PTHREAD_STACK_MIN'
+    // if check is done for precaution.
+    if (minstack_size > (size_t)os::vm_page_size() + PTHREAD_STACK_MIN) {
+      tls_size = minstack_size - os::vm_page_size() - PTHREAD_STACK_MIN;
+    }
+  }
+
+  log_info(os, thread)("Stack size adjustment for TLS is " SIZE_FORMAT,
+                       tls_size);
+  return tls_size;
+}
+
 bool os::create_thread(Thread* thread, ThreadType thr_type,
                        size_t req_stack_size) {
   assert(thread->osthread() == NULL, "caller responsible");
@@ -826,7 +893,7 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
 
   // Calculate stack size if it's not specified by caller.
   size_t stack_size = os::Posix::get_initial_stack_size(thr_type, req_stack_size);
-  // In the Linux NPTL pthread implementation the guard size mechanism
+  // In glibc versions prior to 2.7 the guard size mechanism
   // is not implemented properly. The posix standard requires adding
   // the size of the guard pages to the stack size, instead Linux
   // takes the space out of 'stacksize'. Thus we adapt the requested
@@ -834,17 +901,27 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
   // behaviour. However, be careful not to end up with a size
   // of zero due to overflow. Don't add the guard page in that case.
   size_t guard_size = os::Linux::default_guard_size(thr_type);
-  if (stack_size <= SIZE_MAX - guard_size) {
-    stack_size += guard_size;
+  // Configure glibc guard page. Must happen before calling
+  // get_static_tls_area_size(), which uses the guard_size.
+  pthread_attr_setguardsize(&attr, guard_size);
+
+  size_t stack_adjust_size = 0;
+  if (AdjustStackSizeForTLS) {
+    // Adjust the stack_size for on-stack TLS - see get_static_tls_area_size().
+    stack_adjust_size += get_static_tls_area_size(&attr);
+  } else {
+    stack_adjust_size += guard_size;
+  }
+
+  stack_adjust_size = align_up(stack_adjust_size, os::vm_page_size());
+  if (stack_size <= SIZE_MAX - stack_adjust_size) {
+    stack_size += stack_adjust_size;
   }
   assert(is_aligned(stack_size, os::vm_page_size()), "stack_size not aligned");
 
   int status = pthread_attr_setstacksize(&attr, stack_size);
   assert_status(status == 0, status, "pthread_attr_setstacksize");
 
-  // Configure glibc guard page.
-  pthread_attr_setguardsize(&attr, os::Linux::default_guard_size(thr_type));
-
   ThreadState state;
 
   {
@@ -5145,6 +5222,10 @@ jint os::init_2(void) {
     jdk_misc_signal_init();
   }
 
+  if (AdjustStackSizeForTLS) {
+    get_minstack_init();
+  }
+
   // Check and sets minimum stack sizes against command line options
   if (Posix::set_minimum_stack_sizes() == JNI_ERR) {
     return JNI_ERR;

test/hotspot/jtreg/runtime/TLS/T.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2019, Google Inc. All rights reserved.
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+import java.lang.ProcessBuilder;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+public class T {
+    public static boolean run() {
+        boolean res = false;
+        String echoInput = "foo";
+        ProcessBuilder pb = new ProcessBuilder("echo", echoInput);
+
+        try {
+            // Starting a ProcessBuilder causes the process reaper thread to be
+            // created. The process reaper thread has small stack size. In JDK
+            // 13, the REAPER_DEFAULT_STACKSIZE is 128K. With JDK 8, it is 32K.
+            // Using the process reaper thread can demonstrate the TLS problem.
+            // The reaper thread can fail with StackOverflow in one of the
+            // failure mode with certain TLS sizes. In another observed
+            // failure mode the VM fails to create a thread with error message
+            // 'Failed to start thread - pthread_create failed'.
+            System.out.println("Starting a new process ...");
+            Process process = pb.start();
+            process.waitFor();
+            String echoOutput = output(process.getInputStream());
+            System.out.println("Echo Output: " + echoOutput);
+            if (echoOutput.equals(echoInput)) {
+                res = true;
+            } else {
+                // 'res' is false, fail
+                System.out.println("Unexpected Echo output: " + echoOutput +
+                                   ", expects: " + echoInput);
+            }
+        } catch (Exception e) {
+            System.out.println(e.toString());
+            e.printStackTrace();
+        }
+        return res;
+    }
+
+    private static String output(InputStream inputStream) throws IOException {
+        String s = "";
+        try (BufferedReader br =
+                 new BufferedReader(new InputStreamReader(inputStream))) {
+            s = br.readLine();
+        }
+        return s;
+    }
+}
+

test/hotspot/jtreg/runtime/TLS/exestack-tls.c

@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2019, Google Inc. All rights reserved.
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <jni.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// Declare the thread local variable(s) in the main executable. This can be
+// used to demonstrate the issues associated with the on-stack static TLS blocks
+// that may cause insufficient stack space. The dynamic TLS blocks for shared
+// objects (such as a JNI library) loaded via dlopen are not allocated on stack.
+__thread int tls[128 * 1024];
+
+JNIEnv* create_vm(JavaVM **jvm, char* argTLS) {
+    JNIEnv* env;
+    JavaVMInitArgs args;
+    JavaVMOption options[3];
+    args.version = JNI_VERSION_1_8;
+    args.nOptions = 3;
+    char classpath[4096];
+    snprintf(classpath, sizeof classpath,
+             "-Djava.class.path=%s", getenv("CLASSPATH"));
+    options[0].optionString = classpath;
+    options[1].optionString = "-Xlog:os+thread=info";
+    options[2].optionString = argTLS;
+    args.options = &options[0];
+    args.ignoreUnrecognized = 0;
+    int rv;
+    rv = JNI_CreateJavaVM(jvm, (void**)&env, &args);
+    if (rv < 0) return NULL;
+    return env;
+}
+
+int run(jboolean addTLS) {
+    JavaVM *jvm;
+    jclass testClass;
+    jmethodID runMethod;
+    char* argTLS;
+    int res = -1;
+
+    if (addTLS) {
+      argTLS = "-XX:+AdjustStackSizeForTLS";
+    } else {
+      argTLS = "-XX:-AdjustStackSizeForTLS"; // default
+    }
+    printf("Running test with %s ...\n", argTLS);
+    JNIEnv *env = create_vm(&jvm, argTLS);
+
+    // Run T.run() and check result:
+    // - Expect T.run() to return 'true' when stack size is adjusted for TLS,
+    //   return 0 if so
+    // - Expect T.run() to return 'false' if stack size is not adjusted for
+    //   TLS, return 0 if so
+    // Return -1 (fail) for other cases
+    testClass = (*env)->FindClass(env, "T");
+    runMethod = (*env)->GetStaticMethodID(env, testClass, "run", "()Z");
+    if ((*env)->CallStaticBooleanMethod(env, testClass, runMethod, NULL)) {
+      if (addTLS) {
+        // expect T.run() to return 'true'
+        res = 0;
+      }
+    } else {
+      if (!addTLS) {
+        // expect T.run() to return 'false'
+        res = 0;
+      }
+    }
+
+    if (res == 0) {
+      printf("Test passed with %s\n", argTLS);
+    } else {
+      printf("Test failed with %s\n", argTLS);
+    }
+    return res;
+}
+
+int main(int argc, char **argv) {
+    if (argc == 2 && strcmp(argv[1], "-add_tls") == 0) {
+      return run(JNI_TRUE);
+    } else {
+      return run(JNI_FALSE);
+    }
+}

test/hotspot/jtreg/runtime/TLS/testtls.sh

@@ -0,0 +1,47 @@
+# Copyright (c) 2019, Google Inc. All rights reserved.
+# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#!/bin/sh
+
+#
+# @test testtls.sh
+# @summary Test with extra TLS size.
+# @requires os.family == "linux"
+# @compile T.java
+# @run shell testtls.sh
+#
+
+if [ "${TESTSRC}" = "" ]
+then
+  TESTSRC=${PWD}
+  echo "TESTSRC not set.  Using "${TESTSRC}" as default"
+fi
+echo "TESTSRC=${TESTSRC}"
+## Adding common setup Variables for running shell tests.
+. ${TESTSRC}/../../test_env.sh
+
+LD_LIBRARY_PATH=.:${TESTJAVA}/lib/${VM_TYPE}:/usr/lib:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH
+
+# Test 1) Run with stack size adjusted for TLS
+${TESTNATIVEPATH}/stack-tls -add_tls || exit $?
+# Test 2) Run with no stack size adjustment
+${TESTNATIVEPATH}/stack-tls || exit $?