stop stack walk at signal handler

src/hotspot/os/posix/signals_posix.cpp

@@ -44,6 +44,7 @@
 #include "utilities/checkedCast.hpp"
 #include "utilities/deferredStatic.hpp"
 #include "utilities/events.hpp"
+#include "utilities/nativeCallStack.hpp"
 #include "utilities/ostream.hpp"
 #include "utilities/parseInteger.hpp"
 #include "utilities/vmError.hpp"
@@ -569,6 +570,10 @@ int JVM_HANDLE_XXX_SIGNAL(int sig, siginfo_t* info,
   //  (note: RAII ok here, even with JFR thread crash protection, see below).
   ErrnoPreserver ep;
 
+  // We are called from a signal handler, so stop the stack backtrace here.
+  // See os::is_first_C_frame() in os::get_native_stack().
+  os::FirstNativeFrameMark fnfm;
+
   // Unblock all synchronous error signals (see JDK-8252533)
   PosixSignals::unblock_error_signals();
 

src/hotspot/share/runtime/os.cpp

@@ -1404,11 +1404,18 @@ static bool is_pointer_bad(intptr_t* ptr) {
 // stack is walkable beyond current frame.
 // Returns true if this is not the case, i.e. the frame is possibly
 // the first C frame on the stack.
-bool os::is_first_C_frame(frame* fr) {
+bool os::is_first_C_frame(const frame* fr) {
 
 #ifdef _WINDOWS
   return true; // native stack isn't walkable on windows this way.
 #endif
+
+  // Check for a user-specified first frame, which can be used to prevent
+  // walking through signal handlers, etc.
+  if (FirstNativeFrameMark::is_first_native_frame(*fr)) {
+    return true;
+  }
+
   // Load up sp, fp, sender sp and sender fp, check for reasonable values.
   // Check usp first, because if that's bad the other accessors may fault
   // on some architectures.  Ditto ufp second, etc.
@@ -2640,3 +2647,5 @@ char* os::build_agent_function_name(const char *sym_name, const char *lib_name,
   }
   return agent_entry_name;
 }
+
+THREAD_LOCAL address os::FirstNativeFrameMark::_first_frame_stack_pointer = nullptr;

src/hotspot/share/runtime/os.hpp

@@ -884,7 +884,7 @@ class os: AllStatic {
   // does not require a lookup in the unwind table, which is part of the binary
   // file but may be unsafe to read after a fatal error. So on x86, we can
   // only walk stack if %ebp is used as frame pointer.
-  static bool is_first_C_frame(frame *fr);
+  static bool is_first_C_frame(const frame *fr);
   static frame get_sender_for_C_frame(frame *fr);
 
   // return current frame. pc() and sp() are set to null on failure.
@@ -1100,6 +1100,25 @@ class os: AllStatic {
   static bool set_boot_path(char fileSep, char pathSep);
 
   static bool pd_dll_unload(void* libhandle, char* ebuf, int ebuflen);
+
+ public:
+  class FirstNativeFrameMark {
+   private:
+    static THREAD_LOCAL address _first_frame_stack_pointer;
+    address _saved_stack_pointer;
+
+   public:
+    FirstNativeFrameMark(address sp = os::current_stack_pointer()) {
+      _saved_stack_pointer = _first_frame_stack_pointer;
+      _first_frame_stack_pointer = sp;
+    }
+
+    ~FirstNativeFrameMark() {
+      _first_frame_stack_pointer = _saved_stack_pointer;
+    }
+
+    static inline bool is_first_native_frame(const frame& fr);
+  };
 };
 
 // Note that "PAUSE" is almost always used with synchronization
@@ -1109,4 +1128,6 @@ class os: AllStatic {
 
 extern "C" int SpinPause();
 
+
+
 #endif // SHARE_RUNTIME_OS_HPP

src/hotspot/share/runtime/os.inline.hpp

@@ -59,4 +59,9 @@ inline void* os::resolve_function_descriptor(void* p) {
 }
 #endif
 
+inline bool os::FirstNativeFrameMark::is_first_native_frame(const frame& fr) {
+  address ff_sp = _first_frame_stack_pointer;
+  return ff_sp != nullptr && (address)fr.sp() >= ff_sp;
+}
+
 #endif // SHARE_RUNTIME_OS_INLINE_HPP

test/hotspot/gtest/runtime/test_os.cpp

@@ -1195,3 +1195,17 @@ TEST_VM(os, dll_load_null_error_buf) {
   void* lib = os::dll_load("NoSuchLib", nullptr, 0);
   ASSERT_NULL(lib);
 }
+
+#if !defined(_WINDOWS)
+TEST_VM(os, FirstNativeFrameMark) {
+  {
+    NativeCallStack ncs(0);
+    EXPECT_TRUE(ncs.frames() >= 1) << "expected no less than 1 frame, but saw " << ncs.frames();
+  }
+  {
+    os::FirstNativeFrameMark fnfm;
+    NativeCallStack ncs(0);
+    EXPECT_TRUE(ncs.frames() <= 1) << "expected no more than 1 frame, but saw " << ncs.frames();
+  }
+}
+#endif