8373366: HandshakeState should disallow suspend ops for disabler threads

8375362: Deadlock with unmount of suspended virtual thread interrupting another virtual thread

Reviewed-by: lmesnik, pchilanomate

src/hotspot/share/runtime/handshake.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -521,8 +521,8 @@ HandshakeOperation* HandshakeState::get_op_for_self(bool allow_suspend, bool che
   assert(_lock.owned_by_self(), "Lock must be held");
   assert(allow_suspend || !check_async_exception, "invalid case");
 #if INCLUDE_JVMTI
-  if (allow_suspend && _handshakee->is_disable_suspend()) {
-    // filter out suspend operations while JavaThread is in disable_suspend mode
+  if (allow_suspend && (_handshakee->is_disable_suspend() || _handshakee->is_vthread_transition_disabler())) {
+    // filter out suspend operations while JavaThread can not be suspended
     allow_suspend = false;
   }
 #endif

src/hotspot/share/runtime/javaThread.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -499,7 +499,7 @@ JavaThread::JavaThread(MemTag mem_tag) :
   _suspend_resume_manager(this, &_handshake._lock),
 
   _is_in_vthread_transition(false),
-  DEBUG_ONLY(_is_vthread_transition_disabler(false) COMMA)
+  JVMTI_ONLY(_is_vthread_transition_disabler(false) COMMA)
   DEBUG_ONLY(_is_disabler_at_start(false) COMMA)
 
   _popframe_preserved_args(nullptr),
@@ -1165,11 +1165,13 @@ void JavaThread::set_is_in_vthread_transition(bool val) {
   AtomicAccess::store(&_is_in_vthread_transition, val);
 }
 
-#ifdef ASSERT
+#if INCLUDE_JVMTI
 void JavaThread::set_is_vthread_transition_disabler(bool val) {
   _is_vthread_transition_disabler = val;
 }
+#endif
 
+#ifdef ASSERT
 void JavaThread::set_is_disabler_at_start(bool val) {
   _is_disabler_at_start = val;
 }
@@ -1183,7 +1185,9 @@ void JavaThread::set_is_disabler_at_start(bool val) {
 //
 bool JavaThread::java_suspend(bool register_vthread_SR) {
   // Suspending a vthread transition disabler can cause deadlocks.
-  assert(!is_vthread_transition_disabler(), "no suspend allowed for vthread transition disablers");
+  // The HandshakeState::has_operation does not allow such suspends.
+  // But the suspender thread is an exclusive transition disablers, so there can't be other disabers here.
+  JVMTI_ONLY(assert(!is_vthread_transition_disabler(), "suspender thread is an exclusive transition disabler");)
 
   guarantee(Thread::is_JavaThread_protected(/* target */ this),
             "target JavaThread is not protected in calling context.");

src/hotspot/share/runtime/javaThread.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -734,14 +734,14 @@ public:
 
 private:
   bool _is_in_vthread_transition;                    // thread is in virtual thread mount state transition
-  DEBUG_ONLY(bool _is_vthread_transition_disabler;)  // thread currently disabled vthread transitions
+  JVMTI_ONLY(bool _is_vthread_transition_disabler;)  // thread currently disabled vthread transitions
   DEBUG_ONLY(bool _is_disabler_at_start;)            // thread at process of disabling vthread transitions
 public:
   bool is_in_vthread_transition() const;
   void set_is_in_vthread_transition(bool val);
+  JVMTI_ONLY(bool is_vthread_transition_disabler() const { return _is_vthread_transition_disabler; })
+  JVMTI_ONLY(void set_is_vthread_transition_disabler(bool val);)
 #ifdef ASSERT
-  bool is_vthread_transition_disabler() const       { return _is_vthread_transition_disabler; }
-  void set_is_vthread_transition_disabler(bool val);
   bool is_disabler_at_start() const                 { return _is_disabler_at_start; }
   void set_is_disabler_at_start(bool val);
 #endif

src/hotspot/share/runtime/mountUnmountDisabler.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -129,7 +129,8 @@ bool MountUnmountDisabler::is_start_transition_disabled(JavaThread* thread, oop
   int base_disable_count = notify_jvmti_events() ? 1 : 0;
   return java_lang_Thread::vthread_transition_disable_count(vthread) > 0
          || global_vthread_transition_disable_count() > base_disable_count
-         JVMTI_ONLY(|| (JvmtiVTSuspender::is_vthread_suspended(java_lang_Thread::thread_id(vthread)) || thread->is_suspended()));
+         JVMTI_ONLY(|| (!thread->is_vthread_transition_disabler() &&
+                        (JvmtiVTSuspender::is_vthread_suspended(java_lang_Thread::thread_id(vthread)) || thread->is_suspended())));
 }
 
 void MountUnmountDisabler::start_transition(JavaThread* current, oop vthread, bool is_mount, bool is_thread_end) {
@@ -294,7 +295,7 @@ MountUnmountDisabler::disable_transition_for_one() {
   // carrierThread to float up.
   // This pairs with the release barrier in end_transition().
   OrderAccess::acquire();
-  DEBUG_ONLY(JavaThread::current()->set_is_vthread_transition_disabler(true);)
+  JVMTI_ONLY(JavaThread::current()->set_is_vthread_transition_disabler(true);)
 }
 
 // disable transitions for all virtual threads
@@ -335,7 +336,7 @@ MountUnmountDisabler::disable_transition_for_all() {
   // carrierThread to float up.
   // This pairs with the release barrier in end_transition().
   OrderAccess::acquire();
-  DEBUG_ONLY(thread->set_is_vthread_transition_disabler(true);)
+  JVMTI_ONLY(JavaThread::current()->set_is_vthread_transition_disabler(true);)
   DEBUG_ONLY(thread->set_is_disabler_at_start(false);)
 }
 
@@ -358,14 +359,12 @@ MountUnmountDisabler::enable_transition_for_one() {
   if (java_lang_Thread::vthread_transition_disable_count(_vthread()) == 0) {
     ml.notify_all();
   }
-  DEBUG_ONLY(JavaThread::current()->set_is_vthread_transition_disabler(false);)
+  JVMTI_ONLY(JavaThread::current()->set_is_vthread_transition_disabler(false);)
 }
 
 // enable transitions for all virtual threads
 void
 MountUnmountDisabler::enable_transition_for_all() {
-  JavaThread* thread = JavaThread::current();
-
   // End of the critical section. If some target was unmounted, we need a
   // release barrier before decrementing _global_vthread_transition_disable_count
   // to make sure any memory operations executed by the disabler are visible to
@@ -384,7 +383,7 @@ MountUnmountDisabler::enable_transition_for_all() {
   if (global_vthread_transition_disable_count() == base_disable_count || _is_exclusive) {
     ml.notify_all();
   }
-  DEBUG_ONLY(thread->set_is_vthread_transition_disabler(false);)
+  JVMTI_ONLY(JavaThread::current()->set_is_vthread_transition_disabler(false);)
 }
 
 int MountUnmountDisabler::global_vthread_transition_disable_count() {

src/hotspot/share/runtime/suspendResumeManager.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -130,6 +130,7 @@ void SuspendResumeManager::do_owner_suspend() {
   assert(_state_lock->owned_by_self(), "Lock must be held");
   assert(!_target->has_last_Java_frame() || _target->frame_anchor()->walkable(), "should have walkable stack");
   assert(_target->thread_state() == _thread_blocked, "Caller should have transitioned to _thread_blocked");
+  JVMTI_ONLY(assert(!_target->is_vthread_transition_disabler(), "attempt to suspend a vthread transition disabler");)
 
   while (is_suspended()) {
     log_trace(thread, suspend)("JavaThread:" INTPTR_FORMAT " suspended", p2i(_target));

test/hotspot/jtreg/serviceability/jvmti/vthread/ThreadStateTest2/ThreadStateTest2.java

@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8373366
+ * @summary HandshakeState should disallow suspend ops for disabler threads
+ * @requires vm.continuations
+ * @requires vm.jvmti
+ * @requires vm.compMode != "Xcomp"
+ * @modules java.base/java.lang:+open
+ * @library /test/lib
+ * @run main/othervm/native -agentlib:ThreadStateTest2 ThreadStateTest2
+ */
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadFactory;
+import jdk.test.lib.thread.VThreadScheduler;
+
+/* Testing scenario:
+ * Several threads are involved:
+ *  - VT-0: a virtual thread which is interrupt-friendly and constantly interrupted with JVMTI InterruptThread
+ *  - VT-1: a virtual thread which state is constantly checked with JVMTI GetThreadState
+ *  - VT-2: a virtual thread: in a loop calls JVMTI InterruptThread(VT-0) and GetThreadState(VT-1)
+ *  - main: a platform thread: in a loop invokes native method testSuspendResume which suspends and resumes VT-2
+ * The JVMTI functions above install a MountUnmountDisabler for target virtual thread (VT-0 or VT-1).
+ * The goal is to catch VT-2 in an attempt to self-suspend while in a context of MountUnmountDisabler.
+ * This would mean there is a suspend point while VT-2 is in a context of MountUnmountDisabler.
+ * The InterruptThread implementation does a Java upcall to j.l.Thread::interrupt().
+ * The JavaCallWrapper constructor has such a suspend point.
+ */
+public class ThreadStateTest2 {
+    private static native void setMonitorContendedMode(boolean enable);
+    private static native void testSuspendResume(Thread vthread);
+    private static native void testInterruptThread(Thread vthread);
+    private static native int testGetThreadState(Thread vthread);
+
+    static Thread vthread0;
+    static Thread vthread1;
+    static Thread vthread2;
+    static AtomicBoolean vt2Started = new AtomicBoolean();
+    static AtomicBoolean vt2Finished = new AtomicBoolean();
+
+    static void log(String msg) { System.out.println(msg); }
+
+    // Should handle interruptions from vthread2.
+    final Runnable FOO_0 = () -> {
+        log("VT-0 started");
+        while (!vt2Finished.get()) {
+            try {
+                Thread.sleep(10);
+            } catch (InterruptedException ie) {
+                // ignore
+            }
+        }
+        log("VT-0 finished");
+    };
+
+    // A target for vthread2 to check state with JVMTI GetThreadState.
+    final Runnable FOO_1 = () -> {
+        log("VT-1 started");
+        while (!vt2Finished.get()) {
+            Thread.yield();
+        }
+        log("VT-1 finished");
+    };
+
+    // In a loop execute JVMTI functions on threads vthread0 and vthread1:
+    // InterruptThread(vthread0) and GetThreadState(vthread1).
+    final Runnable FOO_2 = () -> {
+        log("VT-2 started");
+        vt2Started.set(true);
+        for (int i = 0; i < 40; i++) {
+            testInterruptThread(vthread0);
+            int state = testGetThreadState(vthread1);
+            if (state == 2) {
+                break;
+            }
+            Thread.yield();
+        }
+        vt2Finished.set(true);
+        log("VT-2 finished");
+    };
+
+    private void runTest() throws Exception {
+        // Force creation of JvmtiThreadState on vthread start.
+        setMonitorContendedMode(true);
+
+        ExecutorService scheduler = Executors.newFixedThreadPool(2);
+        ThreadFactory factory = VThreadScheduler.virtualThreadBuilder(scheduler).factory();
+
+        vthread0 = factory.newThread(FOO_0);
+        vthread1 = factory.newThread(FOO_1);
+        vthread2 = factory.newThread(FOO_2);
+        vthread0.setName("VT-0");
+        vthread1.setName("VT-1");
+        vthread2.setName("VT-2");
+        vthread0.start();
+        vthread1.start();
+        vthread2.start();
+
+        // Give some time for vthreads to start.
+        while (!vt2Started.get()) {
+            Thread.sleep(1);
+        }
+        while (!vt2Finished.get() /* && tryCount-- > 0 */) {
+            testSuspendResume(vthread2);
+        }
+        vthread0.join();
+        vthread1.join();
+        vthread2.join();
+
+        // Let all carriers go away.
+        scheduler.shutdown();
+        Thread.sleep(20);
+    }
+
+    public static void main(String[] args) throws Exception {
+        ThreadStateTest2 obj = new ThreadStateTest2();
+        obj.runTest();
+    }
+}

test/hotspot/jtreg/serviceability/jvmti/vthread/ThreadStateTest2/libThreadStateTest2.cpp

@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <jni.h>
+#include <jvmti.h>
+#include <stdio.h>
+#include <string.h>
+#include "jvmti_common.hpp"
+
+// set by Agent_OnLoad
+static jvmtiEnv* jvmti = nullptr;
+static jrawMonitorID agent_event_lock = nullptr;
+
+extern "C" {
+
+static void JNICALL
+MonitorContended(jvmtiEnv* jvmti, JNIEnv* jni, jthread thread,
+                 jobject object) {
+}
+
+JNIEXPORT void JNICALL
+Java_ThreadStateTest2_testSuspendResume(JNIEnv* jni, jclass klass, jthread thread) {
+  jvmtiError err;
+  RawMonitorLocker event_locker(jvmti, jni, agent_event_lock);
+
+  LOG("\nMAIN: testSuspendResume: before suspend\n");
+  err = jvmti->SuspendThread(thread);
+  if (err == JVMTI_ERROR_THREAD_NOT_ALIVE) {
+    return;
+  }
+  check_jvmti_status(jni, err, "testSuspendResume error in JVMTI SuspendThread");
+  LOG("\nMAIN: testSuspendResume:  after suspend\n");
+
+  event_locker.wait(1);
+
+  LOG("MAIN: testSuspendResume: before resume\n");
+  err = jvmti->ResumeThread(thread);
+  check_jvmti_status(jni, err, "testSuspendResume error in JVMTI ResumeThread");
+}
+
+JNIEXPORT void JNICALL
+Java_ThreadStateTest2_setMonitorContendedMode(JNIEnv* jni, jclass klass, jboolean enable) {
+  set_event_notification_mode(jvmti, jni, enable ? JVMTI_ENABLE : JVMTI_DISABLE, JVMTI_EVENT_MONITOR_CONTENDED_ENTER, nullptr);
+}
+
+JNIEXPORT void JNICALL
+Java_ThreadStateTest2_testInterruptThread(JNIEnv* jni, jclass klass, jthread vthread) {
+  char* tname = get_thread_name(jvmti, jni, vthread);
+  LOG("VT-2: testInterruptThread: %s\n", tname);
+
+  jvmtiError err = jvmti->InterruptThread(vthread);
+  check_jvmti_status(jni, err, "testInterruptThread error in JVMTI InterruptThread");
+}
+
+JNIEXPORT jint JNICALL
+Java_ThreadStateTest2_testGetThreadState(JNIEnv* jni, jclass klass, jthread vthread) {
+  jint  state = get_thread_state(jvmti, jni, vthread);
+  char* tname = get_thread_name(jvmti, jni, vthread);
+
+  LOG("VT-2: testGetThreadState: %s state: %x\n", tname, state);
+  return state;
+}
+
+JNIEXPORT jint JNICALL
+Agent_OnLoad(JavaVM* jvm, char* options, void* reserved) {
+  jvmtiEventCallbacks callbacks;
+  jvmtiCapabilities caps;
+  jvmtiError err;
+
+  printf("Agent_OnLoad: started\n");
+  if (jvm->GetEnv((void **) (&jvmti), JVMTI_VERSION) != JNI_OK) {
+    LOG("Agent_OnLoad: error in GetEnv");
+    return JNI_ERR;
+  }
+
+  memset(&caps, 0, sizeof(caps));
+  caps.can_suspend = 1;
+  caps.can_signal_thread = 1;
+  caps.can_support_virtual_threads = 1;
+  caps.can_generate_monitor_events = 1;
+
+  err = jvmti->AddCapabilities(&caps);
+  if (err != JVMTI_ERROR_NONE) {
+    LOG("Agent_OnLoad: error in JVMTI AddCapabilities: %d\n", err);
+  }
+  memset(&callbacks, 0, sizeof(callbacks));
+  callbacks.MonitorContendedEnter = &MonitorContended;
+  err = jvmti->SetEventCallbacks(&callbacks, sizeof(jvmtiEventCallbacks));
+  if (err != JVMTI_ERROR_NONE) {
+    LOG("Agent_OnLoad: Error in JVMTI SetEventCallbacks: %d\n", err);
+  }
+  agent_event_lock = create_raw_monitor(jvmti, "agent_event_lock");
+  printf("Agent_OnLoad: finished\n");
+
+  return 0;
+}
+
+} // extern "C"