7185258: [macosx] Deadlock in SunToolKit.realSync()

Reviewed-by: prr

src/java.desktop/macosx/classes/sun/lwawt/macosx/LWCToolkit.java

@@ -107,6 +107,7 @@ import sun.awt.LightweightFrame;
 import sun.awt.PlatformGraphicsInfo;
 import sun.awt.SunToolkit;
 import sun.awt.datatransfer.DataTransferer;
+import sun.awt.dnd.SunDragSourceContextPeer;
 import sun.awt.util.ThreadGroupUtils;
 import sun.java2d.opengl.OGLRenderQueue;
 import sun.lwawt.LWComponentPeer;
@@ -463,6 +464,13 @@ public final class LWCToolkit extends LWToolkit {
 
     @Override
     protected boolean syncNativeQueue(long timeout) {
+        if (SunDragSourceContextPeer.isDragDropInProgress()
+                || EventQueue.isDispatchThread()) {
+            // The java code started the DnD, but the native drag may still not
+            // start, the last attempt to flush the native events,
+            // also do not block EDT for a long time
+            timeout = 50;
+        }
         return nativeSyncQueue(timeout);
     }
 

src/java.desktop/macosx/native/libawt_lwawt/awt/CDragSource.m

@@ -37,6 +37,7 @@
 #import "CDragSource.h"
 #import "DnDUtilities.h"
 #import "ThreadUtilities.h"
+#import "LWCToolkit.h"
 
 
 // When sIsJavaDragging is true Java drag gesture has been recognized and a drag is/has been initialized.
@@ -511,9 +512,9 @@ static BOOL                sNeedsEnter;
     fDragKeyModifiers = [DnDUtilities extractJavaExtKeyModifiersFromJavaExtModifiers:fModifiers];
     fDragMouseModifiers = [DnDUtilities extractJavaExtMouseModifiersFromJavaExtModifiers:fModifiers];
 
-    sNeedsEnter = YES;
-
     @try {
+        sNeedsEnter = YES;
+        AWTToolkit.inDoDragDropLoop = YES;
         // Data dragging:
         if (isFileDrag == FALSE) {
             [view dragImage:dragImage at:dragOrigin offset:dragOffset event:dragEvent pasteboard:pb source:view slideBack:YES];
@@ -561,6 +562,7 @@ static BOOL                sNeedsEnter;
         JNFCallVoidMethod(env, fDragSourceContextPeer, resetHoveringMethod); // Hust reset static variable
     } @finally {
         sNeedsEnter = NO;
+        AWTToolkit.inDoDragDropLoop = NO;
     }
 
     // We have to do this, otherwise AppKit doesn't know we're finished dragging. Yup, it's that bad.
@@ -607,7 +609,7 @@ static BOOL                sNeedsEnter;
 
 - (void)draggedImage:(NSImage *)image beganAt:(NSPoint)screenPoint {
     DLog4(@"[CDragSource draggedImage beganAt]: (%f, %f) %@\n", screenPoint.x, screenPoint.y, self);
-
+    [AWTToolkit eventCountPlusPlus];
     // Initialize static variables:
     sDragOperation = NSDragOperationNone;
     sDraggingLocation = screenPoint;
@@ -615,7 +617,7 @@ static BOOL                sNeedsEnter;
 
 - (void)draggedImage:(NSImage *)image endedAt:(NSPoint)screenPoint operation:(NSDragOperation)operation {
     DLog4(@"[CDragSource draggedImage endedAt:]: (%f, %f) %@\n", screenPoint.x, screenPoint.y, self);
-
+    [AWTToolkit eventCountPlusPlus];
     sDraggingLocation = screenPoint;
     sDragOperation = operation;
 }
@@ -625,6 +627,7 @@ static BOOL                sNeedsEnter;
     JNIEnv* env = [ThreadUtilities getJNIEnv];
 
 JNF_COCOA_ENTER(env);
+    [AWTToolkit eventCountPlusPlus];
     // There are two things we would be interested in:
     // a) mouse pointer has moved
     // b) drag actions (key modifiers) have changed

src/java.desktop/macosx/native/libawt_lwawt/awt/LWCToolkit.h

@@ -39,6 +39,8 @@ extern int gNumberOfButtons;
 extern jint* gButtonDownMasks;
 
 @interface AWTToolkit : NSObject { }
++ (BOOL) inDoDragDropLoop;
++ (void) setInDoDragDropLoop:(BOOL)val;
 + (long) getEventCount;
 + (void) eventCountPlusPlus;
 + (jint) scrollStateWithEvent: (NSEvent*) event;

src/java.desktop/macosx/native/libawt_lwawt/awt/LWCToolkit.m

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -70,13 +70,30 @@ static pthread_cond_t sAppKitStarted_cv = PTHREAD_COND_INITIALIZER;
 @implementation AWTToolkit
 
 static long eventCount;
+static BOOL inDoDragDropLoop;
+
++ (BOOL) inDoDragDropLoop {
+  @synchronized(self) {
+    return inDoDragDropLoop;
+  }
+}
+
++ (void) setInDoDragDropLoop:(BOOL)val {
+  @synchronized(self) {
+    inDoDragDropLoop = val;
+  }
+}
 
 + (long) getEventCount{
+  @synchronized(self) {
     return eventCount;
+  }
 }
 
 + (void) eventCountPlusPlus{
+  @synchronized(self) {
     eventCount++;
+  }
 }
 
 + (jint) scrollStateWithEvent: (NSEvent*) event {
@@ -420,10 +437,16 @@ JNIEXPORT jboolean JNICALL Java_sun_lwawt_macosx_LWCToolkit_nativeSyncQueue
         // immediately after this we will post the second event via
         // [NSApp postEvent] then sometimes the second event will be handled
         // first. The opposite isn't proved, but we use both here to be safer.
-        [theApp postDummyEvent:false];
-        [theApp waitForDummyEvent:timeout / 2.0];
-        [theApp postDummyEvent:true];
-        [theApp waitForDummyEvent:timeout / 2.0];
+
+        // If the native drag is in progress, skip native sync.
+        if (!AWTToolkit.inDoDragDropLoop) {
+            [theApp postDummyEvent:false];
+            [theApp waitForDummyEvent:timeout / 2.0];
+        }
+        if (!AWTToolkit.inDoDragDropLoop) {
+            [theApp postDummyEvent:true];
+            [theApp waitForDummyEvent:timeout / 2.0];
+        }
 
     } else {
         // could happen if we are embedded inside SWT application,

src/java.desktop/share/classes/sun/awt/dnd/SunDragSourceContextPeer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -74,8 +74,8 @@ public abstract class SunDragSourceContextPeer implements DragSourceContextPeer
     private DragSourceContext dragSourceContext;
     private int               sourceActions;
 
-    private static boolean    dragDropInProgress = false;
-    private static boolean    discardingMouseEvents = false;
+    private static volatile boolean dragDropInProgress = false;
+    private static boolean discardingMouseEvents = false;
 
     /*
      * dispatch constants
@@ -381,6 +381,10 @@ public abstract class SunDragSourceContextPeer implements DragSourceContextPeer
         }
     }
 
+    public static boolean isDragDropInProgress() {
+        return dragDropInProgress;
+    }
+
     private static String getExceptionMessage(boolean b) {
         return b ? "Drag and drop in progress" : "No drag in progress";
     }

test/jdk/java/awt/dnd/DragWaitForIdle/DragWaitForIdle.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.awt.Frame;
+import java.awt.Point;
+import java.awt.Robot;
+import java.awt.datatransfer.StringSelection;
+import java.awt.dnd.*;
+import java.awt.event.InputEvent;
+
+import test.java.awt.regtesthelpers.Util;
+
+/**
+ * @test
+ * @key headful
+ * @bug 7185258
+ * @summary Robot.waitForIdle() should not hang forever if dnd is in progress
+ * @library ../../regtesthelpers
+ * @build Util
+ * @run main/othervm DragWaitForIdle
+ */
+public final class DragWaitForIdle {
+
+    public static void main(final String[] args) throws Exception {
+        Frame frame = new Frame();
+        Robot robot = new Robot();
+        robot.setAutoWaitForIdle(true); // key point of the test
+
+        DragGestureListener dragGestureListener = dge -> {
+            dge.startDrag(null, new StringSelection("OK"), new DragSourceAdapter(){});
+        };
+
+        new DragSource().createDefaultDragGestureRecognizer(frame,
+                DnDConstants.ACTION_MOVE, dragGestureListener);
+
+        new DropTarget(frame, new DropTargetAdapter() {
+            public void drop(DropTargetDropEvent dtde) {
+                dtde.acceptDrop(DnDConstants.ACTION_MOVE);
+                dtde.dropComplete(true);
+            }
+        });
+
+        try {
+            frame.setUndecorated(true);
+            frame.setBounds(100, 100, 200, 200);
+            frame.setLocationRelativeTo(null);
+            frame.setVisible(true);
+            robot.waitForIdle();
+            frame.toFront();
+
+            Point startPoint = frame.getLocationOnScreen();
+            Point endPoint = new Point(startPoint);
+            startPoint.translate(50, 50);
+            endPoint.translate(150, 150);
+
+            Util.drag(robot, startPoint, endPoint, InputEvent.BUTTON2_MASK);
+
+            robot.delay(500);
+        } finally {
+            frame.dispose();
+        }
+    }
+}