8048549: [macosx] Disable usage of system menu bar if AWT is embedded in FX

Reviewed-by: anthony, serb

jdk/src/macosx/classes/com/apple/laf/AquaMenuBarUI.java

@@ -26,11 +26,14 @@
 package com.apple.laf;
 
 import java.awt.*;
+import java.security.AccessController;
 
 import javax.swing.*;
 import javax.swing.plaf.ComponentUI;
 import javax.swing.plaf.basic.BasicMenuBarUI;
 
+import sun.lwawt.macosx.LWCToolkit;
+import sun.security.action.GetBooleanAction;
 import sun.security.action.GetPropertyAction;
 
 // MenuBar implementation for Mac L&F
@@ -131,28 +134,20 @@ public class AquaMenuBarUI extends BasicMenuBarUI implements ScreenMenuBarProvid
     ScreenMenuBar fScreenMenuBar;
     boolean useScreenMenuBar = getScreenMenuBarProperty();
 
-    private static String getPrivSysProp(final String propName) {
-        return java.security.AccessController.doPrivileged(new GetPropertyAction(propName));
-    }
-
     static boolean getScreenMenuBarProperty() {
-        final String props[] = new String[]{""};
-
-        boolean useScreenMenuBar = false;
-        try {
-            props[0] = getPrivSysProp(AquaLookAndFeel.sPropertyPrefix + "useScreenMenuBar");
-
-            if (props[0] != null && props[0].equals("true")) useScreenMenuBar = true;
-            else {
-                props[0] = getPrivSysProp(AquaLookAndFeel.sOldPropertyPrefix + "useScreenMenuBar");
-
-                if (props[0] != null && props[0].equals("true")) {
-                    System.err.println(AquaLookAndFeel.sOldPropertyPrefix + "useScreenMenuBar has been deprecated. Please switch to " + AquaLookAndFeel.sPropertyPrefix + "useScreenMenuBar");
-                    useScreenMenuBar = true;
-                }
-            }
-        } catch(final Throwable t) { };
-
-        return useScreenMenuBar;
+        // Do not allow AWT to set the screen menu bar if it's embedded in another UI toolkit
+        if (LWCToolkit.isEmbedded()) return false;
+        if (AccessController.doPrivileged(
+                new GetBooleanAction(AquaLookAndFeel.sPropertyPrefix + "useScreenMenuBar"))) {
+            return true;
+        }
+        if (AccessController.doPrivileged(
+                new GetBooleanAction(AquaLookAndFeel.sOldPropertyPrefix + "useScreenMenuBar"))) {
+                System.err.println(AquaLookAndFeel.sOldPropertyPrefix +
+                        "useScreenMenuBar has been deprecated. Please switch to " +
+                        AquaLookAndFeel.sPropertyPrefix + "useScreenMenuBar");
+                return true;
+        }
+        return false;
     }
 }

jdk/src/macosx/classes/sun/lwawt/macosx/LWCToolkit.java

@@ -792,6 +792,13 @@ public final class LWCToolkit extends LWToolkit {
      */
     native boolean isApplicationActive();
 
+    /**
+     * Returns true if AWT toolkit is embedded, false otherwise.
+     *
+     * @return true if AWT toolkit is embedded, false otherwise
+     */
+    public static native boolean isEmbedded();
+
     /************************
      * Native methods section
      ************************/

jdk/src/macosx/native/sun/awt/LWCToolkit.m

@@ -55,6 +55,9 @@ jint* gButtonDownMasks;
 // we expect an embedder (e.g. SWT) to start it some time later.
 static BOOL forceEmbeddedMode = NO;
 
+// Indicates if awt toolkit is embedded into another UI toolkit
+static BOOL isEmbedded = NO;
+
 // This is the data necessary to have JNI_OnLoad wait for AppKit to start.
 static BOOL sAppKitStarted = NO;
 static pthread_mutex_t sAppKitStarted_mutex = PTHREAD_MUTEX_INITIALIZER;
@@ -325,8 +328,7 @@ static void AWT_NSUncaughtExceptionHandler(NSException *exception) {
     //  and -[NSApplication isRunning] returns YES, AWT is embedded inside another
     //  AppKit Application.
     NSApplication *app = [NSApplicationAWT sharedApplication];
-    BOOL isEmbedded = ![NSApp isKindOfClass:[NSApplicationAWT class]];
-    [ThreadUtilities setAWTEmbedded:isEmbedded];
+    isEmbedded = ![NSApp isKindOfClass:[NSApplicationAWT class]];
 
     if (!isEmbedded) {
         // Install run loop observers and set the AppKit Java thread name
@@ -723,3 +725,14 @@ JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) {
     return JNI_VERSION_1_4;
 }
 
+/*
+ * Class:     sun_lwawt_macosx_LWCToolkit
+ * Method:    isEmbedded
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL
+Java_sun_lwawt_macosx_LWCToolkit_isEmbedded
+(JNIEnv *env, jclass klass) {
+    return isEmbedded ? JNI_TRUE : JNI_FALSE;
+}
+

jdk/src/macosx/native/sun/osxapp/ThreadUtilities.h

@@ -129,8 +129,6 @@ __attribute__((visibility("default")))
 + (JNIEnv*)getJNIEnvUncached;
 + (void)detachCurrentThread;
 + (void)setAppkitThreadGroup:(jobject)group;
-+ (void)setAWTEmbedded:(BOOL)embedded;
-+ (BOOL)isAWTEmbedded;
 
 //Wrappers for the corresponding JNFRunLoop methods with a check for main thread
 + (void)performOnMainThreadWaiting:(BOOL)wait block:(void (^)())block;

jdk/src/macosx/native/sun/osxapp/ThreadUtilities.m

@@ -34,7 +34,6 @@
 JavaVM *jvm = NULL;
 static JNIEnv *appKitEnv = NULL;
 static jobject appkitThreadGroup = NULL;
-static BOOL awtEmbedded = NO;
 
 static inline void attachCurrentThread(void** env) {
     if ([NSThread isMainThread]) {
@@ -88,14 +87,6 @@ AWT_ASSERT_APPKIT_THREAD;
     }
 }
 
-+ (void)setAWTEmbedded:(BOOL)embedded {
-    awtEmbedded = embedded;
-}
-
-+ (BOOL)isAWTEmbedded {
-    return awtEmbedded;
-}
-
 @end