8313424: JavaFX controls in the title bar

buildSrc/win.gradle

@@ -324,9 +324,9 @@ WIN.glass.rcFlags = [
 WIN.glass.ccFlags = [ccFlags].flatten()
 WIN.glass.linker = linker
 WIN.glass.linkFlags = (IS_STATIC_BUILD ? [linkFlags] : [linkFlags, "delayimp.lib", "gdi32.lib", "urlmon.lib", "Comdlg32.lib",
-        "winmm.lib", "imm32.lib", "shell32.lib", "Uiautomationcore.lib", "dwmapi.lib",
+        "winmm.lib", "imm32.lib", "shell32.lib", "Uiautomationcore.lib", "dwmapi.lib", "uxtheme.lib",
         "/DELAYLOAD:user32.dll", "/DELAYLOAD:urlmon.dll", "/DELAYLOAD:winmm.dll", "/DELAYLOAD:shell32.dll",
-        "/DELAYLOAD:Uiautomationcore.dll", "/DELAYLOAD:dwmapi.dll"]).flatten()
+        "/DELAYLOAD:Uiautomationcore.dll", "/DELAYLOAD:dwmapi.dll", "/DELAYLOAD:uxtheme.dll"]).flatten()
 WIN.glass.lib = "glass"
 
 WIN.decora = [:]

modules/javafx.base/src/test/java/test/util/ReflectionUtils.java

@@ -26,10 +26,14 @@
 package test.util;
 
 import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.util.function.Function;
 
 public final class ReflectionUtils {
 
+    private ReflectionUtils() {}
+
     /**
      * Returns the value of a potentially private field of the specified object.
      * The field can be declared on any of the object's inherited classes.
@@ -61,4 +65,41 @@ public static Object getFieldValue(Object object, String fieldName) {
 
         throw new AssertionError("Field not found: " + fieldName);
     }
+
+    /**
+     * Invokes the specified method on the object, and returns a value.
+     * The method can be declared on any of the object's inherited classes.
+     *
+     * @param object the object on which the method will be invoked
+     * @param methodName the method name
+     * @param args the arguments
+     * @return the return value
+     */
+    public static Object invokeMethod(Object object, String methodName, Class<?>[] parameterTypes, Object... args) {
+        Function<Class<?>, Method> getMethod = cls -> {
+            try {
+                var method = cls.getDeclaredMethod(methodName, parameterTypes);
+                method.setAccessible(true);
+                return method;
+            } catch (NoSuchMethodException e) {
+                return null;
+            }
+        };
+
+        Class<?> cls = object.getClass();
+        while (cls != null) {
+            Method method = getMethod.apply(cls);
+            if (method != null) {
+                try {
+                    return method.invoke(object, args);
+                } catch (IllegalAccessException | InvocationTargetException e) {
+                    throw new AssertionError(e);
+                }
+            }
+
+            cls = cls.getSuperclass();
+        }
+
+        throw new AssertionError("Method not found: " + methodName);
+    }
 }

modules/javafx.graphics/src/main/java/com/sun/glass/events/MouseEvent.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,8 @@
  */
 package com.sun.glass.events;
 
+import com.sun.glass.ui.WindowControlsOverlay;
+import javafx.stage.StageStyle;
 import java.lang.annotation.Native;
 
 public class MouseEvent {
@@ -49,4 +51,33 @@ public class MouseEvent {
      * This identifier is required for internal purposes.
      */
     @Native final static public int WHEEL           = 228;
+
+    /**
+     * Non-client events are only natively produced on the Windows platform as a result of
+     * handling the {@code WM_NCHITTEST} message for an {@link StageStyle#EXTENDED} window.
+     * <p>
+     * They are never sent to applications, but are processed by {@link WindowControlsOverlay}.
+     */
+    @Native final static public int NC_DOWN         = 230;
+    @Native final static public int NC_UP           = 231;
+    @Native final static public int NC_DRAG         = 232;
+    @Native final static public int NC_MOVE         = 233;
+    @Native final static public int NC_ENTER        = 234;
+    @Native final static public int NC_EXIT         = 235;
+
+    public static boolean isNonClientEvent(int event) {
+        return event >= NC_DOWN && event <= NC_EXIT;
+    }
+
+    public static int toNonClientEvent(int event) {
+        return switch (event) {
+            case DOWN -> NC_DOWN;
+            case UP -> NC_UP;
+            case DRAG -> NC_DRAG;
+            case MOVE -> NC_MOVE;
+            case ENTER -> NC_ENTER;
+            case EXIT -> NC_EXIT;
+            default -> event;
+        };
+    }
 }

modules/javafx.graphics/src/main/java/com/sun/glass/ui/Application.java

@@ -715,6 +715,12 @@ public final boolean supportsUnifiedWindows() {
         return _supportsUnifiedWindows();
     }
 
+    protected abstract boolean _supportsExtendedWindows();
+    public final boolean supportsExtendedWindows() {
+        checkEventThread();
+        return _supportsExtendedWindows();
+    }
+
     protected boolean _supportsSystemMenu() {
         // Overridden in subclasses
         return false;

modules/javafx.graphics/src/main/java/com/sun/glass/ui/NonClientHandler.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.glass.ui;
+
+import javafx.stage.StageStyle;
+
+/**
+ * A non-client handler is used in some implementations of windows with the {@link StageStyle#EXTENDED} style.
+ * It can inspect a mouse event before it is sent to FX, and decide to consume it if it affects
+ * a non-client part of the window (for example, minimize/maximize/close buttons).
+ */
+public interface NonClientHandler {
+
+    /**
+     * Handles the event.
+     *
+     * @return {@code true} if the event was handled, {@code false} otherwise
+     */
+    boolean handleMouseEvent(int type, int button, int x, int y, int xAbs, int yAbs, int clickCount);
+}

modules/javafx.graphics/src/main/java/com/sun/glass/ui/View.java

@@ -26,7 +26,7 @@
 
 import com.sun.glass.events.MouseEvent;
 import com.sun.glass.events.ViewEvent;
-
+import javafx.scene.Node;
 import java.lang.annotation.Native;
 import java.lang.ref.WeakReference;
 import java.security.AccessController;
@@ -69,8 +69,9 @@ public boolean handleKeyEvent(View view, long time, int action,
                 int keyCode, char[] keyChars, int modifiers) {
             return false;
         }
-        public void handleMenuEvent(View view, int x, int y, int xAbs,
+        public boolean handleMenuEvent(View view, int x, int y, int xAbs,
                 int yAbs, boolean isKeyboardTrigger) {
+            return false;
         }
         public void handleMouseEvent(View view, long time, int type, int button,
                                      int x, int y, int xAbs, int yAbs,
@@ -365,6 +366,18 @@ public void handleSwipeGestureEvent(View view, long time, int type,
                                             int yAbs) {
         }
 
+        /**
+         * Returns the draggable area node at the specified coordinates, or {@code null}
+         * if the specified coordinates do not intersect with a draggable area.
+         *
+         * @param x the X coordinate
+         * @param y the Y coordinate
+         * @return the draggable area node, or {@code null}
+         */
+        public Node pickDragAreaNode(double x, double y) {
+            return null;
+        }
+
         public Accessible getSceneAccessible() {
             return null;
         }
@@ -395,6 +408,7 @@ protected void _finishInputMethodComposition(long ptr) {
      */
     private volatile long ptr; // Native handle (NSView*, or internal structure pointer)
     private Window window; // parent window
+    private NonClientHandler nonClientHandler;
     private EventHandler eventHandler;
 
     private int width = -1;     // not set
@@ -494,6 +508,12 @@ void setWindow(Window window) {
         this.window = window;
         _setParent(this.ptr, window == null ? 0L : window.getNativeHandle());
         this.isValid = this.ptr != 0 && window != null;
+
+        if (this.isValid && window.isExtendedWindow()) {
+            this.nonClientHandler = window.getNonClientHandler();
+        } else {
+            this.nonClientHandler = null;
+        }
     }
 
     // package private
@@ -565,10 +585,11 @@ private void handleMouseEvent(long time, int type, int button, int x, int y,
         }
     }
 
-    private void handleMenuEvent(int x, int y, int xAbs, int yAbs, boolean isKeyboardTrigger) {
+    protected boolean handleMenuEvent(int x, int y, int xAbs, int yAbs, boolean isKeyboardTrigger) {
         if (shouldHandleEvent()) {
-            this.eventHandler.handleMenuEvent(this, x, y, xAbs, yAbs, isKeyboardTrigger);
+            return this.eventHandler.handleMenuEvent(this, x, y, xAbs, yAbs, isKeyboardTrigger);
         }
+        return false;
     }
 
     public void handleBeginTouchEvent(View view, long time, int modifiers,
@@ -913,15 +934,6 @@ protected void notifyMenu(int x, int y, int xAbs, int yAbs, boolean isKeyboardTr
     protected void notifyMouse(int type, int button, int x, int y, int xAbs,
                                int yAbs, int modifiers, boolean isPopupTrigger,
                                boolean isSynthesized) {
-        // gznote: optimize - only call for undecorated Windows!
-        if (this.window != null) {
-            // handled by window (programmatical move/resize)
-            if (this.window.handleMouseEvent(type, button, x, y, xAbs, yAbs)) {
-                // The evnet has been processed by Glass
-                return;
-            }
-        }
-
         long now = System.nanoTime();
         if (type == MouseEvent.DOWN) {
             View lastClickedView = View.lastClickedView == null ? null : View.lastClickedView.get();
@@ -945,6 +957,22 @@ protected void notifyMouse(int type, int button, int x, int y, int xAbs,
             lastClickedTime = now;
         }
 
+        // If we have a non-client handler, we give it the first chance to handle the event.
+        // Note that a full-screen window has no non-client area, and thus the non-client handler
+        // is not notified.
+        // Some implementations (like GTK) can fire synthesized events when they receive a mouse
+        // button event on the resize border. These events, even though happening on non-client
+        // regions, must not be processed by the non-client handler. For example, if a mouse click
+        // happens on the resize border that straddles the window close button, we don't want the
+        // close button to act on this click, because we just started a resize-drag operation.
+        boolean handled = !isSynthesized && !inFullscreen && nonClientHandler != null
+            && nonClientHandler.handleMouseEvent(type, button, x, y, xAbs, yAbs, clickCount);
+
+        // We never send non-client events to the application.
+        if (handled || MouseEvent.isNonClientEvent(type)) {
+            return;
+        }
+
         handleMouseEvent(now, type, button, x, y, xAbs, yAbs,
                          modifiers, isPopupTrigger, isSynthesized);