Cocoa: Fix clearing of unrelated window style bits

Whenever GLFW changed the window style mask, a new mask was created
from scratch based on the attributes set on the GLFW window object.
This caused us to potentially clear unrelated window style bits.

This was always wrong but became a critical issue when Cocoa began
throwing an exception if an application cleared the
NSWindowStyleMaskFullScreen while the window is in macOS fullscreen.

This commit reworks all style mask editing so it only changes the
relevant bits, preserving all others.

This is only a narrow bug fix to prevent crashes, intended for the
stable branch.  Our interaction with macOS fullscreen is still very
poor.  The next step after this is a set of patches that improve the
interaction between the current API and macOS fullscreen.

Fixes glfw#1886
Fixes glfw#2110

Author: elmindreda

Diff for: CONTRIBUTORS.md

@@ -84,6 +84,7 @@ video tutorials.
  - heromyth
  - Lucas Hinderberger
  - Paul Holden
+ - Hajime Hoshi
  - Warren Hu
  - Charles Huber
  - Brent Huisman
@@ -149,6 +150,7 @@ video tutorials.
  - James Murphy
  - Julian Møller
  - ndogxj
+ - F. Nedelec
  - n3rdopolis
  - Kristian Nielsen
  - Kamil Nowakowski

Diff for: README.md

@@ -255,6 +255,10 @@ information on what to include when reporting a bug.
  - [Cocoa] Bugfix: A connected Apple AirPlay would emit a useless error (#1791)
  - [Cocoa] Bugfix: The EGL and OSMesa libraries were not unloaded on termination
  - [Cocoa] Bugfix: `GLFW_MAXIMIZED` was always true when `GLFW_RESIZABLE` was false
+ - [Cocoa] Bugfix: Changing `GLFW_DECORATED` in macOS fullscreen would abort
+   application (#1886)
+ - [Cocoa] Bugfix: Setting a monitor from macOS fullscreen would abort
+   application (#2110)
  - [X11] Bugfix: The CMake files did not check for the XInput headers (#1480)
  - [X11] Bugfix: Key names were not updated when the keyboard layout changed
    (#1462,#1528)

Diff for: src/cocoa_window.m

@@ -31,26 +31,6 @@
 #include <float.h>
 #include <string.h>
 
-// Returns the style mask corresponding to the window settings
-//
-static NSUInteger getStyleMask(_GLFWwindow* window)
-{
-    NSUInteger styleMask = NSWindowStyleMaskMiniaturizable;
-
-    if (window->monitor || !window->decorated)
-        styleMask |= NSWindowStyleMaskBorderless;
-    else
-    {
-        styleMask |= NSWindowStyleMaskTitled |
-                     NSWindowStyleMaskClosable;
-
-        if (window->resizable)
-            styleMask |= NSWindowStyleMaskResizable;
-    }
-
-    return styleMask;
-}
-
 // Returns whether the cursor is in the content area of the specified window
 //
 static GLFWbool cursorInContentArea(_GLFWwindow* window)
@@ -809,9 +789,21 @@ static GLFWbool createNativeWindow(_GLFWwindow* window,
     else
         contentRect = NSMakeRect(0, 0, wndconfig->width, wndconfig->height);
 
+    NSUInteger styleMask = NSWindowStyleMaskMiniaturizable;
+
+    if (window->monitor || !window->decorated)
+        styleMask |= NSWindowStyleMaskBorderless;
+    else
+    {
+        styleMask |= (NSWindowStyleMaskTitled | NSWindowStyleMaskClosable);
+
+        if (window->resizable)
+            styleMask |= NSWindowStyleMaskResizable;
+    }
+
     window->ns.object = [[GLFWWindow alloc]
         initWithContentRect:contentRect
-                  styleMask:getStyleMask(window)
+                  styleMask:styleMask
                     backing:NSBackingStoreBuffered
                       defer:NO];
 
@@ -1241,9 +1233,10 @@ void _glfwSetWindowMonitorCocoa(_GLFWwindow* window,
         {
             const NSRect contentRect =
                 NSMakeRect(xpos, _glfwTransformYCocoa(ypos + height - 1), width, height);
+            const NSUInteger styleMask = [window->ns.object styleMask];
             const NSRect frameRect =
                 [window->ns.object frameRectForContentRect:contentRect
-                                                 styleMask:getStyleMask(window)];
+                                                 styleMask:styleMask];
 
             [window->ns.object setFrame:frameRect display:YES];
         }
@@ -1260,7 +1253,27 @@ void _glfwSetWindowMonitorCocoa(_GLFWwindow* window,
     // TODO: Solve this in a less terrible way
     _glfwPollEventsCocoa();
 
-    const NSUInteger styleMask = getStyleMask(window);
+    NSUInteger styleMask = [window->ns.object styleMask];
+
+    if (window->monitor)
+    {
+        styleMask &= ~(NSWindowStyleMaskTitled | NSWindowStyleMaskClosable);
+        styleMask |= NSWindowStyleMaskBorderless;
+    }
+    else
+    {
+        if (window->decorated)
+        {
+            styleMask &= ~NSWindowStyleMaskBorderless;
+            styleMask |= (NSWindowStyleMaskTitled | NSWindowStyleMaskClosable);
+        }
+
+        if (window->resizable)
+            styleMask |= NSWindowStyleMaskResizable;
+        else
+            styleMask &= ~NSWindowStyleMaskResizable;
+    }
+
     [window->ns.object setStyleMask:styleMask];
     // HACK: Changing the style mask can cause the first responder to be cleared
     [window->ns.object makeFirstResponder:window->ns.view];
@@ -1391,17 +1404,18 @@ void _glfwSetWindowResizableCocoa(_GLFWwindow* window, GLFWbool enabled)
 {
     @autoreleasepool {
 
-    [window->ns.object setStyleMask:getStyleMask(window)];
-
+    const NSUInteger styleMask = [window->ns.object styleMask];
     if (enabled)
     {
+        [window->ns.object setStyleMask:(styleMask | NSWindowStyleMaskResizable)];
         const NSWindowCollectionBehavior behavior =
             NSWindowCollectionBehaviorFullScreenPrimary |
             NSWindowCollectionBehaviorManaged;
         [window->ns.object setCollectionBehavior:behavior];
     }
     else
     {
+        [window->ns.object setStyleMask:(styleMask & ~NSWindowStyleMaskResizable)];
         const NSWindowCollectionBehavior behavior =
             NSWindowCollectionBehaviorFullScreenNone;
         [window->ns.object setCollectionBehavior:behavior];
@@ -1413,8 +1427,22 @@ void _glfwSetWindowResizableCocoa(_GLFWwindow* window, GLFWbool enabled)
 void _glfwSetWindowDecoratedCocoa(_GLFWwindow* window, GLFWbool enabled)
 {
     @autoreleasepool {
-    [window->ns.object setStyleMask:getStyleMask(window)];
+
+    NSUInteger styleMask = [window->ns.object styleMask];
+    if (enabled)
+    {
+        styleMask |= (NSWindowStyleMaskTitled | NSWindowStyleMaskClosable);
+        styleMask &= ~NSWindowStyleMaskBorderless;
+    }
+    else
+    {
+        styleMask |= NSWindowStyleMaskBorderless;
+        styleMask &= ~(NSWindowStyleMaskTitled | NSWindowStyleMaskClosable);
+    }
+
+    [window->ns.object setStyleMask:styleMask];
     [window->ns.object makeFirstResponder:window->ns.view];
+
     } // autoreleasepool
 }