8341382: EXCEPTION_ACCESS_VIOLATION in awt.dll after JDK-8185862

Reviewed-by: kizune, prr, vdyakov

Author: Alexander Zvegintsev

src/java.desktop/windows/native/libawt/windows/Devices.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -117,47 +117,78 @@ static BOOL IsValidMonitor(HMONITOR hMon)
     return TRUE;
 }
 
-// Callback for CountMonitors below
-static BOOL WINAPI clb_fCountMonitors(HMONITOR hMon, HDC hDC, LPRECT rRect, LPARAM lpMonitorCounter)
+
+// Callback for CollectMonitors below
+static BOOL WINAPI clb_fCollectMonitors(HMONITOR hMon, HDC hDC, LPRECT rRect, LPARAM lpMonitorData)
 {
-    if (IsValidMonitor(hMon)) {
-        (*((int *)lpMonitorCounter))++;
+    MonitorData* pMonitorData = (MonitorData *)lpMonitorData;
+
+    if (!IsValidMonitor(hMon)) {
+        return TRUE;
+    }
+
+    if (pMonitorData->monitorCounter == pMonitorData->monitorLimit) {
+        TRY;
+
+        int newMonitorLimit = pMonitorData->monitorLimit * 2;
+        HMONITOR* newMonitors =
+            (HMONITOR*)SAFE_SIZE_ARRAY_REALLOC(
+                safe_Realloc, pMonitorData->hmpMonitors,
+                newMonitorLimit, sizeof(HMONITOR)
+            );
+        pMonitorData->hmpMonitors = newMonitors;
+        pMonitorData->monitorLimit = newMonitorLimit;
+
+        CATCH_BAD_ALLOC_RET(FALSE);
     }
 
+    pMonitorData->hmpMonitors[pMonitorData->monitorCounter] = hMon;
+    pMonitorData->monitorCounter++;
+
     return TRUE;
 }
 
-int WINAPI CountMonitors(void)
+static HMONITOR* CollectMonitors(int* numScreens)
 {
-    int monitorCounter = 0;
-    ::EnumDisplayMonitors(NULL, NULL, clb_fCountMonitors, (LPARAM)&monitorCounter);
-    return monitorCounter;
-}
+    const int initialMonitorLimit = 4;
 
-// Callback for CollectMonitors below
-static BOOL WINAPI clb_fCollectMonitors(HMONITOR hMon, HDC hDC, LPRECT rRect, LPARAM lpMonitorData)
-{
-    MonitorData* pMonitorData = (MonitorData *)lpMonitorData;
-    if ((pMonitorData->monitorCounter < pMonitorData->monitorLimit) && (IsValidMonitor(hMon))) {
-        pMonitorData->hmpMonitors[pMonitorData->monitorCounter] = hMon;
-        pMonitorData->monitorCounter++;
+    *numScreens = 0;
+
+    MonitorData data;
+    data.monitorCounter = 0;
+    data.monitorLimit = initialMonitorLimit;
+
+    TRY;
+
+    data.hmpMonitors = (HMONITOR*)SAFE_SIZE_ARRAY_ALLOC(safe_Malloc,
+                            initialMonitorLimit, sizeof(HMONITOR));
+    CATCH_BAD_ALLOC_RET(NULL);
+
+    if (!::EnumDisplayMonitors(NULL, NULL, clb_fCollectMonitors, (LPARAM)&data)) {
+        free(data.hmpMonitors);
+        return NULL;
     }
 
-    return TRUE;
+    *numScreens = data.monitorCounter;
+    return data.hmpMonitors;
 }
 
-static int WINAPI CollectMonitors(HMONITOR* hmpMonitors, int nNum)
+int WINAPI CountMonitors()
 {
-    if (NULL != hmpMonitors) {
-        MonitorData monitorData;
-        monitorData.monitorCounter = 0;
-        monitorData.monitorLimit = nNum;
-        monitorData.hmpMonitors = hmpMonitors;
-        ::EnumDisplayMonitors(NULL, NULL, clb_fCollectMonitors, (LPARAM)&monitorData);
-        return monitorData.monitorCounter;
-    } else {
-        return 0;
+    int numScreens = 0;
+    HMONITOR* monHds = CollectMonitors(&numScreens);
+    free(monHds);
+    return numScreens;
+}
+
+static BOOL AreSameMonitorInfo(LPMONITORINFOEX oldInfo, LPMONITORINFOEX newInfo)
+{
+    if (oldInfo == NULL || newInfo == NULL) {
+        return FALSE;
     }
+
+    return oldInfo->dwFlags == newInfo->dwFlags
+            && ::lstrcmp(oldInfo->szDevice, newInfo->szDevice) == 0;
 }
 
 BOOL WINAPI MonitorBounds(HMONITOR hmMonitor, RECT* rpBounds)
@@ -206,17 +237,26 @@ BOOL Devices::UpdateInstance(JNIEnv *env)
 {
     J2dTraceLn(J2D_TRACE_INFO, "Devices::UpdateInstance");
 
-    int numScreens = CountMonitors();
-    HMONITOR *monHds = (HMONITOR *)SAFE_SIZE_ARRAY_ALLOC(safe_Malloc,
-            numScreens, sizeof(HMONITOR));
-    if (numScreens != CollectMonitors(monHds, numScreens)) {
+    int numScreens = 0;
+    HMONITOR *monHds = CollectMonitors(&numScreens);
+    if (monHds == NULL) {
         J2dRlsTraceLn(J2D_TRACE_ERROR,
-                      "Devices::UpdateInstance: Failed to get all "\
+                      "Devices::UpdateInstance: Failed to get "\
                       "monitor handles.");
         free(monHds);
         return FALSE;
     }
 
+    if (numScreens == 0) {
+        CriticalSection::Lock l(arrayLock);
+        if (theInstance != NULL) {
+            J2dRlsTraceLn(J2D_TRACE_ERROR,
+                          "Devices::UpdateInstance: No valid monitor handles.");
+            free(monHds);
+            return FALSE;
+        }
+    }
+
     Devices *newDevices = new Devices(numScreens);
     // This way we know that the array will not be disposed of
     // at least until we replaced it with a new one.
@@ -242,18 +282,26 @@ BOOL Devices::UpdateInstance(JNIEnv *env)
         theInstance = newDevices;
 
         if (oldDevices) {
-            // Invalidate the devices with indexes out of the new set of
-            // devices. This doesn't cover all cases when the device
-            // might should be invalidated (like if it's not the last device
-            // that was removed), but it will have to do for now.
             int oldNumScreens = oldDevices->GetNumDevices();
-            int newNumScreens = theInstance->GetNumDevices();
-            J2dTraceLn(J2D_TRACE_VERBOSE, "  Invalidating removed devices");
-            for (int i = newNumScreens; i < oldNumScreens; i++) {
-                // removed device, needs to be invalidated
+            J2dTraceLn(J2D_TRACE_VERBOSE, "  Invalidating changed devices");
+            for (int i = 0; i < oldNumScreens; i++) {
+                AwtWin32GraphicsDevice *oldDevice =
+                    oldDevices->GetDevice(i, FALSE);
+                AwtWin32GraphicsDevice *newDevice =
+                    theInstance->GetDevice(i, FALSE);
+                BOOL changed = (newDevice == NULL)
+                    || !AreSameMonitorInfo(
+                            (LPMONITORINFOEX) oldDevice->GetMonitorInfo(),
+                            (LPMONITORINFOEX) newDevice->GetMonitorInfo());
+
+                if (!changed) {
+                    newDevice->TransferJavaDevice(env, oldDevice);
+                    continue;
+                }
+
                 J2dTraceLn(J2D_TRACE_WARNING,
-                           "Devices::UpdateInstance: device removed: %d", i);
-                oldDevices->GetDevice(i)->Invalidate(env);
+                           "Devices::UpdateInstance: device changed: %d", i);
+                oldDevice->Invalidate(env);
             }
             // Now that we have a new array in place, remove this (possibly the
             // last) reference to the old instance.
@@ -346,6 +394,12 @@ AwtWin32GraphicsDevice *Devices::GetDevice(int index, BOOL adjust)
     J2dTraceLn(J2D_TRACE_INFO,
                "Devices::GetDevice index=%d adjust?=%d",
                index, adjust);
+    if (numDevices <= 0) {
+        J2dTraceLn(J2D_TRACE_WARNING,
+                   "Devices::GetDevice: "\
+                   "no devices, returning NULL.");
+        return NULL;
+    }
     if (index < 0 || index >= numDevices) {
         if (!adjust) {
             J2dTraceLn(J2D_TRACE_WARNING,

src/java.desktop/windows/native/libawt/windows/Devices.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -47,8 +47,11 @@ static BOOL                     UpdateInstance(JNIEnv *env);
        class InstanceAccess {
        public:
            INLINE   InstanceAccess() { devices = Devices::GetInstance(); }
-           INLINE  ~InstanceAccess() { devices->Release(); }
+           INLINE  ~InstanceAccess() { if (devices != NULL) devices->Release(); }
            Devices* operator->()     { return devices; }
+           INLINE AwtWin32GraphicsDevice* Device(int index, BOOL adjust = TRUE) {
+               return devices == NULL ? NULL : devices->GetDevice(index, adjust);
+           }
         private:
            Devices* devices;
            // prevent bad things like copying or getting address of

src/java.desktop/windows/native/libawt/windows/awt_Toolkit.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1996, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -65,7 +65,7 @@
 #include <java_awt_Toolkit.h>
 #include <java_awt_event_InputMethodEvent.h>
 
-extern void initScreens(JNIEnv *env);
+extern BOOL initScreens(JNIEnv *env);
 extern "C" void awt_dnd_initialize();
 extern "C" void awt_dnd_uninitialize();
 extern "C" void awt_clipboard_uninitialize(JNIEnv *env);
@@ -157,6 +157,78 @@ extern "C" JNIEXPORT jboolean JNICALL AWTIsHeadless() {
 }
 
 #define IDT_AWT_MOUSECHECK 0x101
+#define IDT_AWT_DISPLAYCHANGE 0x102
+
+#define AWT_DISPLAYCHANGE_RETRY_DELAY 250
+#define AWT_DISPLAYCHANGE_RETRY_LIMIT 20
+
+class DisplayChangeHandler {
+public:
+    static BOOL Handle(JNIEnv *env, HWND hWnd) {
+        // Reinitialize screens
+        if (!initScreens(env)) {
+            OnDisplayChangeFailed(hWnd);
+            return FALSE;
+        }
+
+        OnDisplayChangeSucceeded(hWnd);
+
+        // Notify Java side - call WToolkit.displayChanged()
+        jclass clazz = env->FindClass("sun/awt/windows/WToolkit");
+        DASSERT(clazz != NULL);
+        if (!clazz) throw std::bad_alloc();
+        env->CallStaticVoidMethod(clazz, AwtToolkit::displayChangeMID);
+
+        return !env->ExceptionCheck();
+    }
+
+    static void Reset(HWND hWnd) {
+        ::KillTimer(hWnd, IDT_AWT_DISPLAYCHANGE);
+        retryCount = 0;
+    }
+
+    static void ScheduleFromSessionChange(HWND hWnd) {
+        if (!recoveryPending) {
+            return;
+        }
+        Reset(hWnd);
+        Schedule(hWnd);
+    }
+
+private:
+    static void OnDisplayChangeFailed(HWND hWnd) {
+        recoveryPending = TRUE;
+        Schedule(hWnd);
+    }
+
+    static void OnDisplayChangeSucceeded(HWND hWnd) {
+        recoveryPending = FALSE;
+        Reset(hWnd);
+    }
+
+    static void Schedule(HWND hWnd) {
+        if (retryCount >= AWT_DISPLAYCHANGE_RETRY_LIMIT) {
+            Reset(hWnd);
+            J2dRlsTraceLn(J2D_TRACE_ERROR,
+                          "AwtToolkit: Display change retry limit exceeded.");
+            return;
+        }
+
+        retryCount++;
+        if (::SetTimer(hWnd, IDT_AWT_DISPLAYCHANGE,
+                       AWT_DISPLAYCHANGE_RETRY_DELAY, NULL) == 0) {
+            Reset(hWnd);
+            J2dRlsTraceLn(J2D_TRACE_ERROR,
+                          "AwtToolkit: Failed to schedule display change retry.");
+        }
+    }
+
+    static int retryCount;
+    static BOOL recoveryPending;
+};
+
+int DisplayChangeHandler::retryCount = 0;
+BOOL DisplayChangeHandler::recoveryPending = FALSE;
 
 static LPCTSTR szAwtToolkitClassName = TEXT("SunAwtToolkit");
 
@@ -1004,6 +1076,14 @@ LRESULT CALLBACK AwtToolkit::WndProc(HWND hWnd, UINT message,
       }
 
       case WM_TIMER: {
+          if (wParam == IDT_AWT_DISPLAYCHANGE) {
+              if (DisplayChangeHandler::Handle(env, hWnd)) {
+                  GetInstance().m_displayChanged = TRUE;
+                  ::PostMessage(HWND_BROADCAST, WM_PALETTEISCHANGING, NULL, NULL);
+              }
+              return 0;
+          }
+
           // 6479820. Should check if a window is in manual resizing process: skip
           // sending any MouseExit/Enter events while inside resize-loop.
           // Note that window being in manual moving process could still
@@ -1245,18 +1325,11 @@ LRESULT CALLBACK AwtToolkit::WndProc(HWND hWnd, UINT message,
           return tk.m_inputMethodData;
       }
       case WM_DISPLAYCHANGE: {
-          // Reinitialize screens
-          initScreens(env);
-
-          // Notify Java side - call WToolkit.displayChanged()
-          jclass clazz = env->FindClass("sun/awt/windows/WToolkit");
-          DASSERT(clazz != NULL);
-          if (!clazz) throw std::bad_alloc();
-          env->CallStaticVoidMethod(clazz, AwtToolkit::displayChangeMID);
-
-          GetInstance().m_displayChanged = TRUE;
-
-          ::PostMessage(HWND_BROADCAST, WM_PALETTEISCHANGING, NULL, NULL);
+          DisplayChangeHandler::Reset(hWnd);
+          if (DisplayChangeHandler::Handle(env, hWnd)) {
+              GetInstance().m_displayChanged = TRUE;
+              ::PostMessage(HWND_BROADCAST, WM_PALETTEISCHANGING, NULL, NULL);
+          }
           break;
       }
       /* Session management */
@@ -1341,6 +1414,9 @@ LRESULT CALLBACK AwtToolkit::WndProc(HWND hWnd, UINT message,
                                               activate
                                               ? JNI_TRUE
                                               : JNI_FALSE, reason);
+              if (activate) {
+                  DisplayChangeHandler::ScheduleFromSessionChange(hWnd);
+              }
           }
           break;
       }