tdf#115420 WIN clean up WinSalFrames DC handling

We still don't return a SalGraphics object from AcquireGraphics
without a valid DC. But internally we keep the WinSalGraphics
objects around, so we now have to verify the DC before using it.

In the end this also fixes the leak of the threaded SalGraphics
of the frame.

Change-Id: I267c96c04b7d00cb66a6c84c63d1373ebe0f529f
Reviewed-on: https://gerrit.libreoffice.org/50908
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Jan-Marek Glogowski <glogow@fbihome.de>

2 files changed, 147 insertions, 161 deletions

diff --git a/vcl/inc/win/salframe.h b/vcl/inc/win/salframe.h
index 002b6731c72b..77902a40034e 100644
--- a/vcl/inc/win/salframe.h
+++ b/vcl/inc/win/salframe.h
@@ -82,6 +82,12 @@ public:
     bool                    mbPropertiesStored;     // has values stored in the window property store
 
     void updateScreenNumber();
+
+private:
+    void ImplSetParentFrame( HWND hNewParentWnd, bool bAsChild );
+    bool InitFrameGraphicsDC( WinSalGraphics *pGraphics, HDC hDC, HWND hWnd );
+    bool ReleaseFrameGraphicsDC( WinSalGraphics* pGraphics );
+
 public:
     WinSalFrame();
     virtual ~WinSalFrame() override;
diff --git a/vcl/win/window/salframe.cxx b/vcl/win/window/salframe.cxx
index 74fb14120c6c..75784736c4e8 100644
--- a/vcl/win/window/salframe.cxx
+++ b/vcl/win/window/salframe.cxx
@@ -907,6 +907,34 @@ void WinSalFrame::updateScreenNumber()
     }
 }
 
+bool WinSalFrame::ReleaseFrameGraphicsDC( WinSalGraphics* pGraphics )
+{
+    assert( pGraphics );
+    SalData* pSalData = GetSalData();
+    HDC hDC = pGraphics->getHDC();
+    if ( !hDC )
+        return FALSE;
+    if ( pGraphics->getDefPal() )
+        SelectPalette( hDC, pGraphics->getDefPal(), TRUE );
+    pGraphics->DeInitGraphics();
+    // we don't want to run the WinProc in the main thread directly
+    // so we don't hit the mbNoYieldLock assert
+    if ( !pSalData->mpInstance->IsMainThread() )
+    {
+        assert( pGraphics == mpThreadGraphics );
+        SendMessageW( pSalData->mpInstance->mhComWnd, SAL_MSG_RELEASEDC,
+            reinterpret_cast<WPARAM>(mhWnd), reinterpret_cast<LPARAM>(hDC) );
+        pSalData->mnCacheDCInUse--;
+    }
+    else
+    {
+        assert( pGraphics == mpLocalGraphics );
+        ReleaseDC( mhWnd, hDC );
+    }
+    pGraphics->setHDC(nullptr);
+    return TRUE;
+}
+
 WinSalFrame::~WinSalFrame()
 {
     SalData* pSalData = GetSalData();
@@ -921,18 +949,18 @@ WinSalFrame::~WinSalFrame()
         *ppFrame = mpNextFrame;
     mpNextFrame = nullptr;
 
-    // Release Cache DC
-    if ( mpThreadGraphics &&
-         mpThreadGraphics->getHDC() )
-        ReleaseGraphics( mpThreadGraphics );
+    // destroy the thread SalGraphics
+    if ( mpThreadGraphics )
+    {
+        ReleaseFrameGraphicsDC( mpThreadGraphics );
+        delete mpThreadGraphics;
+        mpThreadGraphics = nullptr;
+    }
 
-    // destroy saved DC
+    // destroy the local SalGraphics
     if ( mpLocalGraphics )
     {
-        if ( mpLocalGraphics->getDefPal() )
-            SelectPalette( mpLocalGraphics->getHDC(), mpLocalGraphics->getDefPal(), TRUE );
-        mpLocalGraphics->DeInitGraphics();
-        ReleaseDC( mhWnd, mpLocalGraphics->getHDC() );
+        ReleaseFrameGraphicsDC( mpLocalGraphics );
         delete mpLocalGraphics;
         mpLocalGraphics = nullptr;
     }
@@ -962,16 +990,50 @@ WinSalFrame::~WinSalFrame()
     }
 }
 
+bool WinSalFrame::InitFrameGraphicsDC( WinSalGraphics *pGraphics, HDC hDC, HWND hWnd )
+{
+    SalData* pSalData = GetSalData();
+    assert( pGraphics );
+    if ( !pSalData->mpInstance->IsMainThread() )
+        assert( pGraphics == mpThreadGraphics );
+    else
+        assert( pGraphics == mpLocalGraphics );
+    pGraphics->setHWND( hWnd );
+
+    HDC hCurrentDC = pGraphics->getHDC();
+    assert( !hCurrentDC || (hCurrentDC == hDC) );
+    if ( hCurrentDC )
+        return TRUE;
+    pGraphics->setHDC( hDC );
+
+    if ( !hDC )
+        return FALSE;
+
+    if ( pSalData->mhDitherPal )
+    {
+        pGraphics->setDefPal(SelectPalette( hDC, pSalData->mhDitherPal, TRUE ));
+        RealizePalette( hDC );
+    }
+    pGraphics->InitGraphics();
+
+    if ( pGraphics == mpThreadGraphics )
+        pSalData->mnCacheDCInUse++;
+    return TRUE;
+}
+
 SalGraphics* WinSalFrame::AcquireGraphics()
 {
-    if ( mbGraphics )
+    if ( mbGraphics || !mhWnd )
         return nullptr;
 
+    SalData* pSalData = GetSalData();
+    WinSalGraphics *pGraphics = nullptr;
+    HDC hDC = 0;
+
     // Other threads get an own DC, because Windows modify in the
     // other case our DC (changing clip region), when they send a
     // WM_ERASEBACKGROUND message
-    SalData* pSalData = GetSalData();
-    if ( pSalData->mnAppThreadId != GetCurrentThreadId() )
+    if ( !pSalData->mpInstance->IsMainThread() )
     {
         // We use only three CacheDC's for all threads, because W9x is limited
         // to max. 5 Cache DC's per thread
@@ -979,80 +1041,30 @@ SalGraphics* WinSalFrame::AcquireGraphics()
             return nullptr;
 
         if ( !mpThreadGraphics )
-        {
             mpThreadGraphics = new WinSalGraphics(WinSalGraphics::WINDOW, true, mhWnd, this);
-            mpThreadGraphics->setHDC(nullptr);
-        }
-
-        HDC hDC = reinterpret_cast<HDC>(static_cast<sal_IntPtr>(SendMessageW( pSalData->mpInstance->mhComWnd,
-                                        SAL_MSG_GETDC,
-                                        reinterpret_cast<WPARAM>(mhWnd), 0 )));
-        if ( hDC )
-        {
-            mpThreadGraphics->setHDC(hDC);
-            if ( pSalData->mhDitherPal )
-            {
-                mpThreadGraphics->setDefPal(SelectPalette( hDC, pSalData->mhDitherPal, TRUE ));
-                RealizePalette( hDC );
-            }
-            mpThreadGraphics->InitGraphics();
-            mbGraphics = TRUE;
-
-            pSalData->mnCacheDCInUse++;
-            return mpThreadGraphics;
-        }
-        else
-            return nullptr;
+        pGraphics = mpThreadGraphics;
+        assert( !pGraphics->getHDC() );
+        hDC = reinterpret_cast<HDC>(static_cast<sal_IntPtr>(SendMessageW( pSalData->mpInstance->mhComWnd,
+                                    SAL_MSG_GETDC, reinterpret_cast<WPARAM>(mhWnd), 0 )));
     }
     else
     {
         if ( !mpLocalGraphics )
-        {
-            HDC hDC = GetDC( mhWnd );
-            if ( hDC )
-            {
-                mpLocalGraphics = new WinSalGraphics(WinSalGraphics::WINDOW, true, mhWnd, this);
-                mpLocalGraphics->setHDC(hDC);
-                if ( pSalData->mhDitherPal )
-                {
-                    mpLocalGraphics->setDefPal(SelectPalette( hDC, pSalData->mhDitherPal, TRUE ));
-                    RealizePalette( hDC );
-                }
-                mpLocalGraphics->InitGraphics();
-                mbGraphics = TRUE;
-            }
-        }
-        else
-            mbGraphics = TRUE;
-
-        return mpLocalGraphics;
+            mpLocalGraphics = new WinSalGraphics(WinSalGraphics::WINDOW, true, mhWnd, this);
+        pGraphics = mpLocalGraphics;
+        hDC = pGraphics->getHDC();
+        if ( !hDC )
+            hDC = GetDC( mhWnd );
     }
+
+    mbGraphics = InitFrameGraphicsDC( pGraphics, hDC, mhWnd );
+    return mbGraphics ? pGraphics : nullptr;
 }
 
 void WinSalFrame::ReleaseGraphics( SalGraphics* pGraphics )
 {
     if ( mpThreadGraphics == pGraphics )
-    {
-        if ( mpThreadGraphics->getHDC() )
-        {
-            SalData* pSalData = GetSalData();
-            if ( mpThreadGraphics->getDefPal() )
-                SelectPalette( mpThreadGraphics->getHDC(), mpThreadGraphics->getDefPal(), TRUE );
-            mpThreadGraphics->DeInitGraphics();
-            // we don't want to run the WinProc in the main thread directly
-            // so we don't hit the mbNoYieldLock assert
-            if ( !pSalData->mpInstance->IsMainThread() )
-                SendMessageW( pSalData->mpInstance->mhComWnd,
-                              SAL_MSG_RELEASEDC,
-                              reinterpret_cast<WPARAM>(mhWnd),
-                              reinterpret_cast<LPARAM>(mpThreadGraphics->getHDC()) );
-            else
-                ReleaseDC( mhWnd, mpThreadGraphics->getHDC() );
-            mpThreadGraphics->setHDC(nullptr);
-            pSalData->mnCacheDCInUse--;
-        }
-    }
-
+        ReleaseFrameGraphicsDC( mpThreadGraphics );
     mbGraphics = FALSE;
 }
 
@@ -1436,10 +1448,10 @@ void WinSalFrame::SetPosSize( long nX, long nY, long nWidth, long nHeight,
         CallCallback( nEvent, nullptr );
 }
 
-static void ImplSetParentFrame( WinSalFrame* pThis, HWND hNewParentWnd, bool bAsChild )
+void WinSalFrame::ImplSetParentFrame( HWND hNewParentWnd, bool bAsChild )
 {
     // save hwnd, will be overwritten in WM_CREATE during createwindow
-    HWND hWndOld = pThis->mhWnd;
+    HWND hWndOld = mhWnd;
     HWND hWndOldParent = ::GetParent( hWndOld );
     SalData* pSalData = GetSalData();
 
@@ -1454,7 +1466,7 @@ static void ImplSetParentFrame( WinSalFrame* pThis, HWND hNewParentWnd, bool bAs
     while( pFrame )
     {
         HWND hWndParent = ::GetParent( pFrame->mhWnd );
-        if( pThis->mhWnd == hWndParent )
+        if( mhWnd == hWndParent )
             children.push_back( pFrame );
         pFrame = pFrame->mpNextFrame;
     }
@@ -1464,13 +1476,13 @@ static void ImplSetParentFrame( WinSalFrame* pThis, HWND hNewParentWnd, bool bAs
     while( pObject )
     {
         HWND hWndParent = ::GetParent( pObject->mhWnd );
-        if( pThis->mhWnd == hWndParent )
+        if( mhWnd == hWndParent )
             systemChildren.push_back( pObject );
         pObject = pObject->mpNextObject;
     }
 
-    bool bNeedGraphics = pThis->mbGraphics;
-    bool bNeedCacheDC  = FALSE;
+    // to recreate the DCs, if they were destroyed
+    bool bHadLocalGraphics = FALSE, bHadThreadGraphics = FALSE;
 
     HFONT   hFont   = nullptr;
     HPEN    hPen    = nullptr;
@@ -1478,100 +1490,67 @@ static void ImplSetParentFrame( WinSalFrame* pThis, HWND hNewParentWnd, bool bAs
 
     int oldCount = pSalData->mnCacheDCInUse;
 
-    // Release Cache DC
-    if ( pThis->mpThreadGraphics &&
-         pThis->mpThreadGraphics->getHDC() )
+    // release the thread DC
+    if ( mpThreadGraphics )
     {
         // save current gdi objects before hdc is gone
-        hFont   = static_cast<HFONT>(GetCurrentObject( pThis->mpThreadGraphics->getHDC(), OBJ_FONT));
-        hPen    = static_cast<HPEN>(GetCurrentObject( pThis->mpThreadGraphics->getHDC(), OBJ_PEN));
-        hBrush  = static_cast<HBRUSH>(GetCurrentObject( pThis->mpThreadGraphics->getHDC(), OBJ_BRUSH));
-        pThis->ReleaseGraphics( pThis->mpThreadGraphics );
+        HDC hDC = mpThreadGraphics->getHDC();
+        if ( hDC )
+        {
+            hFont  = static_cast<HFONT>(GetCurrentObject( hDC, OBJ_FONT ));
+            hPen   = static_cast<HPEN>(GetCurrentObject( hDC, OBJ_PEN ));
+            hBrush = static_cast<HBRUSH>(GetCurrentObject( hDC, OBJ_BRUSH ));
+        }
 
-        // recreate cache dc only if it was destroyed
-        bNeedCacheDC  = TRUE;
+        bHadThreadGraphics = ReleaseFrameGraphicsDC( mpThreadGraphics );
+        assert( (bHadThreadGraphics && hDC) || (!bHadThreadGraphics && !hDC) );
     }
 
-    // destroy saved DC
-    if ( pThis->mpLocalGraphics )
-    {
-        if ( pThis->mpLocalGraphics->getDefPal() )
-            SelectPalette( pThis->mpLocalGraphics->getHDC(), pThis->mpLocalGraphics->getDefPal(), TRUE );
-        pThis->mpLocalGraphics->DeInitGraphics();
-        ReleaseDC( pThis->mhWnd, pThis->mpLocalGraphics->getHDC() );
-    }
+    // release the local DC
+    if ( mpLocalGraphics )
+        bHadLocalGraphics = ReleaseFrameGraphicsDC( mpLocalGraphics );
 
     // create a new hwnd with the same styles
     HWND hWndParent = hNewParentWnd;
     // forward to main thread
     HWND hWnd = reinterpret_cast<HWND>(static_cast<sal_IntPtr>(SendMessageW( pSalData->mpInstance->mhComWnd,
                                         bAsChild ? SAL_MSG_RECREATECHILDHWND : SAL_MSG_RECREATEHWND,
-                                        reinterpret_cast<WPARAM>(hWndParent), reinterpret_cast<LPARAM>(pThis->mhWnd) )));
+                                        reinterpret_cast<WPARAM>(hWndParent), reinterpret_cast<LPARAM>(mhWnd) )));
 
     // succeeded ?
     SAL_WARN_IF( !IsWindow( hWnd ), "vcl", "WinSalFrame::SetParent not successful");
 
-    // recreate DCs
-    if( bNeedGraphics )
+    // re-create thread DC
+    if( bHadThreadGraphics )
     {
-        if( pThis->mpThreadGraphics )
-        {
-            pThis->mpThreadGraphics->setHWND(hWnd);
-
-            if( bNeedCacheDC )
-            {
-                // re-create cached DC
-                HDC hDC = reinterpret_cast<HDC>(static_cast<sal_IntPtr>(SendMessageW( pSalData->mpInstance->mhComWnd,
-                                                SAL_MSG_GETDC,
-                                                reinterpret_cast<WPARAM>(hWnd), 0 )));
-                if ( hDC )
-                {
-                    pThis->mpThreadGraphics->setHDC(hDC);
-                    if ( pSalData->mhDitherPal )
-                    {
-                        pThis->mpThreadGraphics->setDefPal(SelectPalette( hDC, pSalData->mhDitherPal, TRUE ));
-                        RealizePalette( hDC );
-                    }
-                    pThis->mpThreadGraphics->InitGraphics();
-
-                    // re-select saved gdi objects
-                    if( hFont )
-                        SelectObject( hDC, hFont );
-                    if( hPen )
-                        SelectObject( hDC, hPen );
-                    if( hBrush )
-                        SelectObject( hDC, hBrush );
-
-                    pThis->mbGraphics = TRUE;
-
-                    pSalData->mnCacheDCInUse++;
-
-                    SAL_WARN_IF( oldCount != pSalData->mnCacheDCInUse, "vcl", "WinSalFrame::SetParent() hDC count corrupted");
-                }
-            }
-        }
-
-        if( pThis->mpLocalGraphics )
+        HDC hDC = reinterpret_cast<HDC>(static_cast<sal_IntPtr>(
+                    SendMessageW( pSalData->mpInstance->mhComWnd,
+                        SAL_MSG_GETDC, reinterpret_cast<WPARAM>(hWnd), 0 )));
+        InitFrameGraphicsDC( mpThreadGraphics, hDC, hWnd );
+        if ( hDC )
         {
-            // re-create DC
-            pThis->mpLocalGraphics->setHWND(hWnd);
-            pThis->mpLocalGraphics->setHDC( GetDC( hWnd ) );
-            if ( GetSalData()->mhDitherPal )
-            {
-                pThis->mpLocalGraphics->setDefPal(SelectPalette( pThis->mpLocalGraphics->getHDC(), GetSalData()->mhDitherPal, TRUE ));
-                RealizePalette( pThis->mpLocalGraphics->getHDC() );
-            }
-            pThis->mpLocalGraphics->InitGraphics();
-            pThis->mbGraphics = TRUE;
+            // re-select saved gdi objects
+            if( hFont )
+                SelectObject( hDC, hFont );
+            if( hPen )
+                SelectObject( hDC, hPen );
+            if( hBrush )
+                SelectObject( hDC, hBrush );
+
+            SAL_WARN_IF( oldCount != pSalData->mnCacheDCInUse, "vcl", "WinSalFrame::SetParent() hDC count corrupted");
         }
     }
 
+    // re-create local DC
+    if( bHadLocalGraphics )
+        InitFrameGraphicsDC( mpLocalGraphics, GetDC( hWnd ), hWnd );
+
     // TODO: add SetParent() call for SalObjects
     SAL_WARN_IF( !systemChildren.empty(), "vcl", "WinSalFrame::SetParent() parent of living system child window will be destroyed!");
 
     // reparent children before old parent is destroyed
     for (auto & child : children)
-        ImplSetParentFrame( child, hWnd, false );
+        child->ImplSetParentFrame( hWnd, false );
 
     children.clear();
     systemChildren.clear();
@@ -1584,7 +1563,7 @@ static void ImplSetParentFrame( WinSalFrame* pThis, HWND hNewParentWnd, bool bAs
 void WinSalFrame::SetParent( SalFrame* pNewParent )
 {
     WinSalFrame::mbInReparent = TRUE;
-    ImplSetParentFrame( this, static_cast<WinSalFrame*>(pNewParent)->mhWnd, false );
+    ImplSetParentFrame( static_cast<WinSalFrame*>(pNewParent)->mhWnd, false );
     WinSalFrame::mbInReparent = FALSE;
 }
 
@@ -1596,7 +1575,7 @@ bool WinSalFrame::SetPluginParent( SystemParentData* pNewParent )
     }
 
     WinSalFrame::mbInReparent = TRUE;
-    ImplSetParentFrame( this, pNewParent->hWnd, true );
+    ImplSetParentFrame( pNewParent->hWnd, true );
     WinSalFrame::mbInReparent = FALSE;
     return true;
 }
@@ -3774,9 +3753,11 @@ static bool ImplHandlePaintMsg( HWND hWnd )
     {
         // clip region must be set, as we don't get a proper
         // bounding rectangle otherwise
-        bool bHasClipRegion = pFrame->mpLocalGraphics && pFrame->mpLocalGraphics->getRegion();
+        WinSalGraphics *pGraphics = pFrame->mpLocalGraphics;
+        bool bHasClipRegion = pGraphics &&
+            pGraphics->getHDC() && pGraphics->getRegion();
         if ( bHasClipRegion )
-            SelectClipRgn( pFrame->mpLocalGraphics->getHDC(), nullptr );
+            SelectClipRgn( pGraphics->getHDC(), nullptr );
 
         // according to Windows documentation one shall check first if
         // there really is a paint-region
@@ -3793,8 +3774,7 @@ static bool ImplHandlePaintMsg( HWND hWnd )
 
         // reset clip region
         if ( bHasClipRegion )
-            SelectClipRgn( pFrame->mpLocalGraphics->getHDC(),
-                           pFrame->mpLocalGraphics->getRegion() );
+            SelectClipRgn( pGraphics->getHDC(), pGraphics->getRegion() );
 
         // try painting
         if ( bHasPaintRegion )
@@ -4098,7 +4078,7 @@ static void ImplHandleForcePalette( HWND hWnd )
     if ( hPal )
     {
         WinSalFrame* pFrame = ProcessOrDeferMessage( hWnd, SAL_MSG_FORCEPALETTE );
-        if ( pFrame && pFrame->mpLocalGraphics )
+        if ( pFrame && pFrame->mpLocalGraphics && pFrame->mpLocalGraphics->getHDC() )
         {
             WinSalGraphics* pGraphics = pFrame->mpLocalGraphics;
             if ( pGraphics && pGraphics->getDefPal() )
@@ -4182,7 +4162,7 @@ static LRESULT ImplHandlePalette( bool bFrame, HWND hWnd, UINT nMsg,
     while ( pTempFrame )
     {
         pGraphics = pTempFrame->mpLocalGraphics;
-        if ( pGraphics && pGraphics->getDefPal() )
+        if ( pGraphics && pGraphics->getHDC() && pGraphics->getDefPal() )
         {
             SelectPalette( pGraphics->getHDC(),
                            pGraphics->getDefPal(),
@@ -4195,7 +4175,7 @@ static LRESULT ImplHandlePalette( bool bFrame, HWND hWnd, UINT nMsg,
     WinSalFrame* pFrame = nullptr;
     if ( bFrame )
         pFrame = GetWindowPtr( hWnd );
-    if ( pFrame && pFrame->mpLocalGraphics )
+    if ( pFrame && pFrame->mpLocalGraphics && pFrame->mpLocalGraphics->getHDC() )
     {
         hDC = pFrame->mpLocalGraphics->getHDC();
         bStdDC = TRUE;
@@ -4233,7 +4213,7 @@ static LRESULT ImplHandlePalette( bool bFrame, HWND hWnd, UINT nMsg,
         if ( pTempFrame != pFrame )
         {
             pGraphics = pTempFrame->mpLocalGraphics;
-            if ( pGraphics && pGraphics->getDefPal() )
+            if ( pGraphics && pGraphics->getHDC() && pGraphics->getDefPal() )
             {
                 SelectPalette( pGraphics->getHDC(), hPal, TRUE );
                 if ( RealizePalette( pGraphics->getHDC() ) )
@@ -4250,7 +4230,7 @@ static LRESULT ImplHandlePalette( bool bFrame, HWND hWnd, UINT nMsg,
         while ( pTempFrame )
         {
             pGraphics = pTempFrame->mpLocalGraphics;
-            if ( pGraphics && pGraphics->getDefPal() )
+            if ( pGraphics && pGraphics->getHDC() && pGraphics->getDefPal() )
             {
                 InvalidateRect( pTempFrame->mhWnd, nullptr, FALSE );
                 UpdateWindow( pTempFrame->mhWnd );