tdf#118786 WIN just assert in Yield

This reminds me - again - that Jenkins doesn't run make check.

It turns out InSendMessage() also returns true, if you process a
nested SendMessage in the same thread. Therefore we have to
remove the SalComWndProc assert and just keep the one in the
Yield call.

Why? Because there seem to be no way to get the information
ReplyMessage has access to, so we could detect the caller /
origin of the send message and implement proper nested call
checks. The alternative would be to change all call sites of
SendMessage to:

if ( !pSalData->mpInstance->IsMainThread() )
SendMessage(...)
else
SalComWndProc(...)

which is the same SendMessage already does.

Change-Id: I991d68a64952dc5d47ba51edd8635c9e8c46614c
Reviewed-on: https://gerrit.libreoffice.org/59538
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>
(cherry picked from commit bf0a63dc93a746a008fb1656457c77de8df693ba)
Reviewed-on: https://gerrit.libreoffice.org/59565
(cherry picked from commit a42c65176f2791cf5e48578a8898bf03185adc89)
Reviewed-on: https://gerrit.libreoffice.org/60726
Reviewed-by: Michael Stahl <Michael.Stahl@cib.de>

2 files changed, 11 insertions, 13 deletions

diff --git a/vcl/inc/win/salinst.h b/vcl/inc/win/salinst.h
index 9f5dbdf955b1..89a12c895bf9 100644
--- a/vcl/inc/win/salinst.h
+++ b/vcl/inc/win/salinst.h
@@ -39,7 +39,7 @@ public:
     SalYieldMutex*      mpSalYieldMutex;
 
     osl::Condition      maWaitingYieldCond;
-    bool                mbNoYieldLock;
+    unsigned            m_nNoYieldLock;
 
 public:
     WinSalInstance();
diff --git a/vcl/win/app/salinst.cxx b/vcl/win/app/salinst.cxx
index f6e17bcf7faa..288c5c2cc6e0 100644
--- a/vcl/win/app/salinst.cxx
+++ b/vcl/win/app/salinst.cxx
@@ -138,7 +138,7 @@ void SalYieldMutex::doAcquire( sal_uInt32 nLockCount )
     WinSalInstance* pInst = GetSalData()->mpInstance;
     if ( pInst && pInst->IsMainThread() )
     {
-        if ( pInst->mbNoYieldLock )
+        if ( pInst->m_nNoYieldLock )
             return;
         // tdf#96887 If this is the main thread, then we must wait for two things:
         // - the mpSalYieldMutex being freed
@@ -169,7 +169,7 @@ void SalYieldMutex::doAcquire( sal_uInt32 nLockCount )
 sal_uInt32 SalYieldMutex::doRelease( const bool bUnlockAll )
 {
     WinSalInstance* pInst = GetSalData()->mpInstance;
-    if ( pInst && pInst->mbNoYieldLock && pInst->IsMainThread() )
+    if ( pInst && pInst->m_nNoYieldLock && pInst->IsMainThread() )
         return 1;
 
     sal_uInt32 nCount = comphelper::GenericSolarMutex::doRelease( bUnlockAll );
@@ -184,7 +184,7 @@ bool SalYieldMutex::tryToAcquire()
     WinSalInstance* pInst = GetSalData()->mpInstance;
     if ( pInst )
     {
-        if ( pInst->mbNoYieldLock && pInst->IsMainThread() )
+        if ( pInst->m_nNoYieldLock && pInst->IsMainThread() )
             return true;
         else
             return comphelper::GenericSolarMutex::tryToAcquire();
@@ -218,7 +218,7 @@ void ImplSalYieldMutexRelease()
 
 bool SalYieldMutex::IsCurrentThread() const
 {
-    if ( !GetSalData()->mpInstance->mbNoYieldLock )
+    if ( !GetSalData()->mpInstance->m_nNoYieldLock )
         // For the Windows backend, the LO identifier is the system thread ID
         return m_nThreadId == GetCurrentThreadId();
     else
@@ -431,7 +431,7 @@ void DestroySalInstance( SalInstance* pInst )
 
 WinSalInstance::WinSalInstance()
     : mhComWnd( nullptr )
-    , mbNoYieldLock( false )
+    , m_nNoYieldLock( 0 )
 {
     mpSalYieldMutex = new SalYieldMutex();
     mpSalYieldMutex->acquire();
@@ -477,7 +477,7 @@ bool ImplSalYield( bool bWait, bool bHandleAllCurrentEvents )
     bool bWasMsg = false, bOneEvent = false, bWasTimeoutMsg = false;
     ImplSVData *const pSVData = ImplGetSVData();
     WinSalTimer* pTimer = static_cast<WinSalTimer*>( pSVData->maSchedCtx.mpSalTimer );
-    const bool bNoYieldLock = GetSalData()->mpInstance->mbNoYieldLock;
+    const bool bNoYieldLock = (GetSalData()->mpInstance->m_nNoYieldLock > 0);
 
     assert( !bNoYieldLock );
     if ( bNoYieldLock )
@@ -578,10 +578,9 @@ bool WinSalInstance::DoYield(bool bWait, bool bHandleAllCurrentEvents)
     case salmsg: \
         if (bIsOtherThreadMessage) \
         { \
-            assert( !pInst->mbNoYieldLock ); \
-            pInst->mbNoYieldLock = true; \
+            ++pInst->m_nNoYieldLock; \
             function; \
-            pInst->mbNoYieldLock = false; \
+            --pInst->m_nNoYieldLock; \
         } \
         else \
         { \
@@ -594,10 +593,9 @@ bool WinSalInstance::DoYield(bool bWait, bool bHandleAllCurrentEvents)
     case salmsg: \
         if (bIsOtherThreadMessage) \
         { \
-            assert( !pInst->mbNoYieldLock ); \
-            pInst->mbNoYieldLock = true; \
+            ++pInst->m_nNoYieldLock; \
             nRet = reinterpret_cast<LRESULT>( function ); \
-            pInst->mbNoYieldLock = false; \
+            --pInst->m_nNoYieldLock; \
         } \
         else \
         { \