Unlock scheduler when calling ProcessEventsToIdle

When processing the events, solar mutex will be temporarily released,
and then it's possible that another thread will acquire solar mutex
and try to lock scheduler. This will result in deadlock.

Seen in CppunitTest_sw_core_undo that deadlocked locally on Windows
frequently.

Main thread call stack:

win32u.dll!NtUserMsgWaitForMultipleObjectsEx()
user32.dll!RealMsgWaitForMultipleObjectsEx()
sal3.dll!osl_waitCondition(void * Condition, const TimeValue * pTimeout) Line 82
vclplug_winlo.dll!osl::Condition::wait(const TimeValue * pTimeout) Line 120
vclplug_winlo.dll!SalYieldMutex::doAcquire(unsigned long nLockCount) Line 159
comphelper.dll!comphelper::SolarMutex::acquire(unsigned long nLockCount) Line 87
vclplug_winlo.dll!ImplSalYieldMutexAcquireWithWait(unsigned long nCount) Line 204
vclplug_winlo.dll!WinSalTimer::ImplHandleElapsedTimer() Line 163
vclplug_winlo.dll!WinSalTimer::ImplHandle_WM_TIMER(unsigned __int64 aWPARAM) Line 197
vclplug_winlo.dll!SalComWndProc(HWND__ * __formal, unsigned int nMsg, unsigned __int64 wParam, __int64 lParam, bool & rDef) Line 638
vclplug_winlo.dll!SalComWndProcW(HWND__ * hWnd, unsigned int nMsg, unsigned __int64 wParam, __int64 lParam) Line 665
user32.dll!UserCallWinProcCheckWow()
user32.dll!DispatchMessageWorker()
vclplug_winlo.dll!ImplSalDispatchMessage(const tagMSG * pMsg) Line 426
vclplug_winlo.dll!ImplSalYield(bool bWait, bool bHandleAllCurrentEvents) Line 457
vclplug_winlo.dll!WinSalInstance::DoYield(bool bWait, bool bHandleAllCurrentEvents) Line 528
vcllo.dll!ImplYield(bool i_bWait, bool i_bAllEvents) Line 447
vcllo.dll!Application::Reschedule(bool i_bAllEvents) Line 461
vcllo.dll!Scheduler::ProcessEventsToIdle() Line 466
vcllo.dll!Scheduler::ImplDeInitScheduler() Line 129
vcllo.dll!DeInitVCL() Line 483
vclbootstrapprotector.dll!`anonymous namespace'::Protector::~Protector() Line 34
vclbootstrapprotector.dll!`anonymous namespace'::Protector::`scalar deleting destructor'(unsigned int)
cppunitd_dll.dll!CppUnit::ProtectorChain::pop() Line 56
cppunitd_dll.dll!CppUnit::TestResult::popProtector() Line 197
cppunittester.exe!`anonymous namespace'::ProtectedFixtureFunctor::run() Line 332
cppunittester.exe!sal_main() Line 473
cppunittester.exe!main(int argc, char * * argv) Line 380
cppunittester.exe!invoke_main() Line 79
cppunittester.exe!__scrt_common_main_seh() Line 288
cppunittester.exe!__scrt_common_main() Line 331
cppunittester.exe!mainCRTStartup() Line 17
kernel32.dll!BaseThreadInitThunk()
ntdll.dll!RtlUserThreadStart()

Clipboard thread call stack:

ntdll.dll!NtWaitForAlertByThreadId()
ntdll.dll!RtlpWaitOnAddressWithTimeout()
ntdll.dll!RtlpWaitOnAddress()
ntdll.dll!RtlpWaitOnCriticalSection()
ntdll.dll!RtlpEnterCriticalSectionContended()
ntdll.dll!RtlEnterCriticalSection()
sal3.dll!osl_acquireMutex(_oslMutexImpl * Mutex) Line 66
vcllo.dll!osl::Mutex::acquire() Line 57
vcllo.dll!SchedulerMutex::acquire(unsigned long nLockCount) Line 213
vcllo.dll!Scheduler::Lock(unsigned long nLockCount) Line 237
vcllo.dll!SchedulerGuard::SchedulerGuard() Line 60
vcllo.dll!Task::~Task() Line 660
vcllo.dll!Timer::~Timer() Line 61
vcllo.dll!Idle::~Idle()
svxcorelo.dll!SdrPaintView::~SdrPaintView() Line 189
svxcorelo.dll!SdrSnapView::~SdrSnapView() Line 200
svxcorelo.dll!SdrMarkView::~SdrMarkView() Line 193
svxcorelo.dll!SdrEditView::~SdrEditView() Line 125
svxcorelo.dll!SdrPolyEditView::~SdrPolyEditView() Line 58
svxcorelo.dll!SdrGlueEditView::~SdrGlueEditView() Line 40
svxcorelo.dll!SdrObjEditView::~SdrObjEditView() Line 95
svxcorelo.dll!SdrExchangeView::~SdrExchangeView()
svxcorelo.dll!SdrDragView::~SdrDragView() Line 65
svxcorelo.dll!SdrCreateView::~SdrCreateView() Line 200
svxcorelo.dll!SdrView::~SdrView() Line 159
swlo.dll!SdrView::`scalar deleting destructor'(unsigned int)
swlo.dll!std::default_delete<SdrView>::operator()(SdrView * _Ptr) Line 2537
swlo.dll!std::unique_ptr<SdrView,std::default_delete<SdrView>>::~unique_ptr<SdrView,std::default_delete<SdrView>>() Line 2649
swlo.dll!SwDrawFrameFormat::MakeGraphic(ImageMap * __formal) Line 7491
swlo.dll!OutHTML_FrameFormatAsImage(Writer & rWrt, const SwFrameFormat & rFrameFormat) Line 1746
swlo.dll!SwHTMLWriter::OutFrameFormat(AllHtmlFlags nMode, const SwFrameFormat & rFrameFormat, const SdrObject * pSdrObject) Line 507
swlo.dll!SwHTMLWriter::OutFlyFrame(unsigned __int64 nNdIdx, long nContentIdx, HtmlPosition nPos, HTMLOutContext * pContext) Line 401
swlo.dll!OutHTML_SwTextNode(Writer & rWrt, const SwContentNode & rNode) Line 2509
swlo.dll!SwHTMLWriter::Out_SwDoc(SwPaM * pPam) Line 834
swlo.dll!SwHTMLWriter::WriteStream() Line 471
swlo.dll!Writer::Write(SwPaM & rPaM, SvStream & rStrm, const rtl::OUString * pFName) Line 272
swlo.dll!SwWriter::Write(const tools::SvRef<Writer> & rxWriter, const rtl::OUString * pRealFileName) Line 861
swlo.dll!SwTransferable::WriteObject(tools::SvRef<SotStorageStream> & xStream, void * pObject, unsigned long nObjectType, const com::sun::star::datatransfer::DataFlavor & __formal) Line 823
vcllo.dll!TransferableHelper::SetObject(void * pUserObject, unsigned long nUserObjectId, const com::sun::star::datatransfer::DataFlavor & rFlavor) Line 879
swlo.dll!SwTransferable::GetData(const com::sun::star::datatransfer::DataFlavor & rFlavor, const rtl::OUString & rDestDoc) Line 619
vcllo.dll!TransferableHelper::getTransferData2(const com::sun::star::datatransfer::DataFlavor & rFlavor, const rtl::OUString & rDestDoc) Line 389
vcllo.dll!TransferableHelper::getTransferData(const com::sun::star::datatransfer::DataFlavor & rFlavor) Line 287
vclplug_winlo.dll!CXTDataObject::renderSynthesizedHtmlAndSetupStgMedium(tagFORMATETC & fetc, tagSTGMEDIUM & stgmedium) Line 516
vclplug_winlo.dll!CXTDataObject::renderSynthesizedFormatAndSetupStgMedium(tagFORMATETC & fetc, tagSTGMEDIUM & stgmedium) Line 419
vclplug_winlo.dll!`CXTDataObject::GetData'::`1'::catch$1() Line 269
vcruntime140_1d.dll!_CallSettingFrame_LookupContinuationIndex() Line 98
vcruntime140_1d.dll!__FrameHandler4::CxxCallCatchBlock(_EXCEPTION_RECORD * pExcept) Line 1367
ntdll.dll!RcConsolidateFrames()
vclplug_winlo.dll!CXTDataObject::GetData(tagFORMATETC * pFormatetc, tagSTGMEDIUM * pmedium) Line 261
vclplug_winlo.dll!CXNotifyingDataObject::GetData(tagFORMATETC * pFormatetc, tagSTGMEDIUM * pmedium) Line 86
ole32.dll!HandleFromHandle(IDataObject * pDataObj, tagFORMATETC * pformatetc, tagSTGMEDIUM * pmedium) Line 2048
ole32.dll!RenderCurrentFormat(HWND__ * hClipWnd, unsigned int cf, tagSTGMEDIUM * pmedium, IDataObject * pDataObj) Line 4057
ole32.dll!RenderFormat(HWND__ * hClipWnd, unsigned int cf, IDataObject * pDataObj, void * pvMTADataObject) Line 4191
ole32.dll!ClipboardWndProc(HWND__ * hWnd, unsigned int msg, unsigned __int64 wParam, __int64 lParam) Line 683
user32.dll!UserCallWinProcCheckWow()
user32.dll!DispatchClientMessage()
user32.dll!__fnDWORD()
ntdll.dll!KiUserCallbackDispatcherContinue()
win32u.dll!NtUserGetMessage()
user32.dll!GetMessageW()
vclplug_winlo.dll!CMtaOleClipboard::run() Line 654
vclplug_winlo.dll!CMtaOleClipboard::oleThreadProc(void * pParam) Line 676
ucrtbased.dll!thread_start<unsigned int (__cdecl*)(void *),1>(void * const parameter) Line 97
kernel32.dll!BaseThreadInitThunk()
ntdll.dll!RtlUserThreadStart()

Main thread hold solar mutex when it locked scheduler in
ImplDeInitScheduler; then it released solar mutex in
WinSalInstance::DoYield, and finally tried to acquire it again in
WinSalTimer::ImplHandleElapsedTimer.

At the same time, clipboard thread acquired solar mutex in
TransferableHelper::getTransferData2. Then it attempted to lock
scheduler in Task::~Task.

Change-Id: I2ac87c8ac298104043f5b9b5d6760bea0f408f54
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/107218
Tested-by: Mike Kaganski <mike.kaganski@collabora.com>
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>

2 files changed, 20 insertions, 0 deletions

diff --git a/vcl/inc/schedulerimpl.hxx b/vcl/inc/schedulerimpl.hxx
index ffb08084252c..d95fb0157880 100644
--- a/vcl/inc/schedulerimpl.hxx
+++ b/vcl/inc/schedulerimpl.hxx
@@ -65,6 +65,25 @@ public:
     }
 };
 
+class SchedulerGuardReleaser final
+{
+public:
+    SchedulerGuardReleaser()
+    {
+        Scheduler::Lock();
+        m_nLockCount = Scheduler::Unlock(true) - 1;
+    }
+
+    ~SchedulerGuardReleaser()
+    {
+        if (m_nLockCount)
+            Scheduler::Lock(m_nLockCount);
+    }
+
+private:
+    sal_uInt32 m_nLockCount;
+};
+
 #endif // INCLUDED_VCL_INC_SCHEDULERIMPL_HXX
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/vcl/source/app/svapp.cxx b/vcl/source/app/svapp.cxx
index f91375e77edd..0b30a0b6b688 100644
--- a/vcl/source/app/svapp.cxx
+++ b/vcl/source/app/svapp.cxx
@@ -462,6 +462,7 @@ bool Application::Reschedule( bool i_bAllEvents )
 
 void Scheduler::ProcessEventsToIdle()
 {
+    SchedulerGuardReleaser aReleaser;
     int nSanity = 1;
     while( Application::Reschedule( true ) )
     {