Unlock the mutex to avoid deadlock

The request thread was calling OleComponent::CloseObject, which executed
on main thread; OleEmbeddedObject::m_aMutex as locked by request thread:

emboleobj.dll!OleComponent::CloseObject() Line 1004
emboleobj.dll!OleComponent::Dispose() Line 451
emboleobj.dll!OleComponent::close(unsigned char bDeliverOwnership) Line 1513
emboleobj.dll!OleEmbeddedObject::GetRidOfComponent() Line 235
emboleobj.dll!OleEmbeddedObject::Dispose() Line 269
emboleobj.dll!OleEmbeddedObject::close(unsigned char bDeliverOwnership) Line 498
comphelper.dll!comphelper::EmbeddedObjectContainer::CloseEmbeddedObjects() Line 208
sfxlo.dll!SfxObjectShell::~SfxObjectShell() Line 322
swlo.dll!SwDocShell::~SwDocShell() Line 380
swlo.dll!SwDocShell::`vector deleting destructor'(unsigned int)
cppuhelper3MSC.dll!cppu::OWeakObject::release() Line 229
sfxlo.dll!rtl::Reference<SfxObjectShell>::~Reference<SfxObjectShell>() Line 126
sfxlo.dll!IMPL_SfxBaseModel_DataContainer::~IMPL_SfxBaseModel_DataContainer() Line 265
sfxlo.dll!IMPL_SfxBaseModel_DataContainer::`scalar deleting destructor'(unsigned int)
sfxlo.dll!std::_Destroy_in_place<IMPL_SfxBaseModel_DataContainer>(IMPL_SfxBaseModel_DataContainer & _Obj) Line 293
sfxlo.dll!std::_Ref_count_obj2<IMPL_SfxBaseModel_DataContainer>::_Destroy() Line 2113
sfxlo.dll!std::_Ref_count_base::_Decref() Line 1164
sfxlo.dll!std::_Ptr_base<IMPL_SfxBaseModel_DataContainer>::_Decref() Line 1380
sfxlo.dll!std::shared_ptr<IMPL_SfxBaseModel_DataContainer>::~shared_ptr<IMPL_SfxBaseModel_DataContainer>() Line 1685
sfxlo.dll!std::shared_ptr<IMPL_SfxBaseModel_DataContainer>::reset() Line 1732
sfxlo.dll!SfxBaseModel::dispose() Line 794
swlo.dll!SwXTextDocument::dispose() Line 561
sfxlo.dll!SfxBaseModel::close(unsigned char bDeliverOwnership) Line 1523
swlo.dll!SwXTextDocument::close(unsigned char bDeliverOwnership) Line 574
sfxlo.dll!SfxBaseModel::dispose() Line 745
swlo.dll!SwXTextDocument::dispose() Line 561
mscx_uno.dll!`anonymous namespace'::cpp_call(bridges::cpp_uno::shared::UnoInterfaceProxy * pThis, bridges::cpp_uno::shared::VtableSlot aVtableSlot, _typelib_TypeDescriptionReference * pReturnTypeRef, long nParams, _typelib_MethodParameter * pParams, void * pUnoReturn, void * * pUnoArgs, _uno_Any * * ppUnoExc) Line 214
mscx_uno.dll!unoInterfaceProxyDispatch(_uno_Interface * pUnoI, const _typelib_TypeDescription * pMemberTD, void * pReturn, void * * pArgs, _uno_Any * * ppException) Line 430
binaryurplo.dll!binaryurp::IncomingRequest::execute_throw(binaryurp::BinaryAny * returnValue, std::vector<binaryurp::BinaryAny,std::allocator<binaryurp::BinaryAny>> * outArguments) Line 239
binaryurplo.dll!binaryurp::IncomingRequest::execute() Line 79
binaryurplo.dll!request(void * pThreadSpecificData) Line 84
cppu3.dll!cppu_threadpool::JobQueue::enter(const void * nDisposeId, bool bReturnWhenNoJob) Line 101
cppu3.dll!cppu_threadpool::ORequestThread::run() Line 165
cppu3.dll!threadFunc(void * param) Line 190
sal3.dll!oslWorkerWrapperFunction(void * pData) Line 67

Main thread was acquiring OleEmbeddedObject::m_aMutex, which deadlocked:

sal3.dll!osl_acquireMutex(_oslMutexImpl * Mutex) Line 65
emboleobj.dll!osl::Mutex::acquire() Line 63
emboleobj.dll!osl::ResettableGuard<osl::Mutex>::reset() Line 250
emboleobj.dll!OleEmbeddedObject::setVisualAreaSize(__int64 nAspect, const com::sun::star::awt::Size & aSize) Line 148
swlo.dll!SwWrtShell::CalcAndSetScale(svt::EmbeddedObjectRef & xObj, const SwRect * pFlyPrtRect, const SwRect * pFlyFrameRect, const bool bNoTextFramePrtAreaChanged) Line 777
swlo.dll!SwContentNotify::ImplDestroy() Line 926
swlo.dll!SwContentNotify::~SwContentNotify() Line 1037
swlo.dll!SwNoTextFrame::MakeAll(OutputDevice * pRenderContext) Line 584
swlo.dll!SwFrame::OptPrepareMake() Line 412
swlo.dll!SwFrame::OptCalc() Line 1110
swlo.dll!SwLayAction::FormatContent_(const SwContentFrame * pContent, const SwPageFrame * pPage) Line 1960
swlo.dll!SwLayAction::FormatFlyContent(const SwFlyFrame * pFly) Line 1985
swlo.dll!SwObjectFormatter::FormatObj_(SwAnchoredObject & _rAnchoredObj) Line 312
swlo.dll!SwObjectFormatterTextFrame::DoFormatObj(SwAnchoredObject & _rAnchoredObj, const bool _bCheckForMovedFwd) Line 133
swlo.dll!SwObjectFormatter::FormatObjsAtFrame_(SwTextFrame * _pMasterTextFrame) Line 414
swlo.dll!SwObjectFormatterTextFrame::DoFormatObjs() Line 348
swlo.dll!SwObjectFormatter::FormatObjsAtFrame(SwFrame & _rAnchorFrame, const SwPageFrame & _rPageFrame, SwLayAction * _pLayAction) Line 160
swlo.dll!SwLayAction::FormatContent(SwPageFrame * pPage) Line 1793
swlo.dll!SwLayAction::InternalAction(OutputDevice * pRenderContext) Line 605
swlo.dll!SwLayAction::Action(OutputDevice * pRenderContext) Line 389
swlo.dll!SwLayIdle::SwLayIdle(SwRootFrame * pRt, SwViewShellImp * pI) Line 2363
swlo.dll!SwViewShell::LayoutIdle() Line 827
swlo.dll!sw::DocumentTimerManager::DoIdleJobs(Timer * __formal) Line 176
swlo.dll!sw::DocumentTimerManager::LinkStubDoIdleJobs(void * instance, Timer * data) Line 156
vcllo.dll!Link<Timer *,void>::Call(Timer * data) Line 111
vcllo.dll!Timer::Invoke() Line 75
vcllo.dll!Scheduler::CallbackTaskScheduling() Line 509
vcllo.dll!SalTimer::CallCallback() Line 53
vclplug_winlo.dll!WinSalTimer::ImplHandleElapsedTimer() Line 169
vclplug_winlo.dll!ImplSalYield(bool bWait, bool bHandleAllCurrentEvents) Line 525
vclplug_winlo.dll!WinSalInstance::DoYield(bool bWait, bool bHandleAllCurrentEvents) Line 581
vcllo.dll!ImplYield(bool i_bWait, bool i_bAllEvents) Line 385
vcllo.dll!Application::Yield() Line 473
vcllo.dll!Application::Execute() Line 361
sofficeapp.dll!desktop::Desktop::Main() Line 1652
vcllo.dll!ImplSVMain() Line 229
vcllo.dll!SVMain() Line 262
sofficeapp.dll!soffice_main() Line 110
soffice.bin!sal_main() Line 51
soffice.bin!main(int argc, char * * argv) Line 49
soffice.bin!invoke_main() Line 79
soffice.bin!__scrt_common_main_seh() Line 288
soffice.bin!__scrt_common_main() Line 331
soffice.bin!mainCRTStartup(void * __formal) Line 17

Change-Id: Iaf5133c488d4f26f43530ea19584e99cce12d81e
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/171855
Tested-by: Jenkins
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>
(cherry picked from commit 29f59dad0e23385a23143008509c15ebae4c2b2d)
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/171996
Reviewed-by: Michael Stahl <michael.stahl@allotropia.de>

5 files changed, 33 insertions, 26 deletions

diff --git a/embeddedobj/source/inc/oleembobj.hxx b/embeddedobj/source/inc/oleembobj.hxx
index 087a3c829527..62fea5d35702 100644
--- a/embeddedobj/source/inc/oleembobj.hxx
+++ b/embeddedobj/source/inc/oleembobj.hxx
@@ -217,7 +217,7 @@ protected:
     css::uno::Reference< css::io::XStream > GetNewFilledTempStream_Impl(
                                     const css::uno::Reference< css::io::XInputStream >& xInStream );
 #ifdef _WIN32
-    void SwitchComponentToRunningState_Impl();
+    void SwitchComponentToRunningState_Impl(osl::ResettableMutexGuard& guard);
 #endif
     void MakeEventListenerNotification_Impl( const OUString& aEventName );
 #ifdef _WIN32
@@ -231,7 +231,7 @@ protected:
                         const css::uno::Sequence< css::embed::VerbDescriptor >& aVerbList );
 #endif
 
-    void Dispose();
+    void Dispose(osl::ResettableMutexGuard* guard = nullptr);
 
     void SwitchOwnPersistence(
                 const css::uno::Reference< css::embed::XStorage >& xNewParentStorage,
@@ -242,7 +242,7 @@ protected:
                 const css::uno::Reference< css::embed::XStorage >& xNewParentStorage,
                 const OUString& aNewName );
 
-    void GetRidOfComponent();
+    void GetRidOfComponent(osl::ResettableMutexGuard* guard);
 
     /// @throws css::uno::Exception
     void StoreToLocation_Impl(
@@ -272,6 +272,7 @@ protected:
 
     css::uno::Reference< css::io::XStream > TryToRetrieveCachedVisualRepresentation_Impl(
                     const css::uno::Reference< css::io::XStream >& xStream,
+                    osl::ResettableMutexGuard& rGuard,
                     bool bAllowRepair50 = false )
         noexcept;
 #ifdef _WIN32
diff --git a/embeddedobj/source/msole/oleembed.cxx b/embeddedobj/source/msole/oleembed.cxx
index 835ef9cd4885..f61d8056fdeb 100644
--- a/embeddedobj/source/msole/oleembed.cxx
+++ b/embeddedobj/source/msole/oleembed.cxx
@@ -66,7 +66,7 @@ using namespace ::com::sun::star;
 
 #ifdef _WIN32
 
-void OleEmbeddedObject::SwitchComponentToRunningState_Impl()
+void OleEmbeddedObject::SwitchComponentToRunningState_Impl(osl::ResettableMutexGuard& guard)
 {
     if ( !m_pOleComponent )
     {
@@ -78,12 +78,12 @@ void OleEmbeddedObject::SwitchComponentToRunningState_Impl()
     }
     catch( const embed::UnreachableStateException& )
     {
-        GetRidOfComponent();
+        GetRidOfComponent(&guard);
         throw;
     }
     catch( const embed::WrongStateException& )
     {
-        GetRidOfComponent();
+        GetRidOfComponent(&guard);
         throw;
     }
 }
@@ -518,7 +518,7 @@ void SAL_CALL OleEmbeddedObject::changeState( sal_Int32 nNewState )
                     // it can be created during loading to detect type of object
                     CreateOleComponentAndLoad_Impl( m_pOleComponent );
 
-                    SwitchComponentToRunningState_Impl();
+                    SwitchComponentToRunningState_Impl(aGuard);
                     m_nObjectState = embed::EmbedStates::RUNNING;
                     aGuard.clear();
                     StateChangeNotification_Impl( false, nOldState, m_nObjectState );
diff --git a/embeddedobj/source/msole/olemisc.cxx b/embeddedobj/source/msole/olemisc.cxx
index efed07fa187d..13fc558a529e 100644
--- a/embeddedobj/source/msole/olemisc.cxx
+++ b/embeddedobj/source/msole/olemisc.cxx
@@ -221,7 +221,7 @@ void OleEmbeddedObject::StateChangeNotification_Impl( bool bBeforeChange, sal_In
 }
 #endif
 
-void OleEmbeddedObject::GetRidOfComponent()
+void OleEmbeddedObject::GetRidOfComponent(osl::ResettableMutexGuard* guard)
 {
 #ifdef _WIN32
     if ( m_pOleComponent )
@@ -232,6 +232,9 @@ void OleEmbeddedObject::GetRidOfComponent()
         m_pOleComponent->removeCloseListener( m_xClosePreventer );
         try
         {
+            std::optional<osl::ResettableMutexGuardScopedReleaser> oReleaser;
+            if (guard)
+                oReleaser.emplace(*guard);
             m_pOleComponent->close( false );
         }
         catch( const uno::Exception& )
@@ -245,11 +248,13 @@ void OleEmbeddedObject::GetRidOfComponent()
         m_pOleComponent->disconnectEmbeddedObject();
         m_pOleComponent.clear();
     }
+#else
+    (void)guard;
 #endif
 }
 
 
-void OleEmbeddedObject::Dispose()
+void OleEmbeddedObject::Dispose(osl::ResettableMutexGuard* guard)
 {
     if ( m_pInterfaceContainer )
     {
@@ -266,7 +271,7 @@ void OleEmbeddedObject::Dispose()
 
     if ( m_pOleComponent )
         try {
-            GetRidOfComponent();
+            GetRidOfComponent(guard);
         } catch( const uno::Exception& )
         {
             m_bDisposed = true;
@@ -449,7 +454,7 @@ void SAL_CALL OleEmbeddedObject::close( sal_Bool bDeliverOwnership )
     }
     // end wrapping related part ====================
 
-    ::osl::MutexGuard aGuard( m_aMutex );
+    osl::ResettableMutexGuard aGuard(m_aMutex);
     if ( m_bDisposed )
         throw lang::DisposedException(); // TODO
 
@@ -495,7 +500,7 @@ void SAL_CALL OleEmbeddedObject::close( sal_Bool bDeliverOwnership )
         }
     }
 
-    Dispose();
+    Dispose(&aGuard);
 }
 
 
diff --git a/embeddedobj/source/msole/olepersist.cxx b/embeddedobj/source/msole/olepersist.cxx
index 486c636aee95..65811c31a331 100644
--- a/embeddedobj/source/msole/olepersist.cxx
+++ b/embeddedobj/source/msole/olepersist.cxx
@@ -594,6 +594,7 @@ bool OleEmbeddedObject::HasVisReplInStream()
 
 uno::Reference< io::XStream > OleEmbeddedObject::TryToRetrieveCachedVisualRepresentation_Impl(
         const uno::Reference< io::XStream >& xStream,
+        osl::ResettableMutexGuard& rGuard,
         bool bAllowToRepair50 )
     noexcept
 {
@@ -697,7 +698,7 @@ uno::Reference< io::XStream > OleEmbeddedObject::TryToRetrieveCachedVisualRepres
 
 #ifdef _WIN32
                                     // retry to create the component after recovering
-                                    GetRidOfComponent();
+                                    GetRidOfComponent(&rGuard);
 
                                     try
                                     {
@@ -706,12 +707,12 @@ uno::Reference< io::XStream > OleEmbeddedObject::TryToRetrieveCachedVisualRepres
                                     }
                                     catch( const uno::Exception& )
                                     {
-                                        GetRidOfComponent();
+                                        GetRidOfComponent(&rGuard);
                                     }
 #endif
                                 }
 
-                                xResult = TryToRetrieveCachedVisualRepresentation_Impl( xStream );
+                                xResult = TryToRetrieveCachedVisualRepresentation_Impl( xStream, rGuard );
                             }
                         }
                     }
@@ -1173,7 +1174,7 @@ void OleEmbeddedObject::StoreToLocation_Impl(
             // is not changed by StoreTo action
 
             uno::Reference< io::XStream > xTmpCVRepresentation =
-                        TryToRetrieveCachedVisualRepresentation_Impl( xTargetStream );
+                        TryToRetrieveCachedVisualRepresentation_Impl( xTargetStream, rGuard );
 
             // the locally retrieved representation is always preferable
             if ( xTmpCVRepresentation.is() )
@@ -1204,7 +1205,7 @@ void OleEmbeddedObject::StoreToLocation_Impl(
         if ( bStoreVis )
         {
             if ( !xCachedVisualRepresentation.is() )
-                xCachedVisualRepresentation = TryToRetrieveCachedVisualRepresentation_Impl( xTargetStream );
+                xCachedVisualRepresentation = TryToRetrieveCachedVisualRepresentation_Impl( xTargetStream, rGuard );
 
             SAL_WARN_IF( !xCachedVisualRepresentation.is(), "embeddedobj.ole", "No representation is available!" );
 
@@ -1228,7 +1229,7 @@ void OleEmbeddedObject::StoreToLocation_Impl(
         {
             // the removed representation could be cached by this method
             if ( !xCachedVisualRepresentation.is() )
-                xCachedVisualRepresentation = TryToRetrieveCachedVisualRepresentation_Impl( xTargetStream );
+                xCachedVisualRepresentation = TryToRetrieveCachedVisualRepresentation_Impl( xTargetStream, rGuard );
 
             if (!m_bStreamReadOnly)
                 RemoveVisualCache_Impl(xTargetStream);
@@ -1294,7 +1295,7 @@ void SAL_CALL OleEmbeddedObject::setPersistentEntry(
     // the only exception is object initialized from a stream,
     // the class ID will be detected from the stream
 
-    ::osl::MutexGuard aGuard( m_aMutex );
+    osl::ResettableMutexGuard aGuard( m_aMutex );
     if ( m_bDisposed )
         throw lang::DisposedException(); // TODO
 
@@ -1378,7 +1379,7 @@ void SAL_CALL OleEmbeddedObject::setPersistentEntry(
             {
                 // TODO/LATER: detect classID of the object if possible
                 // means that the object inprocess server could not be successfully instantiated
-                GetRidOfComponent();
+                GetRidOfComponent(&aGuard);
             }
 
             m_nObjectState = embed::EmbedStates::LOADED;
@@ -1753,14 +1754,14 @@ void SAL_CALL OleEmbeddedObject::storeOwn()
                 InsertVisualCache_Impl( m_xObjectStream, m_xCachedVisualRepresentation );
             else
             {
-                m_xCachedVisualRepresentation = TryToRetrieveCachedVisualRepresentation_Impl( m_xObjectStream );
+                m_xCachedVisualRepresentation = TryToRetrieveCachedVisualRepresentation_Impl( m_xObjectStream, aGuard );
                 SAL_WARN_IF( !m_xCachedVisualRepresentation.is(), "embeddedobj.ole", "No representation is available!" );
             }
         }
         else
         {
             if ( !m_xCachedVisualRepresentation.is() )
-                m_xCachedVisualRepresentation = TryToRetrieveCachedVisualRepresentation_Impl( m_xObjectStream );
+                m_xCachedVisualRepresentation = TryToRetrieveCachedVisualRepresentation_Impl( m_xObjectStream, aGuard );
             RemoveVisualCache_Impl( m_xObjectStream );
         }
 
@@ -1875,7 +1876,7 @@ void SAL_CALL OleEmbeddedObject::breakLink( const uno::Reference< embed::XStorag
     }
     // end wrapping related part ====================
 
-    ::osl::MutexGuard aGuard( m_aMutex );
+    osl::ResettableMutexGuard aGuard( m_aMutex );
     if ( m_bDisposed )
         throw lang::DisposedException(); // TODO
 
@@ -1927,7 +1928,7 @@ void SAL_CALL OleEmbeddedObject::breakLink( const uno::Reference< embed::XStorag
     }
 
     try {
-        GetRidOfComponent();
+        GetRidOfComponent(&aGuard);
     }
     catch (const uno::Exception&)
     {
diff --git a/embeddedobj/source/msole/olevisual.cxx b/embeddedobj/source/msole/olevisual.cxx
index 83dc0903e0bb..539933278e45 100644
--- a/embeddedobj/source/msole/olevisual.cxx
+++ b/embeddedobj/source/msole/olevisual.cxx
@@ -346,7 +346,7 @@ embed::VisualRepresentation SAL_CALL OleEmbeddedObject::getPreferredVisualRepres
     if ( !m_xCachedVisualRepresentation.is() && ( !m_bVisReplInitialized || m_bVisReplInStream )
       && m_nObjectState == embed::EmbedStates::LOADED )
     {
-        m_xCachedVisualRepresentation = TryToRetrieveCachedVisualRepresentation_Impl( m_xObjectStream, true );
+        m_xCachedVisualRepresentation = TryToRetrieveCachedVisualRepresentation_Impl( m_xObjectStream, aGuard, true );
         SetVisReplInStream( m_xCachedVisualRepresentation.is() );
     }
 
@@ -388,7 +388,7 @@ embed::VisualRepresentation SAL_CALL OleEmbeddedObject::getPreferredVisualRepres
     // the cache is used only as a fallback if object is not in loaded state
     if ( !m_xCachedVisualRepresentation.is() && ( !m_bVisReplInitialized || m_bVisReplInStream ) )
     {
-        m_xCachedVisualRepresentation = TryToRetrieveCachedVisualRepresentation_Impl( m_xObjectStream );
+        m_xCachedVisualRepresentation = TryToRetrieveCachedVisualRepresentation_Impl( m_xObjectStream, aGuard );
         SetVisReplInStream( m_xCachedVisualRepresentation.is() );
     }