vcl: fix hangs in SvpSalInstance

Since commit cb8bfa9a32799bcde4e960fa56e388d5f7b2a928 the main thread
will read from the timer pipe until it is empty. But evidently this
introduces the problem that the poll() in another thread will not
return, as the file descriptor will no longer be readable; see
https://paste.debian.net/1011306/ for a reproducer of that rather
under-documented poll behaviour.  So other threads can get stuck
forever in poll, and then the main thread can block in poll too with
no other thread to wake it up.  This is the problem that plagues
UITest_writerperfect_epubexport.

The timer pipe is difficult to fix, since the main thread can block on
either the poll or the subsequent AcquireYieldMutex().

So replace the timer pipe with a condition etc. that is mostly
copied from the OSX AquaSalInstance/SalYieldMutex implementation.

The main thread now does something different than the other threads,
and blocks on a condition_variable with a timeout, while other
threads still block on acquiring the mutex.

Non-main threads can poke the main thread to do a DoYield()
on their behalf, and then get the result back with a blocking
read from a pipe, all while holding the YieldMutex.  This
requires some fudging of the YieldMutex so that the main
thread can borrow the ownership temporarily.

Unfortunately SvpSalInstance, in addition to being directly
instantiable for --headless, has a whole bunch of subclasses:

* HeadlessSalInstance
* AndroidSalInstance
* IosSalInstance
* GtkInstance (in the gtk3 case)
* KDE5SalInstance

Of these GtkInstance overrides everything related to the
DoYield/SalYieldMutex implementation, but the others will be
affected by the change.

This commit will probably break IOS due to me not understanding the
point of the undocumented random #ifdef IOS in svpinst.cxx.

Change-Id: I1bbb143952dda89579e97ac32cd147e5b987573c
Reviewed-on: https://gerrit.libreoffice.org/50237
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Michael Stahl <mstahl@redhat.com>

3 files changed, 214 insertions, 72 deletions

diff --git a/vcl/headless/headlessinst.cxx b/vcl/headless/headlessinst.cxx
index a5f1c6ebd8ea..b49859c03c20 100644
--- a/vcl/headless/headlessinst.cxx
+++ b/vcl/headless/headlessinst.cxx
@@ -90,7 +90,7 @@ SalData::~SalData()
 // This is our main entry point:
 SalInstance *CreateSalInstance()
 {
-    HeadlessSalInstance* pInstance = new HeadlessSalInstance( new SalYieldMutex() );
+    HeadlessSalInstance* pInstance = new HeadlessSalInstance( new SvpSalYieldMutex() );
     new HeadlessSalData( pInstance );
     pInstance->AcquireYieldMutex();
     return pInstance;
diff --git a/vcl/headless/svpdata.cxx b/vcl/headless/svpdata.cxx
index 3bc2807c8cc5..ce72305f85d9 100644
--- a/vcl/headless/svpdata.cxx
+++ b/vcl/headless/svpdata.cxx
@@ -21,7 +21,7 @@ public:
 // plugin factory function
 SalInstance* svp_create_SalInstance()
 {
-    SvpSalInstance* pInstance = new SvpSalInstance( new SalYieldMutex() );
+    SvpSalInstance* pInstance = new SvpSalInstance( new SvpSalYieldMutex() );
     new SvpSalData( pInstance );
     return pInstance;
 }
diff --git a/vcl/headless/svpinst.cxx b/vcl/headless/svpinst.cxx
index 943b824ab412..6d867fcf28a1 100644
--- a/vcl/headless/svpinst.cxx
+++ b/vcl/headless/svpinst.cxx
@@ -62,15 +62,15 @@ static void atfork_child()
 
 #endif
 
-SvpSalInstance::SvpSalInstance( SalYieldMutex *pMutex ) :
-    SalGenericInstance( pMutex )
+SvpSalInstance::SvpSalInstance( SalYieldMutex *pMutex )
+    : SalGenericInstance( pMutex )
 {
     m_aTimeout.tv_sec       = 0;
     m_aTimeout.tv_usec      = 0;
     m_nTimeoutMS            = 0;
 
 #ifndef IOS
-    m_pTimeoutFDS[0] = m_pTimeoutFDS[1] = -1;
+    m_MainThread = osl::Thread::getCurrentIdentifier();
     CreateWakeupPipe(true);
 #endif
     if( s_pDefaultInstance == nullptr )
@@ -93,60 +93,56 @@ SvpSalInstance::~SvpSalInstance()
 
 void SvpSalInstance::CloseWakeupPipe(bool log)
 {
-    if (m_pTimeoutFDS[0] != -1)
+    SvpSalYieldMutex *const pMutex(dynamic_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()));
+    if (!pMutex)
+        return;
+    if (pMutex->m_FeedbackFDs[0] != -1)
     {
         if (log)
         {
-            SAL_INFO("vcl.headless", "CloseWakeupPipe: Closing inherited wakeup pipe: [" << m_pTimeoutFDS[0] << "," << m_pTimeoutFDS[1] << "]");
+            SAL_INFO("vcl.headless", "CloseWakeupPipe: Closing inherited feedback pipe: [" << pMutex->m_FeedbackFDs[0] << "," << pMutex->m_FeedbackFDs[1] << "]");
         }
-        close (m_pTimeoutFDS[0]);
-        close (m_pTimeoutFDS[1]);
-        m_pTimeoutFDS[0] = m_pTimeoutFDS[1] = -1;
+        close (pMutex->m_FeedbackFDs[0]);
+        close (pMutex->m_FeedbackFDs[1]);
+        pMutex->m_FeedbackFDs[0] = pMutex->m_FeedbackFDs[1] = -1;
     }
 }
 
 void SvpSalInstance::CreateWakeupPipe(bool log)
 {
-    if (pipe (m_pTimeoutFDS) == -1)
+    SvpSalYieldMutex *const pMutex(dynamic_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()));
+    if (!pMutex)
+        return;
+    if (pipe (pMutex->m_FeedbackFDs) == -1)
     {
         if (log)
         {
-            SAL_WARN("vcl.headless", "Could not create wakeup pipe: " << strerror(errno));
+            SAL_WARN("vcl.headless", "Could not create feedback pipe: " << strerror(errno));
+            std::abort();
         }
     }
     else
     {
         if (log)
         {
-            SAL_INFO("vcl.headless", "CreateWakeupPipe: Created wakeup pipe: [" << m_pTimeoutFDS[0] << "," << m_pTimeoutFDS[1] << "]");
+            SAL_INFO("vcl.headless", "CreateWakeupPipe: Created feedback pipe: [" << pMutex->m_FeedbackFDs[0] << "," << pMutex->m_FeedbackFDs[1] << "]");
         }
 
-        // initialize 'wakeup' pipe.
         int flags;
 
         // set close-on-exec descriptor flag.
-        if ((flags = fcntl (m_pTimeoutFDS[0], F_GETFD)) != -1)
+        if ((flags = fcntl (pMutex->m_FeedbackFDs[0], F_GETFD)) != -1)
         {
             flags |= FD_CLOEXEC;
-            (void)fcntl(m_pTimeoutFDS[0], F_SETFD, flags);
+            (void) fcntl(pMutex->m_FeedbackFDs[0], F_SETFD, flags);
         }
-        if ((flags = fcntl (m_pTimeoutFDS[1], F_GETFD)) != -1)
+        if ((flags = fcntl (pMutex->m_FeedbackFDs[1], F_GETFD)) != -1)
         {
             flags |= FD_CLOEXEC;
-            (void)fcntl(m_pTimeoutFDS[1], F_SETFD, flags);
+            (void) fcntl(pMutex->m_FeedbackFDs[1], F_SETFD, flags);
         }
 
-        // set non-blocking I/O flag.
-        if ((flags = fcntl(m_pTimeoutFDS[0], F_GETFL)) != -1)
-        {
-            flags |= O_NONBLOCK;
-            (void)fcntl(m_pTimeoutFDS[0], F_SETFL, flags);
-        }
-        if ((flags = fcntl(m_pTimeoutFDS[1], F_GETFL)) != -1)
-        {
-            flags |= O_NONBLOCK;
-            (void)fcntl(m_pTimeoutFDS[1], F_SETFL, flags);
-        }
+        // retain the default blocking I/O for feedback pipe
     }
 }
 
@@ -157,10 +153,29 @@ void SvpSalInstance::TriggerUserEventProcessing()
     Wakeup();
 }
 
-void SvpSalInstance::Wakeup()
+#ifndef NDEBUG
+static bool g_CheckedMutex = false;
+#endif
+
+void SvpSalInstance::Wakeup(SvpRequest const request)
 {
+#ifndef NDEBUG
+    if (!g_CheckedMutex)
+    {
+        assert(dynamic_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()) != nullptr
+            && "This SvpSalInstance function requires use of SvpSalYieldMutex");
+        g_CheckedMutex = true;
+    }
+#endif
 #ifndef IOS
-    OSL_VERIFY(write (m_pTimeoutFDS[1], "", 1) == 1);
+    SvpSalYieldMutex *const pMutex(static_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()));
+    std::unique_lock<std::mutex> g(pMutex->m_WakeUpMainMutex);
+    if (request != SvpRequest::NONE)
+    {
+        pMutex->m_Request = request;
+    }
+    pMutex->m_wakeUpMain = true;
+    pMutex->m_WakeUpMainCond.notify_one();
 #endif
 }
 
@@ -259,72 +274,199 @@ void SvpSalInstance::ProcessEvent( SalUserEvent aEvent )
         const SvpSalFrame* pSvpFrame = static_cast<const SvpSalFrame*>( aEvent.m_pFrame);
         pSvpFrame->PostPaint();
     }
+#ifndef NDEBUG
+    if (!g_CheckedMutex)
+    {
+        assert(dynamic_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()) != nullptr
+            && "This SvpSalInstance function requires use of SvpSalYieldMutex");
+        g_CheckedMutex = true;
+    }
+#endif
+    SvpSalYieldMutex *const pMutex(static_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()));
+    pMutex->m_NonMainWaitingYieldCond.set();
 }
 
-
-bool SvpSalInstance::DoYield(bool bWait, bool bHandleAllCurrentEvents)
+SvpSalYieldMutex::SvpSalYieldMutex()
 {
-    // first, process current user events
-    bool bEvent = DispatchUserEvents( bHandleAllCurrentEvents );
-    if ( !bHandleAllCurrentEvents && bEvent )
-        return true;
+    m_FeedbackFDs[0] = m_FeedbackFDs[1] = -1;
+}
 
-    bEvent = CheckTimeout() || bEvent;
+SvpSalYieldMutex::~SvpSalYieldMutex()
+{
+}
 
-    if (bWait && ! bEvent )
+void SvpSalYieldMutex::doAcquire(sal_uInt32 const nLockCount)
+{
+    SvpSalInstance *const pInst = static_cast<SvpSalInstance *>(GetSalData()->m_pInstance);
+    if (pInst && pInst->IsMainThread())
     {
-        int nTimeoutMS = 0;
-        if (m_aTimeout.tv_sec) // Timer is started.
+        if (m_bNoYieldLock)
+            return;
+
+        do
         {
-            timeval Timeout;
-            // determine remaining timeout.
-            gettimeofday (&Timeout, nullptr);
-            if ( m_aTimeout > Timeout )
+            SvpRequest request = SvpRequest::NONE;
+            {
+                std::unique_lock<std::mutex> g(m_WakeUpMainMutex);
+                if (m_aMutex.tryToAcquire()) {
+                    // if there's a request, the other thread holds m_aMutex
+                    assert(m_Request == SvpRequest::NONE);
+                    m_wakeUpMain = false;
+                    break;
+                }
+                m_WakeUpMainCond.wait(g, [this]() { return m_wakeUpMain; });
+                m_wakeUpMain = false;
+                std::swap(m_Request, request);
+            }
+            if (request != SvpRequest::NONE)
             {
-                int nTimeoutMicroS = m_aTimeout.tv_usec - Timeout.tv_usec;
-                nTimeoutMS = (m_aTimeout.tv_sec - Timeout.tv_sec) * 1000
-                           + nTimeoutMicroS / 1000;
-                if ( nTimeoutMicroS % 1000 )
-                    nTimeoutMS += 1;
+                // nested Yield on behalf of another thread
+                assert(!m_bNoYieldLock);
+                m_bNoYieldLock = true;
+                bool const bEvents = pInst->DoYield(false, request == SvpRequest::MainThreadDispatchAllEvents);
+                m_bNoYieldLock = false;
+                write(m_FeedbackFDs[1], &bEvents, sizeof(bool));
             }
         }
+        while (true);
+    }
+    else
+    {
+        m_aMutex.acquire();
+    }
+    ++m_nCount;
+    SalYieldMutex::doAcquire(nLockCount - 1);
+}
+
+sal_uInt32 SvpSalYieldMutex::doRelease(bool const bUnlockAll)
+{
+    SvpSalInstance *const pInst = static_cast<SvpSalInstance *>(GetSalData()->m_pInstance);
+    if (pInst && pInst->IsMainThread())
+    {
+        if (m_bNoYieldLock)
+            return 1;
         else
-            nTimeoutMS = -1; // wait until something happens
+            return SalYieldMutex::doRelease(bUnlockAll);
+    }
+    sal_uInt32 nCount;
+    {
+        // read m_nCount before doRelease
+        bool const isReleased(bUnlockAll || m_nCount == 1);
+        nCount = comphelper::GenericSolarMutex::doRelease( bUnlockAll );
+        if (isReleased) {
+            std::unique_lock<std::mutex> g(m_WakeUpMainMutex);
+            m_wakeUpMain = true;
+            m_WakeUpMainCond.notify_one();
+        }
+    }
+    return nCount;
+}
 
-        DoReleaseYield(nTimeoutMS);
+bool SvpSalYieldMutex::IsCurrentThread() const
+{
+    if (GetSalData()->m_pInstance->IsMainThread() && m_bNoYieldLock)
+    {
+        return true;
     }
-    else if ( bEvent )
+    else
     {
-        // Drain the wakeup pipe
-        int buffer;
-        while (read (m_pTimeoutFDS[0], &buffer, sizeof(buffer)) > 0);
+        return SalYieldMutex::IsCurrentThread();
     }
+}
 
-    return bEvent;
+bool SvpSalInstance::IsMainThread() const
+{
+    return osl::Thread::getCurrentIdentifier() == m_MainThread;
 }
 
-void SvpSalInstance::DoReleaseYield( int nTimeoutMS )
+bool SvpSalInstance::DoYield(bool bWait, bool bHandleAllCurrentEvents)
 {
-    // poll
-    struct pollfd aPoll;
-    aPoll.fd = m_pTimeoutFDS[0];
-    aPoll.events = POLLIN;
-    aPoll.revents = 0;
+#ifndef NDEBUG
+    if (!g_CheckedMutex)
+    {
+        assert(dynamic_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()) != nullptr
+            && "This SvpSalInstance function requires use of SvpSalYieldMutex");
+        g_CheckedMutex = true;
+    }
+#endif
 
-    // release yield mutex
-    sal_uInt32 nAcquireCount = ReleaseYieldMutexAll();
+    // first, process current user events
+    bool bEvent = DispatchUserEvents( bHandleAllCurrentEvents );
+    if ( !bHandleAllCurrentEvents && bEvent )
+        return true;
 
-    (void)poll( &aPoll, 1, nTimeoutMS );
+    bEvent = CheckTimeout() || bEvent;
 
-    // acquire yield mutex again
-    AcquireYieldMutex( nAcquireCount );
+    SvpSalYieldMutex *const pMutex(static_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()));
 
-    // clean up pipe
-    if( (aPoll.revents & POLLIN) != 0 )
+    if (IsMainThread())
+    {
+        if (bWait && ! bEvent)
+        {
+            int nTimeoutMS = 0;
+            if (m_aTimeout.tv_sec) // Timer is started.
+            {
+                timeval Timeout;
+                // determine remaining timeout.
+                gettimeofday (&Timeout, nullptr);
+                if (m_aTimeout > Timeout)
+                {
+                    int nTimeoutMicroS = m_aTimeout.tv_usec - Timeout.tv_usec;
+                    nTimeoutMS = (m_aTimeout.tv_sec - Timeout.tv_sec) * 1000
+                               + nTimeoutMicroS / 1000;
+                    if ( nTimeoutMicroS % 1000 )
+                        nTimeoutMS += 1;
+                }
+            }
+            else
+                nTimeoutMS = -1; // wait until something happens
+
+            sal_uInt32 nAcquireCount = ReleaseYieldMutexAll();
+            {
+                std::unique_lock<std::mutex> g(pMutex->m_WakeUpMainMutex);
+                // wait for doRelease() or Wakeup() to set the condition
+                if (nTimeoutMS == -1)
+                {
+                    pMutex->m_WakeUpMainCond.wait(g,
+                            [pMutex]() { return pMutex->m_wakeUpMain; });
+                }
+                else
+                {
+                    pMutex->m_WakeUpMainCond.wait_for(g,
+                            std::chrono::milliseconds(nTimeoutMS),
+                            [pMutex]() { return pMutex->m_wakeUpMain; });
+                }
+                // here no need to check m_Request because Acquire will do it
+            }
+            AcquireYieldMutex( nAcquireCount );
+        }
+        else if (bEvent)
+        {
+            pMutex->m_NonMainWaitingYieldCond.set(); // wake up other threads
+        }
+    }
+    else // !IsMainThread()
     {
-        int buffer;
-        while (read (m_pTimeoutFDS[0], &buffer, sizeof(buffer)) > 0);
+        Wakeup(bHandleAllCurrentEvents
+                ? SvpRequest::MainThreadDispatchAllEvents
+                : SvpRequest::MainThreadDispatchOneEvent);
+
+        bool bDidWork(false);
+        // blocking read (for synchronisation)
+        auto const nRet = read(pMutex->m_FeedbackFDs[0], &bDidWork, sizeof(bool));
+        assert(nRet == 1); (void) nRet;
+
+        if (!bDidWork && bWait)
+        {
+            // block & release YieldMutex until the main thread does something
+            pMutex->m_NonMainWaitingYieldCond.reset();
+            sal_uInt32 nAcquireCount = ReleaseYieldMutexAll();
+            pMutex->m_NonMainWaitingYieldCond.wait();
+            AcquireYieldMutex( nAcquireCount );
+        }
     }
+
+    return bEvent;
 }
 
 bool SvpSalInstance::AnyInput( VclInputFlags nType )