Improve COM library management in fpicker

Inspired from:
author	Julien Nabet <serval2412@yahoo.fr>	2020-05-29 23:58:59 +0200
committer	Mike Kaganski <mike.kaganski@collabora.com>	2020-06-03 23:04:27 +0200
commit	95e5d37b6e62eb39f2d5337e124e86b3d0c3f399 (patch)
tree	5488826b390c04a2b32d2f9aa43f49a64f52db3d
parent	3d3cb4328ece843b3e31b8411f9d16bbedb57a7b (diff)
Improve COM library management in ADO

Change-Id: If0b136cdcc89baa6bc90912d42b3ba07fa6c0efb
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/95498
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>
Tested-by: Jenkins

3 files changed, 65 insertions, 10 deletions

diff --git a/fpicker/source/win32/VistaFilePickerImpl.cxx b/fpicker/source/win32/VistaFilePickerImpl.cxx
index c84fe5f3a72d..3b8443a40c65 100644
--- a/fpicker/source/win32/VistaFilePickerImpl.cxx
+++ b/fpicker/source/win32/VistaFilePickerImpl.cxx
@@ -190,6 +190,7 @@ VistaFilePickerImpl::VistaFilePickerImpl()
     , m_hParentWindow(choose_parent_window())
     , m_sDirectory   ()
     , m_sFilename    ()
+    , mnNbCallCoInitializeExForReinit(0)
 {
 }
 
@@ -208,15 +209,7 @@ void VistaFilePickerImpl::before()
     // osl::Thread class initializes COm already in MTA mode because it's needed
     // by VCL and UNO so. There is no way to change that from outside...
     // but we need a STA environment...
-    // So we make it by try-and-error...
-    // If first CoInitializeEx will fail... we uninitialize COM initialize it new .-)
-
-    m_hLastResult = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED);
-    if ( FAILED(m_hLastResult) )
-    {
-        CoUninitialize();
-        m_hLastResult = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED);
-    }
+    m_hLastResult = o3tl::safeCoInitializeEx(COINIT_APARTMENTTHREADED, mnNbCallCoInitializeExForReinit);
 }
 
 
@@ -324,7 +317,7 @@ void VistaFilePickerImpl::doRequest(const RequestRef& rRequest)
 
 void VistaFilePickerImpl::after()
 {
-    CoUninitialize();
+    o3tl::safeCoUninitializeReinit(COINIT_MULTITHREADED, mnNbCallCoInitializeExForReinit);
 }
 
 
diff --git a/fpicker/source/win32/VistaFilePickerImpl.hxx b/fpicker/source/win32/VistaFilePickerImpl.hxx
index e5d2d88e4391..0fee8587c89e 100644
--- a/fpicker/source/win32/VistaFilePickerImpl.hxx
+++ b/fpicker/source/win32/VistaFilePickerImpl.hxx
@@ -36,6 +36,7 @@
 #include <cppuhelper/basemutex.hxx>
 #include <osl/thread.hxx>
 #include <osl/conditn.hxx>
+#include <o3tl/safeCoInitUninit.hxx>
 #include <rtl/ustring.hxx>
 
 namespace fpicker{
@@ -326,6 +327,9 @@ class VistaFilePickerImpl : private ::cppu::BaseMutex
 
 
         OUString m_sFilename;
+
+        // to put back all the inits with COINIT_MULTITHREADED if needed
+        int mnNbCallCoInitializeExForReinit;
 };
 
 } // namespace vista
diff --git a/include/o3tl/safeCoInitUninit.hxx b/include/o3tl/safeCoInitUninit.hxx
new file mode 100644
index 000000000000..0ceb4a9746a2
--- /dev/null
+++ b/include/o3tl/safeCoInitUninit.hxx
@@ -0,0 +1,58 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#pragma once
+
+#if defined _WIN32
+#include <prewin.h>
+
+// for CoInitializeEx / CoUninitialize
+#include <combaseapi.h>
+
+#include <postwin.h>
+
+// for std::abort
+#include <cstdlib>
+
+namespace o3tl
+{
+// Helpers for safe calls to CoInitializeEx and CoUninitialize in MSVC
+// Indeed if a thread has been already initialized with a concurrency model
+// (in LO case COINIT_APARTMENTTHREADED or COINIT_MULTITHREADED)
+// CoInitializeEx can't succeed without calling first CoUninitialize
+// also, CoUninitialize must be called the number of times CoInitializeEx has been called
+inline HRESULT safeCoInitializeEx(DWORD dwCoInit, int& nbReinit)
+{
+    HRESULT hr;
+    while ((hr = CoInitializeEx(nullptr, dwCoInit)) == RPC_E_CHANGED_MODE)
+    {
+        // so we're in RPC_E_CHANGED_MODE case
+        // the pb was it was already initialized with a different concurrency model
+        // close this init
+        CoUninitialize();
+        // and increment counter for dtr part
+        ++nbReinit;
+
+        // and keep on the loop if there were multi initializations
+    }
+    if (FAILED(hr))
+        std::abort();
+    return hr;
+}
+
+inline void safeCoUninitializeReinit(DWORD dwCoInit, int nbReinit)
+{
+    CoUninitialize();
+    // Put back all the inits, if there were, before the use of the caller to safeCoInitializeEx
+    for (int i = 0; i < nbReinit; ++i)
+        CoInitializeEx(nullptr, dwCoInit);
+}
+}
+#endif
+/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */