opencl: OpenCLZone, detect CL device change and disable CL on crash

Guard OpenCL calls with OpenCLZone, so if a OpenCL call crashes we
detect this and disable OpenCL so next time the user doesn't encounter
the crash at the same calculation because he has a broken OpenCL
drivers. Similar has been implemented for OpenGL with good results.

Additionaly we persistently remember a known good OpenCL device ID and
driver version so we can match this and perform calculation tests when
they change. This is to ensure that the selected OpenCL device performs
as we expect. In this commit the calculation tests aren't included yet.

Remove complex static initializer in opencl wrapper library.

Reviewed-on: https://gerrit.libreoffice.org/27064
Reviewed-by: Tomaž Vajngerl <quikee@gmail.com>
Tested-by: Tomaž Vajngerl <quikee@gmail.com>
(cherry picked from commit f41eb66302208f384a475fb20c98b6d1b0676cb6)

Change-Id: I1a8b81ee31298731efcf63dc6a476955afc035e9
Reviewed-on: https://gerrit.libreoffice.org/27103
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Michael Meeks <michael.meeks@collabora.com>

12 files changed, 214 insertions, 53 deletions

diff --git a/include/opencl/OpenCLZone.hxx b/include/opencl/OpenCLZone.hxx
new file mode 100644
index 000000000000..1fbc666f4df0
--- /dev/null
+++ b/include/opencl/OpenCLZone.hxx
@@ -0,0 +1,52 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#ifndef INCLUDED_OPENCL_INC_OPENCL_ZONE_HXX
+#define INCLUDED_OPENCL_INC_OPENCL_ZONE_HXX
+
+#include <opencl/opencldllapi.h>
+
+// FIXME: post back-port, templatize me and share with OpenGLZone.
+class OPENCL_DLLPUBLIC OpenCLZone
+{
+    /// how many times have we entered a CL zone
+    static volatile sal_uInt64 gnEnterCount;
+    /// how many times have we left a new CL zone
+    static volatile sal_uInt64 gnLeaveCount;
+
+    static void enter()
+    {
+        gnEnterCount++;
+    }
+    static void leave()
+    {
+        gnLeaveCount--;
+    }
+public:
+    OpenCLZone()
+    {
+        gnEnterCount++;
+    }
+
+    ~OpenCLZone()
+    {
+        gnLeaveCount++;
+    }
+
+    static bool isInZone()
+    {
+        return gnEnterCount != gnLeaveCount;
+    }
+
+    static void hardDisable();
+};
+
+#endif // INCLUDED_OPENCL_INC_OPENCL_ZONE_HXX
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/include/opencl/openclwrapper.hxx b/include/opencl/openclwrapper.hxx
index 5205aaab0767..2121f0e11c1e 100644
--- a/include/opencl/openclwrapper.hxx
+++ b/include/opencl/openclwrapper.hxx
@@ -66,11 +66,13 @@ OPENCL_DLLPUBLIC const std::vector<OpenCLPlatformInfo>& fillOpenCLInfo();
  *
  * @param pDeviceId the id of the opencl device of type cl_device_id, NULL means use software calculation
  * @param bAutoSelect use the algorithm to select the best OpenCL device
+ * @param rOutSelectedDeviceVersionIDString returns the selected device's version string.
  *
  * @return returns true if there is a valid opencl device that has been set up
  */
 OPENCL_DLLPUBLIC bool switchOpenCLDevice(const OUString* pDeviceId, bool bAutoSelect,
-                                         bool bForceEvaluation);
+                                         bool bForceEvaluation,
+                                         OUString& rOutSelectedDeviceVersionIDString);
 
 OPENCL_DLLPUBLIC void getOpenCLDeviceInfo(size_t& rDeviceId, size_t& rPlatformId);
 
diff --git a/officecfg/registry/schema/org/openoffice/Office/Common.xcs b/officecfg/registry/schema/org/openoffice/Office/Common.xcs
index 8662cfc72810..d9e81e5973a2 100644
--- a/officecfg/registry/schema/org/openoffice/Office/Common.xcs
+++ b/officecfg/registry/schema/org/openoffice/Office/Common.xcs
@@ -5641,6 +5641,12 @@
         </info>
         <value oor:separator=";">Linux//Advanced Micro Devices, Inc\.//1445\.5 \(sse2,avx\);//Advanced Micro Devices, Inc\.//;//Intel\(R\) Corporation//;//NVIDIA Corporation//</value>
       </prop>
+      <prop oor:name="SelectedOpenCLDeviceIdentifier" oor:type="xs:string" oor:nillable="false">
+        <info>
+          <desc>The OpenCL device identifier of a device that was successfully selected and passed the OpenCL calculation tests.</desc>
+        </info>
+        <value>none</value>
+      </prop>
       <prop oor:name="UseSwInterpreter" oor:type="xs:boolean" oor:nillable="false">
         <info>
           <desc>Determines whether Software Interpreter can be used to speed
diff --git a/opencl/Library_opencl.mk b/opencl/Library_opencl.mk
index 4529da7129d3..3d9b0325564c 100644
--- a/opencl/Library_opencl.mk
+++ b/opencl/Library_opencl.mk
@@ -38,6 +38,7 @@ $(eval $(call gb_Library_use_libraries,opencl,\
     comphelper \
     cppu \
     sal \
+    salhelper \
     tl \
 ))
 
@@ -46,6 +47,7 @@ $(eval $(call gb_Library_add_exception_objects,opencl,\
     opencl/source/openclwrapper \
     opencl/source/opencl_device \
     opencl/source/platforminfo \
+    opencl/source/OpenCLZone \
 ))
 
 ifeq ($(OS),LINUX)
diff --git a/opencl/inc/opencl_device_selection.h b/opencl/inc/opencl_device_selection.h
index e2bda73af8f3..7d46f29a34b1 100644
--- a/opencl/inc/opencl_device_selection.h
+++ b/opencl/inc/opencl_device_selection.h
@@ -24,6 +24,8 @@
 #include <tools/stream.hxx>
 #include <rtl/math.hxx>
 
+#include <opencl/OpenCLZone.hxx>
+
 #include <vector>
 
 enum ds_status
@@ -127,6 +129,8 @@ inline bool getDeviceInfoBool(cl_device_id aDeviceId, cl_device_info aDeviceInfo
 
 inline ds_status initDSProfile(std::unique_ptr<ds_profile>& rProfile, OString rVersion)
 {
+    OpenCLZone zone;
+
     int numDevices;
     cl_uint numPlatforms;
     std::vector<cl_platform_id> platforms;
diff --git a/opencl/source/OpenCLZone.cxx b/opencl/source/OpenCLZone.cxx
new file mode 100644
index 000000000000..dc3a9522380b
--- /dev/null
+++ b/opencl/source/OpenCLZone.cxx
@@ -0,0 +1,46 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#include <opencl/OpenCLZone.hxx>
+
+#include <memory>
+
+#include <officecfg/Office/Common.hxx>
+#include <com/sun/star/util/XFlushable.hpp>
+#include <com/sun/star/configuration/theDefaultProvider.hpp>
+
+// FIXME: templatize me vs. OpenGLZone.
+
+sal_uInt64 volatile OpenCLZone::gnEnterCount = 0;
+sal_uInt64 volatile OpenCLZone::gnLeaveCount = 0;
+
+/**
+ * Called from a signal handler if we get
+ * a crash or hang in some CL code.
+ */
+void OpenCLZone::hardDisable()
+{
+    // protect ourselves from double calling etc.
+    static bool bDisabled = false;
+    if (!bDisabled)
+    {
+        bDisabled = true;
+
+        std::shared_ptr<comphelper::ConfigurationChanges> xChanges(comphelper::ConfigurationChanges::create());
+        officecfg::Office::Common::Misc::UseOpenCL::set(false, xChanges);
+        xChanges->commit();
+
+        // Force synchronous config write
+        auto xConfProvider = css::configuration::theDefaultProvider::get(comphelper::getProcessComponentContext());
+        css::uno::Reference<css::util::XFlushable> xFlushable(xConfProvider, css::uno::UNO_QUERY_THROW);
+        xFlushable->flush();
+    }
+}
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/opencl/source/opencl_device.cxx b/opencl/source/opencl_device.cxx
index 85fea920e379..d87ead84eb77 100644
--- a/opencl/source/opencl_device.cxx
+++ b/opencl/source/opencl_device.cxx
@@ -30,6 +30,8 @@
 #include <opencl/platforminfo.hxx>
 #include <sal/log.hxx>
 
+#include <opencl/OpenCLZone.hxx>
+
 #include "opencl_device.hxx"
 
 #define INPUTSIZE  15360
@@ -200,14 +202,21 @@ ds_status evaluateScoreForDevice(ds_device& rDevice, std::unique_ptr<LibreOffice
         /* Evaluating an OpenCL device */
         SAL_INFO("opencl.device", "Device: \"" << rDevice.sDeviceName << "\" (OpenCL) evaluation...");
         cl_int clStatus;
+
         /* Check for 64-bit float extensions */
-        size_t aDevExtInfoSize = 0;
-        clStatus = clGetDeviceInfo(rDevice.aDeviceID, CL_DEVICE_EXTENSIONS, 0, nullptr, &aDevExtInfoSize);
-        DS_CHECK_STATUS(clStatus, "evaluateScoreForDevice::clGetDeviceInfo");
+        std::unique_ptr<char[]> aExtInfo;
+        {
+            size_t aDevExtInfoSize = 0;
+
+            OpenCLZone zone;
+            clStatus = clGetDeviceInfo(rDevice.aDeviceID, CL_DEVICE_EXTENSIONS, 0, nullptr, &aDevExtInfoSize);
+            DS_CHECK_STATUS(clStatus, "evaluateScoreForDevice::clGetDeviceInfo");
+
+            aExtInfo.reset(new char[aDevExtInfoSize]);
+            clStatus = clGetDeviceInfo(rDevice.aDeviceID, CL_DEVICE_EXTENSIONS, sizeof(char) * aDevExtInfoSize, aExtInfo.get(), nullptr);
+            DS_CHECK_STATUS(clStatus, "evaluateScoreForDevice::clGetDeviceInfo");
+        }
 
-        std::unique_ptr<char[]> aExtInfo(new char[aDevExtInfoSize]);
-        clStatus = clGetDeviceInfo(rDevice.aDeviceID, CL_DEVICE_EXTENSIONS, sizeof(char) * aDevExtInfoSize, aExtInfo.get(), nullptr);
-        DS_CHECK_STATUS(clStatus, "evaluateScoreForDevice::clGetDeviceInfo");
         bool bKhrFp64Flag = false;
         bool bAmdFp64Flag = false;
         const char* buildOption = nullptr;
@@ -244,6 +253,8 @@ ds_status evaluateScoreForDevice(ds_device& rDevice, std::unique_ptr<LibreOffice
         {
             /* 64-bit float support present */
 
+            OpenCLZone zone;
+
             /* Create context and command queue */
             cl_context  clContext = clCreateContext(nullptr, 1, &rDevice.aDeviceID, nullptr, nullptr, &clStatus);
             DS_CHECK_STATUS(clStatus, "evaluateScoreForDevice::clCreateContext");
diff --git a/opencl/source/openclwrapper.cxx b/opencl/source/openclwrapper.cxx
index 384e9eb03310..1af142c1476c 100644
--- a/opencl/source/openclwrapper.cxx
+++ b/opencl/source/openclwrapper.cxx
@@ -21,6 +21,7 @@
 #include <rtl/ustring.hxx>
 #include <sal/config.h>
 #include <sal/log.hxx>
+#include <opencl/OpenCLZone.hxx>
 
 #include <memory>
 #include <unicode/regex.h>
@@ -82,15 +83,19 @@ OString generateMD5(const void* pData, size_t length)
 
 OString getCacheFolder()
 {
-    OUString url("${$BRAND_BASE_DIR/" LIBO_ETC_FOLDER "/" SAL_CONFIGFILE("bootstrap") ":UserInstallation}/cache/");
-    rtl::Bootstrap::expandMacros(url);
+    static OString aCacheFolder;
 
-    osl::Directory::create(url);
+    if (aCacheFolder.isEmpty())
+    {
+        OUString url("${$BRAND_BASE_DIR/" LIBO_ETC_FOLDER "/" SAL_CONFIGFILE("bootstrap") ":UserInstallation}/cache/");
+        rtl::Bootstrap::expandMacros(url);
 
-    return rtl::OUStringToOString(url, RTL_TEXTENCODING_UTF8);
-}
+        osl::Directory::create(url);
 
-OString maCacheFolder = getCacheFolder();
+        aCacheFolder = rtl::OUStringToOString(url, RTL_TEXTENCODING_UTF8);
+    }
+    return aCacheFolder;
+}
 
 }
 
@@ -132,8 +137,7 @@ OString createFileName(cl_device_id deviceId, const char* clFileName)
     OString aString = OString(deviceName) + driverVersion + platformVersion;
     OString aHash = generateMD5(aString.getStr(), aString.getLength());
 
-    return maCacheFolder + fileName + "-" +
-        aHash + ".bin";
+    return getCacheFolder() + fileName + "-" + aHash + ".bin";
 }
 
 std::vector<std::shared_ptr<osl::File> > binaryGenerated( const char * clFileName, cl_context context )
@@ -267,6 +271,8 @@ bool initOpenCLAttr( OpenCLEnv * env )
 
 void releaseOpenCLEnv( GPUEnv *gpuInfo )
 {
+    OpenCLZone zone;
+
     if ( !bIsInited )
     {
         return;
@@ -319,7 +325,7 @@ bool buildProgram(const char* buildOption, GPUEnv* gpuInfo, int idx)
             return false;
         }
 
-        OString aBuildLogFileURL = maCacheFolder + "kernel-build.log";
+        OString aBuildLogFileURL = getCacheFolder() + "kernel-build.log";
         osl::File aBuildLogFile(rtl::OStringToOUString(aBuildLogFileURL, RTL_TEXTENCODING_UTF8));
         osl::FileBase::RC status = aBuildLogFile.open(
                 osl_File_OpenFlag_Write | osl_File_OpenFlag_Create );
@@ -410,6 +416,8 @@ namespace {
 
 void checkDeviceForDoubleSupport(cl_device_id deviceId, bool& bKhrFp64, bool& bAmdFp64)
 {
+    OpenCLZone zone;
+
     bKhrFp64 = false;
     bAmdFp64 = false;
 
@@ -442,6 +450,8 @@ void checkDeviceForDoubleSupport(cl_device_id deviceId, bool& bKhrFp64, bool& bA
 
 bool initOpenCLRunEnv( GPUEnv *gpuInfo )
 {
+    OpenCLZone zone;
+
     bool bKhrFp64 = false;
     bool bAmdFp64 = false;
 
@@ -684,7 +694,7 @@ void findDeviceInfoFromDeviceId(cl_device_id aDeviceId, size_t& rDeviceId, size_
 
 }
 
-bool switchOpenCLDevice(const OUString* pDevice, bool bAutoSelect, bool bForceEvaluation)
+bool switchOpenCLDevice(const OUString* pDevice, bool bAutoSelect, bool bForceEvaluation, OUString& rOutSelectedDeviceVersionIDString)
 {
     if(fillOpenCLInfo().empty())
         return false;
@@ -716,54 +726,62 @@ bool switchOpenCLDevice(const OUString* pDevice, bool bAutoSelect, bool bForceEv
         return pDeviceId != nullptr;
     }
 
+    cl_context context;
     cl_platform_id platformId;
-    cl_int nState = clGetDeviceInfo(pDeviceId, CL_DEVICE_PLATFORM,
-            sizeof(platformId), &platformId, nullptr);
-
-    cl_context_properties cps[3];
-    cps[0] = CL_CONTEXT_PLATFORM;
-    cps[1] = reinterpret_cast<cl_context_properties>(platformId);
-    cps[2] = 0;
-    cl_context context = clCreateContext( cps, 1, &pDeviceId, nullptr, nullptr, &nState );
-    if (nState != CL_SUCCESS)
-        SAL_WARN("opencl", "clCreateContext failed: " << errorString(nState));
+    cl_command_queue command_queue[OPENCL_CMDQUEUE_SIZE];
 
-    if(nState != CL_SUCCESS || context == nullptr)
     {
-        if(context != nullptr)
-            clReleaseContext(context);
+        OpenCLZone zone;
+        cl_int nState = clGetDeviceInfo(pDeviceId, CL_DEVICE_PLATFORM,
+                                        sizeof(platformId), &platformId, nullptr);
+
+        cl_context_properties cps[3];
+        cps[0] = CL_CONTEXT_PLATFORM;
+        cps[1] = reinterpret_cast<cl_context_properties>(platformId);
+        cps[2] = 0;
+        context = clCreateContext( cps, 1, &pDeviceId, nullptr, nullptr, &nState );
+        if (nState != CL_SUCCESS)
+            SAL_WARN("opencl", "clCreateContext failed: " << errorString(nState));
 
-        SAL_WARN("opencl", "failed to set/switch opencl device");
-        return false;
-    }
-    SAL_INFO("opencl", "Created context " << context << " for platform " << platformId << ", device " << pDeviceId);
+        if(nState != CL_SUCCESS || context == nullptr)
+        {
+            if(context != nullptr)
+                clReleaseContext(context);
 
-    cl_command_queue command_queue[OPENCL_CMDQUEUE_SIZE];
-    for (int i = 0; i < OPENCL_CMDQUEUE_SIZE; ++i)
-    {
-        command_queue[i] = clCreateCommandQueue(
-            context, pDeviceId, 0, &nState);
-        if (nState != CL_SUCCESS)
-            SAL_WARN("opencl", "clCreateCommandQueue failed: " << errorString(nState));
+            SAL_WARN("opencl", "failed to set/switch opencl device");
+            return false;
+        }
+        SAL_INFO("opencl", "Created context " << context << " for platform " << platformId << ", device " << pDeviceId);
 
-        if (command_queue[i] == nullptr || nState != CL_SUCCESS)
+        for (int i = 0; i < OPENCL_CMDQUEUE_SIZE; ++i)
         {
-            // Release all command queues created so far.
-            for (int j = 0; j <= i; ++j)
+            command_queue[i] = clCreateCommandQueue(
+                context, pDeviceId, 0, &nState);
+            if (nState != CL_SUCCESS)
+                SAL_WARN("opencl", "clCreateCommandQueue failed: " << errorString(nState));
+
+            if (command_queue[i] == nullptr || nState != CL_SUCCESS)
             {
-                if (command_queue[j])
+                // Release all command queues created so far.
+                for (int j = 0; j <= i; ++j)
                 {
-                    clReleaseCommandQueue(command_queue[j]);
-                    command_queue[j] = nullptr;
+                    if (command_queue[j])
+                    {
+                        clReleaseCommandQueue(command_queue[j]);
+                        command_queue[j] = nullptr;
+                    }
                 }
+
+                clReleaseContext(context);
+                SAL_WARN("opencl", "failed to set/switch opencl device");
+                return false;
             }
 
-            clReleaseContext(context);
-            SAL_WARN("opencl", "failed to set/switch opencl device");
-            return false;
+            SAL_INFO("opencl", "Created command queue " << command_queue[i] << " for context " << context);
         }
 
-        SAL_INFO("opencl", "Created command queue " << command_queue[i] << " for context " << context);
+        OString sDeviceID = getDeviceInfoString(pDeviceId, CL_DEVICE_VENDOR) + " " + getDeviceInfoString(pDeviceId, CL_DRIVER_VERSION);
+        rOutSelectedDeviceVersionIDString = OStringToOUString(sDeviceID, RTL_TEXTENCODING_UTF8);
     }
 
     setOpenCLCmdQueuePosition(0); // Call this just to avoid the method being deleted from unused function deleter.
diff --git a/sc/source/core/tool/formulagroup.cxx b/sc/source/core/tool/formulagroup.cxx
index f2f8acf1aa7f..abe92e89ec31 100644
--- a/sc/source/core/tool/formulagroup.cxx
+++ b/sc/source/core/tool/formulagroup.cxx
@@ -32,7 +32,7 @@
 #include <vector>
 
 #if HAVE_FEATURE_OPENCL
-#include <opencl/openclwrapper.hxx>
+#  include <opencl/openclwrapper.hxx>
 #endif
 
 namespace sc {
@@ -331,7 +331,9 @@ bool FormulaGroupInterpreter::switchOpenCLDevice(const OUString& rDeviceId, bool
 
         return false;
     }
-    bool bSuccess = ::opencl::switchOpenCLDevice(&rDeviceId, bAutoSelect, bForceEvaluation);
+
+    OUString aSelectedCLDeviceVersionID;
+    bool bSuccess = ::opencl::switchOpenCLDevice(&rDeviceId, bAutoSelect, bForceEvaluation, aSelectedCLDeviceVersionID);
     if(!bSuccess)
         return false;
 
@@ -341,6 +343,17 @@ bool FormulaGroupInterpreter::switchOpenCLDevice(const OUString& rDeviceId, bool
     if (bOpenCLEnabled)
     {
         msInstance = new sc::opencl::FormulaGroupInterpreterOpenCL();
+
+        if (aSelectedCLDeviceVersionID != officecfg::Office::Common::Misc::SelectedOpenCLDeviceIdentifier::get())
+        {
+            // perform OpenCL calculation tests
+
+            // save the device
+            std::shared_ptr<comphelper::ConfigurationChanges> xBatch(comphelper::ConfigurationChanges::create());
+            officecfg::Office::Common::Misc::SelectedOpenCLDeviceIdentifier::set(aSelectedCLDeviceVersionID, xBatch);
+            xBatch->commit();
+        }
+
         return msInstance != nullptr;
     }
 
diff --git a/solenv/gbuild/extensions/pre_MergedLibsList.mk b/solenv/gbuild/extensions/pre_MergedLibsList.mk
index 97953eed736a..b33857054965 100644
--- a/solenv/gbuild/extensions/pre_MergedLibsList.mk
+++ b/solenv/gbuild/extensions/pre_MergedLibsList.mk
@@ -35,6 +35,7 @@ MERGE_LIBRARY_LIST := \
 	localebe1 \
 	mcnttype \
 	msfilter \
+	$(if $(filter OPENCL,$(BUILD_TYPE)),opencl) \
 	package2 \
 	sax \
 	sb \
diff --git a/vcl/Library_vcl.mk b/vcl/Library_vcl.mk
index 7f0b95d5bd20..5a3eb10817da 100644
--- a/vcl/Library_vcl.mk
+++ b/vcl/Library_vcl.mk
@@ -85,6 +85,7 @@ $(eval $(call gb_Library_use_libraries,vcl,\
     cppuhelper \
     i18nlangtag \
     i18nutil \
+    $(if $(filter OPENCL,$(BUILD_TYPE)),opencl) \
     cppu \
     sal \
     salhelper \
diff --git a/vcl/source/app/svmain.cxx b/vcl/source/app/svmain.cxx
index 41243b69c352..38a9481c2351 100644
--- a/vcl/source/app/svmain.cxx
+++ b/vcl/source/app/svmain.cxx
@@ -82,6 +82,7 @@
 #include <cppuhelper/implbase.hxx>
 #include "uno/current_context.hxx"
 
+#include <opencl/OpenCLZone.hxx>
 #include "opengl/zone.hxx"
 #include "opengl/watchdog.hxx"
 
@@ -110,6 +111,10 @@ oslSignalAction SAL_CALL VCLExceptionSignal_impl( void* /*pData*/, oslSignalInfo
             nVCLException = EXC_SYSTEM;
             if (OpenGLZone::isInZone())
                 OpenGLZone::hardDisable();
+#if HAVE_FEATURE_OPENCL
+            if (OpenCLZone::isInZone())
+                OpenCLZone::hardDisable();
+#endif
         }
 
         // RC