tdf#147364: fix function signature to account to calling convention

First, the integer function result is returned in a 64-bit register (RAX),
and truncation it to sal_Int32 breaks any pointer return value.

Second, using explicit (not vararg) first function double argument would
pass it through XMM0, without also copying it to RCX (which is guaranteed
for varargs).

Ref: https://docs.microsoft.com/en-us/cpp/build/x64-calling-convention
Change-Id: I08212c44d8690d6910068b13c16af2ce899c94f2
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/129808
Tested-by: Jenkins
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>
(cherry picked from commit 5276177877cc53c53653a87f3038eb208a5580ab)
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/129832
Reviewed-by: Michael Stahl <michael.stahl@allotropia.de>

2 files changed, 46 insertions, 46 deletions

diff --git a/basic/qa/basic_coverage/test_declare_from_dll.bas b/basic/qa/basic_coverage/test_declare_from_dll.bas
new file mode 100644
index 000000000000..7bdd1b170492
--- /dev/null
+++ b/basic/qa/basic_coverage/test_declare_from_dll.bas
@@ -0,0 +1,36 @@
+'
+' This file is part of the LibreOffice project.
+'
+' This Source Code Form is subject to the terms of the Mozilla Public
+' License, v. 2.0. If a copy of the MPL was not distributed with this
+' file, You can obtain one at http://mozilla.org/MPL/2.0/.
+'
+
+Option Explicit
+
+' We link to shlwapi in many places, so safe to rely on it here
+Declare Function PathFindExtensionA Lib "Shlwapi" (ByVal pszPath As String) As String
+
+Function doUnitTest() As String
+    TestUtil.TestInit
+    verify_testPathFindExtensionA
+    doUnitTest = TestUtil.GetResult()
+End Function
+
+Sub verify_testPathFindExtensionA()
+    On Error GoTo errorHandler
+
+    ' Only test on Windows
+    If (GetGUIType() <> 1) Then
+        TestUtil.Assert(True) ' The test passed
+        Exit Sub
+    End If
+
+    ' Without the fix for tdf#147364 in place, the next call would crash in 64-bit version with
+    '   *** Exception 0xc0000005 occurred ***
+    TestUtil.AssertEqual(PathFindExtensionA("filename.ext"), ".ext", "PathFindExtensionA(""filename.ext"")")
+
+    Exit Sub
+errorHandler:
+    TestUtil.ReportErrorHandler("verify_testPathFindExtensionA", Err, Error$, Erl)
+End Sub
diff --git a/basic/source/runtime/dllmgr-x64.cxx b/basic/source/runtime/dllmgr-x64.cxx
index dad4bea8794a..b2ddbb890da1 100644
--- a/basic/source/runtime/dllmgr-x64.cxx
+++ b/basic/source/runtime/dllmgr-x64.cxx
@@ -512,8 +512,6 @@ ErrCode call(
     // We fake all calls as being to a varargs function,
     // as this means any floating-point argument among the first four
     // ones will end up in a XMM register where the callee expects it.
-    sal_Int32 (*proc_i)(double d, ...) = reinterpret_cast<sal_Int32 (*)(double, ...)>(proc.proc);
-    double (*proc_d)(double d, ...) = reinterpret_cast<double (*)(double, ...)>(proc.proc);
 
     sal_Int64 iRetVal = 0;
     double dRetVal = 0.0;
@@ -527,55 +525,21 @@ ErrCode call(
     case SbxBOOL:
     case SbxBYTE:
         {
-            auto const st = stack.data();
-            iRetVal =
-                proc_i(*reinterpret_cast<double *>(st + 0),
-                       *reinterpret_cast<double *>(st + 1*8),
-                       *reinterpret_cast<double *>(st + 2*8),
-                       *reinterpret_cast<double *>(st + 3*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 4*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 5*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 6*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 7*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 8*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 9*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 10*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 11*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 12*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 13*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 14*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 15*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 16*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 17*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 18*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 19*8));
+            auto p = reinterpret_cast<sal_Int64 (*)(...)>(proc.proc);
+            auto const st = reinterpret_cast<double *>(stack.data());
+            iRetVal
+                = p(st[0], st[1], st[2], st[3], st[4], st[5], st[6], st[7], st[8], st[9], st[10],
+                    st[11], st[12], st[13], st[14], st[15], st[16], st[17], st[18], st[19]);
             break;
         }
     case SbxSINGLE:
     case SbxDOUBLE:
         {
-            auto const st = stack.data();
-            dRetVal =
-                proc_d(*reinterpret_cast<double *>(st + 0),
-                       *reinterpret_cast<double *>(st + 1*8),
-                       *reinterpret_cast<double *>(st + 2*8),
-                       *reinterpret_cast<double *>(st + 3*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 4*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 5*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 6*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 7*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 8*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 9*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 10*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 11*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 12*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 13*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 14*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 15*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 16*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 17*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 18*8),
-                       *reinterpret_cast<sal_uInt64 *>(st + 19*8));
+            auto p = reinterpret_cast<double (*)(...)>(proc.proc);
+            auto const st = reinterpret_cast<double*>(stack.data());
+            dRetVal
+                = p(st[0], st[1], st[2], st[3], st[4], st[5], st[6], st[7], st[8], st[9], st[10],
+                    st[11], st[12], st[13], st[14], st[15], st[16], st[17], st[18], st[19]);
             break;
         }
     default: