tdf#112536 related: make soffice.bin a proper console application on Win

Being a GUI application on Windows (with related flag in the executable header
- see https://blogs.msdn.microsoft.com/oldnewthing/20090101-00/?p=19643/), OS
would detect the subsystem before launching the application, and won't attach
the parent console or redirected output handles from it to the application.
Also, different hacks to reattach the GUI application to the console later are
unreliable on different Windows versions, and work improperly (the output goes
to the console after the launch command has already returned, which is wrong
in batch files). This makes it extremily difficult to do CLI operations with
LibreOffice on Windows, with error codes/warnings/messages/output missing or
going to wrong consoles.

Making an executable for CUI subsystem, on the other hand, makes Windows to
allocate a console before starting it when the program is run by itself. This
makes the console window to appear on screen unconditionally, even if it's
hidden later when the program has started. This flashing is undesirable.

But we use a wrapper executable on Windows, called soffice.exe, which is what
actually launched by user, and which runs soffice.bin. This allows us to make
soffice.bin the proper console application, and thus make it capable to behave
properly in CLI scenarios, while avoid the console flashing when run from the
soffice.exe (which would suppress the console creation using DETACHED_PROCESS
creation flag to CreateProcessW).

Also creating a new wrapper for console (soffice.com) allows to use command
lines which omit explicit executable extension (no ".bin"), like this:

"C:\Program Files\LibreOffice\program\soffice" --help

which allows to continue using multiple available help resources unchanged,
since .com extension is tried prior to .exe by Windows' cmd.exe.

Change-Id: I089d0f30f860da6cfc781b4383f6598a08a4d238
Reviewed-on: https://gerrit.libreoffice.org/63572
Tested-by: Jenkins
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>

5 files changed, 232 insertions, 224 deletions

diff --git a/desktop/win32/source/loader.cxx b/desktop/win32/source/loader.cxx
index 4425c1e697d9..3960dd2c6f1c 100644
--- a/desktop/win32/source/loader.cxx
+++ b/desktop/win32/source/loader.cxx
@@ -17,9 +17,14 @@
  *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
  */
 
-#include <tools/pathutils.hxx>
 #include "loader.hxx"
 #include <cassert>
+#include <systools/win32/uwinapi.h>
+#include <stdlib.h>
+#include <string>
+#include <vector>
+#include <desktop/exithelper.h>
+#include <tools/pathutils.hxx>
 
 namespace {
 
@@ -34,6 +39,40 @@ void fail()
     TerminateProcess(GetCurrentProcess(), 255);
 }
 
+LPWSTR* GetCommandArgs(int* pArgc) { return CommandLineToArgvW(GetCommandLineW(), pArgc); }
+
+// tdf#120249: quotes in arguments need to be escaped; backslashes before quotes need doubling. See
+// https://docs.microsoft.com/en-us/windows/desktop/api/shellapi/nf-shellapi-commandlinetoargvw
+std::wstring EscapeArg(LPCWSTR sArg)
+{
+    const size_t nOrigSize = wcslen(sArg);
+    LPCWSTR const end = sArg + nOrigSize;
+    std::wstring sResult(L"\"");
+
+    LPCWSTR lastPosQuote = sArg;
+    LPCWSTR posQuote;
+    while ((posQuote = std::find(lastPosQuote, end, L'"')) != end)
+    {
+        LPCWSTR posBackslash = posQuote;
+        while (posBackslash != lastPosQuote && *(posBackslash - 1) == L'\\')
+            --posBackslash;
+
+        sResult.append(lastPosQuote, posBackslash);
+        sResult.append((posQuote - posBackslash) * 2 + 1, L'\\'); // 2n+1 '\' to escape the '"'
+        sResult.append(1, L'"');
+        lastPosQuote = posQuote + 1;
+    }
+
+    LPCWSTR posTrailingBackslashSeq = end;
+    while (posTrailingBackslashSeq != lastPosQuote && *(posTrailingBackslashSeq - 1) == L'\\')
+        --posTrailingBackslashSeq;
+    sResult.append(lastPosQuote, posTrailingBackslashSeq);
+    sResult.append((end - posTrailingBackslashSeq) * 2, L'\\'); // 2n '\' before closing '"'
+    sResult.append(1, L'"');
+
+    return sResult;
+}
+
 }
 
 namespace desktop_win32 {
@@ -50,9 +89,12 @@ void extendLoaderEnvironment(WCHAR * binPath, WCHAR * iniDirectory) {
         *nameEnd++ = *p;
     }
     if (!(nameEnd - name >= 4 && nameEnd[-4] == L'.' &&
-          (nameEnd[-3] == L'E' || nameEnd[-3] == L'e') &&
+         ((nameEnd[-3] == L'E' || nameEnd[-3] == L'e') &&
           (nameEnd[-2] == L'X' || nameEnd[-2] == L'x') &&
-          (nameEnd[-1] == L'E' || nameEnd[-1] == L'e')))
+          (nameEnd[-1] == L'E' || nameEnd[-1] == L'e') ||
+          (nameEnd[-3] == L'C' || nameEnd[-3] == L'c') &&
+          (nameEnd[-2] == L'O' || nameEnd[-2] == L'o') &&
+          (nameEnd[-1] == L'M' || nameEnd[-1] == L'm'))))
     {
         *nameEnd = L'.';
         nameEnd += 4;
@@ -90,6 +132,151 @@ void extendLoaderEnvironment(WCHAR * binPath, WCHAR * iniDirectory) {
     }
 }
 
+int officeloader_impl(bool bAllowConsole)
+{
+    WCHAR szTargetFileName[MAX_PATH] = {};
+    WCHAR szIniDirectory[MAX_PATH];
+    STARTUPINFOW aStartupInfo;
+
+    desktop_win32::extendLoaderEnvironment(szTargetFileName, szIniDirectory);
+
+    ZeroMemory(&aStartupInfo, sizeof(aStartupInfo));
+    aStartupInfo.cb = sizeof(aStartupInfo);
+
+    // Create process with same command line, environment and stdio handles which
+    // are directed to the created pipes
+    GetStartupInfoW(&aStartupInfo);
+
+    DWORD dwExitCode = DWORD(-1);
+
+    BOOL fSuccess = FALSE;
+    LPWSTR lpCommandLine = nullptr;
+    bool bFirst = true;
+    WCHAR cwd[MAX_PATH];
+    DWORD cwdLen = GetCurrentDirectoryW(MAX_PATH, cwd);
+    if (cwdLen >= MAX_PATH)
+    {
+        cwdLen = 0;
+    }
+    std::vector<std::wstring> aEscapedArgs;
+
+    do
+    {
+        if (bFirst)
+        {
+            int argc = 0;
+            LPWSTR* argv = GetCommandArgs(&argc);
+            std::size_t n = 0;
+            for (int i = 0; i < argc; ++i)
+            {
+                std::wstring sEscapedArg = EscapeArg(argv[i]);
+                aEscapedArgs.push_back(sEscapedArg);
+                n += sEscapedArg.length() + 1; // a space between args
+            }
+            LocalFree(argv);
+            n += MY_LENGTH(L" \"-env:OOO_CWD=2") + 4 * cwdLen + MY_LENGTH(L"\"") + 1;
+            // 4 * cwdLen: each char preceded by backslash, each trailing
+            // backslash doubled
+            lpCommandLine = new WCHAR[n];
+        }
+        WCHAR* p = desktop_win32::commandLineAppend(lpCommandLine, aEscapedArgs[0].c_str(),
+                                                    aEscapedArgs[0].length());
+        for (size_t i = 1; i < aEscapedArgs.size(); ++i)
+        {
+            const std::wstring& rArg = aEscapedArgs[i];
+            if (bFirst || EXITHELPER_NORMAL_RESTART == dwExitCode
+                || wcsncmp(rArg.c_str(), MY_STRING(L"\"-env:")) == 0)
+            {
+                p = desktop_win32::commandLineAppend(p, MY_STRING(L" "));
+                p = desktop_win32::commandLineAppend(p, rArg.c_str(), rArg.length());
+            }
+        }
+
+        p = desktop_win32::commandLineAppend(p, MY_STRING(L" \"-env:OOO_CWD="));
+        if (cwdLen == 0)
+        {
+            p = desktop_win32::commandLineAppend(p, MY_STRING(L"0"));
+        }
+        else
+        {
+            p = desktop_win32::commandLineAppend(p, MY_STRING(L"2"));
+            p = desktop_win32::commandLineAppendEncoded(p, cwd);
+        }
+        desktop_win32::commandLineAppend(p, MY_STRING(L"\""));
+        bFirst = false;
+
+        WCHAR szParentProcessId[64]; // This is more than large enough for a 128 bit decimal value
+        BOOL bHeadlessMode(FALSE);
+
+        {
+            // Check command line arguments for "--headless" parameter. We only
+            // set the environment variable "ATTACHED_PARENT_PROCESSID" for the headless
+            // mode as self-destruction of the soffice.bin process can lead to
+            // certain side-effects (log-off can result in data-loss, ".lock" is not deleted.
+            // See 138244 for more information.
+            int argc2;
+            LPWSTR* argv2 = GetCommandArgs(&argc2);
+
+            if (argc2 > 1)
+            {
+                int n;
+
+                for (n = 1; n < argc2; n++)
+                {
+                    if (0 == wcsnicmp(argv2[n], L"-headless", 9)
+                        || 0 == wcsnicmp(argv2[n], L"--headless", 10))
+                    {
+                        bHeadlessMode = TRUE;
+                    }
+                }
+            }
+
+            LocalFree(argv2);
+        }
+
+        if (_ltow(static_cast<long>(GetCurrentProcessId()), szParentProcessId, 10) && bHeadlessMode)
+            SetEnvironmentVariableW(L"ATTACHED_PARENT_PROCESSID", szParentProcessId);
+
+        PROCESS_INFORMATION aProcessInfo;
+
+        fSuccess = CreateProcessW(szTargetFileName, lpCommandLine, nullptr, nullptr, TRUE,
+                                  bAllowConsole ? 0 : DETACHED_PROCESS, nullptr, szIniDirectory,
+                                  &aStartupInfo, &aProcessInfo);
+
+        if (fSuccess)
+        {
+            DWORD dwWaitResult;
+
+            do
+            {
+                // On Windows XP it seems as the desktop calls WaitForInputIdle after "OpenWith" so
+                // we have to do so as if we where processing any messages
+
+                dwWaitResult = MsgWaitForMultipleObjects(1, &aProcessInfo.hProcess, FALSE, INFINITE,
+                                                         QS_ALLEVENTS);
+
+                if (WAIT_OBJECT_0 + 1 == dwWaitResult)
+                {
+                    MSG msg;
+
+                    PeekMessageW(&msg, nullptr, 0, 0, PM_REMOVE);
+                }
+            } while (WAIT_OBJECT_0 + 1 == dwWaitResult);
+
+            dwExitCode = 0;
+            GetExitCodeProcess(aProcessInfo.hProcess, &dwExitCode);
+
+            CloseHandle(aProcessInfo.hProcess);
+            CloseHandle(aProcessInfo.hThread);
+        }
+    } while (fSuccess
+             && (EXITHELPER_CRASH_WITH_RESTART == dwExitCode
+                 || EXITHELPER_NORMAL_RESTART == dwExitCode));
+    delete[] lpCommandLine;
+
+    return fSuccess ? dwExitCode : -1;
+}
+
 }
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/desktop/win32/source/loader.hxx b/desktop/win32/source/loader.hxx
index 059fae479958..dfa8f0c971a0 100644
--- a/desktop/win32/source/loader.hxx
+++ b/desktop/win32/source/loader.hxx
@@ -20,6 +20,7 @@
 #ifndef INCLUDED_DESKTOP_WIN32_SOURCE_LOADER_HXX
 #define INCLUDED_DESKTOP_WIN32_SOURCE_LOADER_HXX
 
+#include <cstddef>
 #define WIN32_LEAN_AND_MEAN
 #include <windows.h>
 #include <string.h>
@@ -78,6 +79,9 @@ inline WCHAR * commandLineAppendEncoded(WCHAR * buffer, WCHAR const * text) {
 // corresponding to the current executable.
 void extendLoaderEnvironment(WCHAR * binPath, WCHAR * iniDirectory);
 
+// Implementation of the process guarding soffice.bin
+int officeloader_impl(bool bAllowConsole);
+
 }
 
 #endif
diff --git a/desktop/win32/source/officeloader/officeloader.cxx b/desktop/win32/source/officeloader/officeloader.cxx
deleted file mode 100644
index e1b3323fbe9e..000000000000
--- a/desktop/win32/source/officeloader/officeloader.cxx
+++ /dev/null
@@ -1,221 +0,0 @@
-/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
-/*
- * This file is part of the LibreOffice project.
- *
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/.
- *
- * This file incorporates work covered by the following license notice:
- *
- *   Licensed to the Apache Software Foundation (ASF) under one or more
- *   contributor license agreements. See the NOTICE file distributed
- *   with this work for additional information regarding copyright
- *   ownership. The ASF licenses this file to you under the Apache
- *   License, Version 2.0 (the "License"); you may not use this file
- *   except in compliance with the License. You may obtain a copy of
- *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
- */
-
-#include <cstddef>
-
-#include <systools/win32/uwinapi.h>
-#include <stdlib.h>
-#include <algorithm>
-#include <string>
-#include <vector>
-#include <desktop/exithelper.h>
-
-#include "../loader.hxx"
-
-static LPWSTR *GetCommandArgs( int *pArgc )
-{
-    return CommandLineToArgvW( GetCommandLineW(), pArgc );
-}
-
-// tdf#120249: quotes in arguments need to be escaped; backslashes before quotes need doubling. See
-// https://docs.microsoft.com/en-us/windows/desktop/api/shellapi/nf-shellapi-commandlinetoargvw
-static std::wstring EscapeArg(LPCWSTR sArg)
-{
-    const size_t nOrigSize = wcslen(sArg);
-    LPCWSTR const end = sArg + nOrigSize;
-    std::wstring sResult(L"\"");
-
-    LPCWSTR lastPosQuote = sArg;
-    LPCWSTR posQuote;
-    while ((posQuote = std::find(lastPosQuote, end, L'"')) != end)
-    {
-        LPCWSTR posBackslash = posQuote;
-        while (posBackslash != lastPosQuote && *(posBackslash - 1) == L'\\')
-            --posBackslash;
-
-        sResult.append(lastPosQuote, posBackslash);
-        sResult.append((posQuote - posBackslash) * 2 + 1, L'\\'); // 2n+1 '\' to escape the '"'
-        sResult.append(1, L'"');
-        lastPosQuote = posQuote + 1;
-    }
-
-    LPCWSTR posTrailingBackslashSeq = end;
-    while (posTrailingBackslashSeq != lastPosQuote && *(posTrailingBackslashSeq - 1) == L'\\')
-        --posTrailingBackslashSeq;
-    sResult.append(lastPosQuote, posTrailingBackslashSeq);
-    sResult.append((end - posTrailingBackslashSeq) * 2, L'\\'); // 2n '\' before closing '"'
-    sResult.append(1, L'"');
-
-    return sResult;
-}
-
-int WINAPI wWinMain( HINSTANCE, HINSTANCE, LPWSTR, int )
-{
-    WCHAR        szTargetFileName[MAX_PATH] = {};
-    WCHAR        szIniDirectory[MAX_PATH];
-    STARTUPINFOW aStartupInfo;
-
-    desktop_win32::extendLoaderEnvironment(szTargetFileName, szIniDirectory);
-
-    ZeroMemory( &aStartupInfo, sizeof(aStartupInfo) );
-    aStartupInfo.cb = sizeof(aStartupInfo);
-
-    // Create process with same command line, environment and stdio handles which
-    // are directed to the created pipes
-    GetStartupInfoW(&aStartupInfo);
-
-    // If this process hasn't its stdio handles set, then check if its parent
-    // has a console (i.e. this process is launched from command line), and if so,
-    // attach to it. It will enable child process to retrieve this console if it needs
-    // to output to console
-    if ((aStartupInfo.dwFlags & STARTF_USESTDHANDLES) == 0)
-        AttachConsole(ATTACH_PARENT_PROCESS);
-
-    DWORD   dwExitCode = DWORD(-1);
-
-    BOOL    fSuccess = FALSE;
-    LPWSTR  lpCommandLine = nullptr;
-    bool bFirst = true;
-    WCHAR cwd[MAX_PATH];
-    DWORD cwdLen = GetCurrentDirectoryW(MAX_PATH, cwd);
-    if (cwdLen >= MAX_PATH) {
-        cwdLen = 0;
-    }
-    std::vector<std::wstring> aEscapedArgs;
-
-    do
-    {
-        if ( bFirst ) {
-            int argc = 0;
-            LPWSTR* argv = GetCommandArgs(&argc);
-            std::size_t n = 0;
-            for (int i = 0; i < argc; ++i) {
-                std::wstring sEscapedArg = EscapeArg(argv[i]);
-                aEscapedArgs.push_back(sEscapedArg);
-                n += sEscapedArg.length() + 1; // a space between args
-            }
-            LocalFree(argv);
-            n += MY_LENGTH(L" \"-env:OOO_CWD=2") + 4 * cwdLen +
-                MY_LENGTH(L"\"") + 1;
-                // 4 * cwdLen: each char preceded by backslash, each trailing
-                // backslash doubled
-            lpCommandLine = new WCHAR[n];
-        }
-        WCHAR* p = desktop_win32::commandLineAppend(lpCommandLine, aEscapedArgs[0].c_str(),
-                                                    aEscapedArgs[0].length());
-        for (size_t i = 1; i < aEscapedArgs.size(); ++i)
-        {
-            const std::wstring& rArg = aEscapedArgs[i];
-            if (bFirst || EXITHELPER_NORMAL_RESTART == dwExitCode
-                || wcsncmp(rArg.c_str(), MY_STRING(L"\"-env:")) == 0)
-            {
-                p = desktop_win32::commandLineAppend(p, MY_STRING(L" "));
-                p = desktop_win32::commandLineAppend(p, rArg.c_str(), rArg.length());
-            }
-        }
-
-        p = desktop_win32::commandLineAppend(p, MY_STRING(L" \"-env:OOO_CWD="));
-        if (cwdLen == 0) {
-            p = desktop_win32::commandLineAppend(p, MY_STRING(L"0"));
-        } else {
-            p = desktop_win32::commandLineAppend(p, MY_STRING(L"2"));
-            p = desktop_win32::commandLineAppendEncoded(p, cwd);
-        }
-        desktop_win32::commandLineAppend(p, MY_STRING(L"\""));
-        bFirst = false;
-
-        WCHAR szParentProcessId[64]; // This is more than large enough for a 128 bit decimal value
-        BOOL  bHeadlessMode( FALSE );
-
-        {
-            // Check command line arguments for "--headless" parameter. We only
-            // set the environment variable "ATTACHED_PARENT_PROCESSID" for the headless
-            // mode as self-destruction of the soffice.bin process can lead to
-            // certain side-effects (log-off can result in data-loss, ".lock" is not deleted.
-            // See 138244 for more information.
-            int    argc2;
-            LPWSTR *argv2 = GetCommandArgs( &argc2 );
-
-            if ( argc2 > 1 )
-            {
-                int n;
-
-                for ( n = 1; n < argc2; n++ )
-                {
-                    if ( 0 == wcsnicmp( argv2[n], L"-headless", 9 ) ||
-                         0 == wcsnicmp( argv2[n], L"--headless", 10 ) )
-                    {
-                        bHeadlessMode = TRUE;
-                    }
-                }
-            }
-
-            LocalFree(argv2);
-        }
-
-        if ( _ltow( static_cast<long>(GetCurrentProcessId()),szParentProcessId, 10 ) && bHeadlessMode )
-            SetEnvironmentVariableW( L"ATTACHED_PARENT_PROCESSID", szParentProcessId );
-
-        PROCESS_INFORMATION aProcessInfo;
-
-        fSuccess = CreateProcessW(
-            szTargetFileName,
-            lpCommandLine,
-            nullptr,
-            nullptr,
-            TRUE,
-            0,
-            nullptr,
-            szIniDirectory,
-            &aStartupInfo,
-            &aProcessInfo );
-
-        if ( fSuccess )
-        {
-            DWORD   dwWaitResult;
-
-            do
-            {
-                // On Windows XP it seems as the desktop calls WaitForInputIdle after "OpenWidth" so we have to do so
-                // as if we where processing any messages
-
-                dwWaitResult = MsgWaitForMultipleObjects( 1, &aProcessInfo.hProcess, FALSE, INFINITE, QS_ALLEVENTS );
-
-                if (  WAIT_OBJECT_0 + 1 == dwWaitResult )
-                {
-                    MSG msg;
-
-                    PeekMessageW( &msg, nullptr, 0, 0, PM_REMOVE );
-                }
-            } while ( WAIT_OBJECT_0 + 1 == dwWaitResult );
-
-            dwExitCode = 0;
-            GetExitCodeProcess( aProcessInfo.hProcess, &dwExitCode );
-
-            CloseHandle( aProcessInfo.hProcess );
-            CloseHandle( aProcessInfo.hThread );
-        }
-    } while ( fSuccess
-              && ( EXITHELPER_CRASH_WITH_RESTART == dwExitCode || EXITHELPER_NORMAL_RESTART == dwExitCode ));
-    delete[] lpCommandLine;
-
-    return fSuccess ? dwExitCode : -1;
-}
-
-/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/desktop/win32/source/officeloader/soffice_com.cxx b/desktop/win32/source/officeloader/soffice_com.cxx
new file mode 100644
index 000000000000..5c6974e66a15
--- /dev/null
+++ b/desktop/win32/source/officeloader/soffice_com.cxx
@@ -0,0 +1,19 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ */
+
+#include "../loader.hxx"
+
+int main(int /*argc*/, char** /*argv*/)
+{
+    // let soffice.bin use soffice.com's console
+    return desktop_win32::officeloader_impl(true);
+}
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/desktop/win32/source/officeloader/soffice_exe.cxx b/desktop/win32/source/officeloader/soffice_exe.cxx
new file mode 100644
index 000000000000..03ff0a546b87
--- /dev/null
+++ b/desktop/win32/source/officeloader/soffice_exe.cxx
@@ -0,0 +1,19 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ */
+
+#include "../loader.hxx"
+
+int WINAPI wWinMain(HINSTANCE, HINSTANCE, LPWSTR, int)
+{
+    // no console for soffice.bin when started by soffice.exe
+    return desktop_win32::officeloader_impl(false);
+}
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */