Qt5 fix missing XCB_ICCCM_WM_HINT_WINDOW_GROUP

This is the application level equivalent of the Qt5 fix for bug
QTBUG-46626 / commit 0de4b32 ("xcb: fix issue with dialogs hidden
by other windows"), which was broken since Qt 5.4 and is just
fixed since Qt 5.12.

It is needed for some window managers, which don't know about the
WM_CLIENT_LEADER property. Both settings are the same, but just
the latter is set by older Qt5 releases. This probably isn't a
real problem, as GNOME or XFCE would use the gtk VCL plugin, but
since I already wrote the code when debugging tdf#129071, there
is also no reason to drop it (except: more code, more bugs...).

This fix is optional and needs development headers for xcb-icccm,
which can actually be compiled into Qt5. If missing configure will
just print a warning, since it's a runtime requirement and we
explicitly drop the linked Qt version symbol, so the potential
build Qt version won't matter.

Change-Id: Ifc5a8f8a40ee13779a911efb53e8b8b868614d0b
Reviewed-on: https://gerrit.libreoffice.org/84299
Tested-by: Jenkins
Reviewed-by: Michael Weghorn <m.weghorn@posteo.de>
Reviewed-by: Jan-Marek Glogowski <glogow@fbihome.de>

4 files changed, 83 insertions, 2 deletions

diff --git a/config_host/config_vclplug.h.in b/config_host/config_vclplug.h.in
index 8ff5a9ce527f..488140b9ee4d 100644
--- a/config_host/config_vclplug.h.in
+++ b/config_host/config_vclplug.h.in
@@ -15,5 +15,6 @@ Settings about which desktops have support enabled.
 #define ENABLE_GSTREAMER_1_0 0
 #define QT5_HAVE_GOBJECT 0
 #define QT5_USING_X11 0
+#define QT5_HAVE_XCB_ICCCM 0
 
 #endif
diff --git a/configure.ac b/configure.ac
index 27e0f7a06839..da3e9b9db02a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -11499,8 +11499,15 @@ then
 
     if test "$USING_X11" = TRUE; then
         PKG_CHECK_MODULES(QT5_XCB,[xcb],,[AC_MSG_ERROR([XCB not found, which is needed for correct app grouping in X11.])])
-        QT5_CFLAGS="$QT5_CFLAGS $QT5_XCB_CFLAGS"
-        QT5_LIBS="$QT5_LIBS $QT5_XCB_LIBS -lQt5X11Extras"
+        PKG_CHECK_MODULES(QT5_XCB_ICCCM,[xcb-icccm],[
+            QT5_HAVE_XCB_ICCCM=1
+            AC_DEFINE(QT5_HAVE_XCB_ICCCM)
+        ],[
+            AC_MSG_WARN([XCB ICCCM not found, which is needed for old Qt versions (< 5.12) on some WMs to correctly group dialogs (like QTBUG-46626)])
+            add_warning "XCB ICCCM not found, which is needed for Qt versions (< 5.12) on some WMs to correctly group dialogs (like QTBUG-46626)"
+        ])
+        QT5_CFLAGS="$QT5_CFLAGS $QT5_XCB_CFLAGS $QT5_XCB_ICCCM_CFLAGS"
+        QT5_LIBS="$QT5_LIBS $QT5_XCB_LIBS $QT5_XCB_ICCCM_LIBS -lQt5X11Extras"
         QT5_USING_X11=1
         AC_DEFINE(QT5_USING_X11)
     fi
diff --git a/vcl/inc/qt5/Qt5Frame.hxx b/vcl/inc/qt5/Qt5Frame.hxx
index e296ce92db29..b5eea5b15b1e 100644
--- a/vcl/inc/qt5/Qt5Frame.hxx
+++ b/vcl/inc/qt5/Qt5Frame.hxx
@@ -134,6 +134,7 @@ class VCLPLUG_QT5_PUBLIC Qt5Frame : public QObject, public SalFrame
 
     void TriggerPaintEvent();
     void TriggerPaintEvent(QRect aRect);
+    void fixICCCMwindowGroup();
 
 public:
     Qt5Frame(Qt5Frame* pParent, SalFrameStyleFlags nSalFrameStyle, bool bUseCairo);
diff --git a/vcl/qt5/Qt5Frame.cxx b/vcl/qt5/Qt5Frame.cxx
index ed5f306d7ab0..8726b11bcf6d 100644
--- a/vcl/qt5/Qt5Frame.cxx
+++ b/vcl/qt5/Qt5Frame.cxx
@@ -36,6 +36,7 @@
 #include <QtCore/QPoint>
 #include <QtCore/QSize>
 #include <QtCore/QThread>
+#include <QtCore/QVersionNumber>
 #include <QtGui/QDragMoveEvent>
 #include <QtGui/QDropEvent>
 #include <QtGui/QIcon>
@@ -51,6 +52,9 @@
 #if QT5_USING_X11
 #include <QtX11Extras/QX11Info>
 #include <xcb/xproto.h>
+#if QT5_HAVE_XCB_ICCCM
+#include <xcb/xcb_icccm.h>
+#endif
 #endif
 
 #include <saldatabasic.hxx>
@@ -63,6 +67,11 @@
 #include <cairo.h>
 #include <headless/svpgdi.hxx>
 
+#if QT5_USING_X11 && QT5_HAVE_XCB_ICCCM
+static bool g_bNeedsWmHintsWindowGroup = true;
+static xcb_atom_t g_aXcbClientLeaderAtom = 0;
+#endif
+
 static void SvpDamageHandler(void* handle, sal_Int32 nExtentsX, sal_Int32 nExtentsY,
                              sal_Int32 nExtentsWidth, sal_Int32 nExtentsHeight)
 {
@@ -197,6 +206,69 @@ Qt5Frame::Qt5Frame(Qt5Frame* pParent, SalFrameStyleFlags nStyle, bool bUseCairo)
         m_aSystemData.platform = SystemEnvData::Platform::Wayland;
 
     SetIcon(SV_ICON_ID_OFFICE);
+
+    fixICCCMwindowGroup();
+}
+
+void Qt5Frame::fixICCCMwindowGroup()
+{
+#if QT5_USING_X11 && QT5_HAVE_XCB_ICCCM
+    // older Qt5 just sets WM_CLIENT_LEADER, but not the XCB_ICCCM_WM_HINT_WINDOW_GROUP
+    // see Qt commit 0de4b326d8 ("xcb: fix issue with dialogs hidden by other windows")
+    // or QTBUG-46626. So LO has to set this itself to help some WMs.
+    if (!g_bNeedsWmHintsWindowGroup)
+        return;
+    g_bNeedsWmHintsWindowGroup = false;
+
+    if (QGuiApplication::platformName() != "xcb")
+        return;
+    if (QVersionNumber::fromString(qVersion()) >= QVersionNumber(5, 12))
+        return;
+
+    xcb_connection_t* conn = QX11Info::connection();
+    xcb_window_t win = asChild()->winId();
+
+    xcb_icccm_wm_hints_t hints;
+
+    xcb_get_property_cookie_t prop_cookie = xcb_icccm_get_wm_hints_unchecked(conn, win);
+    if (!xcb_icccm_get_wm_hints_reply(conn, prop_cookie, &hints, nullptr))
+        return;
+
+    if (hints.flags & XCB_ICCCM_WM_HINT_WINDOW_GROUP)
+        return;
+
+    if (g_aXcbClientLeaderAtom == 0)
+    {
+        const char* const leader_name = "WM_CLIENT_LEADER\0";
+        xcb_intern_atom_cookie_t atom_cookie
+            = xcb_intern_atom(conn, 1, strlen(leader_name), leader_name);
+        xcb_intern_atom_reply_t* atom_reply = xcb_intern_atom_reply(conn, atom_cookie, nullptr);
+        if (!atom_reply)
+            return;
+        g_aXcbClientLeaderAtom = atom_reply->atom;
+        free(atom_reply);
+    }
+
+    g_bNeedsWmHintsWindowGroup = true;
+
+    prop_cookie = xcb_get_property(conn, 0, win, g_aXcbClientLeaderAtom, XCB_ATOM_WINDOW, 0, 1);
+    xcb_get_property_reply_t* prop_reply = xcb_get_property_reply(conn, prop_cookie, nullptr);
+    if (!prop_reply)
+        return;
+
+    if (xcb_get_property_value_length(prop_reply) != 4)
+    {
+        free(prop_reply);
+        return;
+    }
+
+    xcb_window_t leader = *static_cast<xcb_window_t*>(xcb_get_property_value(prop_reply));
+    free(prop_reply);
+
+    hints.flags |= XCB_ICCCM_WM_HINT_WINDOW_GROUP;
+    hints.window_group = leader;
+    xcb_icccm_set_wm_hints(conn, win, &hints);
+#endif
 }
 
 Qt5Frame::~Qt5Frame()