New --with-coredumpctl to obtain core dumps of crashed tests from coredumpctl

...for (Linux) systems that don't store core.* files in the current working
directory.  When enabled, this wraps test execution in `systemd-run --scope
--user --unit=...` with unit values unique per individual test invocation, so
that solenv/bin/gdb-core-bt.sh can query coredumpctl for matching core dumps.
(See the mailing list thread starting at
<https://lists.freedesktop.org/archives/systemd-devel/2023-March/048884.html>
"[systemd-devel] coredumpctl: matching by e.g. env var?" for further details.)

The used --unit=... scheme is a best effort to produce system-wide unique
values, combining the target location path of the given test with a
second-granularity date/time and the current PID.  (In case there would be
multiple invocations of the same test per second, which then hopefully wouldn't
reuse the same PID.  The date/time and PID could be replaced with a
high-resolution system-wide monotonic clock/counter if one were easily
available.  The advantage of the current scheme is that it only uses Posix
features.)  The overall length of the unit value (incl. the appended ".scope"
suffix) must not exceed 256 characters, or else systemd-run would fail with
"Failed to mangle scope name: Invalid argument".

It might look more natural to pass the unit value into gdb-core-bt.sh as a
fourth positional argument rather than via a new LIBO_TEST_UNIT env var.  But
for one, the unit value is most easily computed from within the recipe shell
command lines, where an env var is the most natural fit.  And for another, this
avoids having to tunnel yet another value through the tearDown method in
unotest/source/java/org/openoffice/test/OfficeConnection.java to the given
postprocesscommand.

Change-Id: Idcb20cd1e1141d8ec7f10947e5edc70aa2aa7d32
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/149690
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>

5 files changed, 38 insertions, 4 deletions

diff --git a/solenv/bin/gdb-core-bt.sh b/solenv/bin/gdb-core-bt.sh
index 8557825804ab..0276a70f4f0c 100755
--- a/solenv/bin/gdb-core-bt.sh
+++ b/solenv/bin/gdb-core-bt.sh
@@ -35,6 +35,26 @@ then
             echo
         fi
     done
+    if test -n "$WITH_COREDUMPCTL"; then
+        # Unfortunately `coredumpctl debug` only operates on the most recent core dump matching any
+        # given criteria, not on all core dumps matching those criteria; so get the PIDs of all core
+        # dumps matching the given COREDUMP_USER_UNIT (and for which a core dump is still present)
+        # first, and then iterate over them (though this introduces possibilities for some,
+        # hopefully unlikely and mostly harmless, races, like when core dumps disappear in between,
+        # or multiple matching core dumps have identical PIDs):
+        for i in $($COREDUMPCTL --json=short list COREDUMP_USER_UNIT="$LIBO_TEST_UNIT".scope | \
+                       $JQ -r 'map(select(.corefile=="present"))|map(.pid)|join(" ")')
+        do
+            GDBCOMMANDFILE=$(mktemp)
+            printf 'info registers\nthread apply all backtrace full\n' >"$GDBCOMMANDFILE"
+            PYTHONWARNINGS=default $COREDUMPCTL debug \
+                COREDUMP_USER_UNIT="$LIBO_TEST_UNIT".scope COREDUMP_PID="$i" \
+                --debugger-arguments="-iex 'add-auto-load-safe-path ${INSTDIR?}' \
+                    -x '$GDBCOMMANDFILE' --batch"
+            rm "$GDBCOMMANDFILE"
+            found=x
+        done
+    fi
     if [ -z "$found" -a "$EXITCODE" -ge 128 ]; then
         echo
         echo "No core file identified in directory ${COREDIR}"
diff --git a/solenv/gbuild/CppunitTest.mk b/solenv/gbuild/CppunitTest.mk
index b16e3d0e5417..18af97e7fff1 100644
--- a/solenv/gbuild/CppunitTest.mk
+++ b/solenv/gbuild/CppunitTest.mk
@@ -22,6 +22,15 @@
 gb_CppunitTest_UNITTESTFAILED ?= $(GBUILDDIR)/platform/unittest-failed-default.sh
 gb_CppunitTest_PYTHONDEPS ?= $(call gb_Library_get_target,pyuno_wrapper) $(if $(SYSTEM_PYTHON),,$(call gb_Package_get_target,python3))
 
+ifeq ($(WITH_COREDUMPCTL),)
+gb_CppunitTest_coredumpctl_setup :=
+gb_CppunitTest_coredumpctl_run :=
+else
+gb_CppunitTest_coredumpctl_setup = \
+    export LIBO_TEST_UNIT=$$($(SYSTEMD_ESCAPE) "$1:$$(date -u +%Y%m%d%H%M%S):$$$$" | cut -b -249) &&
+gb_CppunitTest_coredumpctl_run := $(SYSTEMD_RUN) --scope --user --unit="$$LIBO_TEST_UNIT"
+endif
+
 ifneq ($(strip $(CPPUNITTRACE)),)
 ifneq ($(filter gdb,$(CPPUNITTRACE)),)
 # sneak (a) setting the LD_LIBRARY_PATH, and (b) setting malloc debug flags, into the "gdb --args" command line
@@ -126,6 +135,7 @@ else
 		$(if $(gb_CppunitTest__interactive),, \
 			$(if $(value gb_CppunitTest_postprocess), \
 				rm -fr $@.core && mkdir $@.core && cd $@.core &&)) \
+		$(call gb_CppunitTest_coredumpctl_setup,$@) \
 		( \
 		$(if $(gb_CppunitTest_localized),for l in $(WITH_LANG_LIST) ; do \
 			printf 'LO_TEST_LOCALE=%s\n' "$$l" && LO_TEST_LOCALE="$$l" ) \
@@ -142,7 +152,7 @@ else
 			PYTHONDONTWRITEBYTECODE=1) \
 		$(if $(filter gdb,$(CPPUNITTRACE)),\
 			PYTHONWARNINGS=default) \
-		$(ICECREAM_RUN) $(gb_CppunitTest_GDBTRACE) $(gb_CppunitTest_VALGRINDTOOL) $(gb_CppunitTest_RR) \
+		$(ICECREAM_RUN) $(gb_CppunitTest_coredumpctl_run) $(gb_CppunitTest_GDBTRACE) $(gb_CppunitTest_VALGRINDTOOL) $(gb_CppunitTest_RR) \
 			$(gb_CppunitTest_CPPTESTCOMMAND) \
 		$(call gb_CppunitTest_get_linktarget_target,$*) \
 		$(call gb_CppunitTest__make_args) "-env:CPPUNITTESTTARGET=$@" \
diff --git a/solenv/gbuild/JunitTest.mk b/solenv/gbuild/JunitTest.mk
index 401c8ff5cd97..66f90a34f4c8 100644
--- a/solenv/gbuild/JunitTest.mk
+++ b/solenv/gbuild/JunitTest.mk
@@ -41,7 +41,9 @@ else
         rm -rf $(call gb_JunitTest_get_userdir,$*) && \
 		mkdir -p $(call gb_JunitTest_get_userdir,$*)/user && \
 		cp $(SRCDIR)/qadevOOo/qa/registrymodifications.xcu $(call gb_JunitTest_get_userdir,$*)/user/ && \
-        ($(gb_TEST_ENV_VARS) $(ICECREAM_RUN) $(gb_JunitTest_JAVACOMMAND) \
+        $(call gb_CppunitTest_coredumpctl_setup,$@) \
+        ($(gb_TEST_ENV_VARS) $(ICECREAM_RUN) $(gb_CppunitTest_coredumpctl_run) \
+            $(gb_JunitTest_JAVACOMMAND) \
             -classpath "$(T_CP)" \
             $(DEFS) \
             org.junit.runner.JUnitCore \
diff --git a/solenv/gbuild/PythonTest.mk b/solenv/gbuild/PythonTest.mk
index 09f039771895..641ff838e4d2 100644
--- a/solenv/gbuild/PythonTest.mk
+++ b/solenv/gbuild/PythonTest.mk
@@ -48,6 +48,7 @@ else
 		$(if $(gb_CppunitTest__interactive),, \
 			$(if $(value gb_CppunitTest_postprocess), \
 				rm -fr $@.core && mkdir $@.core && cd $@.core &&)) \
+		$(call gb_CppunitTest_coredumpctl_setup,$@) \
 		{ \
 		$(if $(filter gdb,$(gb_PythonTest_GDBTRACE)),,$(gb_PythonTest_PRECOMMAND)) \
 		$(if $(G_SLICE),G_SLICE=$(G_SLICE)) \
@@ -62,7 +63,7 @@ else
 		$(gb_TEST_ENV_VARS) \
 		$(if $(filter gdb,$(CPPUNITTRACE)),\
 			PYTHONWARNINGS=default) \
-		$(ICECREAM_RUN) $(gb_PythonTest_GDBTRACE) $(gb_CppunitTest_VALGRINDTOOL) $(gb_CppunitTest_RR) \
+		$(ICECREAM_RUN) $(gb_CppunitTest_coredumpctl_run) $(gb_PythonTest_GDBTRACE) $(gb_CppunitTest_VALGRINDTOOL) $(gb_CppunitTest_RR) \
 			$(gb_PythonTest_COMMAND) \
 			$(if $(PYTHON_TEST_NAME),$(PYTHON_TEST_NAME),$(MODULES)) \
 		; } \
diff --git a/solenv/gbuild/UITest.mk b/solenv/gbuild/UITest.mk
index 3b78b1356d90..2d5867b32d2e 100644
--- a/solenv/gbuild/UITest.mk
+++ b/solenv/gbuild/UITest.mk
@@ -37,7 +37,7 @@ else
 gb_UITest_SOFFICEARG:=path:$(INSTROOT)/$(LIBO_BIN_FOLDER)/soffice
 endif
 
-gb_UITest_COMMAND = $(ICECREAM_RUN) $(gb_CppunitTest_RR) $(gb_UITest_EXECUTABLE) $(SRCDIR)/uitest/test_main.py
+gb_UITest_COMMAND = $(ICECREAM_RUN) $(gb_CppunitTest_coredumpctl_run) $(gb_CppunitTest_RR) $(gb_UITest_EXECUTABLE) $(SRCDIR)/uitest/test_main.py
 
 gb_TEST_ENV_VARS += LIBO_LANG=C
 
@@ -65,6 +65,7 @@ else
 		    rm -fr $@.core && mkdir -p $(dir $(call gb_UITest_get_target,$*))user/ && mkdir $@.core && cd $@.core && ) \
 		$(if $(gb_UITest_use_config), \
 		    cp $(gb_UITest_use_config) $(dir $(call gb_UITest_get_target,$*))user/. && ) \
+		$(call gb_CppunitTest_coredumpctl_setup,$@) \
 		($(gb_UITest_PRECOMMAND) \
 		$(if $(G_SLICE),G_SLICE=$(G_SLICE)) \
 		$(if $(GLIBCXX_FORCE_NEW),GLIBCXX_FORCE_NEW=$(GLIBCXX_FORCE_NEW)) \