Get make_installer calls to run in parallel

If available, use GNU parallel to run N make_installer.pl scripts in
parallel, to scale packaging LibreOffice up with the rest of gbuild.

* fallback if no GNU parallel found - run make_installer sequentially
as before
* push most of the make_installer.pl input param tweaks from gbuild
down into a shared call_installer.sh script
* call gnu parallel with generated number of
"templ:lang:prodname:ext:pkgfmt:strip-flag" tuples, one for each
package to build (empty templ for non-windows, to save on cmd line
length)
* such that we can run all those in parallel (taking into account
the build's PARALLELISM parameter)
* there's still the main package build running epm sequentially for
umpteen sub-packages from within _one_ make_installer.pl instance, but
that's much harder to parallelize from inside Perl (so we punt on
that here)

Change-Id: Ie7d3084ed60d003d587c5e64dc9fb1809b23e409
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/133957
Tested-by: Jenkins
Reviewed-by: Thorsten Behrens <thorsten.behrens@allotropia.de>

4 files changed, 94 insertions, 46 deletions

diff --git a/config_host.mk.in b/config_host.mk.in
index 4d45a94e5a2c..9a2cd1343a18 100644
--- a/config_host.mk.in
+++ b/config_host.mk.in
@@ -259,6 +259,7 @@ export GIO_LIBS=$(gb_SPACE)@GIO_LIBS@
 export GIT_REFERENCE_SRC=@GIT_REFERENCE_SRC@
 export GIT_LINK_SRC=@GIT_LINK_SRC@
 export GIT_NEEDED_SUBMODULES=@GIT_NEEDED_SUBMODULES@
+export GNUPARALLEL=@GNUPARALLEL@
 export EPOXY_CFLAGS=$(gb_SPACE)@EPOXY_CFLAGS@
 export EPOXY_LIBS=$(gb_SPACE)@EPOXY_LIBS@
 export GLM_CFLAGS=$(gb_SPACE)@GLM_CFLAGS@
diff --git a/configure.ac b/configure.ac
index 2e1e3c8b58d3..f75582843689 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2939,6 +2939,10 @@ if test -z "$COMPRESSIONTOOL"; then
 fi
 AC_SUBST(COMPRESSIONTOOL)
 
+# use GNU parallel for packaging, if available
+AC_PATH_PROG(GNUPARALLEL, parallel)
+AC_SUBST(GNUPARALLEL)
+
 AC_MSG_CHECKING([for GNU or BSD tar])
 for a in $GNUTAR gtar gnutar tar bsdtar /usr/sfw/bin/gtar; do
     $a --version 2> /dev/null | egrep "GNU|bsdtar"  2>&1 > /dev/null
diff --git a/instsetoo_native/CustomTarget_install.mk b/instsetoo_native/CustomTarget_install.mk
index 0dab73ad904c..47b2ffe1a262 100644
--- a/instsetoo_native/CustomTarget_install.mk
+++ b/instsetoo_native/CustomTarget_install.mk
@@ -62,39 +62,18 @@ export LOCAL_COMMON_OUT := $(instsetoo_OUT)
 
 instsetoo_native_WITH_LANG := en-US $(filter-out en-US,$(gb_WITH_LANG))
 
-ifeq (WNT,$(OS))
-define instsetoo_native_msitemplates
-
-TEMPLATE_DIR=$(dir $@)msi_templates \
-&& rm -rf $${TEMPLATE_DIR} \
-&& mkdir -p $${TEMPLATE_DIR}/Binary \
-&& for I in $(SRCDIR)/instsetoo_native/inc_$(1)/windows/msi_templates/*.* ; do $(GREP) -v '^#' "$$I" > $${TEMPLATE_DIR}/`basename $$I` || true ; done \
-&& $(GNUCOPY) $(SRCDIR)/instsetoo_native/inc_common/windows/msi_templates/Binary/*.* $${TEMPLATE_DIR}/Binary
-endef
-else
-instsetoo_native_msitemplates :=
-endif
-
 define instsetoo_native_install_command
-$(call instsetoo_native_msitemplates,$(1))
-$(call gb_Helper_print_on_error, \
-cd $(dir $@) \
-$(foreach pkgformat,$(5),\
-&& $(if $(filter-out archive,$(pkgformat)),ENABLE_STRIP=1) $(PERL) -w $< \
-	-f $(BUILDDIR)/instsetoo_native/util/openoffice.lst \
-	-l $(subst $(WHITESPACE),$(COMMA),$(strip $(2))) \
-	-p $(PRODUCTNAME_WITHOUT_SPACES)$(3) \
-	-u $(instsetoo_OUT) \
-	-packer $(COMPRESSIONTOOL) \
-	-buildid $(if $(filter deb0 rpm0,$(pkgformat)$(LIBO_VERSION_PATCH)),1,$(LIBO_VERSION_PATCH)) \
-	$(if $(filter WNT,$(OS)), \
-		-msitemplate $(dir $@)msi_templates \
-		-msilanguage $(dir $@)win_ulffiles \
-	) \
-	$(4) \
-	-format $(pkgformat) \
-	$(if $(verbose),-verbose,-quiet) \
-),$@.log)
+$(if $(GNUPARALLEL), \
+    $(call gb_Helper_print_on_error, \
+    cd $(dir $@) && \
+    $(GNUPARALLEL) -j $(PARALLELISM) $(SRCDIR)/solenv/bin/call_installer.sh $(if $(verbose),-verbose,-quiet) -- $(1) \
+    ,$@.log) \
+, \
+    $(call gb_Helper_print_on_error, \
+    cd $(dir $@) \
+    $(foreach curpkg,$(1),\
+    && $(SRCDIR)/solenv/bin/call_installer.sh $(if $(verbose),-verbose,-quiet) $(curpkg) \
+    ),$@.log))
 endef
 
 $(call gb_CustomTarget_get_workdir,instsetoo_native/install)/install.phony:
@@ -102,13 +81,13 @@ $(call gb_CustomTarget_get_workdir,instsetoo_native/install)/install.phony:
 	$(call gb_Trace_StartRange,$(subst $(WORKDIR)/,,$@),PRL)
 	rm -rf $(instsetoo_OUT)
 ifeq (TRUE,$(LIBO_TEST_INSTALL))
-	$(call instsetoo_native_install_command,openoffice,en-US,,,archive)
+	$(call instsetoo_native_install_command, "openoffice:en-US:::archive:nostrip")
 	unzip -q -d $(TESTINSTALLDIR) $(instsetoo_OUT)/$(PRODUCTNAME_WITHOUT_SPACES)/archive/install/en-US/LibreOffice*_archive.zip
 	mv $(TESTINSTALLDIR)/LibreOffice*_archive/LibreOffice*/* $(TESTINSTALLDIR)/
 	rmdir $(TESTINSTALLDIR)/LibreOffice*_archive/LibreOffice*
 	rmdir $(TESTINSTALLDIR)/LibreOffice*_archive
 ifeq (ODK,$(filter ODK,$(BUILD_TYPE)))
-	$(call instsetoo_native_install_command,sdkoo,en-US,_SDK,,archive)
+	$(call instsetoo_native_install_command, "sdkoo:en-US:_SDK::archive:nostrip")
 	unzip -q -d $(TESTINSTALLDIR) $(instsetoo_OUT)/$(PRODUCTNAME_WITHOUT_SPACES)_SDK/archive/install/en-US/LibreOffice*_archive_sdk.zip
 	mv $(TESTINSTALLDIR)/LibreOffice*_archive_sdk/LibreOffice*_SDK/sdk \
         $(TESTINSTALLDIR)/
@@ -116,18 +95,24 @@ ifeq (ODK,$(filter ODK,$(BUILD_TYPE)))
 	rmdir $(TESTINSTALLDIR)/LibreOffice*_archive_sdk
 endif
 else # LIBO_TEST_INSTALL
-	$(call instsetoo_native_install_command,openoffice,$(if $(filter WNT,$(OS)),$(instsetoo_native_WITH_LANG),en-US),,,$(PKGFORMAT))
-ifeq (ODK,$(filter ODK,$(BUILD_TYPE)))
-	$(call instsetoo_native_install_command,sdkoo,en-US,_SDK,,$(PKGFORMAT))
-endif
-ifeq (HELP,$(filter HELP,$(BUILD_TYPE))$(filter MACOSX,$(OS)))
-	$(foreach lang,$(gb_HELP_LANGS),\
-		$(call instsetoo_native_install_command,ooohelppack,$(lang),,-helppack,$(PKGFORMAT)))
-endif
-ifneq (WNT,$(OS))
-	$(foreach lang,$(instsetoo_native_WITH_LANG),\
-		$(call instsetoo_native_install_command,ooolangpack,$(lang),,-languagepack,$(PKGFORMAT)))
-endif
+	$(call instsetoo_native_install_command, \
+		$(foreach pkgformat,$(PKGFORMAT),\
+			$(if $(filter WNT,$(OS)), \
+				"openoffice:$(subst $(WHITESPACE),$(COMMA),$(strip $(instsetoo_native_WITH_LANG))):::$(pkgformat):$(if $(filter-out archive,$(pkgformat)),strip,nostrip)" \
+				$(if $(filter ODK,$(BUILD_TYPE)), \
+					"sdkoo:en-US:_SDK::$(pkgformat):nostrip") \
+				$(if $(filter HELP,$(BUILD_TYPE)), \
+					$(foreach lang,$(gb_HELP_LANGS), \
+						"ooohelppack:$(lang)::-helppack:$(pkgformat):nostrip" )) \
+			, \
+				":en-US:::$(pkgformat):$(if $(filter-out archive,$(pkgformat)),strip,nostrip)" \
+				$(if $(filter ODK,$(BUILD_TYPE)), \
+					":en-US:_SDK::$(pkgformat):nostrip") \
+				$(if $(and $(filter HELP,$(BUILD_TYPE)), $(filter-out MACOSX,$(OS))), \
+					$(foreach lang,$(gb_HELP_LANGS), \
+						":$(lang)::-helppack:$(pkgformat):nostrip" )) \
+				$(foreach lang,$(instsetoo_native_WITH_LANG), \
+					":$(lang)::-languagepack:$(pkgformat):nostrip" ) )))
 endif # LIBO_TEST_INSTALL
 	touch $@
 	$(call gb_Trace_EndRange,$(subst $(WORKDIR)/,,$@),PRL)
diff --git a/solenv/bin/call_installer.sh b/solenv/bin/call_installer.sh
new file mode 100755
index 000000000000..f1db809f7faf
--- /dev/null
+++ b/solenv/bin/call_installer.sh
@@ -0,0 +1,58 @@
+#!/bin/sh
+
+# unpack parameters
+VERBOSITY=$1; shift
+MSITEMPL=$(echo "$@" | cut -d ':' -f 1)
+LANG=$(echo "$@" | cut -d ':' -f 2)
+PRODNAME=$(echo "$@" | cut -d ':' -f 3)
+EXTENSION=$(echo "$@" | cut -d ':' -f 4)
+PKGFORMAT=$(echo "$@" | cut -d ':' -f 5)
+STRIP=$(echo "$@" | cut -d ':' -f 6)
+
+# need to hack buildid?
+if [ "${PKGFORMAT}${LIBO_VERSION_PATCH}" = "deb0" ] || \
+   [ "${PKGFORMAT}${LIBO_VERSION_PATCH}" = "rpm0" ] ; then
+    LIBO_VERSION_PATCH=1
+fi
+
+# switch to verbose?
+if [ "${VERBOSITY}" = "-verbose" ] ; then
+    set -x
+fi
+
+# populate MSI template dirs for Windows
+if [ -n "${MSITEMPL}" ]; then
+    TEMPLATE_DIR="${WORKDIR}/CustomTarget/instsetoo_native/install/msi_templates"
+    rm -rf "${TEMPLATE_DIR}" && \
+    mkdir -p "${TEMPLATE_DIR}/Binary" && \
+    for I in "${SRCDIR}/instsetoo_native/inc_${MSITEMPL}/windows/msi_templates/"*.* ; do \
+        "${GREP}" -v '^#' "$I" > "${TEMPLATE_DIR}/$(basename "$I")" || true ; \
+    done && \
+    "${GNUCOPY}" "${SRCDIR}/instsetoo_native/inc_common/windows/msi_templates/Binary/"*.* "${TEMPLATE_DIR}/Binary" || exit 1
+fi
+
+# add extra params for Windows
+EXTRA_PARAMS=
+if [ "${OS}" = "WNT" ] ; then
+    EXTRA_PARAMS="${EXTRA_PARAMS} -msitemplate ${WORKDIR}/CustomTarget/instsetoo_native/install/msi_templates"
+    EXTRA_PARAMS="${EXTRA_PARAMS} -msilanguage ${WORKDIR}/CustomTarget/instsetoo_native/install/win_ulffiles"
+fi
+
+# need to strip?
+if [ "${STRIP}" = "strip" ] ; then
+    export ENABLE_STRIP=1
+fi
+
+# shellcheck disable=SC2086
+# shellcheck disable=SC2154
+${PERL} -w "${SRCDIR}"/solenv/bin/make_installer.pl \
+    -f "${BUILDDIR}"/instsetoo_native/util/openoffice.lst \
+    -l "${LANG}" \
+    -p "${PRODUCTNAME_WITHOUT_SPACES}${PRODNAME}" \
+    -u "${instsetoo_OUT}" \
+    -packer "${COMPRESSIONTOOL}" \
+    -buildid "${LIBO_VERSION_PATCH}" \
+    ${EXTRA_PARAMS:+$EXTRA_PARAMS} \
+    ${EXTENSION:+"$EXTENSION"} \
+    -format "${PKGFORMAT}" \
+    "${VERBOSITY}"