gbuild: add support for auto-creationa nd reuse of binary package.

many so-called 'external' libraries are built using the
UnpackedTarball/ExternalProject pattern, and their build is quite
stable... they can go month without any changes.
Yet some buildbot, that need to do full build, build them over and over
again.
This patch introduce the infrastructure to shortcut these build by using
a binary package of the build result

Change-Id: Ib0daf2a9d1a1f76802273c093bae7df8da4a90f8
Reviewed-on: https://gerrit.libreoffice.org/4764
Reviewed-by: Norbert Thiebaud <nthiebaud@gmail.com>
Tested-by: Norbert Thiebaud <nthiebaud@gmail.com>

5 files changed, 256 insertions, 15 deletions

diff --git a/config_host.mk.in b/config_host.mk.in
index 3b9e1a2d3e9e..e848c27b8760 100644
--- a/config_host.mk.in
+++ b/config_host.mk.in
@@ -532,6 +532,7 @@ export TYPO_EXTENSION_PACK=@TYPO_EXTENSION_PACK@
 export UNOWINREG_DLL=@UNOWINREG_DLL@
 export UPD=@UPD@
 export URELIBS=@URELIBS@
+export USE_LIBRARY_BIN_TAR=@USE_LIBRARY_BIN_TAR@
 export USE_XINERAMA=@USE_XINERAMA@
 export use_shl_version=@use_shl_version@
 export UUIDGEN=@UUIDGEN@
diff --git a/configure.ac b/configure.ac
index a5180a1683c5..5894618ab9da 100644
--- a/configure.ac
+++ b/configure.ac
@@ -256,7 +256,7 @@ if test -n "$with_android_ndk"; then
     if test $host_cpu = arm; then
         ANDROID_APP_ABI=armeabi-v7a
         if test -n "$ANDROID_USING_CLANG"; then
-            ANDROIDCFLAGS="-gcc-toolchain $ANDROID_BINUTILS_PREBUILT_ROOT" 
+            ANDROIDCFLAGS="-gcc-toolchain $ANDROID_BINUTILS_PREBUILT_ROOT"
             ANDROIDCFLAGS="$ANDROIDCFLAGS -target armv7-none-linux-androideabi"
             ANDROIDCFLAGS="$ANDROIDCFLAGS -no-canonical-prefixes"
         else
@@ -1204,6 +1204,22 @@ AC_ARG_ENABLE(openssl,
          use only if you are hacking on it.]),
 ,enable_openssl=yes)
 
+AC_ARG_ENABLE(library-bin-tar,
+    AS_HELP_STRING([--enable-library-bin-tar],
+        [Enable the building and reused of tarball of binary build for some 'external' libraries
+        Some libraries can save their build result in a tarball
+        stored in TARFILE_LOCATION. that binary tarball is
+        uniquely identified by the source tarbal,
+        the content of the config_host.mk file and the content
+        of the top-level directory in core for that library
+        If this option is enabled, then if such a tarfile exist, it will be untarred
+        isntead of the source tarfile, and the build step will be skipped for that
+        library.
+        If a proper tarfile does not exist, then the normal source-based
+        build is done for taht library and a proper binary tarfile is created
+        for the next time.]),
+)
+
 dnl ===================================================================
 dnl Optional Packages (--with/without-)
 dnl ===================================================================
@@ -1813,26 +1829,26 @@ AC_ARG_WITH(branding,
     AS_HELP_STRING([--with-branding],
         [Use given path to retrieve branding images set.])
     [
-                          Search for intro.png about.svg flat_logo.svg 
-                          backing_left.png backing_right.png 
-                          backing_rtl_left.png backing_rtl_right.png 
+                          Search for intro.png about.svg flat_logo.svg
+                          backing_left.png backing_right.png
+                          backing_rtl_left.png backing_rtl_right.png
                           backing_space.png. If any is missing, default
                           ones will be used instead.
 
-                          Search also progress.conf for progress 
+                          Search also progress.conf for progress
                           settings on intro screen :
 
-                          PROGRESSBARCOLOR="255,255,255" Set color of 
+                          PROGRESSBARCOLOR="255,255,255" Set color of
                           progress bar. Comma separated RGB decimal values.
-                          PROGRESSSIZE="407,6" Set size of progress bar. 
+                          PROGRESSSIZE="407,6" Set size of progress bar.
                           Comma separated decimal values (width, height).
                           PROGRESSPOSITION="61,317" Set position of progress
                           bar from left,top. Comma separated decimal values.
                           PROGRESSFRAMECOLOR="20,136,3" Set color of progress
                           bar frame. Comma separated RGB decimal values.
-                          PROGRESSTEXTCOLOR="255,255,255" Set color of progress 
+                          PROGRESSTEXTCOLOR="255,255,255" Set color of progress
                           bar text. Comma separated RGB decimal values.
-                          PROGRESSTEXTBASELINE="287" Set vertical position of 
+                          PROGRESSTEXTBASELINE="287" Set vertical position of
                           progress bar text from top. Decimal value.
 
                           Default values will be used if not found.
@@ -1935,6 +1951,17 @@ AC_ARG_WITH(all-tarballs,
         [Download all external tarballs unconditionally]))
 
 dnl ===================================================================
+dnl Do we want to use pre-build binary tarball for recompile
+dnl ===================================================================
+
+if test "$enable_library_bin_tar" = "yes" ; then
+   USE_LIBRARY_BIN_TAR=YES
+else
+   USE_LIBRARY_BIN_TAR=NO
+fi
+AC_SUBST(USE_LIBRARY_BIN_TAR)
+
+dnl ===================================================================
 dnl Test whether build target is Release Build
 dnl ===================================================================
 AC_MSG_CHECKING([whether build target is Release Build])
@@ -3922,7 +3949,7 @@ mingw*)
         AC_MSG_ERROR([Unsupported host_cpu $host_cpu for host_os $host_os])
         ;;
     esac
-    
+
     SOLARLIB="-L${SOLARVER}/${OUTPATH}${PROEXT}/lib"
     ;;
 
diff --git a/solenv/bin/bin_library_info.sh b/solenv/bin/bin_library_info.sh
new file mode 100755
index 000000000000..29d8a4bf4dff
--- /dev/null
+++ b/solenv/bin/bin_library_info.sh
@@ -0,0 +1,184 @@
+#!/usr/bin/env bash
+#
+#    Copyright (C) 2013 Norbert Thiebaud
+#    License: GPLv3
+#
+
+do_help()
+{
+cat <<EOF
+bin_library_info.sh is a tool that create a unique filename for a binary tar file that
+contain the build of the given source tarfile. the unicity is based on the source tarfile which contains
+a md5 already and the calculated sha1 of config_host_.mk and of the tree object associated with the top_level_module
+in git.
+
+syntax: bin_library_info.sh -m|--module <top_level_module> -l|--location <TARFILE_LOCATION> -s|--srcdir <SRCDIR> -b <BUILDDIR> -r|--tarfile <LIBRARY_TARFILE> [ -m|--mode verify|name ]
+
+the default mode is 'name' which just print the assocaited binary tarfile name.
+in 'verify' mode the programe print the name if the assocaited binary tarfile exist
+and print nothing and return an error code if the file does not exist
+
+Note: --location --builddir and --srcdir are optional if they are already in the env in the form of TARFILE_LOCATION and BUILDDIR SRCDIR respectively
+EOF
+
+exit 0;
+}
+
+die()
+{
+    [ $V ] && echo "Error:" "$@"
+    exit -1;
+}
+
+
+get_config_sha()
+{
+    pushd ${SRCDIR?} > /dev/null
+    cat ${BUILDDIR?}/config_host.mk | git hash-object --stdin
+    popd ${SRCDIR?} > /dev/null
+}
+
+get_library_gbuild_sha()
+{
+    local module="$1"
+
+    pushd ${SRCDIR?} > /dev/null
+    git ls-tree HEAD | grep "\t${module?}$" | cut -f 1 | cut -d " " -f 3
+    popd ${SRCDIR?} > /dev/null
+}
+
+
+determine_binary_package_name()
+{
+    local module="$1"
+    local tarball="$2"
+    local csha=""
+    local gsha=""
+    local binfile=""
+
+    csha=$(get_config_sha)
+    gsha=$(get_library_gbuild_sha "${module?}")
+    if [ -n "${csha?}" -a -n "${gsha}" ] ; then
+        binfile="${csha?}_${gsha?}_${tarball?}.${INPATH?}.tar.gz"
+    fi
+    echo "${binfile}"
+
+}
+
+MODULE=""
+SOURCE_TARFILE=""
+MODE="name"
+V=1
+
+while [ "${1}" != "" ]; do
+    parm=${1%%=*}
+    arg=${1#*=}
+    has_arg=
+    if [ "${1}" != "${parm?}" ] ; then
+        has_arg=1
+    else
+        arg=""
+    fi
+
+    case "${parm}" in
+        -h|--help)  # display help
+            do_help
+            exit
+            ;;
+        -b|--builddir)
+            if [ -z "${has_arg}" ] ; then
+                shift;
+                arg="$1"
+            fi
+            BUILDDIR="${arg}"
+            ;;
+        -o|--module)
+            if [ -z "${has_arg}" ] ; then
+                shift;
+                arg="$1"
+            fi
+            MODULE="${arg}"
+            ;;
+
+        -l|--location)
+            if [ -z "${has_arg}" ] ; then
+                shift;
+                arg="$1"
+            fi
+            TARFILE_LOCATION="${arg}"
+            ;;
+        -m|--mode)
+            # test if the binary package exist
+            if [ -z "${has_arg}" ] ; then
+                shift;
+                arg="$1"
+            fi
+            MODE="$arg"
+            ;;
+        -p|--platform)
+            # test if the binary package exist
+            if [ -z "${has_arg}" ] ; then
+                shift;
+                arg="$1"
+            fi
+            INPATH="$arg"
+            ;;
+        -q)
+            V=0
+            ;;
+        -s|--srcdir)  # do not override the local autogen.lastrun if present
+            if [ -z "${has_arg}" ] ; then
+                shift;
+                arg="$1"
+            fi
+            SRCDIR="${arg}"
+            ;;
+
+        -t|--tarfile)
+            if [ -z "${has_arg}" ] ; then
+                shift;
+                arg="$1"
+            fi
+            SOURCE_TARFILE="${arg}"
+            ;;
+        -*)
+            die "Invalid option $1"
+            ;;
+        *)
+            die "Invalid argument $1"
+            ;;
+    esac
+    shift
+done
+
+if [ -z "${MODULE?}" ] ; then
+    die "Missing --module"
+fi
+if [ -z "${TARFILE_LOCATION}" ] ; then
+    die "Missing --location"
+fi
+if [ -z "${SOURCE_TARFILE}" ] ; then
+    die "Missing --tarfile"
+fi
+if [ -z "${SRCDIR}" ] ; then
+    die "Missing --srcdir"
+fi
+
+
+BINARY_TARFILE="$(determine_binary_package_name ${MODULE?} ${SOURCE_TARFILE?})"
+
+if [ -z "${BINARY_TARFILE}" ] ; then
+    exit 2
+fi
+
+if [ "${MODE?}" = "verify" ] ; then
+    if [ -f "${TARFILE_LOCATION?}/${BINARY_TARFILE?}" ] ; then
+        echo "${BINARY_TARFILE?}"
+    else
+        exit 1
+    fi
+else
+    echo "${BINARY_TARFILE?}"
+fi
+
+exit 0
diff --git a/solenv/gbuild/ExternalProject.mk b/solenv/gbuild/ExternalProject.mk
index cd61cd5cf602..9b5466ba86b2 100644
--- a/solenv/gbuild/ExternalProject.mk
+++ b/solenv/gbuild/ExternalProject.mk
@@ -194,9 +194,12 @@ endef
 #
 
 define gb_ExternalProject_run
+$(if $(findstring YES,$(UNPACKED_IS_BIN_TARBALL)),\
+	touch $@,
 $(call gb_Helper_print_on_error,cd $(EXTERNAL_WORKDIR)/$(3) && \
 	$(if $(WRAPPERS),export $(WRAPPERS) &&) \
 	$(2) && touch $@,$(EXTERNAL_WORKDIR)/$(if $(3),$(3)/,)$(if $(4),$(4),$(1).log))
+)
 endef
 
 # vim: set noet sw=4 ts=4:
diff --git a/solenv/gbuild/UnpackedTarball.mk b/solenv/gbuild/UnpackedTarball.mk
index d75cc23e0c72..fdebcb97f576 100644
--- a/solenv/gbuild/UnpackedTarball.mk
+++ b/solenv/gbuild/UnpackedTarball.mk
@@ -130,7 +130,8 @@ define gb_UnpackedTarball__command
 $(call gb_Output_announce,$(2),$(true),PAT,2)
 $(call gb_Helper_abbreviate_dirs,\
 	( \
-		cd $(3) && \
+		cd $(3) \
+		$(if $(UNPACKED_IS_BIN_TARBALL),,&& \
 		$(if $(UNPACKED_PRE_ACTION),\
 			$(UNPACKED_PRE_ACTION) && \
 		) \
@@ -156,6 +157,7 @@ $(call gb_Helper_abbreviate_dirs,\
 			cp -r $(call gb_UnpackedTarball_get_dir,$(2)) $(call gb_UnpackedTarball_get_pristine_dir,$(2)) && \
 		) \
 		touch $(1) \
+		)\
 	) || \
 	( \
 		touch $(call gb_UnpackedTarball_get_preparation_target,$(2)) && \
@@ -230,14 +232,38 @@ $(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_FIX_EOL += $(addprefix $(c
 
 endef
 
+
+# Internal version of set_tarbal, mostly to avoid repeated invocation of $(shel
+define gb_UnpackedTarball_set_tarball_internal
+$(call gb_UnpackedTarget_UnpackedTarget,$(2),$(call gb_UnpackedTarball_get_dir,$(1)),$(3),$(4))
+$(call gb_UnpackedTarball_get_target,$(1)) : $(call gb_UnpackedTarget_get_target,$(2))
+$(call gb_UnpackedTarball_get_clean_target,$(1)) : $(call gb_UnpackedTarget_get_clean_target,$(2))
+$(call gb_UnpackedTarget_get_target,$(2)) : $(call gb_UnpackedTarball_get_preparation_target,$(1))
+$(if $(findstring in,$(5)),
+$(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_IS_BIN_TARBALL := YES
+$(call gb_ExternalProject_get_state_target,$(1),%) : UNPACKED_IS_BIN_TARBALL := YES)
+$(if $(findstring out,$(5)),$(call gb_Module_get_target,$(4)) : $(gb_UnpackedTarget_TARFILE_LOCATION)/$(6)
+$(gb_UnpackedTarget_TARFILE_LOCATION)/$(6) : $(call gb_Module_get_almost_target,$(4))
+	$$(call gb_Output_announce,$(6),$(true),PKB,3)
+	if test ! -f "$$@" ; then cd $(call gb_UnpackedTarball_get_dir,) && $(GNUTAR) -czf "$$@" $(1)/ || $(GNUTAR) -czf "$$@" $(1)/ ; else touch "$$@" ; fi)
+
+endef
+
 # Set tarball name
 #
 # gb_UnpackedTarball_set_tarball unpacked tarball-name
 define gb_UnpackedTarball_set_tarball
-$(call gb_UnpackedTarget_UnpackedTarget,$(2),$(call gb_UnpackedTarball_get_dir,$(1)),$(3))
-$(call gb_UnpackedTarball_get_target,$(1)) : $(call gb_UnpackedTarget_get_target,$(2))
-$(call gb_UnpackedTarball_get_clean_target,$(1)) : $(call gb_UnpackedTarget_get_clean_target,$(2))
-$(call gb_UnpackedTarget_get_target,$(2)) : $(call gb_UnpackedTarball_get_preparation_target,$(1))
+$(if $(findstring YES,$(USE_LIBRARY_BIN_TAR)),
+$(if $(4),
+$(if $(shell "$(SRCDIR)/solenv/bin/bin_library_info.sh" -l "$(gb_UnpackedTarget_TARFILE_LOCATION)" -o "$(4)" -b "$(BUILDDIR)" -s "$(SRCDIR)" -t "$(2)" -m verify -p "$(INPATH)"),
+$(call gb_UnpackedTarball_set_tarball_internal,$(1),$(shell "$(SRCDIR)/solenv/bin/bin_library_info.sh" -l "$(gb_UnpackedTarget_TARFILE_LOCATION)" -o "$(4)" -b "$(BUILDDIR)" -s "$(SRCDIR)" -t "$(2)" -m verify -p "$(INPATH)"),$(3),$(4),in),\
+$(call gb_UnpackedTarball_set_tarball_internal,$(1),$(2),$(3),$(4),out,$(shell "$(SRCDIR)/solenv/bin/bin_library_info.sh" -l "$(gb_UnpackedTarget_TARFILE_LOCATION)" -o "$(4)" -b "$(BUILDDIR)" -s "$(SRCDIR)" -t "$(2)" -m name -p "$(INPATH)")))
+,
+$(call gb_UnpackedTarball_set_tarball_internal,$(1),$(2),$(3),$(4),)
+)
+,
+$(call gb_UnpackedTarball_set_tarball_internal,$(1),$(2),$(3),$(4),)
+)
 
 endef