build nss using their new build system (gyp/ninja-based)

This requires installed ninja, gyp is included as source. This
allows nss be built as a parallel build, unlike the old Makefile
build system.
Since gyp internally uses python, even recursively, this requires
more complicated setup in case our internal python is used. Moreover
gyp itself seems to be kind of deprecated itself and hasn't
been ported to python3 yet, so that needs patching too.
So far only easy Unix-like systems are converted, Windows I'll
do later, the more complicated systems I'll leave to whoever
has access to them.

Change-Id: I358baad7690d2aa6df44bafa9244dc7cc828fc3f
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/90115
Tested-by: Jenkins
Reviewed-by: Luboš Luňák <l.lunak@collabora.com>

9 files changed, 212 insertions, 1 deletions

diff --git a/external/nss/ExternalProject_nss.mk b/external/nss/ExternalProject_nss.mk
index acf418805772..daee76fc8fa9 100644
--- a/external/nss/ExternalProject_nss.mk
+++ b/external/nss/ExternalProject_nss.mk
@@ -34,7 +34,31 @@ $(call gb_ExternalProject_get_state_target,nss,build): $(call gb_ExternalExecuta
 	,nss)
 	$(call gb_Trace_EndRange,nss,EXTERNAL)
 
-else # OS!=WNT
+else ifneq (,$(filter FREEBSD LINUX NETBSD OPENBSD SOLARIS,$(OS))) # non-WNT gyp-based
+
+# The nss build system uses 'python', so make it find our internal python if necessary.
+nss_PYTHON := $(call gb_UnpackedTarball_get_dir,nss)/python
+nss_SETUP_PYTHON := $(call gb_UnpackedTarball_get_dir,nss)/setup-python
+
+$(call gb_ExternalProject_get_state_target,nss,build): $(call gb_ExternalExecutable_get_dependencies,python)
+	$(call gb_Trace_StartRange,nss,EXTERNAL)
+	echo "#! /bin/sh" > $(nss_PYTHON)
+	pythondir=$$($(call gb_ExternalExecutable_get_command,python) -c 'import sys; import os; sys.stdout.write(os.path.dirname(sys.executable))') \
+		&& echo PATH=\"$$pythondir:\$$PATH\" >>$(nss_PYTHON)
+	echo '$(call gb_ExternalExecutable_get_command,python)' \"$$\@\" $(if $(ICECREAM_RUN), | sed 's/$(ICECREAM_RUN)//') >> $(nss_PYTHON)
+	chmod +x $(nss_PYTHON)
+	cp $(SRCDIR)/external/nss/setup-python $(nss_SETUP_PYTHON)
+	chmod +x $(nss_SETUP_PYTHON)
+	$(call gb_ExternalProject_run,build,\
+			COMMA=$(COMMA) \
+			PATH=$(call gb_UnpackedTarball_get_dir,nss):$(call gb_UnpackedTarball_get_dir,gyp):$$PATH \
+			./build.sh -v --disable-tests --enable-libpkix \
+				$(if $(ENABLE_DBGUTIL),,--opt) \
+		&& rm -f $(call gb_UnpackedTarball_get_dir,nss)/dist/out/lib/*.a \
+	,nss)
+	$(call gb_Trace_EndRange,nss,EXTERNAL)
+
+else # platforms which need(?) the old build system (feel free to port to the new system)
 # make sure to specify NSPR_CONFIGURE_OPTS as env (before make command), so nss can append it's own defaults
 # OTOH specify e.g. CC and NSINSTALL as arguments (after make command), so they will overrule nss makefile values
 $(call gb_ExternalProject_get_state_target,nss,build): $(call gb_ExternalExecutable_get_dependencies,python)
diff --git a/external/nss/Module_nss.mk b/external/nss/Module_nss.mk
index eae9e9524480..be4b01e7ae4e 100644
--- a/external/nss/Module_nss.mk
+++ b/external/nss/Module_nss.mk
@@ -11,6 +11,7 @@ $(eval $(call gb_Module_Module,nss))
 
 $(eval $(call gb_Module_add_targets,nss,\
 	UnpackedTarball_nss \
+	UnpackedTarball_gyp \
 	ExternalPackage_nss \
 	ExternalProject_nss \
 ))
diff --git a/external/nss/UnpackedTarball_gyp.mk b/external/nss/UnpackedTarball_gyp.mk
new file mode 100644
index 000000000000..50f7ba9c52b2
--- /dev/null
+++ b/external/nss/UnpackedTarball_gyp.mk
@@ -0,0 +1,18 @@
+# -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*-
+#
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+
+$(eval $(call gb_UnpackedTarball_UnpackedTarball,gyp))
+
+$(eval $(call gb_UnpackedTarball_set_tarball,gyp,$(GYP_TARBALL)))
+
+$(eval $(call gb_UnpackedTarball_add_patches,gyp,\
+	external/nss/gyp-buildsystem.patch.0 \
+))
+
+# vim: set noet sw=4 ts=4:
diff --git a/external/nss/UnpackedTarball_nss.mk b/external/nss/UnpackedTarball_nss.mk
index 0e174879a702..4c6aa3db2030 100644
--- a/external/nss/UnpackedTarball_nss.mk
+++ b/external/nss/UnpackedTarball_nss.mk
@@ -34,6 +34,8 @@ $(eval $(call gb_UnpackedTarball_add_patches,nss,\
 		external/nss/nss-android.patch.1) \
 	external/nss/nss.bzmozilla1238154.patch \
     external/nss/macos-dlopen.patch.0 \
+	external/nss/buildsystem.patch.0 \
+	external/nss/fix-cc-detect-with-ccache.patch.0 \
 ))
 
 ifeq ($(COM_IS_CLANG),TRUE)
diff --git a/external/nss/buildsystem.patch.0 b/external/nss/buildsystem.patch.0
new file mode 100644
index 000000000000..a27408d32669
--- /dev/null
+++ b/external/nss/buildsystem.patch.0
@@ -0,0 +1,32 @@
+--- ./nss/coreconf/nspr.sh.sav	2019-11-19 20:55:30.000000000 +0100
++++ ./nss/coreconf/nspr.sh	2020-02-19 22:25:05.863542364 +0100
+@@ -28,7 +28,7 @@
+ 
+     # These NSPR options are directory-specific, so they don't need to be
+     # included in nspr_opt and changing them doesn't force a rebuild of NSPR.
+-    extra_params=(--prefix="$dist_dir"/$target)
++    extra_params=(--prefix="$dist_dir"/out --libdir="$dist_dir"/out/lib --includedir="$dist_dir"/out/include)
+     if [ "$opt_build" = 1 ]; then
+         extra_params+=(--disable-debug --enable-optimize)
+     fi
+--- ./nss/build.sh.sav	2019-11-19 20:55:30.000000000 +0100
++++ ./nss/build.sh	2020-02-19 22:28:30.976068194 +0100
+@@ -243,10 +243,10 @@
+         exit 3
+     fi
+     # These extra arguments aren't used in determining whether to rebuild.
+-    obj_dir="$dist_dir/$target"
++    obj_dir="$dist_dir/out"
+     gyp_params+=(-Dnss_dist_obj_dir="$obj_dir")
+     if [ "$no_local_nspr" = 0 ]; then
+-        set_nspr_path "$obj_dir/include/nspr:$obj_dir/lib"
++        set_nspr_path "$obj_dir/include:$obj_dir/lib"
+     fi
+ 
+     run_verbose run_scanbuild ${GYP} -f ninja "${gyp_params[@]}" "$cwd/nss.gyp"
+@@ -263,4 +263,4 @@
+     echo "Building NSS requires an installation of ninja: https://ninja-build.org/" 1>&2
+     exit 3
+ fi
+-run_scanbuild "$ninja" -C "$target_dir" "${ninja_params[@]}"
++run_scanbuild setup-python "$ninja" -C "$target_dir" "${ninja_params[@]}"
diff --git a/external/nss/fix-cc-detect-with-ccache.patch.0 b/external/nss/fix-cc-detect-with-ccache.patch.0
new file mode 100644
index 000000000000..6c3ebcfb77ed
--- /dev/null
+++ b/external/nss/fix-cc-detect-with-ccache.patch.0
@@ -0,0 +1,11 @@
+--- ./nss/coreconf/check_cc.py.sav	2020-03-20 17:58:57.275605539 +0100
++++ ./nss/coreconf/check_cc.py	2020-03-20 18:10:56.053688945 +0100
+@@ -11,7 +11,7 @@ def main():
+         cc = os.environ.get('CC', 'cc')
+         try:
+             cc_is_arg = sys.argv[1] in subprocess.check_output(
+-              [cc, '--version'], universal_newlines=True)
++              cc.split(' ') + ['--version'], universal_newlines=True)
+         except OSError:
+             # We probably just don't have CC/cc.
+             cc_is_arg = False
diff --git a/external/nss/gyp-buildsystem.patch.0 b/external/nss/gyp-buildsystem.patch.0
new file mode 100644
index 000000000000..8323d1a7a7c3
--- /dev/null
+++ b/external/nss/gyp-buildsystem.patch.0
@@ -0,0 +1,119 @@
+--- pylib/gyp/msvs_emulation.py.sav	2020-02-19 21:45:13.150161000 +0100
++++ pylib/gyp/msvs_emulation.py	2020-02-21 23:51:56.815937600 +0100
+@@ -138,7 +138,7 @@ def _FindDirectXInstallation():
+   if not dxsdk_dir:
+     # Setup params to pass to and attempt to launch reg.exe.
+     cmd = ['reg.exe', 'query', r'HKLM\Software\Microsoft\DirectX', '/s']
+-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
++    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True)
+     for line in p.communicate()[0].splitlines():
+       if 'InstallPath' in line:
+         dxsdk_dir = line.split('    ')[3] + "\\"
+@@ -321,7 +321,7 @@ class MsvsSettings(object):
+     # first level is globally for the configuration (this is what we consider
+     # "the" config at the gyp level, which will be something like 'Debug' or
+     # 'Release'), VS2015 and later only use this level
+-    if self.vs_version.short_name >= 2015:
++    if int(self.vs_version.short_name) >= 2015:
+       return config
+     # and a second target-specific configuration, which is an
+     # override for the global one. |config| is remapped here to take into
+@@ -485,7 +485,7 @@ class MsvsSettings(object):
+         prefix='/arch:')
+     cflags.extend(['/FI' + f for f in self._Setting(
+         ('VCCLCompilerTool', 'ForcedIncludeFiles'), config, default=[])])
+-    if self.vs_version.project_version >= 12.0:
++    if float(self.vs_version.project_version) >= 12.0:
+       # New flag introduced in VS2013 (project version 12.0) Forces writes to
+       # the program database (PDB) to be serialized through MSPDBSRV.EXE.
+       # https://msdn.microsoft.com/en-us/library/dn502518.aspx
+@@ -1050,7 +1050,7 @@ def GenerateEnvironmentFiles(toplevel_bu
+     args = vs.SetupScript(arch)
+     args.extend(('&&', 'set'))
+     popen = subprocess.Popen(
+-        args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
++        args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,universal_newlines=True)
+     variables, _ = popen.communicate()
+     if popen.returncode != 0:
+       raise Exception('"%s" failed with error %d' % (args, popen.returncode))
+@@ -1071,7 +1071,7 @@ def GenerateEnvironmentFiles(toplevel_bu
+     args = vs.SetupScript(arch)
+     args.extend(('&&',
+       'for', '%i', 'in', '(cl.exe)', 'do', '@echo', 'LOC:%~$PATH:i'))
+-    popen = subprocess.Popen(args, shell=True, stdout=subprocess.PIPE)
++    popen = subprocess.Popen(args, shell=True, stdout=subprocess.PIPE,universal_newlines=True)
+     output, _ = popen.communicate()
+     cl_paths[arch] = _ExtractCLPath(output)
+   return cl_paths
+--- pylib/gyp/generator/ninja.py.sav	2020-02-19 21:45:13.150161000 +0100
++++ pylib/gyp/generator/ninja.py	2020-02-22 00:42:48.090472000 +0100
+@@ -1741,7 +1741,10 @@ def CalculateGeneratorInputInfo(params):
+ def OpenOutput(path, mode='w'):
+   """Open |path| for writing, creating directories if necessary."""
+   gyp.common.EnsureDirExists(path)
+-  return open(path, mode)
++  if sys.version_info[0] < 3:
++    return open(path, mode)
++  else:
++    return open(path, mode, encoding="utf-8")
+ 
+ 
+ def CommandWithWrapper(cmd, wrappers, prog):
+--- pylib/gyp/win_tool.py.sav	2020-02-19 21:45:13.150161000 +0100
++++ pylib/gyp/win_tool.py	2020-03-04 12:46:55.098189500 +0100
+@@ -130,7 +130,7 @@ class WinTool(object):
+     # For that reason, since going through the shell doesn't seem necessary on
+     # non-Windows don't do that there.
+     link = subprocess.Popen(args, shell=sys.platform == 'win32', env=env,
+-                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
++                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT,universal_newlines=True)
+     out, _ = link.communicate()
+     for line in out.splitlines():
+       if (not line.startswith('   Creating library ') and
+@@ -197,8 +197,8 @@ class WinTool(object):
+       # and sometimes doesn't unfortunately.
+       with open(our_manifest, 'r') as our_f:
+         with open(assert_manifest, 'r') as assert_f:
+-          our_data = our_f.read().translate(None, string.whitespace)
+-          assert_data = assert_f.read().translate(None, string.whitespace)
++          our_data = our_f.read().translate(str.maketrans('','', string.whitespace))
++          assert_data = assert_f.read().translate(str.maketrans('','', string.whitespace))
+       if our_data != assert_data:
+         os.unlink(out)
+         def dump(filename):
+@@ -223,7 +223,7 @@ class WinTool(object):
+     tool)."""
+     env = self._GetEnv(arch)
+     popen = subprocess.Popen(args, shell=True, env=env,
+-                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
++                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT,universal_newlines=True)
+     out, _ = popen.communicate()
+     for line in out.splitlines():
+       if line and 'manifest authoring warning 81010002' not in line:
+@@ -255,7 +255,7 @@ class WinTool(object):
+         idl]
+     env = self._GetEnv(arch)
+     popen = subprocess.Popen(args, shell=True, env=env,
+-                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
++                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT,universal_newlines=True)
+     out, _ = popen.communicate()
+     # Filter junk out of stdout, and write filtered versions. Output we want
+     # to filter is pairs of lines that look like this:
+@@ -274,7 +274,7 @@ class WinTool(object):
+     """Filter logo banner from invocations of asm.exe."""
+     env = self._GetEnv(arch)
+     popen = subprocess.Popen(args, shell=True, env=env,
+-                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
++                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT,universal_newlines=True)
+     out, _ = popen.communicate()
+     for line in out.splitlines():
+       if (not line.startswith('Copyright (C) Microsoft Corporation') and
+@@ -289,7 +289,7 @@ class WinTool(object):
+     don't support the /nologo flag."""
+     env = self._GetEnv(arch)
+     popen = subprocess.Popen(args, shell=True, env=env,
+-                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
++                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT,universal_newlines=True)
+     out, _ = popen.communicate()
+     for line in out.splitlines():
+       if (not line.startswith('Microsoft (R) Windows (R) Resource Compiler') and
diff --git a/external/nss/setup-python b/external/nss/setup-python
new file mode 100755
index 000000000000..7f4d367c8602
--- /dev/null
+++ b/external/nss/setup-python
@@ -0,0 +1,3 @@
+#! /bin/sh
+# need to run using our python wrapper to set up python env. vars, for python calls from ninja
+python -c "import sys; import subprocess; args=sys.argv; args.pop(0); args.pop(0); sys.exit(subprocess.call(args))" -- "$@"
diff --git a/external/python3/ExternalPackage_python3.mk b/external/python3/ExternalPackage_python3.mk
index 4dfc3b196f82..9036b6db4581 100644
--- a/external/python3/ExternalPackage_python3.mk
+++ b/external/python3/ExternalPackage_python3.mk
@@ -725,6 +725,7 @@ $(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/p
 	Lib/multiprocessing/forkserver.py \
 	Lib/multiprocessing/heap.py \
 	Lib/multiprocessing/managers.py \
+	Lib/multiprocessing/pool.py \
 	Lib/multiprocessing/popen_fork.py \
 	Lib/multiprocessing/popen_forkserver.py \
 	Lib/multiprocessing/popen_spawn_posix.py \