Create MAR updates from msi rather than from archive

...as the former is more convenient for release engineering (see the discussion
in the comments at
<https://gerrit.libreoffice.org/c/core/+/162157/1#message-8d7ebbcc64a87ee8e4a073ae1a05e3b74f5a3d6a>
"Also enable --with-package-format=archive for LibreOfficeWin64.conf").

Instead of ONLINEUPDATE_MAR_OLDARCHIVE and ONLINEUPDATE_MAR_OLDMETADATA make
variables, the create-partial-info target now only needs an
ONLINEUPDATE_MAR_OLDMSI make variable.

TODO:  There are two issues when comparing the content of msi files (extracted
with msiexec /a), which the old code comparing the content of archives had tried
to somewhat (but not fully) address with the metadata files that I had invented
(and now reverted):  For one, msiexec /a also extracts content that would
normally be installed somewhere else in the system (e.g., it extracts Fonts,
System, and System64 directories).  Differences in those directories will cause
a MAR update to create those directories in the installation directory, rather
than to update the corresponding files in their actual locations.  For another,
optional components are not recognized as such, but their content must be added
to the MAR file as add/patch-if, not as plain add/patch.  To work around that,
for now *all* files are added as add/patch-if, conditional on the files
themselves.  Thus, addition of files will cause a MAR update to miss them.

(As they now exclusively operate on msi files, the create-update-info and
create-partial-info targets are no longer meaningful for non-Windows platforms,
so drop the non-Windows bin/update/create_full_mar_for_languages.py part.)

Change-Id: Ifb55b5e7d1a201b4f50a27cb449a634b96c2e29b
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/162399
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <stephan.bergmann@allotropia.de>
(cherry picked from commit 1dd40d6f488c46a0946ea773cb7a741f34bbb49f)
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/162374
Reviewed-by: Michael Stahl <michael.stahl@allotropia.de>

4 files changed, 26 insertions, 146 deletions

diff --git a/bin/update/create_full_mar.py b/bin/update/create_full_mar.py
index b4f53c48f163..1c5714e69b67 100755
--- a/bin/update/create_full_mar.py
+++ b/bin/update/create_full_mar.py
@@ -3,7 +3,6 @@
 import sys
 import glob
 import os
-import re
 import subprocess
 import json
 import argparse
@@ -38,35 +37,20 @@ def main():
     target_dir = update_path.get_update_dir()
     temp_dir = update_path.get_current_build_dir()
 
-    tar_file_glob = os.path.join(update_path.get_workdir(), "installation", product_name, "archive", "install", "*", f'{product_name}_*_archive*')
-    tar_files = glob.glob(tar_file_glob)
-    if len(tar_files) != 1:
-        raise Exception(f'`{tar_file_glob}` does not match exactly one file')
-    tar_file = tar_files[0]
+    msi_file_glob = os.path.join(update_path.get_workdir(), "installation", product_name, "msi", "install", "*", f'{product_name}_*.msi')
+    msi_files = glob.glob(msi_file_glob)
+    if len(msi_files) != 1:
+        raise Exception(f'`{msi_file_glob}` does not match exactly one file')
+    msi_file = msi_files[0]
 
-    uncompress_dir = uncompress_file_to_dir(tar_file, temp_dir)
-
-    metadatafile = os.path.join(
-        update_path.get_workdir(), 'installation', product_name, 'archive', 'install', 'metadata')
-    ifsfile = os.path.join(update_path.get_mar_dir(), 'ifs')
-    with open(metadatafile) as meta, open(ifsfile, 'w') as ifs:
-        for l in meta:
-            m = re.fullmatch('(skip|cond) (.*)', l.rstrip())
-            if m and m.group(2).startswith(f'{product_name}/'):
-                path = m.group(2)[len(f'{product_name}/'):]
-                if m.group(1) == 'skip':
-                    os.remove(os.path.join(uncompress_dir, path))
-                else:
-                    ifs.write(f'"{path}" "{path}"\n')
+    uncompress_dir = uncompress_file_to_dir(msi_file, temp_dir)
 
     mar_file = make_complete_mar_name(target_dir, filename_prefix)
     path = os.path.join(
         workdir, 'UnpackedTarball/onlineupdate/tools/update-packaging/make_full_update.sh')
     os.putenv('MOZ_PRODUCT_VERSION', version)
     os.putenv('MAR_CHANNEL_ID', 'LOOnlineUpdater')
-    subprocess.call([
-        path, convert_to_native(mar_file), convert_to_native(uncompress_dir),
-        convert_to_native(ifsfile)])
+    subprocess.call([path, convert_to_native(mar_file), convert_to_native(uncompress_dir)])
 
     sign_mar_file(target_dir, certificate_path, certificate_name, mar_file, filename_prefix)
 
diff --git a/bin/update/create_full_mar_for_languages.py b/bin/update/create_full_mar_for_languages.py
deleted file mode 100755
index d431ecaf6d1a..000000000000
--- a/bin/update/create_full_mar_for_languages.py
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-import os
-import subprocess
-import json
-
-from tools import uncompress_file_to_dir, get_file_info
-
-from path import UpdaterPath
-from signing import sign_mar_file
-
-
-def make_complete_mar_name(target_dir, filename_prefix, language):
-    filename = filename_prefix + "_" + language + "_complete_langpack.mar"
-    return os.path.join(target_dir, filename)
-
-
-def create_lang_infos(mar_file_name, language, url):
-    data = {'lang': language,
-            'complete': get_file_info(mar_file_name, url)
-            }
-    return data
-
-
-def main():
-    if len(sys.argv) < 8:
-        print(
-            "Usage: create_full_mar_for_languages.py $PRODUCTNAME $WORKDIR $TARGETDIR $TEMPDIR $FILENAMEPREFIX $CERTIFICATEPATH $CERTIFICATENAME $BASEURL $VERSION")
-        sys.exit(1)
-
-    certificate_path = sys.argv[4]
-    certificate_name = sys.argv[5]
-    base_url = sys.argv[6]
-    filename_prefix = sys.argv[3]
-    workdir = sys.argv[2]
-    product_name = sys.argv[1]
-    version = sys.argv[7]
-
-    updater_path = UpdaterPath(workdir)
-    target_dir = updater_path.get_update_dir()
-    temp_dir = updater_path.get_language_dir()
-
-    language_pack_dir = os.path.join(workdir, "installation", product_name + "_languagepack", "archive", "install")
-    language_packs = os.listdir(language_pack_dir)
-    lang_infos = []
-    for language in language_packs:
-        if language == 'log':
-            continue
-
-        language_dir = os.path.join(language_pack_dir, language)
-        language_file = os.path.join(language_dir, os.listdir(language_dir)[0])
-
-        directory = uncompress_file_to_dir(language_file, os.path.join(temp_dir, language))
-
-        mar_file_name = make_complete_mar_name(target_dir, filename_prefix, language)
-
-        os.putenv('MOZ_PRODUCT_VERSION', version)
-        os.putenv('MAR_CHANNEL_ID', 'LOOnlineUpdater')
-        subprocess.call([os.path.join(workdir, 'UnpackedTarball/onlineupdate/tools/update-packaging/make_full_update.sh'), mar_file_name, directory])
-
-        sign_mar_file(target_dir, certificate_path, certificate_name, mar_file_name, filename_prefix)
-
-        lang_infos.append(create_lang_infos(mar_file_name, language, base_url))
-
-    with open(os.path.join(target_dir, "complete_lang_info.json"), "w") as language_info_file:
-        json.dump({'languages': lang_infos}, language_info_file, indent=4)
-
-
-if __name__ == '__main__':
-    main()
diff --git a/bin/update/create_partial_update.py b/bin/update/create_partial_update.py
index 88a4553d6a36..8c49bd9159fd 100755
--- a/bin/update/create_partial_update.py
+++ b/bin/update/create_partial_update.py
@@ -29,10 +29,9 @@ def main():
     base_url = sys.argv[6]
     product_name = sys.argv[7]
     version = sys.argv[8]
-    old_archive = sys.argv[9]
-    old_metadata = sys.argv[10]
+    old_msi = sys.argv[9]
 
-    old_uncompress_dir = uncompress_file_to_dir(old_archive, updater_path.get_previous_build_dir())
+    old_uncompress_dir = uncompress_file_to_dir(old_msi, updater_path.get_previous_build_dir())
     versionini = os.path.join(old_uncompress_dir, 'program', 'version.ini') #TODO: Linux, macOS
     old_build_id = None
     with open(versionini) as f:
@@ -44,33 +43,12 @@ def main():
     if old_build_id is None:
         raise Exception(f'Cannot find buildid in {versionini}')
 
-    #TODO: check for problematic changes between old and new metadata
-    with open(old_metadata) as meta:
-        for l in meta:
-            m = re.fullmatch('skip (.*)', l.rstrip())
-            if m and m.group(1).startswith(f'{product_name}/'):
-                path = m.group(1)[len(f'{product_name}/'):]
-                os.remove(os.path.join(old_uncompress_dir, path))
-
-    new_tar_file_glob = os.path.join(updater_path.get_workdir(), "installation", product_name, "archive", "install", "*", f'{product_name}_*_archive*')
-    new_tar_files = glob.glob(new_tar_file_glob)
-    if len(new_tar_files) != 1:
-        raise Exception(f'`{new_tar_file_glob}` does not match exactly one file')
-    new_tar_file = new_tar_files[0]
-    new_uncompress_dir = uncompress_file_to_dir(new_tar_file, updater_path.get_current_build_dir())
-
-    new_metadata = os.path.join(
-        updater_path.get_workdir(), 'installation', product_name, 'archive', 'install', 'metadata')
-    ifsfile = os.path.join(updater_path.get_mar_dir(), 'ifs')
-    with open(new_metadata) as meta, open(ifsfile, 'w') as ifs:
-        for l in meta:
-            m = re.fullmatch('(skip|cond) (.*)', l.rstrip())
-            if m and m.group(2).startswith(f'{product_name}/'):
-                path = m.group(2)[len(f'{product_name}/'):]
-                if m.group(1) == 'skip':
-                    os.remove(os.path.join(new_uncompress_dir, path))
-                else:
-                    ifs.write(f'"{path}" "{path}"\n')
+    new_msi_file_glob = os.path.join(updater_path.get_workdir(), "installation", product_name, "msi", "install", "*", f'{product_name}_*.msi')
+    new_msi_files = glob.glob(new_msi_file_glob)
+    if len(new_msi_files) != 1:
+        raise Exception(f'`{new_msi_file_glob}` does not match exactly one file')
+    new_msi_file = new_msi_files[0]
+    new_uncompress_dir = uncompress_file_to_dir(new_msi_file, updater_path.get_current_build_dir())
 
     update_dir = updater_path.get_update_dir()
 
@@ -80,8 +58,7 @@ def main():
     os.putenv('MOZ_PRODUCT_VERSION', version)
     os.putenv('MAR_CHANNEL_ID', 'LOOnlineUpdater')
     subprocess.call([os.path.join(workdir, 'UnpackedTarball/onlineupdate/tools/update-packaging/make_incremental_update.sh'), convert_to_native(mar_file),
-                     convert_to_native(old_uncompress_dir), convert_to_native(new_uncompress_dir),
-                     convert_to_native(ifsfile)])
+                     convert_to_native(old_uncompress_dir), convert_to_native(new_uncompress_dir)])
 
     sign_mar_file(update_dir, certificate_path, certificate_name, mar_file, mar_name_prefix)
 
diff --git a/bin/update/tools.py b/bin/update/tools.py
index ab38d10f4b57..6bc3f7971fc8 100644
--- a/bin/update/tools.py
+++ b/bin/update/tools.py
@@ -1,30 +1,20 @@
 import os
 import hashlib
-import zipfile
-import tarfile
+import subprocess
 
+from path import convert_to_native
 
-def uncompress_file_to_dir(compressed_file, uncompress_dir):
-    extension = os.path.splitext(compressed_file)[1]
 
+def uncompress_file_to_dir(compressed_file, uncompress_dir):
     os.makedirs(uncompress_dir, exist_ok=True)
 
-    if extension == '.gz':
-        with tarfile.open(compressed_file) as tar:
-            tar.extractall(uncompress_dir)
-    elif extension == '.zip':
-        with zipfile.ZipFile(compressed_file) as zip_file:
-            zip_file.extractall(uncompress_dir)
-
-        uncompress_dir = os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0])
-        if " " in os.listdir(uncompress_dir)[0]:
-            print("replacing whitespace in directory name")
-            os.rename(os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0]),
-                      os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0].replace(" ", "_")))
-    else:
-        print("Error: unknown extension " + extension)
-
-    return os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0])
+    if subprocess.call([
+            'msiexec', '/a', convert_to_native(compressed_file).replace('/', '\\'),
+            '/quiet',
+            'TARGETDIR=' + convert_to_native(uncompress_dir).replace('/', '\\')]) != 0:
+        raise Exception(f'msiexec failed')
+
+    return uncompress_dir
 
 
 BUF_SIZE = 1048576