From 084c64982ef6187292fd73e6deaa4448e0b6f3de Mon Sep 17 00:00:00 2001 From: Stephan Bergmann Date: Mon, 22 Jan 2024 11:49:45 +0100 Subject: Create MAR updates from msi rather than from archive ...as the former is more convenient for release engineering (see the discussion in the comments at "Also enable --with-package-format=archive for LibreOfficeWin64.conf"). Instead of ONLINEUPDATE_MAR_OLDARCHIVE and ONLINEUPDATE_MAR_OLDMETADATA make variables, the create-partial-info target now only needs an ONLINEUPDATE_MAR_OLDMSI make variable. TODO: There are two issues when comparing the content of msi files (extracted with msiexec /a), which the old code comparing the content of archives had tried to somewhat (but not fully) address with the metadata files that I had invented (and now reverted): For one, msiexec /a also extracts content that would normally be installed somewhere else in the system (e.g., it extracts Fonts, System, and System64 directories). Differences in those directories will cause a MAR update to create those directories in the installation directory, rather than to update the corresponding files in their actual locations. For another, optional components are not recognized as such, but their content must be added to the MAR file as add/patch-if, not as plain add/patch. To work around that, for now *all* files are added as add/patch-if, conditional on the files themselves. Thus, addition of files will cause a MAR update to miss them. (As they now exclusively operate on msi files, the create-update-info and create-partial-info targets are no longer meaningful for non-Windows platforms, so drop the non-Windows bin/update/create_full_mar_for_languages.py part.) Change-Id: Ifb55b5e7d1a201b4f50a27cb449a634b96c2e29b Reviewed-on: https://gerrit.libreoffice.org/c/core/+/162399 Tested-by: Jenkins Reviewed-by: Stephan Bergmann --- bin/update/create_full_mar.py | 30 +++--------- bin/update/create_full_mar_for_languages.py | 71 ----------------------------- bin/update/create_partial_update.py | 41 ++++------------- bin/update/tools.py | 30 ++++-------- 4 files changed, 26 insertions(+), 146 deletions(-) delete mode 100755 bin/update/create_full_mar_for_languages.py (limited to 'bin') diff --git a/bin/update/create_full_mar.py b/bin/update/create_full_mar.py index 1c2b249db3b5..d43c0804c9a7 100755 --- a/bin/update/create_full_mar.py +++ b/bin/update/create_full_mar.py @@ -2,7 +2,6 @@ import glob import os -import re import subprocess import json import argparse @@ -37,35 +36,20 @@ def main(): target_dir = update_path.get_update_dir() temp_dir = update_path.get_current_build_dir() - tar_file_glob = os.path.join(update_path.get_workdir(), "installation", product_name, "archive", "install", "*", f'{product_name}_*_archive*') - tar_files = glob.glob(tar_file_glob) - if len(tar_files) != 1: - raise Exception(f'`{tar_file_glob}` does not match exactly one file') - tar_file = tar_files[0] + msi_file_glob = os.path.join(update_path.get_workdir(), "installation", product_name, "msi", "install", "*", f'{product_name}_*.msi') + msi_files = glob.glob(msi_file_glob) + if len(msi_files) != 1: + raise Exception(f'`{msi_file_glob}` does not match exactly one file') + msi_file = msi_files[0] - uncompress_dir = uncompress_file_to_dir(tar_file, temp_dir) - - metadatafile = os.path.join( - update_path.get_workdir(), 'installation', product_name, 'archive', 'install', 'metadata') - ifsfile = os.path.join(update_path.get_mar_dir(), 'ifs') - with open(metadatafile) as meta, open(ifsfile, 'w') as ifs: - for l in meta: - m = re.fullmatch('(skip|cond) (.*)', l.rstrip()) - if m and m.group(2).startswith(f'{product_name}/'): - path = m.group(2)[len(f'{product_name}/'):] - if m.group(1) == 'skip': - os.remove(os.path.join(uncompress_dir, path)) - else: - ifs.write(f'"{path}" "{path}"\n') + uncompress_dir = uncompress_file_to_dir(msi_file, temp_dir) mar_file = make_complete_mar_name(target_dir, filename_prefix) path = os.path.join( workdir, 'UnpackedTarball/onlineupdate/tools/update-packaging/make_full_update.sh') os.putenv('MOZ_PRODUCT_VERSION', version) os.putenv('MAR_CHANNEL_ID', 'LOOnlineUpdater') - subprocess.call([ - path, convert_to_native(mar_file), convert_to_native(uncompress_dir), - convert_to_native(ifsfile)]) + subprocess.call([path, convert_to_native(mar_file), convert_to_native(uncompress_dir)]) sign_mar_file(target_dir, certificate_path, certificate_name, mar_file, filename_prefix) diff --git a/bin/update/create_full_mar_for_languages.py b/bin/update/create_full_mar_for_languages.py deleted file mode 100755 index d431ecaf6d1a..000000000000 --- a/bin/update/create_full_mar_for_languages.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -import subprocess -import json - -from tools import uncompress_file_to_dir, get_file_info - -from path import UpdaterPath -from signing import sign_mar_file - - -def make_complete_mar_name(target_dir, filename_prefix, language): - filename = filename_prefix + "_" + language + "_complete_langpack.mar" - return os.path.join(target_dir, filename) - - -def create_lang_infos(mar_file_name, language, url): - data = {'lang': language, - 'complete': get_file_info(mar_file_name, url) - } - return data - - -def main(): - if len(sys.argv) < 8: - print( - "Usage: create_full_mar_for_languages.py $PRODUCTNAME $WORKDIR $TARGETDIR $TEMPDIR $FILENAMEPREFIX $CERTIFICATEPATH $CERTIFICATENAME $BASEURL $VERSION") - sys.exit(1) - - certificate_path = sys.argv[4] - certificate_name = sys.argv[5] - base_url = sys.argv[6] - filename_prefix = sys.argv[3] - workdir = sys.argv[2] - product_name = sys.argv[1] - version = sys.argv[7] - - updater_path = UpdaterPath(workdir) - target_dir = updater_path.get_update_dir() - temp_dir = updater_path.get_language_dir() - - language_pack_dir = os.path.join(workdir, "installation", product_name + "_languagepack", "archive", "install") - language_packs = os.listdir(language_pack_dir) - lang_infos = [] - for language in language_packs: - if language == 'log': - continue - - language_dir = os.path.join(language_pack_dir, language) - language_file = os.path.join(language_dir, os.listdir(language_dir)[0]) - - directory = uncompress_file_to_dir(language_file, os.path.join(temp_dir, language)) - - mar_file_name = make_complete_mar_name(target_dir, filename_prefix, language) - - os.putenv('MOZ_PRODUCT_VERSION', version) - os.putenv('MAR_CHANNEL_ID', 'LOOnlineUpdater') - subprocess.call([os.path.join(workdir, 'UnpackedTarball/onlineupdate/tools/update-packaging/make_full_update.sh'), mar_file_name, directory]) - - sign_mar_file(target_dir, certificate_path, certificate_name, mar_file_name, filename_prefix) - - lang_infos.append(create_lang_infos(mar_file_name, language, base_url)) - - with open(os.path.join(target_dir, "complete_lang_info.json"), "w") as language_info_file: - json.dump({'languages': lang_infos}, language_info_file, indent=4) - - -if __name__ == '__main__': - main() diff --git a/bin/update/create_partial_update.py b/bin/update/create_partial_update.py index 88a4553d6a36..8c49bd9159fd 100755 --- a/bin/update/create_partial_update.py +++ b/bin/update/create_partial_update.py @@ -29,10 +29,9 @@ def main(): base_url = sys.argv[6] product_name = sys.argv[7] version = sys.argv[8] - old_archive = sys.argv[9] - old_metadata = sys.argv[10] + old_msi = sys.argv[9] - old_uncompress_dir = uncompress_file_to_dir(old_archive, updater_path.get_previous_build_dir()) + old_uncompress_dir = uncompress_file_to_dir(old_msi, updater_path.get_previous_build_dir()) versionini = os.path.join(old_uncompress_dir, 'program', 'version.ini') #TODO: Linux, macOS old_build_id = None with open(versionini) as f: @@ -44,33 +43,12 @@ def main(): if old_build_id is None: raise Exception(f'Cannot find buildid in {versionini}') - #TODO: check for problematic changes between old and new metadata - with open(old_metadata) as meta: - for l in meta: - m = re.fullmatch('skip (.*)', l.rstrip()) - if m and m.group(1).startswith(f'{product_name}/'): - path = m.group(1)[len(f'{product_name}/'):] - os.remove(os.path.join(old_uncompress_dir, path)) - - new_tar_file_glob = os.path.join(updater_path.get_workdir(), "installation", product_name, "archive", "install", "*", f'{product_name}_*_archive*') - new_tar_files = glob.glob(new_tar_file_glob) - if len(new_tar_files) != 1: - raise Exception(f'`{new_tar_file_glob}` does not match exactly one file') - new_tar_file = new_tar_files[0] - new_uncompress_dir = uncompress_file_to_dir(new_tar_file, updater_path.get_current_build_dir()) - - new_metadata = os.path.join( - updater_path.get_workdir(), 'installation', product_name, 'archive', 'install', 'metadata') - ifsfile = os.path.join(updater_path.get_mar_dir(), 'ifs') - with open(new_metadata) as meta, open(ifsfile, 'w') as ifs: - for l in meta: - m = re.fullmatch('(skip|cond) (.*)', l.rstrip()) - if m and m.group(2).startswith(f'{product_name}/'): - path = m.group(2)[len(f'{product_name}/'):] - if m.group(1) == 'skip': - os.remove(os.path.join(new_uncompress_dir, path)) - else: - ifs.write(f'"{path}" "{path}"\n') + new_msi_file_glob = os.path.join(updater_path.get_workdir(), "installation", product_name, "msi", "install", "*", f'{product_name}_*.msi') + new_msi_files = glob.glob(new_msi_file_glob) + if len(new_msi_files) != 1: + raise Exception(f'`{new_msi_file_glob}` does not match exactly one file') + new_msi_file = new_msi_files[0] + new_uncompress_dir = uncompress_file_to_dir(new_msi_file, updater_path.get_current_build_dir()) update_dir = updater_path.get_update_dir() @@ -80,8 +58,7 @@ def main(): os.putenv('MOZ_PRODUCT_VERSION', version) os.putenv('MAR_CHANNEL_ID', 'LOOnlineUpdater') subprocess.call([os.path.join(workdir, 'UnpackedTarball/onlineupdate/tools/update-packaging/make_incremental_update.sh'), convert_to_native(mar_file), - convert_to_native(old_uncompress_dir), convert_to_native(new_uncompress_dir), - convert_to_native(ifsfile)]) + convert_to_native(old_uncompress_dir), convert_to_native(new_uncompress_dir)]) sign_mar_file(update_dir, certificate_path, certificate_name, mar_file, mar_name_prefix) diff --git a/bin/update/tools.py b/bin/update/tools.py index 71fcc5cc2f69..1c5588cb5470 100644 --- a/bin/update/tools.py +++ b/bin/update/tools.py @@ -1,30 +1,20 @@ import os import hashlib -import zipfile -import tarfile +import subprocess +from path import convert_to_native -def uncompress_file_to_dir(compressed_file, uncompress_dir): - extension = os.path.splitext(compressed_file)[1] +def uncompress_file_to_dir(compressed_file, uncompress_dir): os.makedirs(uncompress_dir, exist_ok=True) - if extension == '.gz': - with tarfile.open(compressed_file) as tar: - tar.extractall(uncompress_dir) - elif extension == '.zip': - with zipfile.ZipFile(compressed_file) as zip_file: - zip_file.extractall(uncompress_dir) - - uncompress_dir = os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0]) - if " " in os.listdir(uncompress_dir)[0]: - print("replacing whitespace in directory name") - os.rename(os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0]), - os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0].replace(" ", "_"))) - else: - print("Error: unknown extension " + extension) - - return os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0]) + if subprocess.call([ + 'msiexec', '/a', convert_to_native(compressed_file).replace('/', '\\'), + '/quiet', + 'TARGETDIR=' + convert_to_native(uncompress_dir).replace('/', '\\')]) != 0: + raise Exception(f'msiexec failed') + + return uncompress_dir BUF_SIZE = 1048576 -- cgit