diff options
| -rw-r--r-- | Makefile.gbuild | 2 | ||||
| -rw-r--r-- | bin/update/common.sh | 222 | ||||
| -rwxr-xr-x | bin/update/create_full_mar.py | 9 | ||||
| -rwxr-xr-x | bin/update/make_full_update.sh | 122 | ||||
| -rwxr-xr-x | bin/update/make_incremental_update.sh | 318 | ||||
| -rw-r--r-- | bin/update/tools.py | 2 | ||||
| -rw-r--r-- | download.lst | 4 | ||||
| -rwxr-xr-x | external/onlineupdate/generate-sources.sh | 3 | ||||
| -rw-r--r-- | external/onlineupdate/lo.patch | 133 |
9 files changed, 146 insertions, 669 deletions
diff --git a/Makefile.gbuild b/Makefile.gbuild index 2dfbce13cba0..d86a0cd9ce69 100644 --- a/Makefile.gbuild +++ b/Makefile.gbuild @@ -41,7 +41,7 @@ create-update-info: rm -rf $(MAR_DIR) || true rm -rf $(UPDATE_DIR) || true mkdir -p $(MAR_DIR)/language - MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_full_mar.py "$(PRODUCTNAME)" "$(WORKDIR)" "$(MAR_NAME_PREFIX)" "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)" + MAR=$(INSTDIR)/program/mar $(if $(filter WNT,$(OS)),$(shell cygpath -u $(SRCDIR)/bin/update/create_full_mar.py),$(SRCDIR)/bin/update/create_full_mar.py) "$(PRODUCTNAME)" "$(WORKDIR)" "$(MAR_NAME_PREFIX)" "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)" '$(LIBO_VERSION_MAJOR).$(LIBO_VERSION_MINOR).$(LIBO_VERSION_MICRO).$(LIBO_VERSION_PATCH)' $(if $(filter WNT,$(OS)),, \ MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_full_mar_for_languages.py "$(PRODUCTNAME)" "$(WORKDIR)" "$(MAR_NAME_PREFIX)" "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)" \ ) diff --git a/bin/update/common.sh b/bin/update/common.sh deleted file mode 100644 index dcdbea8bb815..000000000000 --- a/bin/update/common.sh +++ /dev/null @@ -1,222 +0,0 @@ -#!/usr/bin/env bash -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# -# Code shared by update packaging scripts. -# Author: Darin Fisher -# - -# ----------------------------------------------------------------------------- -# By default just assume that these tools exist on our path -MAR=${MAR:-mar} -BZIP2=${BZIP2:-bzip2} -MBSDIFF=${MBSDIFF:-mbsdiff} - -# ----------------------------------------------------------------------------- -# Helper routines - -notice() { - echo "$*" 1>&2 -} - -get_file_size() { - info=($(ls -ln "$1")) - echo ${info[4]} -} - -check_externals() { - - # check whether we can call the mar executable - "$MAR" --version > /dev/null 2>&1 - if [ $? != 0 ]; then - notice "Could not find a valid mar executable in the path or in the MAR environment variable" - exit 1 - fi - - # check whether we can access the bzip2 executable - "$BZIP2" --help > /dev/null 2>&1 - if [ $? != 0 ]; then - notice "Could not find a valid bzip2 executable in the PATH or in the BZIP2 environment variable" - exit 1 - fi -} - -copy_perm() { - reference="$1" - target="$2" - - if [ -x "$reference" ]; then - chmod 0755 "$target" - else - chmod 0644 "$target" - fi -} - -make_add_instruction() { - f="$1" - filev2="$2" - # The third param will be an empty string when a file add instruction is only - # needed in the version 2 manifest. This only happens when the file has an - # add-if-not instruction in the version 3 manifest. This is due to the - # precomplete file prior to the version 3 manifest having a remove instruction - # for this file so the file is removed before applying a complete update. - filev3="$3" - - # Used to log to the console - if [ $4 ]; then - forced=" (forced)" - else - forced= - fi - - is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') - if [ $is_extension = "1" ]; then - # Use the subdirectory of the extensions folder as the file to test - # before performing this add instruction. - testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') - notice " add-if \"$testdir\" \"$f\"" - echo "add-if \"$testdir\" \"$f\"" >> $filev2 - if [ ! $filev3 = "" ]; then - echo "add-if \"$testdir\" \"$f\"" >> $filev3 - fi - else - notice " add \"$f\"$forced" - echo "add \"$f\"" >> $filev2 - if [ ! $filev3 = "" ]; then - echo "add \"$f\"" >> $filev3 - fi - fi -} - -check_for_add_if_not_update() { - add_if_not_file_chk="$1" - - if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \ - `basename $add_if_not_file_chk` = "update-settings.ini" ]; then - ## "true" *giggle* - return 0; - fi - ## 'false'... because this is bash. Oh yay! - return 1; -} - -check_for_add_to_manifestv2() { - add_if_not_file_chk="$1" - - if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then - ## "true" *giggle* - return 0; - fi - ## 'false'... because this is bash. Oh yay! - return 1; -} - -make_add_if_not_instruction() { - f="$1" - filev3="$2" - - notice " add-if-not \"$f\" \"$f\"" - echo "add-if-not \"$f\" \"$f\"" >> $filev3 -} - -make_patch_instruction() { - f="$1" - filev2="$2" - filev3="$3" - - is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') - if [ $is_extension = "1" ]; then - # Use the subdirectory of the extensions folder as the file to test - # before performing this add instruction. - testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') - notice " patch-if \"$testdir\" \"$f.patch\" \"$f\"" - echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2 - echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3 - else - notice " patch \"$f.patch\" \"$f\"" - echo "patch \"$f.patch\" \"$f\"" >> $filev2 - echo "patch \"$f.patch\" \"$f\"" >> $filev3 - fi -} - -append_remove_instructions() { - dir="$1" - filev2="$2" - filev3="$3" - - if [ -f "$dir/removed-files" ]; then - listfile="$dir/removed-files" - elif [ -f "$dir/Contents/Resources/removed-files" ]; then - listfile="$dir/Contents/Resources/removed-files" - fi - if [ -n "$listfile" ]; then - # Map spaces to pipes so that we correctly handle filenames with spaces. - files=($(cat "$listfile" | tr " " "|" | sort -r)) - num_files=${#files[*]} - for ((i=0; $i<$num_files; i=$i+1)); do - # Map pipes back to whitespace and remove carriage returns - f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r') - # Trim whitespace - f=$(echo $f) - # Exclude blank lines. - if [ -n "$f" ]; then - # Exclude comments - if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then - if [ $(echo "$f" | grep -c '\/$') = 1 ]; then - notice " rmdir \"$f\"" - echo "rmdir \"$f\"" >> $filev2 - echo "rmdir \"$f\"" >> $filev3 - elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then - # Remove the * - f=$(echo "$f" | sed -e 's:\*$::') - notice " rmrfdir \"$f\"" - echo "rmrfdir \"$f\"" >> $filev2 - echo "rmrfdir \"$f\"" >> $filev3 - else - notice " remove \"$f\"" - echo "remove \"$f\"" >> $filev2 - echo "remove \"$f\"" >> $filev3 - fi - fi - fi - done - fi -} - -# List all files in the current directory, stripping leading "./" -# Pass a variable name and it will be filled as an array. -list_files() { - count=0 - - find . -type f \ - ! -name "update.manifest" \ - ! -name "updatev2.manifest" \ - ! -name "updatev3.manifest" \ - ! -name "temp-dirlist" \ - ! -name "temp-filelist" \ - | sed 's/\.\/\(.*\)/\1/' \ - | sort -r > "temp-filelist" - while read file; do - eval "${1}[$count]=\"$file\"" - (( count++ )) - done < "temp-filelist" - rm "temp-filelist" -} - -# List all directories in the current directory, stripping leading "./" -list_dirs() { - count=0 - - find . -type d \ - ! -name "." \ - ! -name ".." \ - | sed 's/\.\/\(.*\)/\1/' \ - | sort -r > "temp-dirlist" - while read dir; do - eval "${1}[$count]=\"$dir\"" - (( count++ )) - done < "temp-dirlist" - rm "temp-dirlist" -} diff --git a/bin/update/create_full_mar.py b/bin/update/create_full_mar.py index f3ccc972afce..81442d337f40 100755 --- a/bin/update/create_full_mar.py +++ b/bin/update/create_full_mar.py @@ -9,8 +9,6 @@ from tools import uncompress_file_to_dir, get_file_info, make_complete_mar_name from signing import sign_mar_file from path import UpdaterPath, convert_to_unix, convert_to_native -current_dir_path = os.path.dirname(os.path.realpath(convert_to_unix(__file__))) - def main(): parser = argparse.ArgumentParser() @@ -20,6 +18,7 @@ def main(): parser.add_argument('certificate_path') parser.add_argument('certificate_name') parser.add_argument('base_url') + parser.add_argument('version') args = parser.parse_args() certificate_path = args.certificate_path @@ -28,6 +27,7 @@ def main(): filename_prefix = args.filename_prefix workdir = args.workdir product_name = args.product_name + version = args.version update_path = UpdaterPath(workdir) update_path.ensure_dir_exist() @@ -41,7 +41,10 @@ def main(): uncompress_dir = uncompress_file_to_dir(tar_file, temp_dir) mar_file = make_complete_mar_name(target_dir, filename_prefix) - path = os.path.join(current_dir_path, 'make_full_update.sh') + path = os.path.join( + workdir, 'UnpackedTarball/onlineupdate/tools/update-packaging/make_full_update.sh') + os.putenv('MOZ_PRODUCT_VERSION', version) + os.putenv('MAR_CHANNEL_ID', 'LOOnlineUpdater') subprocess.call([path, convert_to_native(mar_file), convert_to_native(uncompress_dir)]) sign_mar_file(target_dir, certificate_path, certificate_name, mar_file, filename_prefix) diff --git a/bin/update/make_full_update.sh b/bin/update/make_full_update.sh deleted file mode 100755 index 4140ecae6d14..000000000000 --- a/bin/update/make_full_update.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env bash -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# -# This tool generates full update packages for the update system. -# Author: Darin Fisher -# - -. $(dirname "$0")/common.sh - -# ----------------------------------------------------------------------------- - -print_usage() { - notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY" -} - -if [ $# = 0 ]; then - print_usage - exit 1 -fi - -if [ $1 = -h ]; then - print_usage - notice "" - notice "The contents of DIRECTORY will be stored in ARCHIVE." - notice "" - notice "Options:" - notice " -h show this help text" - notice "" - exit 1 -fi - -check_externals -# ----------------------------------------------------------------------------- - -archive="$1" -targetdir="$2" -# Prevent the workdir from being inside the targetdir so it isn't included in -# the update mar. -if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then - # Remove the / - targetdir=$(echo "$targetdir" | sed -e 's:\/$::') -fi -workdir="$targetdir.work" -updatemanifestv2="$workdir/updatev2.manifest" -updatemanifestv3="$workdir/updatev3.manifest" -targetfiles="updatev2.manifest updatev3.manifest" - -mkdir -p "$workdir" -echo "updatev2.manifest" >> $workdir/files.txt -echo "updatev3.manifest" >> $workdir/files.txt - -# Generate a list of all files in the target directory. -pushd "$targetdir" -if test $? -ne 0 ; then - exit 1 -fi - -# if [ ! -f "precomplete" ]; then -# if [ ! -f "Contents/Resources/precomplete" ]; then -# notice "precomplete file is missing!" -# exit 1 -# fi -# fi - -list_files files - -popd - -# Add the type of update to the beginning of the update manifests. -> $updatemanifestv2 -> $updatemanifestv3 -notice "" -notice "Adding type instruction to update manifests" -notice " type complete" -echo "type \"complete\"" >> $updatemanifestv2 -echo "type \"complete\"" >> $updatemanifestv3 - -notice "" -notice "Adding file add instructions to update manifests" -num_files=${#files[*]} - -for ((i=0; $i<$num_files; i=$i+1)); do - f="${files[$i]}" - - if check_for_add_if_not_update "$f"; then - make_add_if_not_instruction "$f" "$updatemanifestv3" - if check_for_add_to_manifestv2 "$f"; then - make_add_instruction "$f" "$updatemanifestv2" "" 1 - fi - else - make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" - fi - - dir=$(dirname "$f") - mkdir -p "$workdir/$dir" - $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f" - copy_perm "$targetdir/$f" "$workdir/$f" - - targetfiles="$targetfiles \"$f\"" - echo $f >> $workdir/files.txt -done - -# Append remove instructions for any dead files. -notice "" -notice "Adding file and directory remove instructions from file 'removed-files'" -append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3" - -$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" -$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" - -eval "$MAR -C \"$workdir\" -c output.mar -f $workdir/files.txt" -mv -f "$workdir/output.mar" "$archive" - -# cleanup -rm -fr "$workdir" - -notice "" -notice "Finished" -notice "" diff --git a/bin/update/make_incremental_update.sh b/bin/update/make_incremental_update.sh deleted file mode 100755 index 31bddabdb082..000000000000 --- a/bin/update/make_incremental_update.sh +++ /dev/null @@ -1,318 +0,0 @@ -#!/usr/bin/env bash -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# -# This tool generates incremental update packages for the update system. -# Author: Darin Fisher -# - -. $(dirname "$0")/common.sh - -# ----------------------------------------------------------------------------- - -print_usage() { - notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR" - notice "" - notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE." - notice "" - notice "Options:" - notice " -h show this help text" - notice " -f clobber this file in the installation" - notice " Must be a path to a file to clobber in the partial update." - notice "" -} - -check_for_forced_update() { - force_list="$1" - forced_file_chk="$2" - - local f - - if [ "$forced_file_chk" = "precomplete" ]; then - ## "true" *giggle* - return 0; - fi - - if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then - ## "true" *giggle* - return 0; - fi - - if [ "$forced_file_chk" = "removed-files" ]; then - ## "true" *giggle* - return 0; - fi - - if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then - ## "true" *giggle* - return 0; - fi - - if [ "${forced_file_chk##*.}" = "chk" ]; then - ## "true" *giggle* - return 0; - fi - - for f in $force_list; do - #echo comparing $forced_file_chk to $f - if [ "$forced_file_chk" = "$f" ]; then - ## "true" *giggle* - return 0; - fi - done - ## 'false'... because this is bash. Oh yay! - return 1; -} - -if [ $# = 0 ]; then - print_usage - exit 1 -fi - -requested_forced_updates='Contents/MacOS/firefox' - -while getopts "hf:" flag -do - case "$flag" in - h) print_usage; exit 0 - ;; - f) requested_forced_updates="$requested_forced_updates $OPTARG" - ;; - ?) print_usage; exit 1 - ;; - esac -done - -# ----------------------------------------------------------------------------- - -let arg_start=$OPTIND-1 -shift $arg_start - -archive="$1" -olddir="$2" -newdir="$3" -# Prevent the workdir from being inside the targetdir so it isn't included in -# the update mar. -if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then - # Remove the / - newdir=$(echo "$newdir" | sed -e 's:\/$::') -fi -workdir="$newdir.work" -updatemanifestv2="$workdir/updatev2.manifest" -updatemanifestv3="$workdir/updatev3.manifest" - -mkdir -p "$workdir" -echo "updatev2.manifest" >> $workdir/files.txt -echo "updatev3.manifest" >> $workdir/files.txt - -# Generate a list of all files in the target directory. -pushd "$olddir" -if test $? -ne 0 ; then - exit 1 -fi - -list_files oldfiles -list_dirs olddirs - -popd - -pushd "$newdir" -if test $? -ne 0 ; then - exit 1 -fi - -# if [ ! -f "precomplete" ]; then -# if [ ! -f "Contents/Resources/precomplete" ]; then -# notice "precomplete file is missing!" -# exit 1 -# fi -# fi - -list_dirs newdirs -list_files newfiles - -popd - -# Add the type of update to the beginning of the update manifests. -notice "" -notice "Adding type instruction to update manifests" -> $updatemanifestv2 -> $updatemanifestv3 -notice " type partial" -echo "type \"partial\"" >> $updatemanifestv2 -echo "type \"partial\"" >> $updatemanifestv3 - -notice "" -notice "Adding file patch and add instructions to update manifests" - -num_oldfiles=${#oldfiles[*]} -remove_array= -num_removes=0 - -for ((i=0; $i<$num_oldfiles; i=$i+1)); do - f="${oldfiles[$i]}" - - # If this file exists in the new directory as well, then check if it differs. - if [ -f "$newdir/$f" ]; then - - if check_for_add_if_not_update "$f"; then - # The full workdir may not exist yet, so create it if necessary. - mkdir -p `dirname "$workdir/$f"` - $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" - copy_perm "$newdir/$f" "$workdir/$f" - make_add_if_not_instruction "$f" "$updatemanifestv3" - echo $f >> $workdir/files.txt - continue 1 - fi - - if check_for_forced_update "$requested_forced_updates" "$f"; then - # The full workdir may not exist yet, so create it if necessary. - mkdir -p `dirname "$workdir/$f"` - $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" - copy_perm "$newdir/$f" "$workdir/$f" - make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1 - echo $f >> $workdir/files.txt - continue 1 - fi - - if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then - # Compute both the compressed binary diff and the compressed file, and - # compare the sizes. Then choose the smaller of the two to package. - dir=$(dirname "$workdir/$f") - mkdir -p "$dir" - notice "diffing \"$f\"" - # MBSDIFF_HOOK represents the communication interface with funsize and, - # if enabled, caches the intermediate patches for future use and - # compute avoidance - # - # An example of MBSDIFF_HOOK env variable could look like this: - # export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user" - # where myscript.sh has the following usage: - # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \ - # PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL - # - # Note: patches are bzipped stashed in funsize to gain more speed - - # if service is not enabled then default to old behavior - if [ -z "$MBSDIFF_HOOK" ]; then - $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" - $BZIP2 -z9 "$workdir/$f.patch" - else - # if service enabled then check patch existence for retrieval - if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then - notice "file \"$f\" found in funsize, diffing skipped" - else - # if not found already - compute it and cache it for future use - $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" - $BZIP2 -z9 "$workdir/$f.patch" - $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2" - fi - fi - $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" - copy_perm "$newdir/$f" "$workdir/$f" - patchfile="$workdir/$f.patch.bz2" - patchsize=$(get_file_size "$patchfile") - fullsize=$(get_file_size "$workdir/$f") - - if [ $patchsize -lt $fullsize ]; then - make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" - mv -f "$patchfile" "$workdir/$f.patch" - rm -f "$workdir/$f" - echo $f.patch >> $workdir/files.txt - else - make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" - rm -f "$patchfile" - echo $f >> $workdir/files.txt - fi - fi - else - # remove instructions are added after add / patch instructions for - # consistency with make_incremental_updates.py - remove_array[$num_removes]=$f - (( num_removes++ )) - fi -done - -# Newly added files -notice "" -notice "Adding file add instructions to update manifests" -num_newfiles=${#newfiles[*]} - -for ((i=0; $i<$num_newfiles; i=$i+1)); do - f="${newfiles[$i]}" - - # If we've already tested this file, then skip it - for ((j=0; $j<$num_oldfiles; j=$j+1)); do - if [ "$f" = "${oldfiles[j]}" ]; then - continue 2 - fi - done - - dir=$(dirname "$workdir/$f") - mkdir -p "$dir" - - $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" - copy_perm "$newdir/$f" "$workdir/$f" - - if check_for_add_if_not_update "$f"; then - make_add_if_not_instruction "$f" "$updatemanifestv3" - else - make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" - fi - - - echo $f >> $workdir/files.txt -done - -notice "" -notice "Adding file remove instructions to update manifests" -for ((i=0; $i<$num_removes; i=$i+1)); do - f="${remove_array[$i]}" - notice " remove \"$f\"" - echo "remove \"$f\"" >> $updatemanifestv2 - echo "remove \"$f\"" >> $updatemanifestv3 -done - -# Add remove instructions for any dead files. -notice "" -notice "Adding file and directory remove instructions from file 'removed-files'" -append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3" - -notice "" -notice "Adding directory remove instructions for directories that no longer exist" -num_olddirs=${#olddirs[*]} - -for ((i=0; $i<$num_olddirs; i=$i+1)); do - f="${olddirs[$i]}" - # If this dir doesn't exist in the new directory remove it. - if [ ! -d "$newdir/$f" ]; then - notice " rmdir $f/" - echo "rmdir \"$f/\"" >> $updatemanifestv2 - echo "rmdir \"$f/\"" >> $updatemanifestv3 - fi -done - -$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" -$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" - -mar_command="$MAR" -if [[ -n $PRODUCT_VERSION ]] -then - mar_command="$mar_command -V $PRODUCT_VERSION" -fi -if [[ -n $CHANNEL_ID ]] -then - mar_command="$mar_command -H $CHANNEL_ID" -fi -mar_command="$mar_command -C \"$workdir\" -c output.mar -f $workdir/files.txt" -eval "$mar_command" -mv -f "$workdir/output.mar" "$archive" - -# cleanup -rm -fr "$workdir" - -notice "" -notice "Finished" -notice "" diff --git a/bin/update/tools.py b/bin/update/tools.py index 35e635cf8336..ab38d10f4b57 100644 --- a/bin/update/tools.py +++ b/bin/update/tools.py @@ -41,7 +41,7 @@ def get_hash(file_path): def get_file_info(mar_file, url): filesize = os.path.getsize(mar_file) data = {'hash': get_hash(mar_file), - 'hashFunction': 'sha512', + 'hash_function': 'sha512', 'size': filesize, 'url': url + os.path.basename(mar_file)} diff --git a/download.lst b/download.lst index bd5ded0d9793..568b404f03f3 100644 --- a/download.lst +++ b/download.lst @@ -498,8 +498,8 @@ OFFICEOTRON_JAR := 8249374c274932a21846fa7629c2aa9b-officeotron-0.7.4-master.jar # three static lines # so that git cherry-pick # will not run into conflicts -ONLINEUPDATE_SHA256SUM := cfd00948425424d7934f5f3e26c830eda9ebbf2d18e13656bd20e4f6cd99e4ac -ONLINEUPDATE_TARBALL := onlineupdate-c003be8b9727672e7d30972983b375f4c200233f.tar.xz +ONLINEUPDATE_SHA256SUM := 37206cf981e8409d048b59ac5839621ea107ff49af72beb9d7769a2f41da8d90 +ONLINEUPDATE_TARBALL := onlineupdate-c003be8b9727672e7d30972983b375f4c200233f-2.tar.xz # three static lines # so that git cherry-pick # will not run into conflicts diff --git a/external/onlineupdate/generate-sources.sh b/external/onlineupdate/generate-sources.sh index bc39603c156b..8f11faf5a65d 100755 --- a/external/onlineupdate/generate-sources.sh +++ b/external/onlineupdate/generate-sources.sh @@ -185,6 +185,9 @@ copyto toolkit/mozapps/update/common/updateutils_win.cpp toolkit/mozapps/update/ copyto toolkit/mozapps/update/common/updateutils_win.h toolkit/mozapps/update/common/updateutils_win.h copyto toolkit/mozapps/update/updater/crctable.h toolkit/mozapps/update/updater/crctable.h copyto toolkit/xre/nsWindowsRestart.cpp toolkit/xre/nsWindowsRestart.cpp +copyto tools/update-packaging/common.sh tools/update-packaging/common.sh +copyto tools/update-packaging/make_full_update.sh tools/update-packaging/make_full_update.sh +copyto tools/update-packaging/make_incremental_update.sh tools/update-packaging/make_incremental_update.sh copyto xpcom/base/nsAlgorithm.h xpcom/base/nsAlgorithm.h copyto xpcom/base/nsAutoRef.h xpcom/base/nsAutoRef.h copyto xpcom/base/nsWindowsHelpers.h xpcom/base/nsWindowsHelpers.h diff --git a/external/onlineupdate/lo.patch b/external/onlineupdate/lo.patch index a8b92830419c..0021c3a69fe7 100644 --- a/external/onlineupdate/lo.patch +++ b/external/onlineupdate/lo.patch @@ -1,3 +1,96 @@ +--- onlineupdate/source/libmar/tool/mar.c ++++ onlineupdate/source/libmar/tool/mar.c +@@ -14,7 +14,9 @@ + # include <windows.h> + # include <direct.h> + # define chdir _chdir ++# define PATH_MAX MAX_PATH + #else ++# include <limits.h> + # include <unistd.h> + #endif + +@@ -39,7 +41,7 @@ + printf("Create a MAR file:\n"); + printf( + " mar -H MARChannelID -V ProductVersion [-C workingDir] " +- "-c archive.mar [files...]\n"); ++ "-c archive.mar [files...|-f files]\n"); + + printf("Extract a MAR file:\n"); + printf(" mar [-C workingDir] -x archive.mar\n"); +@@ -244,6 +246,8 @@ + + switch (argv[1][1]) { + case 'c': { ++ int numfiles; ++ char** files; + struct ProductInformationBlock infoBlock; + if (!productVersion) { + fprintf(stderr, +@@ -256,9 +260,61 @@ + "<mar-channel-id>`).\n"); + return -1; + } ++ if (argc == 5 && !strcmp(argv[3], "-f")) { ++ FILE* in; ++ in = fopen(argv[4], "r"); ++ if (!in) { ++ fprintf(stderr, ++ "ERROR: Cannot open file `%s` for reading.\n", argv[4]); ++ return -1; ++ } ++ numfiles = 0; ++ files = malloc(sizeof(char*) * 10000); /*TODO*/ ++ if (!files) { ++ fprintf(stderr, ++ "ERROR: Cannot allocate memory"); ++ return -1; ++ } ++ for (;;) { ++ char buf[PATH_MAX + 1]; ++ size_t len; ++ if (!fgets(buf, PATH_MAX + 1, in)) { ++ if (feof(in)) { ++ break; ++ } ++ fprintf(stderr, ++ "ERROR: Cannot read from file `%s`.\n", argv[4]); ++ return -1; ++ } ++ len = strlen(buf); ++ if (len != 0 && buf[len - 1] == '\n') { ++ buf[len - 1] = '\0'; ++ } else if (!feof(in)) { ++ fprintf(stderr, ++ "ERROR: Too long line in file `%s`.\n", argv[4]); ++ return -1; ++ } ++ if (numfiles == 10000) { /*TODO*/ ++ fprintf(stderr, ++ "ERROR: Too many lines in file `%s`.\n", argv[4]); ++ return -1; ++ } ++ files[numfiles] = strdup(buf); ++ if (!files[numfiles]) { ++ fprintf(stderr, ++ "ERROR: Cannot allocate memory"); ++ return -1; ++ } ++ ++numfiles; ++ } ++ fclose(in); ++ } else { ++ numfiles = argc - 3; ++ files = argv + 3; ++ } + infoBlock.MARChannelID = MARChannelID; + infoBlock.productVersion = productVersion; +- return mar_create(argv[2], argc - 3, argv + 3, &infoBlock); ++ return mar_create(argv[2], numfiles, files, &infoBlock); + } + case 'i': { + if (!productVersion) { --- onlineupdate/source/service/serviceinstall.cpp +++ onlineupdate/source/service/serviceinstall.cpp @@ -25,7 +25,7 @@ @@ -130,3 +223,43 @@ LOG_WARN(("Install directory updater could not be determined.")); result = FALSE; } +--- tools/update-packaging/make_full_update.sh ++++ tools/update-packaging/make_full_update.sh +@@ -53,9 +53,10 @@ + fi + workdir="$targetdir.work" + updatemanifestv3="$workdir/updatev3.manifest" +-targetfiles="updatev3.manifest" + + mkdir -p "$workdir" ++ ++printf 'updatev3.manifest\n' >"$workdir/files.txt" + + # Generate a list of all files in the target directory. + pushd "$targetdir" +@@ -66,7 +67,6 @@ + if [ ! -f "precomplete" ]; then + if [ ! -f "Contents/Resources/precomplete" ]; then + notice "precomplete file is missing!" +- exit 1 + fi + fi + +@@ -99,7 +99,7 @@ + $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f" + copy_perm "$targetdir/$f" "$workdir/$f" + +- targetfiles="$targetfiles \"$f\"" ++ printf '%s\n' "$f" >>"$workdir/files.txt" + done + + # Append remove instructions for any dead files. +@@ -110,7 +110,7 @@ + $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3" + + mar_command="$mar_command -C \"$workdir\" -c output.mar" +-eval "$mar_command $targetfiles" ++eval "$mar_command -f $workdir/files.txt" + mv -f "$workdir/output.mar" "$archive" + + # cleanup |