diff options
Diffstat (limited to 'bin')
| -rw-r--r-- | bin/update/common.sh | 222 | ||||
| -rwxr-xr-x | bin/update/create_full_mar.py | 9 | ||||
| -rwxr-xr-x | bin/update/make_full_update.sh | 122 | ||||
| -rwxr-xr-x | bin/update/make_incremental_update.sh | 318 | ||||
| -rw-r--r-- | bin/update/tools.py | 2 |
5 files changed, 7 insertions, 666 deletions
diff --git a/bin/update/common.sh b/bin/update/common.sh deleted file mode 100644 index dcdbea8bb815..000000000000 --- a/bin/update/common.sh +++ /dev/null @@ -1,222 +0,0 @@ -#!/usr/bin/env bash -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# -# Code shared by update packaging scripts. -# Author: Darin Fisher -# - -# ----------------------------------------------------------------------------- -# By default just assume that these tools exist on our path -MAR=${MAR:-mar} -BZIP2=${BZIP2:-bzip2} -MBSDIFF=${MBSDIFF:-mbsdiff} - -# ----------------------------------------------------------------------------- -# Helper routines - -notice() { - echo "$*" 1>&2 -} - -get_file_size() { - info=($(ls -ln "$1")) - echo ${info[4]} -} - -check_externals() { - - # check whether we can call the mar executable - "$MAR" --version > /dev/null 2>&1 - if [ $? != 0 ]; then - notice "Could not find a valid mar executable in the path or in the MAR environment variable" - exit 1 - fi - - # check whether we can access the bzip2 executable - "$BZIP2" --help > /dev/null 2>&1 - if [ $? != 0 ]; then - notice "Could not find a valid bzip2 executable in the PATH or in the BZIP2 environment variable" - exit 1 - fi -} - -copy_perm() { - reference="$1" - target="$2" - - if [ -x "$reference" ]; then - chmod 0755 "$target" - else - chmod 0644 "$target" - fi -} - -make_add_instruction() { - f="$1" - filev2="$2" - # The third param will be an empty string when a file add instruction is only - # needed in the version 2 manifest. This only happens when the file has an - # add-if-not instruction in the version 3 manifest. This is due to the - # precomplete file prior to the version 3 manifest having a remove instruction - # for this file so the file is removed before applying a complete update. - filev3="$3" - - # Used to log to the console - if [ $4 ]; then - forced=" (forced)" - else - forced= - fi - - is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') - if [ $is_extension = "1" ]; then - # Use the subdirectory of the extensions folder as the file to test - # before performing this add instruction. - testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') - notice " add-if \"$testdir\" \"$f\"" - echo "add-if \"$testdir\" \"$f\"" >> $filev2 - if [ ! $filev3 = "" ]; then - echo "add-if \"$testdir\" \"$f\"" >> $filev3 - fi - else - notice " add \"$f\"$forced" - echo "add \"$f\"" >> $filev2 - if [ ! $filev3 = "" ]; then - echo "add \"$f\"" >> $filev3 - fi - fi -} - -check_for_add_if_not_update() { - add_if_not_file_chk="$1" - - if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \ - `basename $add_if_not_file_chk` = "update-settings.ini" ]; then - ## "true" *giggle* - return 0; - fi - ## 'false'... because this is bash. Oh yay! - return 1; -} - -check_for_add_to_manifestv2() { - add_if_not_file_chk="$1" - - if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then - ## "true" *giggle* - return 0; - fi - ## 'false'... because this is bash. Oh yay! - return 1; -} - -make_add_if_not_instruction() { - f="$1" - filev3="$2" - - notice " add-if-not \"$f\" \"$f\"" - echo "add-if-not \"$f\" \"$f\"" >> $filev3 -} - -make_patch_instruction() { - f="$1" - filev2="$2" - filev3="$3" - - is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') - if [ $is_extension = "1" ]; then - # Use the subdirectory of the extensions folder as the file to test - # before performing this add instruction. - testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') - notice " patch-if \"$testdir\" \"$f.patch\" \"$f\"" - echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2 - echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3 - else - notice " patch \"$f.patch\" \"$f\"" - echo "patch \"$f.patch\" \"$f\"" >> $filev2 - echo "patch \"$f.patch\" \"$f\"" >> $filev3 - fi -} - -append_remove_instructions() { - dir="$1" - filev2="$2" - filev3="$3" - - if [ -f "$dir/removed-files" ]; then - listfile="$dir/removed-files" - elif [ -f "$dir/Contents/Resources/removed-files" ]; then - listfile="$dir/Contents/Resources/removed-files" - fi - if [ -n "$listfile" ]; then - # Map spaces to pipes so that we correctly handle filenames with spaces. - files=($(cat "$listfile" | tr " " "|" | sort -r)) - num_files=${#files[*]} - for ((i=0; $i<$num_files; i=$i+1)); do - # Map pipes back to whitespace and remove carriage returns - f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r') - # Trim whitespace - f=$(echo $f) - # Exclude blank lines. - if [ -n "$f" ]; then - # Exclude comments - if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then - if [ $(echo "$f" | grep -c '\/$') = 1 ]; then - notice " rmdir \"$f\"" - echo "rmdir \"$f\"" >> $filev2 - echo "rmdir \"$f\"" >> $filev3 - elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then - # Remove the * - f=$(echo "$f" | sed -e 's:\*$::') - notice " rmrfdir \"$f\"" - echo "rmrfdir \"$f\"" >> $filev2 - echo "rmrfdir \"$f\"" >> $filev3 - else - notice " remove \"$f\"" - echo "remove \"$f\"" >> $filev2 - echo "remove \"$f\"" >> $filev3 - fi - fi - fi - done - fi -} - -# List all files in the current directory, stripping leading "./" -# Pass a variable name and it will be filled as an array. -list_files() { - count=0 - - find . -type f \ - ! -name "update.manifest" \ - ! -name "updatev2.manifest" \ - ! -name "updatev3.manifest" \ - ! -name "temp-dirlist" \ - ! -name "temp-filelist" \ - | sed 's/\.\/\(.*\)/\1/' \ - | sort -r > "temp-filelist" - while read file; do - eval "${1}[$count]=\"$file\"" - (( count++ )) - done < "temp-filelist" - rm "temp-filelist" -} - -# List all directories in the current directory, stripping leading "./" -list_dirs() { - count=0 - - find . -type d \ - ! -name "." \ - ! -name ".." \ - | sed 's/\.\/\(.*\)/\1/' \ - | sort -r > "temp-dirlist" - while read dir; do - eval "${1}[$count]=\"$dir\"" - (( count++ )) - done < "temp-dirlist" - rm "temp-dirlist" -} diff --git a/bin/update/create_full_mar.py b/bin/update/create_full_mar.py index f3ccc972afce..81442d337f40 100755 --- a/bin/update/create_full_mar.py +++ b/bin/update/create_full_mar.py @@ -9,8 +9,6 @@ from tools import uncompress_file_to_dir, get_file_info, make_complete_mar_name from signing import sign_mar_file from path import UpdaterPath, convert_to_unix, convert_to_native -current_dir_path = os.path.dirname(os.path.realpath(convert_to_unix(__file__))) - def main(): parser = argparse.ArgumentParser() @@ -20,6 +18,7 @@ def main(): parser.add_argument('certificate_path') parser.add_argument('certificate_name') parser.add_argument('base_url') + parser.add_argument('version') args = parser.parse_args() certificate_path = args.certificate_path @@ -28,6 +27,7 @@ def main(): filename_prefix = args.filename_prefix workdir = args.workdir product_name = args.product_name + version = args.version update_path = UpdaterPath(workdir) update_path.ensure_dir_exist() @@ -41,7 +41,10 @@ def main(): uncompress_dir = uncompress_file_to_dir(tar_file, temp_dir) mar_file = make_complete_mar_name(target_dir, filename_prefix) - path = os.path.join(current_dir_path, 'make_full_update.sh') + path = os.path.join( + workdir, 'UnpackedTarball/onlineupdate/tools/update-packaging/make_full_update.sh') + os.putenv('MOZ_PRODUCT_VERSION', version) + os.putenv('MAR_CHANNEL_ID', 'LOOnlineUpdater') subprocess.call([path, convert_to_native(mar_file), convert_to_native(uncompress_dir)]) sign_mar_file(target_dir, certificate_path, certificate_name, mar_file, filename_prefix) diff --git a/bin/update/make_full_update.sh b/bin/update/make_full_update.sh deleted file mode 100755 index 4140ecae6d14..000000000000 --- a/bin/update/make_full_update.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env bash -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# -# This tool generates full update packages for the update system. -# Author: Darin Fisher -# - -. $(dirname "$0")/common.sh - -# ----------------------------------------------------------------------------- - -print_usage() { - notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY" -} - -if [ $# = 0 ]; then - print_usage - exit 1 -fi - -if [ $1 = -h ]; then - print_usage - notice "" - notice "The contents of DIRECTORY will be stored in ARCHIVE." - notice "" - notice "Options:" - notice " -h show this help text" - notice "" - exit 1 -fi - -check_externals -# ----------------------------------------------------------------------------- - -archive="$1" -targetdir="$2" -# Prevent the workdir from being inside the targetdir so it isn't included in -# the update mar. -if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then - # Remove the / - targetdir=$(echo "$targetdir" | sed -e 's:\/$::') -fi -workdir="$targetdir.work" -updatemanifestv2="$workdir/updatev2.manifest" -updatemanifestv3="$workdir/updatev3.manifest" -targetfiles="updatev2.manifest updatev3.manifest" - -mkdir -p "$workdir" -echo "updatev2.manifest" >> $workdir/files.txt -echo "updatev3.manifest" >> $workdir/files.txt - -# Generate a list of all files in the target directory. -pushd "$targetdir" -if test $? -ne 0 ; then - exit 1 -fi - -# if [ ! -f "precomplete" ]; then -# if [ ! -f "Contents/Resources/precomplete" ]; then -# notice "precomplete file is missing!" -# exit 1 -# fi -# fi - -list_files files - -popd - -# Add the type of update to the beginning of the update manifests. -> $updatemanifestv2 -> $updatemanifestv3 -notice "" -notice "Adding type instruction to update manifests" -notice " type complete" -echo "type \"complete\"" >> $updatemanifestv2 -echo "type \"complete\"" >> $updatemanifestv3 - -notice "" -notice "Adding file add instructions to update manifests" -num_files=${#files[*]} - -for ((i=0; $i<$num_files; i=$i+1)); do - f="${files[$i]}" - - if check_for_add_if_not_update "$f"; then - make_add_if_not_instruction "$f" "$updatemanifestv3" - if check_for_add_to_manifestv2 "$f"; then - make_add_instruction "$f" "$updatemanifestv2" "" 1 - fi - else - make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" - fi - - dir=$(dirname "$f") - mkdir -p "$workdir/$dir" - $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f" - copy_perm "$targetdir/$f" "$workdir/$f" - - targetfiles="$targetfiles \"$f\"" - echo $f >> $workdir/files.txt -done - -# Append remove instructions for any dead files. -notice "" -notice "Adding file and directory remove instructions from file 'removed-files'" -append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3" - -$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" -$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" - -eval "$MAR -C \"$workdir\" -c output.mar -f $workdir/files.txt" -mv -f "$workdir/output.mar" "$archive" - -# cleanup -rm -fr "$workdir" - -notice "" -notice "Finished" -notice "" diff --git a/bin/update/make_incremental_update.sh b/bin/update/make_incremental_update.sh deleted file mode 100755 index 31bddabdb082..000000000000 --- a/bin/update/make_incremental_update.sh +++ /dev/null @@ -1,318 +0,0 @@ -#!/usr/bin/env bash -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# -# This tool generates incremental update packages for the update system. -# Author: Darin Fisher -# - -. $(dirname "$0")/common.sh - -# ----------------------------------------------------------------------------- - -print_usage() { - notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR" - notice "" - notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE." - notice "" - notice "Options:" - notice " -h show this help text" - notice " -f clobber this file in the installation" - notice " Must be a path to a file to clobber in the partial update." - notice "" -} - -check_for_forced_update() { - force_list="$1" - forced_file_chk="$2" - - local f - - if [ "$forced_file_chk" = "precomplete" ]; then - ## "true" *giggle* - return 0; - fi - - if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then - ## "true" *giggle* - return 0; - fi - - if [ "$forced_file_chk" = "removed-files" ]; then - ## "true" *giggle* - return 0; - fi - - if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then - ## "true" *giggle* - return 0; - fi - - if [ "${forced_file_chk##*.}" = "chk" ]; then - ## "true" *giggle* - return 0; - fi - - for f in $force_list; do - #echo comparing $forced_file_chk to $f - if [ "$forced_file_chk" = "$f" ]; then - ## "true" *giggle* - return 0; - fi - done - ## 'false'... because this is bash. Oh yay! - return 1; -} - -if [ $# = 0 ]; then - print_usage - exit 1 -fi - -requested_forced_updates='Contents/MacOS/firefox' - -while getopts "hf:" flag -do - case "$flag" in - h) print_usage; exit 0 - ;; - f) requested_forced_updates="$requested_forced_updates $OPTARG" - ;; - ?) print_usage; exit 1 - ;; - esac -done - -# ----------------------------------------------------------------------------- - -let arg_start=$OPTIND-1 -shift $arg_start - -archive="$1" -olddir="$2" -newdir="$3" -# Prevent the workdir from being inside the targetdir so it isn't included in -# the update mar. -if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then - # Remove the / - newdir=$(echo "$newdir" | sed -e 's:\/$::') -fi -workdir="$newdir.work" -updatemanifestv2="$workdir/updatev2.manifest" -updatemanifestv3="$workdir/updatev3.manifest" - -mkdir -p "$workdir" -echo "updatev2.manifest" >> $workdir/files.txt -echo "updatev3.manifest" >> $workdir/files.txt - -# Generate a list of all files in the target directory. -pushd "$olddir" -if test $? -ne 0 ; then - exit 1 -fi - -list_files oldfiles -list_dirs olddirs - -popd - -pushd "$newdir" -if test $? -ne 0 ; then - exit 1 -fi - -# if [ ! -f "precomplete" ]; then -# if [ ! -f "Contents/Resources/precomplete" ]; then -# notice "precomplete file is missing!" -# exit 1 -# fi -# fi - -list_dirs newdirs -list_files newfiles - -popd - -# Add the type of update to the beginning of the update manifests. -notice "" -notice "Adding type instruction to update manifests" -> $updatemanifestv2 -> $updatemanifestv3 -notice " type partial" -echo "type \"partial\"" >> $updatemanifestv2 -echo "type \"partial\"" >> $updatemanifestv3 - -notice "" -notice "Adding file patch and add instructions to update manifests" - -num_oldfiles=${#oldfiles[*]} -remove_array= -num_removes=0 - -for ((i=0; $i<$num_oldfiles; i=$i+1)); do - f="${oldfiles[$i]}" - - # If this file exists in the new directory as well, then check if it differs. - if [ -f "$newdir/$f" ]; then - - if check_for_add_if_not_update "$f"; then - # The full workdir may not exist yet, so create it if necessary. - mkdir -p `dirname "$workdir/$f"` - $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" - copy_perm "$newdir/$f" "$workdir/$f" - make_add_if_not_instruction "$f" "$updatemanifestv3" - echo $f >> $workdir/files.txt - continue 1 - fi - - if check_for_forced_update "$requested_forced_updates" "$f"; then - # The full workdir may not exist yet, so create it if necessary. - mkdir -p `dirname "$workdir/$f"` - $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" - copy_perm "$newdir/$f" "$workdir/$f" - make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1 - echo $f >> $workdir/files.txt - continue 1 - fi - - if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then - # Compute both the compressed binary diff and the compressed file, and - # compare the sizes. Then choose the smaller of the two to package. - dir=$(dirname "$workdir/$f") - mkdir -p "$dir" - notice "diffing \"$f\"" - # MBSDIFF_HOOK represents the communication interface with funsize and, - # if enabled, caches the intermediate patches for future use and - # compute avoidance - # - # An example of MBSDIFF_HOOK env variable could look like this: - # export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user" - # where myscript.sh has the following usage: - # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \ - # PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL - # - # Note: patches are bzipped stashed in funsize to gain more speed - - # if service is not enabled then default to old behavior - if [ -z "$MBSDIFF_HOOK" ]; then - $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" - $BZIP2 -z9 "$workdir/$f.patch" - else - # if service enabled then check patch existence for retrieval - if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then - notice "file \"$f\" found in funsize, diffing skipped" - else - # if not found already - compute it and cache it for future use - $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" - $BZIP2 -z9 "$workdir/$f.patch" - $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2" - fi - fi - $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" - copy_perm "$newdir/$f" "$workdir/$f" - patchfile="$workdir/$f.patch.bz2" - patchsize=$(get_file_size "$patchfile") - fullsize=$(get_file_size "$workdir/$f") - - if [ $patchsize -lt $fullsize ]; then - make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" - mv -f "$patchfile" "$workdir/$f.patch" - rm -f "$workdir/$f" - echo $f.patch >> $workdir/files.txt - else - make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" - rm -f "$patchfile" - echo $f >> $workdir/files.txt - fi - fi - else - # remove instructions are added after add / patch instructions for - # consistency with make_incremental_updates.py - remove_array[$num_removes]=$f - (( num_removes++ )) - fi -done - -# Newly added files -notice "" -notice "Adding file add instructions to update manifests" -num_newfiles=${#newfiles[*]} - -for ((i=0; $i<$num_newfiles; i=$i+1)); do - f="${newfiles[$i]}" - - # If we've already tested this file, then skip it - for ((j=0; $j<$num_oldfiles; j=$j+1)); do - if [ "$f" = "${oldfiles[j]}" ]; then - continue 2 - fi - done - - dir=$(dirname "$workdir/$f") - mkdir -p "$dir" - - $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" - copy_perm "$newdir/$f" "$workdir/$f" - - if check_for_add_if_not_update "$f"; then - make_add_if_not_instruction "$f" "$updatemanifestv3" - else - make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" - fi - - - echo $f >> $workdir/files.txt -done - -notice "" -notice "Adding file remove instructions to update manifests" -for ((i=0; $i<$num_removes; i=$i+1)); do - f="${remove_array[$i]}" - notice " remove \"$f\"" - echo "remove \"$f\"" >> $updatemanifestv2 - echo "remove \"$f\"" >> $updatemanifestv3 -done - -# Add remove instructions for any dead files. -notice "" -notice "Adding file and directory remove instructions from file 'removed-files'" -append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3" - -notice "" -notice "Adding directory remove instructions for directories that no longer exist" -num_olddirs=${#olddirs[*]} - -for ((i=0; $i<$num_olddirs; i=$i+1)); do - f="${olddirs[$i]}" - # If this dir doesn't exist in the new directory remove it. - if [ ! -d "$newdir/$f" ]; then - notice " rmdir $f/" - echo "rmdir \"$f/\"" >> $updatemanifestv2 - echo "rmdir \"$f/\"" >> $updatemanifestv3 - fi -done - -$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" -$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" - -mar_command="$MAR" -if [[ -n $PRODUCT_VERSION ]] -then - mar_command="$mar_command -V $PRODUCT_VERSION" -fi -if [[ -n $CHANNEL_ID ]] -then - mar_command="$mar_command -H $CHANNEL_ID" -fi -mar_command="$mar_command -C \"$workdir\" -c output.mar -f $workdir/files.txt" -eval "$mar_command" -mv -f "$workdir/output.mar" "$archive" - -# cleanup -rm -fr "$workdir" - -notice "" -notice "Finished" -notice "" diff --git a/bin/update/tools.py b/bin/update/tools.py index 35e635cf8336..ab38d10f4b57 100644 --- a/bin/update/tools.py +++ b/bin/update/tools.py @@ -41,7 +41,7 @@ def get_hash(file_path): def get_file_info(mar_file, url): filesize = os.path.getsize(mar_file) data = {'hash': get_hash(mar_file), - 'hashFunction': 'sha512', + 'hash_function': 'sha512', 'size': filesize, 'url': url + os.path.basename(mar_file)} |