Fix `make create-update-info` (for Windows, at least)

* The shell scripts that had once been copied from Mozilla to bin/update/*.sh
are now included in
onlineupdate-c003be8b9727672e7d30972983b375f4c200233f-2.tar.xz (which is still
generated from the same
<https://github.com/mozilla/gecko-dev/commit/c003be8b9727672e7d30972983b375f4c200233f>
that was used for the original tarball in
3a445cb49795fabe0d8caaf12bfc38eb9e12d5fc "Turn onlineupdate into
external/onlineupdate").

* The additional modifications in external/onlineupdate/lo.patch are:

** Allowing to pass the list of files into the mar tool via -f instead of on the
command line, to avoid "command line too long" errors on Windows, inspired by
the modifications once made directly in our old downstream sources with
4165dd4e465a86ba6abe9afb3abfda5ef72493ea "add a way to create mar file from
file", 8e4d49340bd235a7db8fde1d24dd1d63ddc4d571 "use the new file based
approach for the mar creation", and fb13ed6955cd66017e5348b915af371a184ea633
"add the manifest file to the mar file".  (To keep things simple for now, it
still uses a hard-coded maximum of 10000 lines in the file, marked with a
TODO.)

** Not failing when "precomplete file is missing!"  (There is a comment

// Applications aren't required to have a precomplete manifest. The mar
// generation scripts enforce the presence of a precomplete manifest.

in
workdir/UnpackedTarball/onlineupdate/onlineupdate/source/update/updater/updater.cpp
and it appears to be OK that we don't have such a  precomplete manifest file
and just skip that test.)

* In the Makefile.gbuild create-update-info, the create_full_mar.py script needs
to be called with a Unix pathname on Windows, or else the

#!/usr/bin/env python3

shebang in that script would get confused.

* The related Makefile.gbuild targets upload-update-info and create-partial-info
have not been addressed yet.

Change-Id: Iab4e083ddbe99e07d846e202f20c6031e2983e1b
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/161656
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <stephan.bergmann@allotropia.de>

9 files changed, 146 insertions, 669 deletions

diff --git a/Makefile.gbuild b/Makefile.gbuild
index 2dfbce13cba0..d86a0cd9ce69 100644
--- a/Makefile.gbuild
+++ b/Makefile.gbuild
@@ -41,7 +41,7 @@ create-update-info:
 	rm -rf $(MAR_DIR) || true
 	rm -rf $(UPDATE_DIR) || true
 	mkdir -p $(MAR_DIR)/language
-	MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_full_mar.py "$(PRODUCTNAME)" "$(WORKDIR)" "$(MAR_NAME_PREFIX)" "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)"
+	MAR=$(INSTDIR)/program/mar $(if $(filter WNT,$(OS)),$(shell cygpath -u $(SRCDIR)/bin/update/create_full_mar.py),$(SRCDIR)/bin/update/create_full_mar.py) "$(PRODUCTNAME)" "$(WORKDIR)" "$(MAR_NAME_PREFIX)" "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)" '$(LIBO_VERSION_MAJOR).$(LIBO_VERSION_MINOR).$(LIBO_VERSION_MICRO).$(LIBO_VERSION_PATCH)'
 	$(if $(filter WNT,$(OS)),, \
 		MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_full_mar_for_languages.py "$(PRODUCTNAME)" "$(WORKDIR)" "$(MAR_NAME_PREFIX)" "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)" \
 	)
diff --git a/bin/update/common.sh b/bin/update/common.sh
deleted file mode 100644
index dcdbea8bb815..000000000000
--- a/bin/update/common.sh
+++ /dev/null
@@ -1,222 +0,0 @@
-#!/usr/bin/env bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#
-# Code shared by update packaging scripts.
-# Author: Darin Fisher
-#
-
-# -----------------------------------------------------------------------------
-# By default just assume that these tools exist on our path
-MAR=${MAR:-mar}
-BZIP2=${BZIP2:-bzip2}
-MBSDIFF=${MBSDIFF:-mbsdiff}
-
-# -----------------------------------------------------------------------------
-# Helper routines
-
-notice() {
-  echo "$*" 1>&2
-}
-
-get_file_size() {
-  info=($(ls -ln "$1"))
-  echo ${info[4]}
-}
-
-check_externals() {
-
-    # check whether we can call the mar executable
-    "$MAR" --version > /dev/null 2>&1
-    if [ $? != 0 ]; then
-        notice "Could not find a valid mar executable in the path or in the MAR environment variable"
-        exit 1
-    fi
-
-    # check whether we can access the bzip2 executable
-    "$BZIP2" --help > /dev/null 2>&1
-    if [ $? != 0 ]; then
-        notice "Could not find a valid bzip2 executable in the PATH or in the BZIP2 environment variable"
-        exit 1
-    fi
-}
-
-copy_perm() {
-  reference="$1"
-  target="$2"
-
-  if [ -x "$reference" ]; then
-    chmod 0755 "$target"
-  else
-    chmod 0644 "$target"
-  fi
-}
-
-make_add_instruction() {
-  f="$1"
-  filev2="$2"
-  # The third param will be an empty string when a file add instruction is only
-  # needed in the version 2 manifest. This only happens when the file has an
-  # add-if-not instruction in the version 3 manifest. This is due to the
-  # precomplete file prior to the version 3 manifest having a remove instruction
-  # for this file so the file is removed before applying a complete update.
-  filev3="$3"
-
-  # Used to log to the console
-  if [ $4 ]; then
-    forced=" (forced)"
-  else
-    forced=
-  fi
-
-  is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
-  if [ $is_extension = "1" ]; then
-    # Use the subdirectory of the extensions folder as the file to test
-    # before performing this add instruction.
-    testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
-    notice "     add-if \"$testdir\" \"$f\""
-    echo "add-if \"$testdir\" \"$f\"" >> $filev2
-    if [ ! $filev3 = "" ]; then
-      echo "add-if \"$testdir\" \"$f\"" >> $filev3
-    fi
-  else
-    notice "        add \"$f\"$forced"
-    echo "add \"$f\"" >> $filev2
-    if [ ! $filev3 = "" ]; then
-      echo "add \"$f\"" >> $filev3
-    fi
-  fi
-}
-
-check_for_add_if_not_update() {
-  add_if_not_file_chk="$1"
-
-  if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \
-       `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
-    ## "true" *giggle*
-    return 0;
-  fi
-  ## 'false'... because this is bash. Oh yay!
-  return 1;
-}
-
-check_for_add_to_manifestv2() {
-  add_if_not_file_chk="$1"
-
-  if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
-    ## "true" *giggle*
-    return 0;
-  fi
-  ## 'false'... because this is bash. Oh yay!
-  return 1;
-}
-
-make_add_if_not_instruction() {
-  f="$1"
-  filev3="$2"
-
-  notice " add-if-not \"$f\" \"$f\""
-  echo "add-if-not \"$f\" \"$f\"" >> $filev3
-}
-
-make_patch_instruction() {
-  f="$1"
-  filev2="$2"
-  filev3="$3"
-
-  is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
-  if [ $is_extension = "1" ]; then
-    # Use the subdirectory of the extensions folder as the file to test
-    # before performing this add instruction.
-    testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
-    notice "   patch-if \"$testdir\" \"$f.patch\" \"$f\""
-    echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2
-    echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3
-  else
-    notice "      patch \"$f.patch\" \"$f\""
-    echo "patch \"$f.patch\" \"$f\"" >> $filev2
-    echo "patch \"$f.patch\" \"$f\"" >> $filev3
-  fi
-}
-
-append_remove_instructions() {
-  dir="$1"
-  filev2="$2"
-  filev3="$3"
-
-  if [ -f "$dir/removed-files" ]; then
-    listfile="$dir/removed-files"
-  elif [ -f "$dir/Contents/Resources/removed-files" ]; then
-    listfile="$dir/Contents/Resources/removed-files"
-  fi
-  if [ -n "$listfile" ]; then
-    # Map spaces to pipes so that we correctly handle filenames with spaces.
-    files=($(cat "$listfile" | tr " " "|"  | sort -r))
-    num_files=${#files[*]}
-    for ((i=0; $i<$num_files; i=$i+1)); do
-      # Map pipes back to whitespace and remove carriage returns
-      f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r')
-      # Trim whitespace
-      f=$(echo $f)
-      # Exclude blank lines.
-      if [ -n "$f" ]; then
-        # Exclude comments
-        if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
-          if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
-            notice "      rmdir \"$f\""
-            echo "rmdir \"$f\"" >> $filev2
-            echo "rmdir \"$f\"" >> $filev3
-          elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
-            # Remove the *
-            f=$(echo "$f" | sed -e 's:\*$::')
-            notice "    rmrfdir \"$f\""
-            echo "rmrfdir \"$f\"" >> $filev2
-            echo "rmrfdir \"$f\"" >> $filev3
-          else
-            notice "     remove \"$f\""
-            echo "remove \"$f\"" >> $filev2
-            echo "remove \"$f\"" >> $filev3
-          fi
-        fi
-      fi
-    done
-  fi
-}
-
-# List all files in the current directory, stripping leading "./"
-# Pass a variable name and it will be filled as an array.
-list_files() {
-  count=0
-
-  find . -type f \
-    ! -name "update.manifest" \
-    ! -name "updatev2.manifest" \
-    ! -name "updatev3.manifest" \
-    ! -name "temp-dirlist" \
-    ! -name "temp-filelist" \
-    | sed 's/\.\/\(.*\)/\1/' \
-    | sort -r > "temp-filelist"
-  while read file; do
-    eval "${1}[$count]=\"$file\""
-    (( count++ ))
-  done < "temp-filelist"
-  rm "temp-filelist"
-}
-
-# List all directories in the current directory, stripping leading "./"
-list_dirs() {
-  count=0
-
-  find . -type d \
-    ! -name "." \
-    ! -name ".." \
-    | sed 's/\.\/\(.*\)/\1/' \
-    | sort -r > "temp-dirlist"
-  while read dir; do
-    eval "${1}[$count]=\"$dir\""
-    (( count++ ))
-  done < "temp-dirlist"
-  rm "temp-dirlist"
-}
diff --git a/bin/update/create_full_mar.py b/bin/update/create_full_mar.py
index f3ccc972afce..81442d337f40 100755
--- a/bin/update/create_full_mar.py
+++ b/bin/update/create_full_mar.py
@@ -9,8 +9,6 @@ from tools import uncompress_file_to_dir, get_file_info, make_complete_mar_name
 from signing import sign_mar_file
 from path import UpdaterPath, convert_to_unix, convert_to_native
 
-current_dir_path = os.path.dirname(os.path.realpath(convert_to_unix(__file__)))
-
 
 def main():
     parser = argparse.ArgumentParser()
@@ -20,6 +18,7 @@ def main():
     parser.add_argument('certificate_path')
     parser.add_argument('certificate_name')
     parser.add_argument('base_url')
+    parser.add_argument('version')
     args = parser.parse_args()
 
     certificate_path = args.certificate_path
@@ -28,6 +27,7 @@ def main():
     filename_prefix = args.filename_prefix
     workdir = args.workdir
     product_name = args.product_name
+    version = args.version
 
     update_path = UpdaterPath(workdir)
     update_path.ensure_dir_exist()
@@ -41,7 +41,10 @@ def main():
     uncompress_dir = uncompress_file_to_dir(tar_file, temp_dir)
 
     mar_file = make_complete_mar_name(target_dir, filename_prefix)
-    path = os.path.join(current_dir_path, 'make_full_update.sh')
+    path = os.path.join(
+        workdir, 'UnpackedTarball/onlineupdate/tools/update-packaging/make_full_update.sh')
+    os.putenv('MOZ_PRODUCT_VERSION', version)
+    os.putenv('MAR_CHANNEL_ID', 'LOOnlineUpdater')
     subprocess.call([path, convert_to_native(mar_file), convert_to_native(uncompress_dir)])
 
     sign_mar_file(target_dir, certificate_path, certificate_name, mar_file, filename_prefix)
diff --git a/bin/update/make_full_update.sh b/bin/update/make_full_update.sh
deleted file mode 100755
index 4140ecae6d14..000000000000
--- a/bin/update/make_full_update.sh
+++ /dev/null
@@ -1,122 +0,0 @@
-#!/usr/bin/env bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#
-# This tool generates full update packages for the update system.
-# Author: Darin Fisher
-#
-
-. $(dirname "$0")/common.sh
-
-# -----------------------------------------------------------------------------
-
-print_usage() {
-  notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY"
-}
-
-if [ $# = 0 ]; then
-  print_usage
-  exit 1
-fi
-
-if [ $1 = -h ]; then
-  print_usage
-  notice ""
-  notice "The contents of DIRECTORY will be stored in ARCHIVE."
-  notice ""
-  notice "Options:"
-  notice "  -h  show this help text"
-  notice ""
-  exit 1
-fi
-
-check_externals
-# -----------------------------------------------------------------------------
-
-archive="$1"
-targetdir="$2"
-# Prevent the workdir from being inside the targetdir so it isn't included in
-# the update mar.
-if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
-  # Remove the /
-  targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
-fi
-workdir="$targetdir.work"
-updatemanifestv2="$workdir/updatev2.manifest"
-updatemanifestv3="$workdir/updatev3.manifest"
-targetfiles="updatev2.manifest updatev3.manifest"
-
-mkdir -p "$workdir"
-echo "updatev2.manifest" >> $workdir/files.txt
-echo "updatev3.manifest" >> $workdir/files.txt
-
-# Generate a list of all files in the target directory.
-pushd "$targetdir"
-if test $? -ne 0 ; then
-  exit 1
-fi
-
-# if [ ! -f "precomplete" ]; then
-#   if [ ! -f "Contents/Resources/precomplete" ]; then
-#     notice "precomplete file is missing!"
-#     exit 1
-#   fi
-# fi
-
-list_files files
-
-popd
-
-# Add the type of update to the beginning of the update manifests.
-> $updatemanifestv2
-> $updatemanifestv3
-notice ""
-notice "Adding type instruction to update manifests"
-notice "       type complete"
-echo "type \"complete\"" >> $updatemanifestv2
-echo "type \"complete\"" >> $updatemanifestv3
-
-notice ""
-notice "Adding file add instructions to update manifests"
-num_files=${#files[*]}
-
-for ((i=0; $i<$num_files; i=$i+1)); do
-  f="${files[$i]}"
-
-  if check_for_add_if_not_update "$f"; then
-    make_add_if_not_instruction "$f" "$updatemanifestv3"
-    if check_for_add_to_manifestv2 "$f"; then
-      make_add_instruction "$f" "$updatemanifestv2" "" 1
-    fi
-  else
-    make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
-  fi
-
-  dir=$(dirname "$f")
-  mkdir -p "$workdir/$dir"
-  $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
-  copy_perm "$targetdir/$f" "$workdir/$f"
-
-  targetfiles="$targetfiles \"$f\""
-  echo $f >> $workdir/files.txt
-done
-
-# Append remove instructions for any dead files.
-notice ""
-notice "Adding file and directory remove instructions from file 'removed-files'"
-append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
-
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
-
-eval "$MAR -C \"$workdir\" -c output.mar -f $workdir/files.txt"
-mv -f "$workdir/output.mar" "$archive"
-
-# cleanup
-rm -fr "$workdir"
-
-notice ""
-notice "Finished"
-notice ""
diff --git a/bin/update/make_incremental_update.sh b/bin/update/make_incremental_update.sh
deleted file mode 100755
index 31bddabdb082..000000000000
--- a/bin/update/make_incremental_update.sh
+++ /dev/null
@@ -1,318 +0,0 @@
-#!/usr/bin/env bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#
-# This tool generates incremental update packages for the update system.
-# Author: Darin Fisher
-#
-
-. $(dirname "$0")/common.sh
-
-# -----------------------------------------------------------------------------
-
-print_usage() {
-  notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR"
-  notice ""
-  notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE."
-  notice ""
-  notice "Options:"
-  notice "  -h  show this help text"
-  notice "  -f  clobber this file in the installation"
-  notice "      Must be a path to a file to clobber in the partial update."
-  notice ""
-}
-
-check_for_forced_update() {
-  force_list="$1"
-  forced_file_chk="$2"
-
-  local f
-
-  if [ "$forced_file_chk" = "precomplete" ]; then
-    ## "true" *giggle*
-    return 0;
-  fi
-
-  if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then
-    ## "true" *giggle*
-    return 0;
-  fi
-
-  if [ "$forced_file_chk" = "removed-files" ]; then
-    ## "true" *giggle*
-    return 0;
-  fi
-
-  if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then
-    ## "true" *giggle*
-    return 0;
-  fi
-
-  if [ "${forced_file_chk##*.}" = "chk" ]; then
-    ## "true" *giggle*
-    return 0;
-  fi
-
-  for f in $force_list; do
-    #echo comparing $forced_file_chk to $f
-    if [ "$forced_file_chk" = "$f" ]; then
-      ## "true" *giggle*
-      return 0;
-    fi
-  done
-  ## 'false'... because this is bash. Oh yay!
-  return 1;
-}
-
-if [ $# = 0 ]; then
-  print_usage
-  exit 1
-fi
-
-requested_forced_updates='Contents/MacOS/firefox'
-
-while getopts "hf:" flag
-do
-   case "$flag" in
-      h) print_usage; exit 0
-      ;;
-      f) requested_forced_updates="$requested_forced_updates $OPTARG"
-      ;;
-      ?) print_usage; exit 1
-      ;;
-   esac
-done
-
-# -----------------------------------------------------------------------------
-
-let arg_start=$OPTIND-1
-shift $arg_start
-
-archive="$1"
-olddir="$2"
-newdir="$3"
-# Prevent the workdir from being inside the targetdir so it isn't included in
-# the update mar.
-if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then
-  # Remove the /
-  newdir=$(echo "$newdir" | sed -e 's:\/$::')
-fi
-workdir="$newdir.work"
-updatemanifestv2="$workdir/updatev2.manifest"
-updatemanifestv3="$workdir/updatev3.manifest"
-
-mkdir -p "$workdir"
-echo "updatev2.manifest" >> $workdir/files.txt
-echo "updatev3.manifest" >> $workdir/files.txt
-
-# Generate a list of all files in the target directory.
-pushd "$olddir"
-if test $? -ne 0 ; then
-  exit 1
-fi
-
-list_files oldfiles
-list_dirs olddirs
-
-popd
-
-pushd "$newdir"
-if test $? -ne 0 ; then
-  exit 1
-fi
-
-# if [ ! -f "precomplete" ]; then
-#   if [ ! -f "Contents/Resources/precomplete" ]; then
-#     notice "precomplete file is missing!"
-#     exit 1
-#   fi
-# fi
-
-list_dirs newdirs
-list_files newfiles
-
-popd
-
-# Add the type of update to the beginning of the update manifests.
-notice ""
-notice "Adding type instruction to update manifests"
-> $updatemanifestv2
-> $updatemanifestv3
-notice "       type partial"
-echo "type \"partial\"" >> $updatemanifestv2
-echo "type \"partial\"" >> $updatemanifestv3
-
-notice ""
-notice "Adding file patch and add instructions to update manifests"
-
-num_oldfiles=${#oldfiles[*]}
-remove_array=
-num_removes=0
-
-for ((i=0; $i<$num_oldfiles; i=$i+1)); do
-  f="${oldfiles[$i]}"
-
-  # If this file exists in the new directory as well, then check if it differs.
-  if [ -f "$newdir/$f" ]; then
-
-    if check_for_add_if_not_update "$f"; then
-      # The full workdir may not exist yet, so create it if necessary.
-      mkdir -p `dirname "$workdir/$f"`
-      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
-      copy_perm "$newdir/$f" "$workdir/$f"
-      make_add_if_not_instruction "$f" "$updatemanifestv3"
-      echo $f >> $workdir/files.txt
-      continue 1
-    fi
-
-    if check_for_forced_update "$requested_forced_updates" "$f"; then
-      # The full workdir may not exist yet, so create it if necessary.
-      mkdir -p `dirname "$workdir/$f"`
-      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
-      copy_perm "$newdir/$f" "$workdir/$f"
-      make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
-      echo $f >> $workdir/files.txt
-      continue 1
-    fi
-
-    if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
-      # Compute both the compressed binary diff and the compressed file, and
-      # compare the sizes.  Then choose the smaller of the two to package.
-      dir=$(dirname "$workdir/$f")
-      mkdir -p "$dir"
-      notice "diffing \"$f\""
-      # MBSDIFF_HOOK represents the communication interface with funsize and,
-      # if enabled, caches the intermediate patches for future use and
-      # compute avoidance
-      #
-      # An example of MBSDIFF_HOOK env variable could look like this:
-      # export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user"
-      # where myscript.sh has the following usage:
-      # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \
-      #   PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL
-      #
-      # Note: patches are bzipped stashed in funsize to gain more speed
-
-      # if service is not enabled then default to old behavior
-      if [ -z "$MBSDIFF_HOOK" ]; then
-        $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
-        $BZIP2 -z9 "$workdir/$f.patch"
-      else
-        # if service enabled then check patch existence for retrieval
-        if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
-          notice "file \"$f\" found in funsize, diffing skipped"
-        else
-          # if not found already - compute it and cache it for future use
-          $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
-          $BZIP2 -z9 "$workdir/$f.patch"
-          $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
-        fi
-      fi
-      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
-      copy_perm "$newdir/$f" "$workdir/$f"
-      patchfile="$workdir/$f.patch.bz2"
-      patchsize=$(get_file_size "$patchfile")
-      fullsize=$(get_file_size "$workdir/$f")
-
-      if [ $patchsize -lt $fullsize ]; then
-        make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
-        mv -f "$patchfile" "$workdir/$f.patch"
-        rm -f "$workdir/$f"
-        echo $f.patch >> $workdir/files.txt
-      else
-        make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
-        rm -f "$patchfile"
-        echo $f >> $workdir/files.txt
-      fi
-    fi
-  else
-    # remove instructions are added after add / patch instructions for
-    # consistency with make_incremental_updates.py
-    remove_array[$num_removes]=$f
-    (( num_removes++ ))
-  fi
-done
-
-# Newly added files
-notice ""
-notice "Adding file add instructions to update manifests"
-num_newfiles=${#newfiles[*]}
-
-for ((i=0; $i<$num_newfiles; i=$i+1)); do
-  f="${newfiles[$i]}"
-
-  # If we've already tested this file, then skip it
-  for ((j=0; $j<$num_oldfiles; j=$j+1)); do
-    if [ "$f" = "${oldfiles[j]}" ]; then
-      continue 2
-    fi
-  done
-
-  dir=$(dirname "$workdir/$f")
-  mkdir -p "$dir"
-
-  $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
-  copy_perm "$newdir/$f" "$workdir/$f"
-
-  if check_for_add_if_not_update "$f"; then
-    make_add_if_not_instruction "$f" "$updatemanifestv3"
-  else
-    make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
-  fi
-
-
-  echo $f >> $workdir/files.txt
-done
-
-notice ""
-notice "Adding file remove instructions to update manifests"
-for ((i=0; $i<$num_removes; i=$i+1)); do
-  f="${remove_array[$i]}"
-  notice "     remove \"$f\""
-  echo "remove \"$f\"" >> $updatemanifestv2
-  echo "remove \"$f\"" >> $updatemanifestv3
-done
-
-# Add remove instructions for any dead files.
-notice ""
-notice "Adding file and directory remove instructions from file 'removed-files'"
-append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3"
-
-notice ""
-notice "Adding directory remove instructions for directories that no longer exist"
-num_olddirs=${#olddirs[*]}
-
-for ((i=0; $i<$num_olddirs; i=$i+1)); do
-  f="${olddirs[$i]}"
-  # If this dir doesn't exist in the new directory remove it.
-  if [ ! -d "$newdir/$f" ]; then
-    notice "      rmdir $f/"
-    echo "rmdir \"$f/\"" >> $updatemanifestv2
-    echo "rmdir \"$f/\"" >> $updatemanifestv3
-  fi
-done
-
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
-
-mar_command="$MAR"
-if [[ -n $PRODUCT_VERSION ]]
-then
-  mar_command="$mar_command -V $PRODUCT_VERSION"
-fi
-if [[ -n $CHANNEL_ID ]]
-then
-  mar_command="$mar_command -H $CHANNEL_ID"
-fi
-mar_command="$mar_command -C \"$workdir\" -c output.mar -f $workdir/files.txt"
-eval "$mar_command"
-mv -f "$workdir/output.mar" "$archive"
-
-# cleanup
-rm -fr "$workdir"
-
-notice ""
-notice "Finished"
-notice ""
diff --git a/bin/update/tools.py b/bin/update/tools.py
index 35e635cf8336..ab38d10f4b57 100644
--- a/bin/update/tools.py
+++ b/bin/update/tools.py
@@ -41,7 +41,7 @@ def get_hash(file_path):
 def get_file_info(mar_file, url):
     filesize = os.path.getsize(mar_file)
     data = {'hash': get_hash(mar_file),
-            'hashFunction': 'sha512',
+            'hash_function': 'sha512',
             'size': filesize,
             'url': url + os.path.basename(mar_file)}
 
diff --git a/download.lst b/download.lst
index bd5ded0d9793..568b404f03f3 100644
--- a/download.lst
+++ b/download.lst
@@ -498,8 +498,8 @@ OFFICEOTRON_JAR := 8249374c274932a21846fa7629c2aa9b-officeotron-0.7.4-master.jar
 # three static lines
 # so that git cherry-pick
 # will not run into conflicts
-ONLINEUPDATE_SHA256SUM := cfd00948425424d7934f5f3e26c830eda9ebbf2d18e13656bd20e4f6cd99e4ac
-ONLINEUPDATE_TARBALL := onlineupdate-c003be8b9727672e7d30972983b375f4c200233f.tar.xz
+ONLINEUPDATE_SHA256SUM := 37206cf981e8409d048b59ac5839621ea107ff49af72beb9d7769a2f41da8d90
+ONLINEUPDATE_TARBALL := onlineupdate-c003be8b9727672e7d30972983b375f4c200233f-2.tar.xz
 # three static lines
 # so that git cherry-pick
 # will not run into conflicts
diff --git a/external/onlineupdate/generate-sources.sh b/external/onlineupdate/generate-sources.sh
index bc39603c156b..8f11faf5a65d 100755
--- a/external/onlineupdate/generate-sources.sh
+++ b/external/onlineupdate/generate-sources.sh
@@ -185,6 +185,9 @@ copyto toolkit/mozapps/update/common/updateutils_win.cpp toolkit/mozapps/update/
 copyto toolkit/mozapps/update/common/updateutils_win.h toolkit/mozapps/update/common/updateutils_win.h
 copyto toolkit/mozapps/update/updater/crctable.h toolkit/mozapps/update/updater/crctable.h
 copyto toolkit/xre/nsWindowsRestart.cpp toolkit/xre/nsWindowsRestart.cpp
+copyto tools/update-packaging/common.sh tools/update-packaging/common.sh
+copyto tools/update-packaging/make_full_update.sh tools/update-packaging/make_full_update.sh
+copyto tools/update-packaging/make_incremental_update.sh tools/update-packaging/make_incremental_update.sh
 copyto xpcom/base/nsAlgorithm.h xpcom/base/nsAlgorithm.h
 copyto xpcom/base/nsAutoRef.h xpcom/base/nsAutoRef.h
 copyto xpcom/base/nsWindowsHelpers.h xpcom/base/nsWindowsHelpers.h
diff --git a/external/onlineupdate/lo.patch b/external/onlineupdate/lo.patch
index a8b92830419c..0021c3a69fe7 100644
--- a/external/onlineupdate/lo.patch
+++ b/external/onlineupdate/lo.patch
@@ -1,3 +1,96 @@
+--- onlineupdate/source/libmar/tool/mar.c
++++ onlineupdate/source/libmar/tool/mar.c
+@@ -14,7 +14,9 @@
+ #  include <windows.h>
+ #  include <direct.h>
+ #  define chdir _chdir
++#  define PATH_MAX MAX_PATH
+ #else
++#  include <limits.h>
+ #  include <unistd.h>
+ #endif
+ 
+@@ -39,7 +41,7 @@
+   printf("Create a MAR file:\n");
+   printf(
+       "  mar -H MARChannelID -V ProductVersion [-C workingDir] "
+-      "-c archive.mar [files...]\n");
++      "-c archive.mar [files...|-f files]\n");
+ 
+   printf("Extract a MAR file:\n");
+   printf("  mar [-C workingDir] -x archive.mar\n");
+@@ -244,6 +246,8 @@
+ 
+   switch (argv[1][1]) {
+     case 'c': {
++      int numfiles;
++      char** files;
+       struct ProductInformationBlock infoBlock;
+       if (!productVersion) {
+         fprintf(stderr,
+@@ -256,9 +260,61 @@
+                 "<mar-channel-id>`).\n");
+         return -1;
+       }
++      if (argc == 5 && !strcmp(argv[3], "-f")) {
++        FILE* in;
++        in = fopen(argv[4], "r");
++        if (!in) {
++          fprintf(stderr,
++                  "ERROR: Cannot open file `%s` for reading.\n", argv[4]);
++          return -1;
++        }
++        numfiles = 0;
++        files = malloc(sizeof(char*) * 10000); /*TODO*/
++        if (!files) {
++          fprintf(stderr,
++                  "ERROR: Cannot allocate memory");
++          return -1;
++        }
++        for (;;) {
++          char buf[PATH_MAX + 1];
++          size_t len;
++          if (!fgets(buf, PATH_MAX + 1, in)) {
++            if (feof(in)) {
++              break;
++            }
++            fprintf(stderr,
++                    "ERROR: Cannot read from file `%s`.\n", argv[4]);
++            return -1;
++          }
++          len = strlen(buf);
++          if (len != 0 && buf[len - 1] == '\n') {
++            buf[len - 1] = '\0';
++          } else if (!feof(in)) {
++            fprintf(stderr,
++                    "ERROR: Too long line in file `%s`.\n", argv[4]);
++            return -1;
++          }
++          if (numfiles == 10000) { /*TODO*/
++            fprintf(stderr,
++                    "ERROR: Too many lines in file `%s`.\n", argv[4]);
++            return -1;
++          }
++          files[numfiles] = strdup(buf);
++          if (!files[numfiles]) {
++            fprintf(stderr,
++                    "ERROR: Cannot allocate memory");
++            return -1;
++          }
++          ++numfiles;
++        }
++        fclose(in);
++      } else {
++        numfiles = argc - 3;
++        files = argv + 3;
++      }
+       infoBlock.MARChannelID = MARChannelID;
+       infoBlock.productVersion = productVersion;
+-      return mar_create(argv[2], argc - 3, argv + 3, &infoBlock);
++      return mar_create(argv[2], numfiles, files, &infoBlock);
+     }
+     case 'i': {
+       if (!productVersion) {
 --- onlineupdate/source/service/serviceinstall.cpp
 +++ onlineupdate/source/service/serviceinstall.cpp
 @@ -25,7 +25,7 @@
@@ -130,3 +223,43 @@
        LOG_WARN(("Install directory updater could not be determined."));
        result = FALSE;
      }
+--- tools/update-packaging/make_full_update.sh
++++ tools/update-packaging/make_full_update.sh
+@@ -53,9 +53,10 @@
+ fi
+ workdir="$targetdir.work"
+ updatemanifestv3="$workdir/updatev3.manifest"
+-targetfiles="updatev3.manifest"
+ 
+ mkdir -p "$workdir"
++
++printf 'updatev3.manifest\n' >"$workdir/files.txt"
+ 
+ # Generate a list of all files in the target directory.
+ pushd "$targetdir"
+@@ -66,7 +67,6 @@
+ if [ ! -f "precomplete" ]; then
+   if [ ! -f "Contents/Resources/precomplete" ]; then
+     notice "precomplete file is missing!"
+-    exit 1
+   fi
+ fi
+ 
+@@ -99,7 +99,7 @@
+   $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
+   copy_perm "$targetdir/$f" "$workdir/$f"
+ 
+-  targetfiles="$targetfiles \"$f\""
++  printf '%s\n' "$f" >>"$workdir/files.txt"
+ done
+ 
+ # Append remove instructions for any dead files.
+@@ -110,7 +110,7 @@
+ $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
+ 
+ mar_command="$mar_command -C \"$workdir\" -c output.mar"
+-eval "$mar_command $targetfiles"
++eval "$mar_command -f $workdir/files.txt"
+ mv -f "$workdir/output.mar" "$archive"
+ 
+ # cleanup