roytam1/UXP
1
0
Fork 0
mirror of https://github.com/roytam1/UXP.git synced 2026-05-26 13:58:49 +00:00
Code Issues Projects Releases Wiki Activity

Issue #1859 - Part 4: Update mar file generation scripts for use of xz.

Browse Source 
- Update packaging scripts to support both lzma and bzip2
- Update python script to support lzma
- Update test scripts to support lzma to be thorough

This all requires xz to be available in the path on the build system.
This commit is contained in:
Moonchild
2022-01-21 22:22:07 +00:00
committed by roytam1
parent a195efca40
commit 121d56e082
9 changed files with 163 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/update-packaging/common.sh
+7 -2
View File
@@ -9,10 +9,15 @@
#
# -----------------------------------------------------------------------------
# By default just assume that these tools exist on our path
# By default just assume that these tools exist in our path
MAR=${MAR:-mar}
BZIP2=${BZIP2:-bzip2}
MBSDIFF=${MBSDIFF:-mbsdiff}
if [[ -z "${MAR_OLD_FORMAT}" ]]; then
XZ=${XZ:-xz}
else
MAR_OLD_FORMAT=1
BZIP2=${BZIP2:-bzip2}
fi
# -----------------------------------------------------------------------------
# Helper routines
tools/update-packaging/make_full_update.sh
+12 -3
View File
@@ -93,7 +93,11 @@ for ((i=0; $i<$num_files; i=$i+1)); do
dir=$(dirname "$f")
mkdir -p "$workdir/$dir"
$BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
else
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
fi
copy_perm "$targetdir/$f" "$workdir/$f"
targetfiles="$targetfiles \"$f\""
@@ -104,8 +108,13 @@ notice ""
notice "Adding file and directory remove instructions from file 'removed-files'"
append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
else
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
fi
eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
mv -f "$workdir/output.mar" "$archive"
tools/update-packaging/make_incremental_update.sh
+54 -14
View File
@@ -169,7 +169,11 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
if check_for_add_if_not_update "$f"; then
# The full workdir may not exist yet, so create it if necessary.
mkdir -p `dirname "$workdir/$f"`
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
else
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
fi
copy_perm "$newdir/$f" "$workdir/$f"
make_add_if_not_instruction "$f" "$updatemanifestv3"
archivefiles="$archivefiles \"$f\""
@@ -179,7 +183,11 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
if check_for_forced_update "$requested_forced_updates" "$f"; then
# The full workdir may not exist yet, so create it if necessary.
mkdir -p `dirname "$workdir/$f"`
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
else
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
fi
copy_perm "$newdir/$f" "$workdir/$f"
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
archivefiles="$archivefiles \"$f\""
@@ -207,21 +215,44 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
# if service is not enabled then default to old behavior
if [ -z "$MBSDIFF_HOOK" ]; then
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
$BZIP2 -z9 "$workdir/$f.patch"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -z9 "$workdir/$f.patch"
else
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
fi
else
# if service enabled then check patch existence for retrieval
if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
notice "file \"$f\" found in funsize, diffing skipped"
if [[ -n $MAR_OLD_FORMAT ]]; then
if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
notice "file \"$f\" found in funsize, diffing skipped"
else
# if not found already - compute it and cache it for future use
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
$BZIP2 -z9 "$workdir/$f.patch"
$MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
fi
else
# if not found already - compute it and cache it for future use
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
$BZIP2 -z9 "$workdir/$f.patch"
$MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then
notice "file \"$f\" found in funsize, diffing skipped"
else
# if not found already - compute it and cache it for future use
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
$MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"
fi
fi
fi
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
else
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
fi
copy_perm "$newdir/$f" "$workdir/$f"
patchfile="$workdir/$f.patch.bz2"
if [[ -n $MAR_OLD_FORMAT ]]; then
patchfile="$workdir/$f.patch.bz2"
else
patchfile="$workdir/$f.patch.xz"
fi
patchsize=$(get_file_size "$patchfile")
fullsize=$(get_file_size "$workdir/$f")
@@ -262,7 +293,11 @@ for ((i=0; $i<$num_newfiles; i=$i+1)); do
dir=$(dirname "$workdir/$f")
mkdir -p "$dir"
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
else
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
fi
copy_perm "$newdir/$f" "$workdir/$f"
if check_for_add_if_not_update "$f"; then
@@ -303,8 +338,13 @@ for ((i=0; $i<$num_olddirs; i=$i+1)); do
fi
done
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
else
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
fi
mar_command="$MAR"
if [[ -n $MOZ_PRODUCT_VERSION ]]
tools/update-packaging/make_incremental_updates.py
+47 -36
View File
@@ -4,7 +4,7 @@
import os
import shutil
import sha
import hashlib
from os.path import join, getsize
from stat import *
import re
@@ -12,9 +12,9 @@ import sys
import getopt
import time
import datetime
import bz2
import string
import tempfile
import io
class PatchInfo:
""" Represents the meta-data associated with a patch
@@ -107,23 +107,23 @@ class PatchInfo:
""" Create the v2 manifest file in the root of the work_dir """
manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest")
manifest_file = open(manifest_file_path, "wb")
manifest_file.writelines("type \"partial\"\n")
manifest_file.writelines(string.join(self.manifestv2, '\n'))
manifest_file.writelines("\n")
manifest_file.writelines(io.BytesIO(b"type \"partial\"\n"))
manifest_file.writelines(io.BytesIO('\n'.join(self.manifestv2).encode('ascii')))
manifest_file.writelines(io.BytesIO(b"\n"))
manifest_file.close()
bzip_file(manifest_file_path)
xz_file(manifest_file_path)
self.archive_files.append('"updatev2.manifest"')
""" Create the v3 manifest file in the root of the work_dir """
manifest_file_path = os.path.join(self.work_dir,"updatev3.manifest")
manifest_file = open(manifest_file_path, "wb")
manifest_file.writelines("type \"partial\"\n")
manifest_file.writelines(string.join(self.manifestv3, '\n'))
manifest_file.writelines("\n")
manifest_file.writelines(io.BytesIO(b"type \"partial\"\n"))
manifest_file.writelines(io.BytesIO('\n'.join(self.manifestv3).encode('ascii')))
manifest_file.writelines(io.BytesIO(b"\n"))
manifest_file.close()
bzip_file(manifest_file_path)
xz_file(manifest_file_path)
self.archive_files.append('"updatev3.manifest"')
def build_marfile_entry_hash(self, root_path):
@@ -176,8 +176,8 @@ class MarFileEntry:
def calc_file_sha_digest(self, filename):
""" Returns sha digest of given filename"""
file_content = open(filename, 'r').read()
return sha.new(file_content).digest()
file_content = open(filename, 'rb').read()
return hashlib.sha1(file_content).digest()
def sha(self):
""" Returns sha digest of file repreesnted by this _marfile_entry"""
@@ -188,7 +188,7 @@ class MarFileEntry:
def exec_shell_cmd(cmd):
"""Execs shell cmd and raises an exception if the cmd fails"""
if (os.system(cmd)):
raise Exception, "cmd failed "+cmd
raise Exception("cmd failed "+cmd)
def copy_file(src_file_abs_path, dst_file_abs_path):
@@ -199,19 +199,19 @@ def copy_file(src_file_abs_path, dst_file_abs_path):
# Copy the file over
shutil.copy2(src_file_abs_path, dst_file_abs_path)
def bzip_file(filename):
""" Bzip's the file in place. The original file is replaced with a bzip'd version of itself
def xz_file(filename):
""" XZ compresses the file in place. The original file is replaced with the xz compressed version of itself
assumes the path is absolute"""
exec_shell_cmd('bzip2 -z9 "' + filename+'"')
os.rename(filename+".bz2",filename)
exec_shell_cmd('xz --compress --x86 --lzma2 --format=xz --check=crc64 "' + filename+'"')
os.rename(filename+".xz",filename)
def bunzip_file(filename):
""" Bzip's the file in palce. The original file is replaced with a bunzip'd version of itself.
doesn't matter if the filename ends in .bz2 or not"""
if not filename.endswith(".bz2"):
os.rename(filename, filename+".bz2")
filename=filename+".bz2"
exec_shell_cmd('bzip2 -d "' + filename+'"')
def xzunzip_file(filename):
""" xz decompresses the file in palce. The original file is replaced with a xz decompressed version of itself.
doesn't matter if the filename ends in .xz or not"""
if not filename.endswith(".xz"):
os.rename(filename, filename+".xz")
filename=filename+".xz"
exec_shell_cmd('xz -d "' + filename+'"')
def extract_mar(filename, work_dir):
@@ -241,14 +241,14 @@ def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, pa
if not os.path.exists(patch_file_dir):
os.makedirs(patch_file_dir)
# Create bzip'd patch file
# Create xz'd patch file
exec_shell_cmd("mbsdiff "+from_marfile_entry.abs_path+" "+to_marfile_entry.abs_path+" "+patch_file_abs_path)
bzip_file(patch_file_abs_path)
xz_file(patch_file_abs_path)
# Create bzip's full file
# Create xz full file
full_file_abs_path = os.path.join(patch_info.work_dir, to_marfile_entry.name)
shutil.copy2(to_marfile_entry.abs_path, full_file_abs_path)
bzip_file(full_file_abs_path)
xz_file(full_file_abs_path)
if os.path.getsize(patch_file_abs_path) < os.path.getsize(full_file_abs_path):
# Patch is smaller than file. Remove the file and add patch to manifest
@@ -304,13 +304,17 @@ def process_explicit_remove_files(dir_path, patch_info):
list_file_path = os.path.join(dir_path, "Contents/Resources/removed-files")
if (os.path.exists(list_file_path)):
list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist
fd, tmppath = tempfile.mkstemp('', 'tmp', os.getcwd())
os.close(fd)
exec_shell_cmd('xz -k -d --stdout "' + list_file_path + '" > "'+tmppath+'"')
list_file = open(tmppath)
lines = []
for line in list_file:
lines.append(line.strip())
list_file.close()
list_file.close()
os.remove(tmppath)
lines.sort(reverse=True)
for line in lines:
# Exclude any blank and comment lines.
@@ -406,7 +410,7 @@ def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch
patch_info.create_manifest_files()
# And construct the mar
mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ')
mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+' '.join(patch_info.archive_files)
exec_shell_cmd(mar_cmd)
# Copy mar to final destination
@@ -431,11 +435,18 @@ def get_buildid(work_dir):
print 'WARNING: application.ini not found, cannot find build ID'
return ''
file = bz2.BZ2File(ini)
fd, tmppath = tempfile.mkstemp('', 'tmp', os.getcwd())
os.close(fd)
exec_shell_cmd('xz -k -d --stdout "' + ini + '" > "'+tmppath+'"')
file = open(tmppath)
for line in file:
if line.find('BuildID') == 0:
file.close()
os.remove(tmppath)
return line.strip().split('=')[1]
print 'WARNING: cannot find build ID in application.ini'
print('WARNING: cannot find build ID in application.ini')
file.close()
os.remove(tmppath)
return ''
def decode_filename(filepath):
@@ -486,7 +497,7 @@ def create_partial_patches(patches):
extract_mar(from_filename,work_dir_from)
from_decoded = decode_filename(from_filename)
from_buildid = get_buildid(work_dir_from)
from_shasum = sha.sha(open(from_filename).read()).hexdigest()
from_shasum = hashlib.sha1(open(from_filename, "rb").read()).hexdigest()
from_size = str(os.path.getsize(to_filename))
# Extract to mar into to dir
@@ -495,14 +506,14 @@ def create_partial_patches(patches):
extract_mar(to_filename, work_dir_to)
to_decoded = decode_filename(from_filename)
to_buildid = get_buildid(work_dir_to)
to_shasum = sha.sha(open(to_filename).read()).hexdigest()
to_shasum = hashlib.sha1(open(to_filename, 'rb').read()).hexdigest()
to_size = str(os.path.getsize(to_filename))
mar_extract_time = time.time()
partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['update.manifest','updatev2.manifest','updatev3.manifest'],[]),forced_updates,['channel-prefs.js','update-settings.ini'])
partial_buildid = to_buildid
partial_shasum = sha.sha(open(partial_filename).read()).hexdigest()
partial_shasum = hashlib.sha1(open(partial_filename, "rb").read()).hexdigest()
partial_size = str(os.path.getsize(partial_filename))
metadata.append({
tools/update-packaging/test/catmanifest.sh
+2 -2
View File
@@ -9,6 +9,6 @@ mkdir -p "$workdir"
cp "$1" "$workdir"
cd "$workdir"
mar -x "$1"
mv updatev2.manifest updatev2.manifest.bz2
bzip2 -d updatev2.manifest.bz2
mv updatev2.manifest updatev2.manifest.xz
xz -d updatev2.manifest.xz
cat updatev2.manifest
tools/update-packaging/test/common.sh
+2 -2
View File
@@ -10,9 +10,9 @@
# In here to use the local common.sh to allow the full mars to have unfiltered files
# -----------------------------------------------------------------------------
# By default just assume that these tools exist on our path
# By default just assume that these tools exist in our path
MAR=${MAR:-mar}
BZIP2=${BZIP2:-bzip2}
XZ=${XZ:-xz}
MBSDIFF=${MBSDIFF:-mbsdiff}
# -----------------------------------------------------------------------------
tools/update-packaging/test/diffmar.sh
+8 -8
View File
@@ -30,19 +30,19 @@ cp "$2" "$todir"
cd "$fromdir"
mar -x "$1"
rm "$1"
mv updatev2.manifest updatev2.manifest.bz2
bzip2 -d updatev2.manifest.bz2
mv updatev3.manifest updatev3.manifest.bz2
bzip2 -d updatev3.manifest.bz2
mv updatev2.manifest updatev2.manifest.xz
xz -d updatev2.manifest.xz
mv updatev3.manifest updatev3.manifest.xz
xz -d updatev3.manifest.xz
ls $lsargs > files.txt
cd "$todir"
mar -x "$2"
rm "$2"
mv updatev2.manifest updatev2.manifest.bz2
bzip2 -d updatev2.manifest.bz2
mv updatev3.manifest updatev3.manifest.bz2
bzip2 -d updatev3.manifest.bz2
mv updatev2.manifest updatev2.manifest.xz
xz -d updatev2.manifest.xz
mv updatev3.manifest updatev3.manifest.xz
xz -d updatev3.manifest.xz
ls $lsargs > files.txt
echo "diffing $fromdir and $todir"
tools/update-packaging/test/make_full_update.sh
+3 -3
View File
@@ -94,7 +94,7 @@ for ((i=0; $i<$num_files; i=$i+1)); do
dir=$(dirname "$f")
mkdir -p "$workdir/$dir"
$BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
copy_perm "$targetdir/$f" "$workdir/$f"
targetfiles="$targetfiles \"$f\""
@@ -105,8 +105,8 @@ notice ""
notice "Adding file and directory remove instructions from file 'removed-files'"
append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
mv -f "$workdir/output.mar" "$archive"
tools/update-packaging/unwrap_full_update.pl
+28 -8
View File
@@ -13,7 +13,7 @@
use Getopt::Std;
my ($MAR, $BZIP2, $archive, @marentries, @marfiles);
my ($MAR, $XZ, $BZIP2, $MAR_OLD_FORMAT, $archive, @marentries, @marfiles);
if (defined($ENV{"MAR"})) {
$MAR = $ENV{"MAR"};
@@ -22,11 +22,23 @@ else {
$MAR = "mar";
}
if (defined($ENV{"BZIP2"})) {
$BZIP2 = $ENV{"BZIP2"};
if (defined($ENV{"MAR_OLD_FORMAT"})) {
$MAR_OLD_FORMAT = 1;
if (defined($ENV{"BZIP2"})) {
$BZIP2 = $ENV{"BZIP2"};
}
else {
$BZIP2 = "bzip2";
}
}
else {
$BZIP2 = "bzip2";
$MAR_OLD_FORMAT = 0;
if (defined($ENV{"XZ"})) {
$XZ = $ENV{"XZ"};
}
else {
$XZ = "xz";
}
}
sub print_usage
@@ -59,9 +71,17 @@ foreach (@marentries) {
my @splits = split(/\t/,$_);
my $file = $splits[2];
system("mv \"$file\" \"$file.bz2\"") == 0 ||
die "Couldn't mv \"$file\"";
system("\"$BZIP2\" -d \"$file.bz2\"") == 0 ||
die "Couldn't decompress \"$file\"";
if ($MAR_OLD_FORMAT == 1) {
system("mv \"$file\" \"$file.bz2\"") == 0 ||
die "Couldn't mv \"$file\"";
system("\"$BZIP2\" -d \"$file.bz2\"") == 0 ||
die "Couldn't decompress \"$file\"";
}
else {
system("mv \"$file\" \"$file.xz\"") == 0 ||
die "Couldn't mv \"$file\"";
system("\"$XZ\" -d \"$file.xz\"") == 0 ||
die "Couldn't decompress \"$file\"";
}
}