ota_from_target_files.py revision 67369983cf23e12724c135c3850c98326558256b
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 -b (--board_config) <file> 25 Deprecated. 26 27 -k (--package_key) <key> 28 Key to use to sign the package (default is 29 "build/target/product/security/testkey"). 30 31 -i (--incremental_from) <file> 32 Generate an incremental OTA using the given target-files zip as 33 the starting build. 34 35 -w (--wipe_user_data) 36 Generate an OTA package that will wipe the user data partition 37 when installed. 38 39 -n (--no_prereq) 40 Omit the timestamp prereq check normally included at the top of 41 the build scripts (used for developer OTA packages which 42 legitimately need to go back and forth). 43 44 -e (--extra_script) <file> 45 Insert the contents of file at the end of the update script. 46 47""" 48 49import sys 50 51if sys.hexversion < 0x02040000: 52 print >> sys.stderr, "Python 2.4 or newer is required." 53 sys.exit(1) 54 55import copy 56import errno 57import os 58import re 59import sha 60import subprocess 61import tempfile 62import threading 63import time 64import zipfile 65 66import common 67import edify_generator 68 69OPTIONS = common.OPTIONS 70OPTIONS.package_key = "build/target/product/security/testkey" 71OPTIONS.incremental_source = None 72OPTIONS.require_verbatim = set() 73OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 74OPTIONS.patch_threshold = 0.95 75OPTIONS.wipe_user_data = False 76OPTIONS.omit_prereq = False 77OPTIONS.extra_script = None 78OPTIONS.worker_threads = 3 79 80def MostPopularKey(d, default): 81 """Given a dict, return the key corresponding to the largest 82 value. Returns 'default' if the dict is empty.""" 83 x = [(v, k) for (k, v) in d.iteritems()] 84 if not x: return default 85 x.sort() 86 return x[-1][1] 87 88 89def IsSymlink(info): 90 """Return true if the zipfile.ZipInfo object passed in represents a 91 symlink.""" 92 return (info.external_attr >> 16) == 0120777 93 94 95 96class Item: 97 """Items represent the metadata (user, group, mode) of files and 98 directories in the system image.""" 99 ITEMS = {} 100 def __init__(self, name, dir=False): 101 self.name = name 102 self.uid = None 103 self.gid = None 104 self.mode = None 105 self.dir = dir 106 107 if name: 108 self.parent = Item.Get(os.path.dirname(name), dir=True) 109 self.parent.children.append(self) 110 else: 111 self.parent = None 112 if dir: 113 self.children = [] 114 115 def Dump(self, indent=0): 116 if self.uid is not None: 117 print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode) 118 else: 119 print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode) 120 if self.dir: 121 print "%s%s" % (" "*indent, self.descendants) 122 print "%s%s" % (" "*indent, self.best_subtree) 123 for i in self.children: 124 i.Dump(indent=indent+1) 125 126 @classmethod 127 def Get(cls, name, dir=False): 128 if name not in cls.ITEMS: 129 cls.ITEMS[name] = Item(name, dir=dir) 130 return cls.ITEMS[name] 131 132 @classmethod 133 def GetMetadata(cls, input_zip): 134 135 try: 136 # See if the target_files contains a record of what the uid, 137 # gid, and mode is supposed to be. 138 output = input_zip.read("META/filesystem_config.txt") 139 except KeyError: 140 # Run the external 'fs_config' program to determine the desired 141 # uid, gid, and mode for every Item object. Note this uses the 142 # one in the client now, which might not be the same as the one 143 # used when this target_files was built. 144 p = common.Run(["fs_config"], stdin=subprocess.PIPE, 145 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 146 suffix = { False: "", True: "/" } 147 input = "".join(["%s%s\n" % (i.name, suffix[i.dir]) 148 for i in cls.ITEMS.itervalues() if i.name]) 149 output, error = p.communicate(input) 150 assert not error 151 152 for line in output.split("\n"): 153 if not line: continue 154 name, uid, gid, mode = line.split() 155 i = cls.ITEMS.get(name, None) 156 if i is not None: 157 i.uid = int(uid) 158 i.gid = int(gid) 159 i.mode = int(mode, 8) 160 if i.dir: 161 i.children.sort(key=lambda i: i.name) 162 163 # set metadata for the files generated by this script. 164 i = cls.ITEMS.get("system/recovery-from-boot.p", None) 165 if i: i.uid, i.gid, i.mode = 0, 0, 0644 166 i = cls.ITEMS.get("system/etc/install-recovery.sh", None) 167 if i: i.uid, i.gid, i.mode = 0, 0, 0544 168 169 def CountChildMetadata(self): 170 """Count up the (uid, gid, mode) tuples for all children and 171 determine the best strategy for using set_perm_recursive and 172 set_perm to correctly chown/chmod all the files to their desired 173 values. Recursively calls itself for all descendants. 174 175 Returns a dict of {(uid, gid, dmode, fmode): count} counting up 176 all descendants of this node. (dmode or fmode may be None.) Also 177 sets the best_subtree of each directory Item to the (uid, gid, 178 dmode, fmode) tuple that will match the most descendants of that 179 Item. 180 """ 181 182 assert self.dir 183 d = self.descendants = {(self.uid, self.gid, self.mode, None): 1} 184 for i in self.children: 185 if i.dir: 186 for k, v in i.CountChildMetadata().iteritems(): 187 d[k] = d.get(k, 0) + v 188 else: 189 k = (i.uid, i.gid, None, i.mode) 190 d[k] = d.get(k, 0) + 1 191 192 # Find the (uid, gid, dmode, fmode) tuple that matches the most 193 # descendants. 194 195 # First, find the (uid, gid) pair that matches the most 196 # descendants. 197 ug = {} 198 for (uid, gid, _, _), count in d.iteritems(): 199 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 200 ug = MostPopularKey(ug, (0, 0)) 201 202 # Now find the dmode and fmode that match the most descendants 203 # with that (uid, gid), and choose those. 204 best_dmode = (0, 0755) 205 best_fmode = (0, 0644) 206 for k, count in d.iteritems(): 207 if k[:2] != ug: continue 208 if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2]) 209 if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3]) 210 self.best_subtree = ug + (best_dmode[1], best_fmode[1]) 211 212 return d 213 214 def SetPermissions(self, script): 215 """Append set_perm/set_perm_recursive commands to 'script' to 216 set all permissions, users, and groups for the tree of files 217 rooted at 'self'.""" 218 219 self.CountChildMetadata() 220 221 def recurse(item, current): 222 # current is the (uid, gid, dmode, fmode) tuple that the current 223 # item (and all its children) have already been set to. We only 224 # need to issue set_perm/set_perm_recursive commands if we're 225 # supposed to be something different. 226 if item.dir: 227 if current != item.best_subtree: 228 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 229 current = item.best_subtree 230 231 if item.uid != current[0] or item.gid != current[1] or \ 232 item.mode != current[2]: 233 script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode) 234 235 for i in item.children: 236 recurse(i, current) 237 else: 238 if item.uid != current[0] or item.gid != current[1] or \ 239 item.mode != current[3]: 240 script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode) 241 242 recurse(self, (-1, -1, -1, -1)) 243 244 245def CopySystemFiles(input_zip, output_zip=None, 246 substitute=None): 247 """Copies files underneath system/ in the input zip to the output 248 zip. Populates the Item class with their metadata, and returns a 249 list of symlinks. output_zip may be None, in which case the copy is 250 skipped (but the other side effects still happen). substitute is an 251 optional dict of {output filename: contents} to be output instead of 252 certain input files. 253 """ 254 255 symlinks = [] 256 257 for info in input_zip.infolist(): 258 if info.filename.startswith("SYSTEM/"): 259 basefilename = info.filename[7:] 260 if IsSymlink(info): 261 symlinks.append((input_zip.read(info.filename), 262 "/system/" + basefilename)) 263 else: 264 info2 = copy.copy(info) 265 fn = info2.filename = "system/" + basefilename 266 if substitute and fn in substitute and substitute[fn] is None: 267 continue 268 if output_zip is not None: 269 if substitute and fn in substitute: 270 data = substitute[fn] 271 else: 272 data = input_zip.read(info.filename) 273 output_zip.writestr(info2, data) 274 if fn.endswith("/"): 275 Item.Get(fn[:-1], dir=True) 276 else: 277 Item.Get(fn, dir=False) 278 279 symlinks.sort() 280 return symlinks 281 282 283def SignOutput(temp_zip_name, output_zip_name): 284 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 285 pw = key_passwords[OPTIONS.package_key] 286 287 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 288 whole_file=True) 289 290 291def AppendAssertions(script, input_zip): 292 device = GetBuildProp("ro.product.device", input_zip) 293 script.AssertDevice(device) 294 295 296def MakeRecoveryPatch(output_zip, recovery_img, boot_img, info): 297 """Generate a binary patch that creates the recovery image starting 298 with the boot image. (Most of the space in these images is just the 299 kernel, which is identical for the two, so the resulting patch 300 should be efficient.) Add it to the output zip, along with a shell 301 script that is run from init.rc on first boot to actually do the 302 patching and install the new recovery image. 303 304 recovery_img and boot_img should be File objects for the 305 corresponding images. info should be the dictionary returned by 306 common.LoadInfoDict() on the input target_files. 307 308 Returns an Item for the shell script, which must be made 309 executable. 310 """ 311 312 d = Difference(recovery_img, boot_img) 313 _, _, patch = d.ComputePatch() 314 common.ZipWriteStr(output_zip, "recovery/recovery-from-boot.p", patch) 315 Item.Get("system/recovery-from-boot.p", dir=False) 316 317 # Images with different content will have a different first page, so 318 # we check to see if this recovery has already been installed by 319 # testing just the first 2k. 320 HEADER_SIZE = 2048 321 header_sha1 = sha.sha(recovery_img.data[:HEADER_SIZE]).hexdigest() 322 sh = """#!/system/bin/sh 323if ! applypatch -c %(partition_type)s:%(partition_path)srecovery:%(header_size)d:%(header_sha1)s; then 324 log -t recovery "Installing new recovery image" 325 applypatch %(partition_type)s:%(partition_path)sboot:%(boot_size)d:%(boot_sha1)s %(partition_type)s:%(partition_path)srecovery %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p 326else 327 log -t recovery "Recovery image already installed" 328fi 329""" % { 'boot_size': boot_img.size, 330 'boot_sha1': boot_img.sha1, 331 'header_size': HEADER_SIZE, 332 'header_sha1': header_sha1, 333 'recovery_size': recovery_img.size, 334 'recovery_sha1': recovery_img.sha1, 335 'partition_type': info["partition_type"], 336 'partition_path': info.get("partition_path", ""), 337 } 338 common.ZipWriteStr(output_zip, "recovery/etc/install-recovery.sh", sh) 339 return Item.Get("system/etc/install-recovery.sh", dir=False) 340 341 342def WriteFullOTAPackage(input_zip, output_zip, info): 343 # TODO: how to determine this? We don't know what version it will 344 # be installed on top of. For now, we expect the API just won't 345 # change very often. 346 script = edify_generator.EdifyGenerator(3, info) 347 348 metadata = {"post-build": GetBuildProp("ro.build.fingerprint", input_zip), 349 "pre-device": GetBuildProp("ro.product.device", input_zip), 350 "post-timestamp": GetBuildProp("ro.build.date.utc", input_zip), 351 } 352 353 device_specific = common.DeviceSpecificParams( 354 input_zip=input_zip, 355 input_version=GetRecoveryAPIVersion(input_zip), 356 output_zip=output_zip, 357 script=script, 358 input_tmp=OPTIONS.input_tmp, 359 metadata=metadata) 360 361 if not OPTIONS.omit_prereq: 362 ts = GetBuildProp("ro.build.date.utc", input_zip) 363 script.AssertOlderBuild(ts) 364 365 AppendAssertions(script, input_zip) 366 device_specific.FullOTA_Assertions() 367 368 script.ShowProgress(0.5, 0) 369 370 if OPTIONS.wipe_user_data: 371 script.FormatPartition("userdata") 372 373 script.FormatPartition("system") 374 script.Mount("system", "/system") 375 script.UnpackPackageDir("recovery", "/system") 376 script.UnpackPackageDir("system", "/system") 377 378 symlinks = CopySystemFiles(input_zip, output_zip) 379 script.MakeSymlinks(symlinks) 380 381 boot_img = File("boot.img", common.BuildBootableImage( 382 os.path.join(OPTIONS.input_tmp, "BOOT"))) 383 recovery_img = File("recovery.img", common.BuildBootableImage( 384 os.path.join(OPTIONS.input_tmp, "RECOVERY"))) 385 MakeRecoveryPatch(output_zip, recovery_img, boot_img, info) 386 387 Item.GetMetadata(input_zip) 388 Item.Get("system").SetPermissions(script) 389 390 common.CheckSize(boot_img.data, "boot.img") 391 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 392 script.ShowProgress(0.2, 0) 393 394 script.ShowProgress(0.2, 10) 395 script.WriteRawImage("boot", "boot.img") 396 397 script.ShowProgress(0.1, 0) 398 device_specific.FullOTA_InstallEnd() 399 400 if OPTIONS.extra_script is not None: 401 script.AppendExtra(OPTIONS.extra_script) 402 403 script.UnmountAll() 404 script.AddToZip(input_zip, output_zip) 405 WriteMetadata(metadata, output_zip) 406 407 408def WriteMetadata(metadata, output_zip): 409 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata", 410 "".join(["%s=%s\n" % kv 411 for kv in sorted(metadata.iteritems())])) 412 413 414class File(object): 415 def __init__(self, name, data): 416 self.name = name 417 self.data = data 418 self.size = len(data) 419 self.sha1 = sha.sha(data).hexdigest() 420 421 def WriteToTemp(self): 422 t = tempfile.NamedTemporaryFile() 423 t.write(self.data) 424 t.flush() 425 return t 426 427 def AddToZip(self, z): 428 common.ZipWriteStr(z, self.name, self.data) 429 430 431def LoadSystemFiles(z): 432 """Load all the files from SYSTEM/... in a given target-files 433 ZipFile, and return a dict of {filename: File object}.""" 434 out = {} 435 for info in z.infolist(): 436 if info.filename.startswith("SYSTEM/") and not IsSymlink(info): 437 fn = "system/" + info.filename[7:] 438 data = z.read(info.filename) 439 out[fn] = File(fn, data) 440 return out 441 442 443DIFF_PROGRAM_BY_EXT = { 444 ".gz" : "imgdiff", 445 ".zip" : ["imgdiff", "-z"], 446 ".jar" : ["imgdiff", "-z"], 447 ".apk" : ["imgdiff", "-z"], 448 ".img" : "imgdiff", 449 } 450 451 452class Difference(object): 453 def __init__(self, tf, sf): 454 self.tf = tf 455 self.sf = sf 456 self.patch = None 457 458 def ComputePatch(self): 459 """Compute the patch (as a string of data) needed to turn sf into 460 tf. Returns the same tuple as GetPatch().""" 461 462 tf = self.tf 463 sf = self.sf 464 465 ext = os.path.splitext(tf.name)[1] 466 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 467 468 ttemp = tf.WriteToTemp() 469 stemp = sf.WriteToTemp() 470 471 ext = os.path.splitext(tf.name)[1] 472 473 try: 474 ptemp = tempfile.NamedTemporaryFile() 475 if isinstance(diff_program, list): 476 cmd = copy.copy(diff_program) 477 else: 478 cmd = [diff_program] 479 cmd.append(stemp.name) 480 cmd.append(ttemp.name) 481 cmd.append(ptemp.name) 482 p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 483 _, err = p.communicate() 484 if err or p.returncode != 0: 485 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 486 return None 487 diff = ptemp.read() 488 finally: 489 ptemp.close() 490 stemp.close() 491 ttemp.close() 492 493 self.patch = diff 494 return self.tf, self.sf, self.patch 495 496 497 def GetPatch(self): 498 """Return a tuple (target_file, source_file, patch_data). 499 patch_data may be None if ComputePatch hasn't been called, or if 500 computing the patch failed.""" 501 return self.tf, self.sf, self.patch 502 503 504def ComputeDifferences(diffs): 505 """Call ComputePatch on all the Difference objects in 'diffs'.""" 506 print len(diffs), "diffs to compute" 507 508 # Do the largest files first, to try and reduce the long-pole effect. 509 by_size = [(i.tf.size, i) for i in diffs] 510 by_size.sort(reverse=True) 511 by_size = [i[1] for i in by_size] 512 513 lock = threading.Lock() 514 diff_iter = iter(by_size) # accessed under lock 515 516 def worker(): 517 try: 518 lock.acquire() 519 for d in diff_iter: 520 lock.release() 521 start = time.time() 522 d.ComputePatch() 523 dur = time.time() - start 524 lock.acquire() 525 526 tf, sf, patch = d.GetPatch() 527 if sf.name == tf.name: 528 name = tf.name 529 else: 530 name = "%s (%s)" % (tf.name, sf.name) 531 if patch is None: 532 print "patching failed! %s" % (name,) 533 else: 534 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 535 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 536 lock.release() 537 except Exception, e: 538 print e 539 raise 540 541 # start worker threads; wait for them all to finish. 542 threads = [threading.Thread(target=worker) 543 for i in range(OPTIONS.worker_threads)] 544 for th in threads: 545 th.start() 546 while threads: 547 threads.pop().join() 548 549 550def GetBuildProp(property, z): 551 """Return the fingerprint of the build of a given target-files 552 ZipFile object.""" 553 bp = z.read("SYSTEM/build.prop") 554 if not property: 555 return bp 556 m = re.search(re.escape(property) + r"=(.*)\n", bp) 557 if not m: 558 raise common.ExternalError("couldn't find %s in build.prop" % (property,)) 559 return m.group(1).strip() 560 561 562def GetRecoveryAPIVersion(zip): 563 """Returns the version of the recovery API. Version 0 is the older 564 amend code (no separate binary).""" 565 try: 566 version = zip.read("META/recovery-api-version.txt") 567 return int(version) 568 except KeyError: 569 try: 570 # version one didn't have the recovery-api-version.txt file, but 571 # it did include an updater binary. 572 zip.getinfo("OTA/bin/updater") 573 return 1 574 except KeyError: 575 return 0 576 577 578def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip, info): 579 source_version = GetRecoveryAPIVersion(source_zip) 580 target_version = GetRecoveryAPIVersion(target_zip) 581 partition_type = info["partition_type"] 582 partition_path = info.get("partition_path", "") 583 584 if source_version == 0: 585 print ("WARNING: generating edify script for a source that " 586 "can't install it.") 587 script = edify_generator.EdifyGenerator(source_version, info) 588 589 metadata = {"pre-device": GetBuildProp("ro.product.device", source_zip), 590 "post-timestamp": GetBuildProp("ro.build.date.utc", target_zip), 591 } 592 593 device_specific = common.DeviceSpecificParams( 594 source_zip=source_zip, 595 source_version=source_version, 596 target_zip=target_zip, 597 target_version=target_version, 598 output_zip=output_zip, 599 script=script, 600 metadata=metadata) 601 602 print "Loading target..." 603 target_data = LoadSystemFiles(target_zip) 604 print "Loading source..." 605 source_data = LoadSystemFiles(source_zip) 606 607 verbatim_targets = [] 608 patch_list = [] 609 diffs = [] 610 largest_source_size = 0 611 for fn in sorted(target_data.keys()): 612 tf = target_data[fn] 613 assert fn == tf.name 614 sf = source_data.get(fn, None) 615 616 if sf is None or fn in OPTIONS.require_verbatim: 617 # This file should be included verbatim 618 if fn in OPTIONS.prohibit_verbatim: 619 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 620 print "send", fn, "verbatim" 621 tf.AddToZip(output_zip) 622 verbatim_targets.append((fn, tf.size)) 623 elif tf.sha1 != sf.sha1: 624 # File is different; consider sending as a patch 625 diffs.append(Difference(tf, sf)) 626 else: 627 # Target file identical to source. 628 pass 629 630 ComputeDifferences(diffs) 631 632 for diff in diffs: 633 tf, sf, d = diff.GetPatch() 634 if d is None or len(d) > tf.size * OPTIONS.patch_threshold: 635 # patch is almost as big as the file; don't bother patching 636 tf.AddToZip(output_zip) 637 verbatim_targets.append((tf.name, tf.size)) 638 else: 639 common.ZipWriteStr(output_zip, "patch/" + tf.name + ".p", d) 640 patch_list.append((tf.name, tf, sf, tf.size, sha.sha(d).hexdigest())) 641 largest_source_size = max(largest_source_size, sf.size) 642 643 source_fp = GetBuildProp("ro.build.fingerprint", source_zip) 644 target_fp = GetBuildProp("ro.build.fingerprint", target_zip) 645 metadata["pre-build"] = source_fp 646 metadata["post-build"] = target_fp 647 648 script.Mount("system", "/system") 649 script.AssertSomeFingerprint(source_fp, target_fp) 650 651 source_boot = File("/tmp/boot.img", 652 common.BuildBootableImage( 653 os.path.join(OPTIONS.source_tmp, "BOOT"))) 654 target_boot = File("/tmp/boot.img", 655 common.BuildBootableImage( 656 os.path.join(OPTIONS.target_tmp, "BOOT"))) 657 updating_boot = (source_boot.data != target_boot.data) 658 659 source_recovery = File("system/recovery.img", 660 common.BuildBootableImage( 661 os.path.join(OPTIONS.source_tmp, "RECOVERY"))) 662 target_recovery = File("system/recovery.img", 663 common.BuildBootableImage( 664 os.path.join(OPTIONS.target_tmp, "RECOVERY"))) 665 updating_recovery = (source_recovery.data != target_recovery.data) 666 667 # Here's how we divide up the progress bar: 668 # 0.1 for verifying the start state (PatchCheck calls) 669 # 0.8 for applying patches (ApplyPatch calls) 670 # 0.1 for unpacking verbatim files, symlinking, and doing the 671 # device-specific commands. 672 673 AppendAssertions(script, target_zip) 674 device_specific.IncrementalOTA_Assertions() 675 676 script.Print("Verifying current system...") 677 678 script.ShowProgress(0.1, 0) 679 total_verify_size = float(sum([i[2].size for i in patch_list]) + 1) 680 if updating_boot: 681 total_verify_size += source_boot.size 682 so_far = 0 683 684 for fn, tf, sf, size, patch_sha in patch_list: 685 script.PatchCheck("/"+fn, tf.sha1, sf.sha1) 686 so_far += sf.size 687 script.SetProgress(so_far / total_verify_size) 688 689 if updating_boot: 690 d = Difference(target_boot, source_boot) 691 _, _, d = d.ComputePatch() 692 print "boot target: %d source: %d diff: %d" % ( 693 target_boot.size, source_boot.size, len(d)) 694 695 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 696 697 script.PatchCheck("%s:%sboot:%d:%s:%d:%s" % 698 (partition_type, partition_path, 699 source_boot.size, source_boot.sha1, 700 target_boot.size, target_boot.sha1)) 701 so_far += source_boot.size 702 script.SetProgress(so_far / total_verify_size) 703 704 if patch_list or updating_recovery or updating_boot: 705 script.CacheFreeSpaceCheck(largest_source_size) 706 707 device_specific.IncrementalOTA_VerifyEnd() 708 709 script.Comment("---- start making changes here ----") 710 711 if OPTIONS.wipe_user_data: 712 script.Print("Erasing user data...") 713 script.FormatPartition("userdata") 714 715 script.Print("Removing unneeded files...") 716 script.DeleteFiles(["/"+i[0] for i in verbatim_targets] + 717 ["/"+i for i in sorted(source_data) 718 if i not in target_data] + 719 ["/system/recovery.img"]) 720 721 script.ShowProgress(0.8, 0) 722 total_patch_size = float(sum([i[1].size for i in patch_list]) + 1) 723 if updating_boot: 724 total_patch_size += target_boot.size 725 so_far = 0 726 727 script.Print("Patching system files...") 728 for fn, tf, sf, size, _ in patch_list: 729 script.ApplyPatch("/"+fn, "-", tf.size, tf.sha1, sf.sha1, "patch/"+fn+".p") 730 so_far += tf.size 731 script.SetProgress(so_far / total_patch_size) 732 733 if updating_boot: 734 # Produce the boot image by applying a patch to the current 735 # contents of the boot partition, and write it back to the 736 # partition. 737 script.Print("Patching boot image...") 738 script.ApplyPatch("%s:%sboot:%d:%s:%d:%s" 739 % (partition_type, partition_path, 740 source_boot.size, source_boot.sha1, 741 target_boot.size, target_boot.sha1), 742 "-", 743 target_boot.size, target_boot.sha1, 744 source_boot.sha1, "patch/boot.img.p") 745 so_far += target_boot.size 746 script.SetProgress(so_far / total_patch_size) 747 print "boot image changed; including." 748 else: 749 print "boot image unchanged; skipping." 750 751 if updating_recovery: 752 # Is it better to generate recovery as a patch from the current 753 # boot image, or from the previous recovery image? For large 754 # updates with significant kernel changes, probably the former. 755 # For small updates where the kernel hasn't changed, almost 756 # certainly the latter. We pick the first option. Future 757 # complicated schemes may let us effectively use both. 758 # 759 # A wacky possibility: as long as there is room in the boot 760 # partition, include the binaries and image files from recovery in 761 # the boot image (though not in the ramdisk) so they can be used 762 # as fodder for constructing the recovery image. 763 MakeRecoveryPatch(output_zip, target_recovery, target_boot, info) 764 script.DeleteFiles(["/system/recovery-from-boot.p", 765 "/system/etc/install-recovery.sh"]) 766 print "recovery image changed; including as patch from boot." 767 else: 768 print "recovery image unchanged; skipping." 769 770 script.ShowProgress(0.1, 10) 771 772 target_symlinks = CopySystemFiles(target_zip, None) 773 774 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 775 temp_script = script.MakeTemporary() 776 Item.GetMetadata(target_zip) 777 Item.Get("system").SetPermissions(temp_script) 778 779 # Note that this call will mess up the tree of Items, so make sure 780 # we're done with it. 781 source_symlinks = CopySystemFiles(source_zip, None) 782 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 783 784 # Delete all the symlinks in source that aren't in target. This 785 # needs to happen before verbatim files are unpacked, in case a 786 # symlink in the source is replaced by a real file in the target. 787 to_delete = [] 788 for dest, link in source_symlinks: 789 if link not in target_symlinks_d: 790 to_delete.append(link) 791 script.DeleteFiles(to_delete) 792 793 if verbatim_targets: 794 script.Print("Unpacking new files...") 795 script.UnpackPackageDir("system", "/system") 796 797 if updating_recovery: 798 script.Print("Unpacking new recovery...") 799 script.UnpackPackageDir("recovery", "/system") 800 801 script.Print("Symlinks and permissions...") 802 803 # Create all the symlinks that don't already exist, or point to 804 # somewhere different than what we want. Delete each symlink before 805 # creating it, since the 'symlink' command won't overwrite. 806 to_create = [] 807 for dest, link in target_symlinks: 808 if link in source_symlinks_d: 809 if dest != source_symlinks_d[link]: 810 to_create.append((dest, link)) 811 else: 812 to_create.append((dest, link)) 813 script.DeleteFiles([i[1] for i in to_create]) 814 script.MakeSymlinks(to_create) 815 816 # Now that the symlinks are created, we can set all the 817 # permissions. 818 script.AppendScript(temp_script) 819 820 # Do device-specific installation (eg, write radio image). 821 device_specific.IncrementalOTA_InstallEnd() 822 823 if OPTIONS.extra_script is not None: 824 script.AppendExtra(OPTIONS.extra_script) 825 826 script.AddToZip(target_zip, output_zip) 827 WriteMetadata(metadata, output_zip) 828 829 830def main(argv): 831 832 def option_handler(o, a): 833 if o in ("-b", "--board_config"): 834 pass # deprecated 835 elif o in ("-k", "--package_key"): 836 OPTIONS.package_key = a 837 elif o in ("-i", "--incremental_from"): 838 OPTIONS.incremental_source = a 839 elif o in ("-w", "--wipe_user_data"): 840 OPTIONS.wipe_user_data = True 841 elif o in ("-n", "--no_prereq"): 842 OPTIONS.omit_prereq = True 843 elif o in ("-e", "--extra_script"): 844 OPTIONS.extra_script = a 845 elif o in ("--worker_threads"): 846 OPTIONS.worker_threads = int(a) 847 else: 848 return False 849 return True 850 851 args = common.ParseOptions(argv, __doc__, 852 extra_opts="b:k:i:d:wne:", 853 extra_long_opts=["board_config=", 854 "package_key=", 855 "incremental_from=", 856 "wipe_user_data", 857 "no_prereq", 858 "extra_script=", 859 "worker_threads="], 860 extra_option_handler=option_handler) 861 862 if len(args) != 2: 863 common.Usage(__doc__) 864 sys.exit(1) 865 866 if OPTIONS.extra_script is not None: 867 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 868 869 print "unzipping target target-files..." 870 OPTIONS.input_tmp = common.UnzipTemp(args[0]) 871 872 if OPTIONS.device_specific is None: 873 # look for the device-specific tools extension location in the input 874 try: 875 f = open(os.path.join(OPTIONS.input_tmp, "META", "tool-extensions.txt")) 876 ds = f.read().strip() 877 f.close() 878 if ds: 879 ds = os.path.normpath(ds) 880 print "using device-specific extensions in", ds 881 OPTIONS.device_specific = ds 882 except IOError, e: 883 if e.errno == errno.ENOENT: 884 # nothing specified in the file 885 pass 886 else: 887 raise 888 889 info = common.LoadInfoDict() 890 common.LoadMaxSizes(info) 891 if not OPTIONS.max_image_size: 892 print 893 print " WARNING: Failed to load max image sizes; will not enforce" 894 print " image size limits." 895 print 896 897 OPTIONS.target_tmp = OPTIONS.input_tmp 898 input_zip = zipfile.ZipFile(args[0], "r") 899 if OPTIONS.package_key: 900 temp_zip_file = tempfile.NamedTemporaryFile() 901 output_zip = zipfile.ZipFile(temp_zip_file, "w", 902 compression=zipfile.ZIP_DEFLATED) 903 else: 904 output_zip = zipfile.ZipFile(args[1], "w", 905 compression=zipfile.ZIP_DEFLATED) 906 907 if OPTIONS.incremental_source is None: 908 WriteFullOTAPackage(input_zip, output_zip, info) 909 else: 910 print "unzipping source target-files..." 911 OPTIONS.source_tmp = common.UnzipTemp(OPTIONS.incremental_source) 912 source_zip = zipfile.ZipFile(OPTIONS.incremental_source, "r") 913 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip, info) 914 915 output_zip.close() 916 if OPTIONS.package_key: 917 SignOutput(temp_zip_file.name, args[1]) 918 temp_zip_file.close() 919 920 common.Cleanup() 921 922 print "done." 923 924 925if __name__ == '__main__': 926 try: 927 main(sys.argv[1:]) 928 except common.ExternalError, e: 929 print 930 print " ERROR: %s" % (e,) 931 print 932 sys.exit(1) 933