ota_from_target_files.py revision 42265390d993664e7797abc12d7e6bd1c2a6dc6b
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 -b (--board_config) <file> 25 Deprecated. 26 27 -k (--package_key) <key> 28 Key to use to sign the package (default is 29 "build/target/product/security/testkey"). 30 31 -i (--incremental_from) <file> 32 Generate an incremental OTA using the given target-files zip as 33 the starting build. 34 35 -w (--wipe_user_data) 36 Generate an OTA package that will wipe the user data partition 37 when installed. 38 39 -n (--no_prereq) 40 Omit the timestamp prereq check normally included at the top of 41 the build scripts (used for developer OTA packages which 42 legitimately need to go back and forth). 43 44 -e (--extra_script) <file> 45 Insert the contents of file at the end of the update script. 46 47 -m (--script_mode) <mode> 48 Specify 'amend' or 'edify' scripts, or 'auto' to pick 49 automatically (this is the default). 50 51""" 52 53import sys 54 55if sys.hexversion < 0x02040000: 56 print >> sys.stderr, "Python 2.4 or newer is required." 57 sys.exit(1) 58 59import copy 60import errno 61import os 62import re 63import sha 64import subprocess 65import tempfile 66import threading 67import time 68import zipfile 69 70import common 71import amend_generator 72import edify_generator 73import both_generator 74 75OPTIONS = common.OPTIONS 76OPTIONS.package_key = "build/target/product/security/testkey" 77OPTIONS.incremental_source = None 78OPTIONS.require_verbatim = set() 79OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 80OPTIONS.patch_threshold = 0.95 81OPTIONS.wipe_user_data = False 82OPTIONS.omit_prereq = False 83OPTIONS.extra_script = None 84OPTIONS.script_mode = 'auto' 85OPTIONS.worker_threads = 3 86 87def MostPopularKey(d, default): 88 """Given a dict, return the key corresponding to the largest 89 value. Returns 'default' if the dict is empty.""" 90 x = [(v, k) for (k, v) in d.iteritems()] 91 if not x: return default 92 x.sort() 93 return x[-1][1] 94 95 96def IsSymlink(info): 97 """Return true if the zipfile.ZipInfo object passed in represents a 98 symlink.""" 99 return (info.external_attr >> 16) == 0120777 100 101 102 103class Item: 104 """Items represent the metadata (user, group, mode) of files and 105 directories in the system image.""" 106 ITEMS = {} 107 def __init__(self, name, dir=False): 108 self.name = name 109 self.uid = None 110 self.gid = None 111 self.mode = None 112 self.dir = dir 113 114 if name: 115 self.parent = Item.Get(os.path.dirname(name), dir=True) 116 self.parent.children.append(self) 117 else: 118 self.parent = None 119 if dir: 120 self.children = [] 121 122 def Dump(self, indent=0): 123 if self.uid is not None: 124 print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode) 125 else: 126 print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode) 127 if self.dir: 128 print "%s%s" % (" "*indent, self.descendants) 129 print "%s%s" % (" "*indent, self.best_subtree) 130 for i in self.children: 131 i.Dump(indent=indent+1) 132 133 @classmethod 134 def Get(cls, name, dir=False): 135 if name not in cls.ITEMS: 136 cls.ITEMS[name] = Item(name, dir=dir) 137 return cls.ITEMS[name] 138 139 @classmethod 140 def GetMetadata(cls): 141 """Run the external 'fs_config' program to determine the desired 142 uid, gid, and mode for every Item object.""" 143 p = common.Run(["fs_config"], stdin=subprocess.PIPE, 144 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 145 suffix = { False: "", True: "/" } 146 input = "".join(["%s%s\n" % (i.name, suffix[i.dir]) 147 for i in cls.ITEMS.itervalues() if i.name]) 148 output, error = p.communicate(input) 149 assert not error 150 151 for line in output.split("\n"): 152 if not line: continue 153 name, uid, gid, mode = line.split() 154 i = cls.ITEMS[name] 155 i.uid = int(uid) 156 i.gid = int(gid) 157 i.mode = int(mode, 8) 158 if i.dir: 159 i.children.sort(key=lambda i: i.name) 160 161 def CountChildMetadata(self): 162 """Count up the (uid, gid, mode) tuples for all children and 163 determine the best strategy for using set_perm_recursive and 164 set_perm to correctly chown/chmod all the files to their desired 165 values. Recursively calls itself for all descendants. 166 167 Returns a dict of {(uid, gid, dmode, fmode): count} counting up 168 all descendants of this node. (dmode or fmode may be None.) Also 169 sets the best_subtree of each directory Item to the (uid, gid, 170 dmode, fmode) tuple that will match the most descendants of that 171 Item. 172 """ 173 174 assert self.dir 175 d = self.descendants = {(self.uid, self.gid, self.mode, None): 1} 176 for i in self.children: 177 if i.dir: 178 for k, v in i.CountChildMetadata().iteritems(): 179 d[k] = d.get(k, 0) + v 180 else: 181 k = (i.uid, i.gid, None, i.mode) 182 d[k] = d.get(k, 0) + 1 183 184 # Find the (uid, gid, dmode, fmode) tuple that matches the most 185 # descendants. 186 187 # First, find the (uid, gid) pair that matches the most 188 # descendants. 189 ug = {} 190 for (uid, gid, _, _), count in d.iteritems(): 191 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 192 ug = MostPopularKey(ug, (0, 0)) 193 194 # Now find the dmode and fmode that match the most descendants 195 # with that (uid, gid), and choose those. 196 best_dmode = (0, 0755) 197 best_fmode = (0, 0644) 198 for k, count in d.iteritems(): 199 if k[:2] != ug: continue 200 if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2]) 201 if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3]) 202 self.best_subtree = ug + (best_dmode[1], best_fmode[1]) 203 204 return d 205 206 def SetPermissions(self, script): 207 """Append set_perm/set_perm_recursive commands to 'script' to 208 set all permissions, users, and groups for the tree of files 209 rooted at 'self'.""" 210 211 self.CountChildMetadata() 212 213 def recurse(item, current): 214 # current is the (uid, gid, dmode, fmode) tuple that the current 215 # item (and all its children) have already been set to. We only 216 # need to issue set_perm/set_perm_recursive commands if we're 217 # supposed to be something different. 218 if item.dir: 219 if current != item.best_subtree: 220 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 221 current = item.best_subtree 222 223 if item.uid != current[0] or item.gid != current[1] or \ 224 item.mode != current[2]: 225 script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode) 226 227 for i in item.children: 228 recurse(i, current) 229 else: 230 if item.uid != current[0] or item.gid != current[1] or \ 231 item.mode != current[3]: 232 script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode) 233 234 recurse(self, (-1, -1, -1, -1)) 235 236 237def CopySystemFiles(input_zip, output_zip=None, 238 substitute=None): 239 """Copies files underneath system/ in the input zip to the output 240 zip. Populates the Item class with their metadata, and returns a 241 list of symlinks. output_zip may be None, in which case the copy is 242 skipped (but the other side effects still happen). substitute is an 243 optional dict of {output filename: contents} to be output instead of 244 certain input files. 245 """ 246 247 symlinks = [] 248 249 for info in input_zip.infolist(): 250 if info.filename.startswith("SYSTEM/"): 251 basefilename = info.filename[7:] 252 if IsSymlink(info): 253 symlinks.append((input_zip.read(info.filename), 254 "/system/" + basefilename)) 255 else: 256 info2 = copy.copy(info) 257 fn = info2.filename = "system/" + basefilename 258 if substitute and fn in substitute and substitute[fn] is None: 259 continue 260 if output_zip is not None: 261 if substitute and fn in substitute: 262 data = substitute[fn] 263 else: 264 data = input_zip.read(info.filename) 265 output_zip.writestr(info2, data) 266 if fn.endswith("/"): 267 Item.Get(fn[:-1], dir=True) 268 else: 269 Item.Get(fn, dir=False) 270 271 symlinks.sort() 272 return symlinks 273 274 275def SignOutput(temp_zip_name, output_zip_name): 276 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 277 pw = key_passwords[OPTIONS.package_key] 278 279 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 280 whole_file=True) 281 282 283def AppendAssertions(script, input_zip): 284 device = GetBuildProp("ro.product.device", input_zip) 285 script.AssertDevice(device) 286 287 288def MakeRecoveryPatch(output_zip, recovery_img, boot_img): 289 """Generate a binary patch that creates the recovery image starting 290 with the boot image. (Most of the space in these images is just the 291 kernel, which is identical for the two, so the resulting patch 292 should be efficient.) Add it to the output zip, along with a shell 293 script that is run from init.rc on first boot to actually do the 294 patching and install the new recovery image. 295 296 recovery_img and boot_img should be File objects for the 297 corresponding images. 298 299 Returns an Item for the shell script, which must be made 300 executable. 301 """ 302 303 d = Difference(recovery_img, boot_img) 304 _, _, patch = d.ComputePatch() 305 common.ZipWriteStr(output_zip, "recovery/recovery-from-boot.p", patch) 306 Item.Get("system/recovery-from-boot.p", dir=False) 307 308 # Images with different content will have a different first page, so 309 # we check to see if this recovery has already been installed by 310 # testing just the first 2k. 311 HEADER_SIZE = 2048 312 header_sha1 = sha.sha(recovery_img.data[:HEADER_SIZE]).hexdigest() 313 sh = """#!/system/bin/sh 314if ! applypatch -c MTD:recovery:%(header_size)d:%(header_sha1)s; then 315 log -t recovery "Installing new recovery image" 316 applypatch MTD:boot:%(boot_size)d:%(boot_sha1)s MTD:recovery %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p 317else 318 log -t recovery "Recovery image already installed" 319fi 320""" % { 'boot_size': boot_img.size, 321 'boot_sha1': boot_img.sha1, 322 'header_size': HEADER_SIZE, 323 'header_sha1': header_sha1, 324 'recovery_size': recovery_img.size, 325 'recovery_sha1': recovery_img.sha1 } 326 common.ZipWriteStr(output_zip, "recovery/etc/install-recovery.sh", sh) 327 return Item.Get("system/etc/install-recovery.sh", dir=False) 328 329 330def WriteFullOTAPackage(input_zip, output_zip): 331 if OPTIONS.script_mode == "auto": 332 script = both_generator.BothGenerator(2) 333 elif OPTIONS.script_mode == "amend": 334 script = amend_generator.AmendGenerator() 335 else: 336 # TODO: how to determine this? We don't know what version it will 337 # be installed on top of. For now, we expect the API just won't 338 # change very often. 339 script = edify_generator.EdifyGenerator(2) 340 341 device_specific = common.DeviceSpecificParams( 342 input_zip=input_zip, 343 input_version=GetRecoveryAPIVersion(input_zip), 344 output_zip=output_zip, 345 script=script, 346 input_tmp=OPTIONS.input_tmp) 347 348 if not OPTIONS.omit_prereq: 349 ts = GetBuildProp("ro.build.date.utc", input_zip) 350 script.AssertOlderBuild(ts) 351 352 AppendAssertions(script, input_zip) 353 device_specific.FullOTA_Assertions() 354 355 script.ShowProgress(0.5, 0) 356 357 if OPTIONS.wipe_user_data: 358 script.FormatPartition("userdata") 359 360 script.FormatPartition("system") 361 script.Mount("MTD", "system", "/system") 362 script.UnpackPackageDir("recovery", "/system") 363 script.UnpackPackageDir("system", "/system") 364 365 symlinks = CopySystemFiles(input_zip, output_zip) 366 script.MakeSymlinks(symlinks) 367 368 boot_img = File("boot.img", common.BuildBootableImage( 369 os.path.join(OPTIONS.input_tmp, "BOOT"))) 370 recovery_img = File("recovery.img", common.BuildBootableImage( 371 os.path.join(OPTIONS.input_tmp, "RECOVERY"))) 372 i = MakeRecoveryPatch(output_zip, recovery_img, boot_img) 373 374 Item.GetMetadata() 375 376 # GetMetadata uses the data in android_filesystem_config.h to assign 377 # the uid/gid/mode of all files. We want to override that for the 378 # recovery patching shell script to make it executable. 379 i.uid = 0 380 i.gid = 0 381 i.mode = 0544 382 Item.Get("system").SetPermissions(script) 383 384 common.CheckSize(boot_img.data, "boot.img") 385 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 386 script.ShowProgress(0.2, 0) 387 388 script.ShowProgress(0.2, 10) 389 script.WriteRawImage("boot", "boot.img") 390 391 script.ShowProgress(0.1, 0) 392 device_specific.FullOTA_InstallEnd() 393 394 if OPTIONS.extra_script is not None: 395 script.AppendExtra(OPTIONS.extra_script) 396 397 script.UnmountAll() 398 script.AddToZip(input_zip, output_zip) 399 400 401class File(object): 402 def __init__(self, name, data): 403 self.name = name 404 self.data = data 405 self.size = len(data) 406 self.sha1 = sha.sha(data).hexdigest() 407 408 def WriteToTemp(self): 409 t = tempfile.NamedTemporaryFile() 410 t.write(self.data) 411 t.flush() 412 return t 413 414 def AddToZip(self, z): 415 common.ZipWriteStr(z, self.name, self.data) 416 417 418def LoadSystemFiles(z): 419 """Load all the files from SYSTEM/... in a given target-files 420 ZipFile, and return a dict of {filename: File object}.""" 421 out = {} 422 for info in z.infolist(): 423 if info.filename.startswith("SYSTEM/") and not IsSymlink(info): 424 fn = "system/" + info.filename[7:] 425 data = z.read(info.filename) 426 out[fn] = File(fn, data) 427 return out 428 429 430DIFF_PROGRAM_BY_EXT = { 431 ".gz" : "imgdiff", 432 ".zip" : ["imgdiff", "-z"], 433 ".jar" : ["imgdiff", "-z"], 434 ".apk" : ["imgdiff", "-z"], 435 ".img" : "imgdiff", 436 } 437 438 439class Difference(object): 440 def __init__(self, tf, sf): 441 self.tf = tf 442 self.sf = sf 443 self.patch = None 444 445 def ComputePatch(self): 446 """Compute the patch (as a string of data) needed to turn sf into 447 tf. Returns the same tuple as GetPatch().""" 448 449 tf = self.tf 450 sf = self.sf 451 452 ext = os.path.splitext(tf.name)[1] 453 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 454 455 ttemp = tf.WriteToTemp() 456 stemp = sf.WriteToTemp() 457 458 ext = os.path.splitext(tf.name)[1] 459 460 try: 461 ptemp = tempfile.NamedTemporaryFile() 462 if isinstance(diff_program, list): 463 cmd = copy.copy(diff_program) 464 else: 465 cmd = [diff_program] 466 cmd.append(stemp.name) 467 cmd.append(ttemp.name) 468 cmd.append(ptemp.name) 469 p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 470 _, err = p.communicate() 471 if err or p.returncode != 0: 472 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 473 return None 474 diff = ptemp.read() 475 finally: 476 ptemp.close() 477 stemp.close() 478 ttemp.close() 479 480 self.patch = diff 481 return self.tf, self.sf, self.patch 482 483 484 def GetPatch(self): 485 """Return a tuple (target_file, source_file, patch_data). 486 patch_data may be None if ComputePatch hasn't been called, or if 487 computing the patch failed.""" 488 return self.tf, self.sf, self.patch 489 490 491def ComputeDifferences(diffs): 492 """Call ComputePatch on all the Difference objects in 'diffs'.""" 493 print len(diffs), "diffs to compute" 494 495 # Do the largest files first, to try and reduce the long-pole effect. 496 by_size = [(i.tf.size, i) for i in diffs] 497 by_size.sort(reverse=True) 498 by_size = [i[1] for i in by_size] 499 500 lock = threading.Lock() 501 diff_iter = iter(by_size) # accessed under lock 502 503 def worker(): 504 try: 505 lock.acquire() 506 for d in diff_iter: 507 lock.release() 508 start = time.time() 509 d.ComputePatch() 510 dur = time.time() - start 511 lock.acquire() 512 513 tf, sf, patch = d.GetPatch() 514 if sf.name == tf.name: 515 name = tf.name 516 else: 517 name = "%s (%s)" % (tf.name, sf.name) 518 if patch is None: 519 print "patching failed! %s" % (name,) 520 else: 521 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 522 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 523 lock.release() 524 except Exception, e: 525 print e 526 raise 527 528 # start worker threads; wait for them all to finish. 529 threads = [threading.Thread(target=worker) 530 for i in range(OPTIONS.worker_threads)] 531 for th in threads: 532 th.start() 533 while threads: 534 threads.pop().join() 535 536 537def GetBuildProp(property, z): 538 """Return the fingerprint of the build of a given target-files 539 ZipFile object.""" 540 bp = z.read("SYSTEM/build.prop") 541 if not property: 542 return bp 543 m = re.search(re.escape(property) + r"=(.*)\n", bp) 544 if not m: 545 raise common.ExternalError("couldn't find %s in build.prop" % (property,)) 546 return m.group(1).strip() 547 548 549def GetRecoveryAPIVersion(zip): 550 """Returns the version of the recovery API. Version 0 is the older 551 amend code (no separate binary).""" 552 try: 553 version = zip.read("META/recovery-api-version.txt") 554 return int(version) 555 except KeyError: 556 try: 557 # version one didn't have the recovery-api-version.txt file, but 558 # it did include an updater binary. 559 zip.getinfo("OTA/bin/updater") 560 return 1 561 except KeyError: 562 return 0 563 564 565def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): 566 source_version = GetRecoveryAPIVersion(source_zip) 567 target_version = GetRecoveryAPIVersion(target_zip) 568 569 if OPTIONS.script_mode == 'amend': 570 script = amend_generator.AmendGenerator() 571 elif OPTIONS.script_mode == 'edify': 572 if source_version == 0: 573 print ("WARNING: generating edify script for a source that " 574 "can't install it.") 575 script = edify_generator.EdifyGenerator(source_version) 576 elif OPTIONS.script_mode == 'auto': 577 if source_version > 0: 578 script = edify_generator.EdifyGenerator(source_version) 579 else: 580 script = amend_generator.AmendGenerator() 581 else: 582 raise ValueError('unknown script mode "%s"' % (OPTIONS.script_mode,)) 583 584 device_specific = common.DeviceSpecificParams( 585 source_zip=source_zip, 586 source_version=source_version, 587 target_zip=target_zip, 588 target_version=target_version, 589 output_zip=output_zip, 590 script=script) 591 592 print "Loading target..." 593 target_data = LoadSystemFiles(target_zip) 594 print "Loading source..." 595 source_data = LoadSystemFiles(source_zip) 596 597 verbatim_targets = [] 598 patch_list = [] 599 diffs = [] 600 largest_source_size = 0 601 for fn in sorted(target_data.keys()): 602 tf = target_data[fn] 603 assert fn == tf.name 604 sf = source_data.get(fn, None) 605 606 if sf is None or fn in OPTIONS.require_verbatim: 607 # This file should be included verbatim 608 if fn in OPTIONS.prohibit_verbatim: 609 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 610 print "send", fn, "verbatim" 611 tf.AddToZip(output_zip) 612 verbatim_targets.append((fn, tf.size)) 613 elif tf.sha1 != sf.sha1: 614 # File is different; consider sending as a patch 615 diffs.append(Difference(tf, sf)) 616 else: 617 # Target file identical to source. 618 pass 619 620 ComputeDifferences(diffs) 621 622 for diff in diffs: 623 tf, sf, d = diff.GetPatch() 624 if d is None or len(d) > tf.size * OPTIONS.patch_threshold: 625 # patch is almost as big as the file; don't bother patching 626 tf.AddToZip(output_zip) 627 verbatim_targets.append((tf.name, tf.size)) 628 else: 629 common.ZipWriteStr(output_zip, "patch/" + tf.name + ".p", d) 630 patch_list.append((tf.name, tf, sf, tf.size)) 631 largest_source_size = max(largest_source_size, sf.size) 632 633 source_fp = GetBuildProp("ro.build.fingerprint", source_zip) 634 target_fp = GetBuildProp("ro.build.fingerprint", target_zip) 635 636 script.Mount("MTD", "system", "/system") 637 script.AssertSomeFingerprint(source_fp, target_fp) 638 639 source_boot = File("/tmp/boot.img", 640 common.BuildBootableImage( 641 os.path.join(OPTIONS.source_tmp, "BOOT"))) 642 target_boot = File("/tmp/boot.img", 643 common.BuildBootableImage( 644 os.path.join(OPTIONS.target_tmp, "BOOT"))) 645 updating_boot = (source_boot.data != target_boot.data) 646 647 source_recovery = File("system/recovery.img", 648 common.BuildBootableImage( 649 os.path.join(OPTIONS.source_tmp, "RECOVERY"))) 650 target_recovery = File("system/recovery.img", 651 common.BuildBootableImage( 652 os.path.join(OPTIONS.target_tmp, "RECOVERY"))) 653 updating_recovery = (source_recovery.data != target_recovery.data) 654 655 # Here's how we divide up the progress bar: 656 # 0.1 for verifying the start state (PatchCheck calls) 657 # 0.8 for applying patches (ApplyPatch calls) 658 # 0.1 for unpacking verbatim files, symlinking, and doing the 659 # device-specific commands. 660 661 AppendAssertions(script, target_zip) 662 device_specific.IncrementalOTA_Assertions() 663 664 script.Print("Verifying current system...") 665 666 script.ShowProgress(0.1, 0) 667 total_verify_size = float(sum([i[2].size for i in patch_list]) + 1) 668 if updating_boot: 669 total_verify_size += source_boot.size 670 so_far = 0 671 672 for fn, tf, sf, size in patch_list: 673 script.PatchCheck("/"+fn, tf.sha1, sf.sha1) 674 so_far += sf.size 675 script.SetProgress(so_far / total_verify_size) 676 677 if updating_boot: 678 d = Difference(target_boot, source_boot) 679 _, _, d = d.ComputePatch() 680 print "boot target: %d source: %d diff: %d" % ( 681 target_boot.size, source_boot.size, len(d)) 682 683 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 684 685 script.PatchCheck("MTD:boot:%d:%s:%d:%s" % 686 (source_boot.size, source_boot.sha1, 687 target_boot.size, target_boot.sha1)) 688 so_far += source_boot.size 689 script.SetProgress(so_far / total_verify_size) 690 691 if patch_list or updating_recovery or updating_boot: 692 script.CacheFreeSpaceCheck(largest_source_size) 693 script.Print("Unpacking patches...") 694 script.UnpackPackageDir("patch", "/tmp/patchtmp") 695 696 device_specific.IncrementalOTA_VerifyEnd() 697 698 script.Comment("---- start making changes here ----") 699 700 if OPTIONS.wipe_user_data: 701 script.Print("Erasing user data...") 702 script.FormatPartition("userdata") 703 704 script.Print("Removing unneeded files...") 705 script.DeleteFiles(["/"+i[0] for i in verbatim_targets] + 706 ["/"+i for i in sorted(source_data) 707 if i not in target_data] + 708 ["/system/recovery.img"]) 709 710 script.ShowProgress(0.8, 0) 711 total_patch_size = float(sum([i[1].size for i in patch_list]) + 1) 712 if updating_boot: 713 total_patch_size += target_boot.size 714 so_far = 0 715 716 script.Print("Patching system files...") 717 for fn, tf, sf, size in patch_list: 718 script.ApplyPatch("/"+fn, "-", tf.size, tf.sha1, 719 sf.sha1, "/tmp/patchtmp/"+fn+".p") 720 so_far += tf.size 721 script.SetProgress(so_far / total_patch_size) 722 723 if updating_boot: 724 # Produce the boot image by applying a patch to the current 725 # contents of the boot partition, and write it back to the 726 # partition. 727 script.Print("Patching boot image...") 728 script.ApplyPatch("MTD:boot:%d:%s:%d:%s" 729 % (source_boot.size, source_boot.sha1, 730 target_boot.size, target_boot.sha1), 731 "-", 732 target_boot.size, target_boot.sha1, 733 source_boot.sha1, "/tmp/patchtmp/boot.img.p") 734 so_far += target_boot.size 735 script.SetProgress(so_far / total_patch_size) 736 print "boot image changed; including." 737 else: 738 print "boot image unchanged; skipping." 739 740 if updating_recovery: 741 # Is it better to generate recovery as a patch from the current 742 # boot image, or from the previous recovery image? For large 743 # updates with significant kernel changes, probably the former. 744 # For small updates where the kernel hasn't changed, almost 745 # certainly the latter. We pick the first option. Future 746 # complicated schemes may let us effectively use both. 747 # 748 # A wacky possibility: as long as there is room in the boot 749 # partition, include the binaries and image files from recovery in 750 # the boot image (though not in the ramdisk) so they can be used 751 # as fodder for constructing the recovery image. 752 recovery_sh_item = MakeRecoveryPatch(output_zip, 753 target_recovery, target_boot) 754 script.DeleteFiles(["/system/recovery-from-boot.p", 755 "/system/etc/install-recovery.sh"]) 756 print "recovery image changed; including as patch from boot." 757 else: 758 print "recovery image unchanged; skipping." 759 760 script.ShowProgress(0.1, 10) 761 762 target_symlinks = CopySystemFiles(target_zip, None) 763 764 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 765 temp_script = script.MakeTemporary() 766 Item.GetMetadata() 767 if updating_recovery: 768 recovery_sh_item.uid = 0 769 recovery_sh_item.gid = 0 770 recovery_sh_item.mode = 0544 771 Item.Get("system").SetPermissions(temp_script) 772 773 # Note that this call will mess up the tree of Items, so make sure 774 # we're done with it. 775 source_symlinks = CopySystemFiles(source_zip, None) 776 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 777 778 # Delete all the symlinks in source that aren't in target. This 779 # needs to happen before verbatim files are unpacked, in case a 780 # symlink in the source is replaced by a real file in the target. 781 to_delete = [] 782 for dest, link in source_symlinks: 783 if link not in target_symlinks_d: 784 to_delete.append(link) 785 script.DeleteFiles(to_delete) 786 787 if verbatim_targets: 788 script.Print("Unpacking new files...") 789 script.UnpackPackageDir("system", "/system") 790 791 if updating_recovery: 792 script.Print("Unpacking new recovery...") 793 script.UnpackPackageDir("recovery", "/system") 794 795 script.Print("Symlinks and permissions...") 796 797 # Create all the symlinks that don't already exist, or point to 798 # somewhere different than what we want. Delete each symlink before 799 # creating it, since the 'symlink' command won't overwrite. 800 to_create = [] 801 for dest, link in target_symlinks: 802 if link in source_symlinks_d: 803 if dest != source_symlinks_d[link]: 804 to_create.append((dest, link)) 805 else: 806 to_create.append((dest, link)) 807 script.DeleteFiles([i[1] for i in to_create]) 808 script.MakeSymlinks(to_create) 809 810 # Now that the symlinks are created, we can set all the 811 # permissions. 812 script.AppendScript(temp_script) 813 814 # Do device-specific installation (eg, write radio image). 815 device_specific.IncrementalOTA_InstallEnd() 816 817 if OPTIONS.extra_script is not None: 818 scirpt.AppendExtra(OPTIONS.extra_script) 819 820 script.AddToZip(target_zip, output_zip) 821 822 823def main(argv): 824 825 def option_handler(o, a): 826 if o in ("-b", "--board_config"): 827 pass # deprecated 828 elif o in ("-k", "--package_key"): 829 OPTIONS.package_key = a 830 elif o in ("-i", "--incremental_from"): 831 OPTIONS.incremental_source = a 832 elif o in ("-w", "--wipe_user_data"): 833 OPTIONS.wipe_user_data = True 834 elif o in ("-n", "--no_prereq"): 835 OPTIONS.omit_prereq = True 836 elif o in ("-e", "--extra_script"): 837 OPTIONS.extra_script = a 838 elif o in ("-m", "--script_mode"): 839 OPTIONS.script_mode = a 840 elif o in ("--worker_threads"): 841 OPTIONS.worker_threads = int(a) 842 else: 843 return False 844 return True 845 846 args = common.ParseOptions(argv, __doc__, 847 extra_opts="b:k:i:d:wne:m:", 848 extra_long_opts=["board_config=", 849 "package_key=", 850 "incremental_from=", 851 "wipe_user_data", 852 "no_prereq", 853 "extra_script=", 854 "script_mode=", 855 "worker_threads="], 856 extra_option_handler=option_handler) 857 858 if len(args) != 2: 859 common.Usage(__doc__) 860 sys.exit(1) 861 862 if OPTIONS.script_mode not in ("amend", "edify", "auto"): 863 raise ValueError('unknown script mode "%s"' % (OPTIONS.script_mode,)) 864 865 if OPTIONS.extra_script is not None: 866 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 867 868 print "unzipping target target-files..." 869 OPTIONS.input_tmp = common.UnzipTemp(args[0]) 870 871 if OPTIONS.device_specific is None: 872 # look for the device-specific tools extension location in the input 873 try: 874 f = open(os.path.join(OPTIONS.input_tmp, "META", "tool-extensions.txt")) 875 ds = f.read().strip() 876 f.close() 877 if ds: 878 ds = os.path.normpath(ds) 879 print "using device-specific extensions in", ds 880 OPTIONS.device_specific = ds 881 except IOError, e: 882 if e.errno == errno.ENOENT: 883 # nothing specified in the file 884 pass 885 else: 886 raise 887 888 common.LoadMaxSizes() 889 if not OPTIONS.max_image_size: 890 print 891 print " WARNING: Failed to load max image sizes; will not enforce" 892 print " image size limits." 893 print 894 895 OPTIONS.target_tmp = OPTIONS.input_tmp 896 input_zip = zipfile.ZipFile(args[0], "r") 897 if OPTIONS.package_key: 898 temp_zip_file = tempfile.NamedTemporaryFile() 899 output_zip = zipfile.ZipFile(temp_zip_file, "w", 900 compression=zipfile.ZIP_DEFLATED) 901 else: 902 output_zip = zipfile.ZipFile(args[1], "w", 903 compression=zipfile.ZIP_DEFLATED) 904 905 if OPTIONS.incremental_source is None: 906 WriteFullOTAPackage(input_zip, output_zip) 907 else: 908 print "unzipping source target-files..." 909 OPTIONS.source_tmp = common.UnzipTemp(OPTIONS.incremental_source) 910 source_zip = zipfile.ZipFile(OPTIONS.incremental_source, "r") 911 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) 912 913 output_zip.close() 914 if OPTIONS.package_key: 915 SignOutput(temp_zip_file.name, args[1]) 916 temp_zip_file.close() 917 918 common.Cleanup() 919 920 print "done." 921 922 923if __name__ == '__main__': 924 try: 925 main(sys.argv[1:]) 926 except common.ExternalError, e: 927 print 928 print " ERROR: %s" % (e,) 929 print 930 sys.exit(1) 931