ota_from_target_files.py revision 9b23f2cd786b46991b7c0198e69264b17875288d
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 -b (--board_config) <file> 25 Deprecated. 26 27 -k (--package_key) <key> Key to use to sign the package (default is 28 the value of default_system_dev_certificate from the input 29 target-files's META/misc_info.txt, or 30 "build/target/product/security/testkey" if that value is not 31 specified). 32 33 For incremental OTAs, the default value is based on the source 34 target-file, not the target build. 35 36 -i (--incremental_from) <file> 37 Generate an incremental OTA using the given target-files zip as 38 the starting build. 39 40 -w (--wipe_user_data) 41 Generate an OTA package that will wipe the user data partition 42 when installed. 43 44 -n (--no_prereq) 45 Omit the timestamp prereq check normally included at the top of 46 the build scripts (used for developer OTA packages which 47 legitimately need to go back and forth). 48 49 -e (--extra_script) <file> 50 Insert the contents of file at the end of the update script. 51 52 -a (--aslr_mode) <on|off> 53 Specify whether to turn on ASLR for the package (on by default). 54 55 -2 (--two_step) 56 Generate a 'two-step' OTA package, where recovery is updated 57 first, so that any changes made to the system partition are done 58 using the new recovery (new kernel, etc.). 59 60""" 61 62import sys 63 64if sys.hexversion < 0x02040000: 65 print >> sys.stderr, "Python 2.4 or newer is required." 66 sys.exit(1) 67 68import copy 69import errno 70import os 71import re 72import subprocess 73import tempfile 74import time 75import zipfile 76 77try: 78 from hashlib import sha1 as sha1 79except ImportError: 80 from sha import sha as sha1 81 82import common 83import edify_generator 84 85OPTIONS = common.OPTIONS 86OPTIONS.package_key = None 87OPTIONS.incremental_source = None 88OPTIONS.require_verbatim = set() 89OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 90OPTIONS.patch_threshold = 0.95 91OPTIONS.wipe_user_data = False 92OPTIONS.omit_prereq = False 93OPTIONS.extra_script = None 94OPTIONS.aslr_mode = True 95OPTIONS.worker_threads = 3 96OPTIONS.two_step = False 97 98def MostPopularKey(d, default): 99 """Given a dict, return the key corresponding to the largest 100 value. Returns 'default' if the dict is empty.""" 101 x = [(v, k) for (k, v) in d.iteritems()] 102 if not x: return default 103 x.sort() 104 return x[-1][1] 105 106 107def IsSymlink(info): 108 """Return true if the zipfile.ZipInfo object passed in represents a 109 symlink.""" 110 return (info.external_attr >> 16) == 0120777 111 112def IsRegular(info): 113 """Return true if the zipfile.ZipInfo object passed in represents a 114 symlink.""" 115 return (info.external_attr >> 28) == 010 116 117class Item: 118 """Items represent the metadata (user, group, mode) of files and 119 directories in the system image.""" 120 ITEMS = {} 121 def __init__(self, name, dir=False): 122 self.name = name 123 self.uid = None 124 self.gid = None 125 self.mode = None 126 self.selabel = None 127 self.capabilities = None 128 self.dir = dir 129 130 if name: 131 self.parent = Item.Get(os.path.dirname(name), dir=True) 132 self.parent.children.append(self) 133 else: 134 self.parent = None 135 if dir: 136 self.children = [] 137 138 def Dump(self, indent=0): 139 if self.uid is not None: 140 print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode) 141 else: 142 print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode) 143 if self.dir: 144 print "%s%s" % (" "*indent, self.descendants) 145 print "%s%s" % (" "*indent, self.best_subtree) 146 for i in self.children: 147 i.Dump(indent=indent+1) 148 149 @classmethod 150 def Get(cls, name, dir=False): 151 if name not in cls.ITEMS: 152 cls.ITEMS[name] = Item(name, dir=dir) 153 return cls.ITEMS[name] 154 155 @classmethod 156 def GetMetadata(cls, input_zip): 157 158 # The target_files contains a record of what the uid, 159 # gid, and mode are supposed to be. 160 output = input_zip.read("META/filesystem_config.txt") 161 162 for line in output.split("\n"): 163 if not line: continue 164 columns = line.split() 165 name, uid, gid, mode = columns[:4] 166 selabel = None 167 capabilities = None 168 169 # After the first 4 columns, there are a series of key=value 170 # pairs. Extract out the fields we care about. 171 for element in columns[4:]: 172 key, value = element.split("=") 173 if key == "selabel": 174 selabel = value 175 if key == "capabilities": 176 capabilities = value 177 178 i = cls.ITEMS.get(name, None) 179 if i is not None: 180 i.uid = int(uid) 181 i.gid = int(gid) 182 i.mode = int(mode, 8) 183 i.selabel = selabel 184 i.capabilities = capabilities 185 if i.dir: 186 i.children.sort(key=lambda i: i.name) 187 188 # set metadata for the files generated by this script. 189 i = cls.ITEMS.get("system/recovery-from-boot.p", None) 190 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0644, None, None 191 i = cls.ITEMS.get("system/etc/install-recovery.sh", None) 192 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0544, None, None 193 194 def CountChildMetadata(self): 195 """Count up the (uid, gid, mode, selabel, capabilities) tuples for 196 all children and determine the best strategy for using set_perm_recursive and 197 set_perm to correctly chown/chmod all the files to their desired 198 values. Recursively calls itself for all descendants. 199 200 Returns a dict of {(uid, gid, dmode, fmode, selabel, capabilities): count} counting up 201 all descendants of this node. (dmode or fmode may be None.) Also 202 sets the best_subtree of each directory Item to the (uid, gid, 203 dmode, fmode, selabel, capabilities) tuple that will match the most 204 descendants of that Item. 205 """ 206 207 assert self.dir 208 d = self.descendants = {(self.uid, self.gid, self.mode, None, self.selabel, self.capabilities): 1} 209 for i in self.children: 210 if i.dir: 211 for k, v in i.CountChildMetadata().iteritems(): 212 d[k] = d.get(k, 0) + v 213 else: 214 k = (i.uid, i.gid, None, i.mode, i.selabel, i.capabilities) 215 d[k] = d.get(k, 0) + 1 216 217 # Find the (uid, gid, dmode, fmode, selabel, capabilities) 218 # tuple that matches the most descendants. 219 220 # First, find the (uid, gid) pair that matches the most 221 # descendants. 222 ug = {} 223 for (uid, gid, _, _, _, _), count in d.iteritems(): 224 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 225 ug = MostPopularKey(ug, (0, 0)) 226 227 # Now find the dmode, fmode, selabel, and capabilities that match 228 # the most descendants with that (uid, gid), and choose those. 229 best_dmode = (0, 0755) 230 best_fmode = (0, 0644) 231 best_selabel = (0, None) 232 best_capabilities = (0, None) 233 for k, count in d.iteritems(): 234 if k[:2] != ug: continue 235 if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2]) 236 if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3]) 237 if k[4] is not None and count >= best_selabel[0]: best_selabel = (count, k[4]) 238 if k[5] is not None and count >= best_capabilities[0]: best_capabilities = (count, k[5]) 239 self.best_subtree = ug + (best_dmode[1], best_fmode[1], best_selabel[1], best_capabilities[1]) 240 241 return d 242 243 def SetPermissions(self, script): 244 """Append set_perm/set_perm_recursive commands to 'script' to 245 set all permissions, users, and groups for the tree of files 246 rooted at 'self'.""" 247 248 self.CountChildMetadata() 249 250 def recurse(item, current): 251 # current is the (uid, gid, dmode, fmode, selabel, capabilities) tuple that the current 252 # item (and all its children) have already been set to. We only 253 # need to issue set_perm/set_perm_recursive commands if we're 254 # supposed to be something different. 255 if item.dir: 256 if current != item.best_subtree: 257 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 258 current = item.best_subtree 259 260 if item.uid != current[0] or item.gid != current[1] or \ 261 item.mode != current[2] or item.selabel != current[4] or \ 262 item.capabilities != current[5]: 263 script.SetPermissions("/"+item.name, item.uid, item.gid, 264 item.mode, item.selabel, item.capabilities) 265 266 for i in item.children: 267 recurse(i, current) 268 else: 269 if item.uid != current[0] or item.gid != current[1] or \ 270 item.mode != current[3] or item.selabel != current[4] or \ 271 item.capabilities != current[5]: 272 script.SetPermissions("/"+item.name, item.uid, item.gid, 273 item.mode, item.selabel, item.capabilities) 274 275 recurse(self, (-1, -1, -1, -1, None, None)) 276 277 278def CopySystemFiles(input_zip, output_zip=None, 279 substitute=None): 280 """Copies files underneath system/ in the input zip to the output 281 zip. Populates the Item class with their metadata, and returns a 282 list of symlinks. output_zip may be None, in which case the copy is 283 skipped (but the other side effects still happen). substitute is an 284 optional dict of {output filename: contents} to be output instead of 285 certain input files. 286 """ 287 288 symlinks = [] 289 290 for info in input_zip.infolist(): 291 if info.filename.startswith("SYSTEM/"): 292 basefilename = info.filename[7:] 293 if IsSymlink(info): 294 symlinks.append((input_zip.read(info.filename), 295 "/system/" + basefilename)) 296 else: 297 info2 = copy.copy(info) 298 fn = info2.filename = "system/" + basefilename 299 if substitute and fn in substitute and substitute[fn] is None: 300 continue 301 if output_zip is not None: 302 if substitute and fn in substitute: 303 data = substitute[fn] 304 else: 305 data = input_zip.read(info.filename) 306 output_zip.writestr(info2, data) 307 if fn.endswith("/"): 308 Item.Get(fn[:-1], dir=True) 309 else: 310 Item.Get(fn, dir=False) 311 312 symlinks.sort() 313 return symlinks 314 315 316def SignOutput(temp_zip_name, output_zip_name): 317 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 318 pw = key_passwords[OPTIONS.package_key] 319 320 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 321 whole_file=True) 322 323 324def AppendAssertions(script, info_dict): 325 device = GetBuildProp("ro.product.device", info_dict) 326 script.AssertDevice(device) 327 328 329def MakeRecoveryPatch(input_tmp, output_zip, recovery_img, boot_img): 330 """Generate a binary patch that creates the recovery image starting 331 with the boot image. (Most of the space in these images is just the 332 kernel, which is identical for the two, so the resulting patch 333 should be efficient.) Add it to the output zip, along with a shell 334 script that is run from init.rc on first boot to actually do the 335 patching and install the new recovery image. 336 337 recovery_img and boot_img should be File objects for the 338 corresponding images. info should be the dictionary returned by 339 common.LoadInfoDict() on the input target_files. 340 341 Returns an Item for the shell script, which must be made 342 executable. 343 """ 344 345 diff_program = ["imgdiff"] 346 path = os.path.join(input_tmp, "SYSTEM", "etc", "recovery-resource.dat") 347 if os.path.exists(path): 348 diff_program.append("-b") 349 diff_program.append(path) 350 bonus_args = "-b /system/etc/recovery-resource.dat" 351 else: 352 bonus_args = "" 353 354 d = common.Difference(recovery_img, boot_img, diff_program=diff_program) 355 _, _, patch = d.ComputePatch() 356 common.ZipWriteStr(output_zip, "recovery/recovery-from-boot.p", patch) 357 Item.Get("system/recovery-from-boot.p", dir=False) 358 359 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 360 recovery_type, recovery_device = common.GetTypeAndDevice("/recovery", OPTIONS.info_dict) 361 362 sh = """#!/system/bin/sh 363if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 364 log -t recovery "Installing new recovery image" 365 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p 366else 367 log -t recovery "Recovery image already installed" 368fi 369""" % { 'boot_size': boot_img.size, 370 'boot_sha1': boot_img.sha1, 371 'recovery_size': recovery_img.size, 372 'recovery_sha1': recovery_img.sha1, 373 'boot_type': boot_type, 374 'boot_device': boot_device, 375 'recovery_type': recovery_type, 376 'recovery_device': recovery_device, 377 'bonus_args': bonus_args, 378 } 379 common.ZipWriteStr(output_zip, "recovery/etc/install-recovery.sh", sh) 380 return Item.Get("system/etc/install-recovery.sh", dir=False) 381 382 383def WriteFullOTAPackage(input_zip, output_zip): 384 # TODO: how to determine this? We don't know what version it will 385 # be installed on top of. For now, we expect the API just won't 386 # change very often. 387 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 388 389 metadata = {"post-build": GetBuildProp("ro.build.fingerprint", 390 OPTIONS.info_dict), 391 "pre-device": GetBuildProp("ro.product.device", 392 OPTIONS.info_dict), 393 "post-timestamp": GetBuildProp("ro.build.date.utc", 394 OPTIONS.info_dict), 395 } 396 397 device_specific = common.DeviceSpecificParams( 398 input_zip=input_zip, 399 input_version=OPTIONS.info_dict["recovery_api_version"], 400 output_zip=output_zip, 401 script=script, 402 input_tmp=OPTIONS.input_tmp, 403 metadata=metadata, 404 info_dict=OPTIONS.info_dict) 405 406 if not OPTIONS.omit_prereq: 407 ts = GetBuildProp("ro.build.date.utc", OPTIONS.info_dict) 408 ts_text = GetBuildProp("ro.build.date", OPTIONS.info_dict) 409 script.AssertOlderBuild(ts, ts_text) 410 411 AppendAssertions(script, OPTIONS.info_dict) 412 device_specific.FullOTA_Assertions() 413 414 # Two-step package strategy (in chronological order, which is *not* 415 # the order in which the generated script has things): 416 # 417 # if stage is not "2/3" or "3/3": 418 # write recovery image to boot partition 419 # set stage to "2/3" 420 # reboot to boot partition and restart recovery 421 # else if stage is "2/3": 422 # write recovery image to recovery partition 423 # set stage to "3/3" 424 # reboot to recovery partition and restart recovery 425 # else: 426 # (stage must be "3/3") 427 # set stage to "" 428 # do normal full package installation: 429 # wipe and install system, boot image, etc. 430 # set up system to update recovery partition on first boot 431 # complete script normally (allow recovery to mark itself finished and reboot) 432 433 recovery_img = common.GetBootableImage("recovery.img", "recovery.img", 434 OPTIONS.input_tmp, "RECOVERY") 435 if OPTIONS.two_step: 436 if not OPTIONS.info_dict.get("multistage_support", None): 437 assert False, "two-step packages not supported by this build" 438 fs = OPTIONS.info_dict["fstab"]["/misc"] 439 assert fs.fs_type.upper() == "EMMC", \ 440 "two-step packages only supported on devices with EMMC /misc partitions" 441 bcb_dev = {"bcb_dev": fs.device} 442 common.ZipWriteStr(output_zip, "recovery.img", recovery_img.data) 443 script.AppendExtra(""" 444if get_stage("%(bcb_dev)s", "stage") == "2/3" then 445""" % bcb_dev) 446 script.WriteRawImage("/recovery", "recovery.img") 447 script.AppendExtra(""" 448set_stage("%(bcb_dev)s", "3/3"); 449reboot_now("%(bcb_dev)s", "recovery"); 450else if get_stage("%(bcb_dev)s", "stage") == "3/3" then 451""" % bcb_dev) 452 453 device_specific.FullOTA_InstallBegin() 454 455 script.ShowProgress(0.5, 0) 456 457 if OPTIONS.wipe_user_data: 458 script.FormatPartition("/data") 459 460 if "selinux_fc" in OPTIONS.info_dict: 461 WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip) 462 463 script.FormatPartition("/system") 464 script.Mount("/system") 465 script.UnpackPackageDir("recovery", "/system") 466 script.UnpackPackageDir("system", "/system") 467 468 symlinks = CopySystemFiles(input_zip, output_zip) 469 script.MakeSymlinks(symlinks) 470 471 boot_img = common.GetBootableImage("boot.img", "boot.img", 472 OPTIONS.input_tmp, "BOOT") 473 MakeRecoveryPatch(OPTIONS.input_tmp, output_zip, recovery_img, boot_img) 474 475 Item.GetMetadata(input_zip) 476 Item.Get("system").SetPermissions(script) 477 478 common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict) 479 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 480 script.ShowProgress(0.2, 0) 481 482 script.ShowProgress(0.2, 10) 483 script.WriteRawImage("/boot", "boot.img") 484 485 script.ShowProgress(0.1, 0) 486 device_specific.FullOTA_InstallEnd() 487 488 if OPTIONS.extra_script is not None: 489 script.AppendExtra(OPTIONS.extra_script) 490 491 script.UnmountAll() 492 493 if OPTIONS.two_step: 494 script.AppendExtra(""" 495set_stage("%(bcb_dev)s", ""); 496""" % bcb_dev) 497 script.AppendExtra("else\n") 498 script.WriteRawImage("/boot", "recovery.img") 499 script.AppendExtra(""" 500set_stage("%(bcb_dev)s", "2/3"); 501reboot_now("%(bcb_dev)s", ""); 502endif; 503endif; 504""" % bcb_dev) 505 script.AddToZip(input_zip, output_zip) 506 WriteMetadata(metadata, output_zip) 507 508def WritePolicyConfig(file_context, output_zip): 509 f = open(file_context, 'r'); 510 basename = os.path.basename(file_context) 511 common.ZipWriteStr(output_zip, basename, f.read()) 512 513 514def WriteMetadata(metadata, output_zip): 515 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata", 516 "".join(["%s=%s\n" % kv 517 for kv in sorted(metadata.iteritems())])) 518 519def LoadSystemFiles(z): 520 """Load all the files from SYSTEM/... in a given target-files 521 ZipFile, and return a dict of {filename: File object}.""" 522 out = {} 523 for info in z.infolist(): 524 if info.filename.startswith("SYSTEM/") and not IsSymlink(info): 525 basefilename = info.filename[7:] 526 fn = "system/" + basefilename 527 data = z.read(info.filename) 528 out[fn] = common.File(fn, data) 529 return out 530 531 532def GetBuildProp(prop, info_dict): 533 """Return the fingerprint of the build of a given target-files info_dict.""" 534 try: 535 return info_dict.get("build.prop", {})[prop] 536 except KeyError: 537 raise common.ExternalError("couldn't find %s in build.prop" % (property,)) 538 539 540def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): 541 source_version = OPTIONS.source_info_dict["recovery_api_version"] 542 target_version = OPTIONS.target_info_dict["recovery_api_version"] 543 544 if source_version == 0: 545 print ("WARNING: generating edify script for a source that " 546 "can't install it.") 547 script = edify_generator.EdifyGenerator(source_version, 548 OPTIONS.target_info_dict) 549 550 metadata = {"pre-device": GetBuildProp("ro.product.device", 551 OPTIONS.source_info_dict), 552 "post-timestamp": GetBuildProp("ro.build.date.utc", 553 OPTIONS.target_info_dict), 554 } 555 556 device_specific = common.DeviceSpecificParams( 557 source_zip=source_zip, 558 source_version=source_version, 559 target_zip=target_zip, 560 target_version=target_version, 561 output_zip=output_zip, 562 script=script, 563 metadata=metadata, 564 info_dict=OPTIONS.info_dict) 565 566 print "Loading target..." 567 target_data = LoadSystemFiles(target_zip) 568 print "Loading source..." 569 source_data = LoadSystemFiles(source_zip) 570 571 verbatim_targets = [] 572 patch_list = [] 573 diffs = [] 574 largest_source_size = 0 575 for fn in sorted(target_data.keys()): 576 tf = target_data[fn] 577 assert fn == tf.name 578 sf = source_data.get(fn, None) 579 580 if sf is None or fn in OPTIONS.require_verbatim: 581 # This file should be included verbatim 582 if fn in OPTIONS.prohibit_verbatim: 583 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 584 print "send", fn, "verbatim" 585 tf.AddToZip(output_zip) 586 verbatim_targets.append((fn, tf.size)) 587 elif tf.sha1 != sf.sha1: 588 # File is different; consider sending as a patch 589 diffs.append(common.Difference(tf, sf)) 590 else: 591 # Target file identical to source. 592 pass 593 594 common.ComputeDifferences(diffs) 595 596 for diff in diffs: 597 tf, sf, d = diff.GetPatch() 598 if d is None or len(d) > tf.size * OPTIONS.patch_threshold: 599 # patch is almost as big as the file; don't bother patching 600 tf.AddToZip(output_zip) 601 verbatim_targets.append((tf.name, tf.size)) 602 else: 603 common.ZipWriteStr(output_zip, "patch/" + tf.name + ".p", d) 604 patch_list.append((tf.name, tf, sf, tf.size, common.sha1(d).hexdigest())) 605 largest_source_size = max(largest_source_size, sf.size) 606 607 source_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.source_info_dict) 608 target_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.target_info_dict) 609 metadata["pre-build"] = source_fp 610 metadata["post-build"] = target_fp 611 612 script.Mount("/system") 613 script.AssertSomeFingerprint(source_fp, target_fp) 614 615 source_boot = common.GetBootableImage( 616 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 617 OPTIONS.source_info_dict) 618 target_boot = common.GetBootableImage( 619 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 620 updating_boot = (not OPTIONS.two_step and 621 (source_boot.data != target_boot.data)) 622 623 source_recovery = common.GetBootableImage( 624 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 625 OPTIONS.source_info_dict) 626 target_recovery = common.GetBootableImage( 627 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 628 updating_recovery = (source_recovery.data != target_recovery.data) 629 630 # Here's how we divide up the progress bar: 631 # 0.1 for verifying the start state (PatchCheck calls) 632 # 0.8 for applying patches (ApplyPatch calls) 633 # 0.1 for unpacking verbatim files, symlinking, and doing the 634 # device-specific commands. 635 636 AppendAssertions(script, OPTIONS.target_info_dict) 637 device_specific.IncrementalOTA_Assertions() 638 639 # Two-step incremental package strategy (in chronological order, 640 # which is *not* the order in which the generated script has 641 # things): 642 # 643 # if stage is not "2/3" or "3/3": 644 # do verification on current system 645 # write recovery image to boot partition 646 # set stage to "2/3" 647 # reboot to boot partition and restart recovery 648 # else if stage is "2/3": 649 # write recovery image to recovery partition 650 # set stage to "3/3" 651 # reboot to recovery partition and restart recovery 652 # else: 653 # (stage must be "3/3") 654 # perform update: 655 # patch system files, etc. 656 # force full install of new boot image 657 # set up system to update recovery partition on first boot 658 # complete script normally (allow recovery to mark itself finished and reboot) 659 660 if OPTIONS.two_step: 661 if not OPTIONS.info_dict.get("multistage_support", None): 662 assert False, "two-step packages not supported by this build" 663 fs = OPTIONS.info_dict["fstab"]["/misc"] 664 assert fs.fs_type.upper() == "EMMC", \ 665 "two-step packages only supported on devices with EMMC /misc partitions" 666 bcb_dev = {"bcb_dev": fs.device} 667 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 668 script.AppendExtra(""" 669if get_stage("%(bcb_dev)s", "stage") == "2/3" then 670""" % bcb_dev) 671 script.AppendExtra("sleep(20);\n"); 672 script.WriteRawImage("/recovery", "recovery.img") 673 script.AppendExtra(""" 674set_stage("%(bcb_dev)s", "3/3"); 675reboot_now("%(bcb_dev)s", "recovery"); 676else if get_stage("%(bcb_dev)s", "stage") != "3/3" then 677""" % bcb_dev) 678 679 script.Print("Verifying current system...") 680 681 device_specific.IncrementalOTA_VerifyBegin() 682 683 script.ShowProgress(0.1, 0) 684 total_verify_size = float(sum([i[2].size for i in patch_list]) + 1) 685 if updating_boot: 686 total_verify_size += source_boot.size 687 so_far = 0 688 689 for fn, tf, sf, size, patch_sha in patch_list: 690 script.PatchCheck("/"+fn, tf.sha1, sf.sha1) 691 so_far += sf.size 692 script.SetProgress(so_far / total_verify_size) 693 694 if updating_boot: 695 d = common.Difference(target_boot, source_boot) 696 _, _, d = d.ComputePatch() 697 print "boot target: %d source: %d diff: %d" % ( 698 target_boot.size, source_boot.size, len(d)) 699 700 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 701 702 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 703 704 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 705 (boot_type, boot_device, 706 source_boot.size, source_boot.sha1, 707 target_boot.size, target_boot.sha1)) 708 so_far += source_boot.size 709 script.SetProgress(so_far / total_verify_size) 710 711 if patch_list or updating_recovery or updating_boot: 712 script.CacheFreeSpaceCheck(largest_source_size) 713 714 device_specific.IncrementalOTA_VerifyEnd() 715 716 if OPTIONS.two_step: 717 script.WriteRawImage("/boot", "recovery.img") 718 script.AppendExtra(""" 719set_stage("%(bcb_dev)s", "2/3"); 720reboot_now("%(bcb_dev)s", ""); 721else 722""" % bcb_dev) 723 724 script.Comment("---- start making changes here ----") 725 726 device_specific.IncrementalOTA_InstallBegin() 727 728 if OPTIONS.two_step: 729 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 730 script.WriteRawImage("/boot", "boot.img") 731 print "writing full boot image (forced by two-step mode)" 732 733 if OPTIONS.wipe_user_data: 734 script.Print("Erasing user data...") 735 script.FormatPartition("/data") 736 737 script.Print("Removing unneeded files...") 738 script.DeleteFiles(["/"+i[0] for i in verbatim_targets] + 739 ["/"+i for i in sorted(source_data) 740 if i not in target_data] + 741 ["/system/recovery.img"]) 742 743 script.ShowProgress(0.8, 0) 744 total_patch_size = float(sum([i[1].size for i in patch_list]) + 1) 745 if updating_boot: 746 total_patch_size += target_boot.size 747 so_far = 0 748 749 script.Print("Patching system files...") 750 deferred_patch_list = [] 751 for item in patch_list: 752 fn, tf, sf, size, _ = item 753 if tf.name == "system/build.prop": 754 deferred_patch_list.append(item) 755 continue 756 script.ApplyPatch("/"+fn, "-", tf.size, tf.sha1, sf.sha1, "patch/"+fn+".p") 757 so_far += tf.size 758 script.SetProgress(so_far / total_patch_size) 759 760 if not OPTIONS.two_step: 761 if updating_boot: 762 # Produce the boot image by applying a patch to the current 763 # contents of the boot partition, and write it back to the 764 # partition. 765 script.Print("Patching boot image...") 766 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 767 % (boot_type, boot_device, 768 source_boot.size, source_boot.sha1, 769 target_boot.size, target_boot.sha1), 770 "-", 771 target_boot.size, target_boot.sha1, 772 source_boot.sha1, "patch/boot.img.p") 773 so_far += target_boot.size 774 script.SetProgress(so_far / total_patch_size) 775 print "boot image changed; including." 776 else: 777 print "boot image unchanged; skipping." 778 779 if updating_recovery: 780 # Recovery is generated as a patch using both the boot image 781 # (which contains the same linux kernel as recovery) and the file 782 # /system/etc/recovery-resource.dat (which contains all the images 783 # used in the recovery UI) as sources. This lets us minimize the 784 # size of the patch, which must be included in every OTA package. 785 # 786 # For older builds where recovery-resource.dat is not present, we 787 # use only the boot image as the source. 788 789 MakeRecoveryPatch(OPTIONS.target_tmp, output_zip, 790 target_recovery, target_boot) 791 script.DeleteFiles(["/system/recovery-from-boot.p", 792 "/system/etc/install-recovery.sh"]) 793 print "recovery image changed; including as patch from boot." 794 else: 795 print "recovery image unchanged; skipping." 796 797 script.ShowProgress(0.1, 10) 798 799 target_symlinks = CopySystemFiles(target_zip, None) 800 801 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 802 temp_script = script.MakeTemporary() 803 Item.GetMetadata(target_zip) 804 Item.Get("system").SetPermissions(temp_script) 805 806 # Note that this call will mess up the tree of Items, so make sure 807 # we're done with it. 808 source_symlinks = CopySystemFiles(source_zip, None) 809 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 810 811 # Delete all the symlinks in source that aren't in target. This 812 # needs to happen before verbatim files are unpacked, in case a 813 # symlink in the source is replaced by a real file in the target. 814 to_delete = [] 815 for dest, link in source_symlinks: 816 if link not in target_symlinks_d: 817 to_delete.append(link) 818 script.DeleteFiles(to_delete) 819 820 if verbatim_targets: 821 script.Print("Unpacking new files...") 822 script.UnpackPackageDir("system", "/system") 823 824 if updating_recovery: 825 script.Print("Unpacking new recovery...") 826 script.UnpackPackageDir("recovery", "/system") 827 828 script.Print("Symlinks and permissions...") 829 830 # Create all the symlinks that don't already exist, or point to 831 # somewhere different than what we want. Delete each symlink before 832 # creating it, since the 'symlink' command won't overwrite. 833 to_create = [] 834 for dest, link in target_symlinks: 835 if link in source_symlinks_d: 836 if dest != source_symlinks_d[link]: 837 to_create.append((dest, link)) 838 else: 839 to_create.append((dest, link)) 840 script.DeleteFiles([i[1] for i in to_create]) 841 script.MakeSymlinks(to_create) 842 843 # Now that the symlinks are created, we can set all the 844 # permissions. 845 script.AppendScript(temp_script) 846 847 # Do device-specific installation (eg, write radio image). 848 device_specific.IncrementalOTA_InstallEnd() 849 850 if OPTIONS.extra_script is not None: 851 script.AppendExtra(OPTIONS.extra_script) 852 853 # Patch the build.prop file last, so if something fails but the 854 # device can still come up, it appears to be the old build and will 855 # get set the OTA package again to retry. 856 script.Print("Patching remaining system files...") 857 for item in deferred_patch_list: 858 fn, tf, sf, size, _ = item 859 script.ApplyPatch("/"+fn, "-", tf.size, tf.sha1, sf.sha1, "patch/"+fn+".p") 860 script.SetPermissions("/system/build.prop", 0, 0, 0644, None, None) 861 862 if OPTIONS.two_step: 863 script.AppendExtra(""" 864set_stage("%(bcb_dev)s", ""); 865endif; 866endif; 867""" % bcb_dev) 868 869 script.AddToZip(target_zip, output_zip) 870 WriteMetadata(metadata, output_zip) 871 872 873def main(argv): 874 875 def option_handler(o, a): 876 if o in ("-b", "--board_config"): 877 pass # deprecated 878 elif o in ("-k", "--package_key"): 879 OPTIONS.package_key = a 880 elif o in ("-i", "--incremental_from"): 881 OPTIONS.incremental_source = a 882 elif o in ("-w", "--wipe_user_data"): 883 OPTIONS.wipe_user_data = True 884 elif o in ("-n", "--no_prereq"): 885 OPTIONS.omit_prereq = True 886 elif o in ("-e", "--extra_script"): 887 OPTIONS.extra_script = a 888 elif o in ("-a", "--aslr_mode"): 889 if a in ("on", "On", "true", "True", "yes", "Yes"): 890 OPTIONS.aslr_mode = True 891 else: 892 OPTIONS.aslr_mode = False 893 elif o in ("--worker_threads"): 894 OPTIONS.worker_threads = int(a) 895 elif o in ("-2", "--two_step"): 896 OPTIONS.two_step = True 897 else: 898 return False 899 return True 900 901 args = common.ParseOptions(argv, __doc__, 902 extra_opts="b:k:i:d:wne:a:2", 903 extra_long_opts=["board_config=", 904 "package_key=", 905 "incremental_from=", 906 "wipe_user_data", 907 "no_prereq", 908 "extra_script=", 909 "worker_threads=", 910 "aslr_mode=", 911 "two_step", 912 ], 913 extra_option_handler=option_handler) 914 915 if len(args) != 2: 916 common.Usage(__doc__) 917 sys.exit(1) 918 919 if OPTIONS.extra_script is not None: 920 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 921 922 print "unzipping target target-files..." 923 OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0]) 924 925 OPTIONS.target_tmp = OPTIONS.input_tmp 926 OPTIONS.info_dict = common.LoadInfoDict(input_zip) 927 928 # If this image was originally labelled with SELinux contexts, make sure we 929 # also apply the labels in our new image. During building, the "file_contexts" 930 # is in the out/ directory tree, but for repacking from target-files.zip it's 931 # in the root directory of the ramdisk. 932 if "selinux_fc" in OPTIONS.info_dict: 933 OPTIONS.info_dict["selinux_fc"] = os.path.join(OPTIONS.input_tmp, "BOOT", "RAMDISK", 934 "file_contexts") 935 936 if OPTIONS.verbose: 937 print "--- target info ---" 938 common.DumpInfoDict(OPTIONS.info_dict) 939 940 if OPTIONS.device_specific is None: 941 OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None) 942 if OPTIONS.device_specific is not None: 943 OPTIONS.device_specific = os.path.normpath(OPTIONS.device_specific) 944 print "using device-specific extensions in", OPTIONS.device_specific 945 946 temp_zip_file = tempfile.NamedTemporaryFile() 947 output_zip = zipfile.ZipFile(temp_zip_file, "w", 948 compression=zipfile.ZIP_DEFLATED) 949 950 if OPTIONS.incremental_source is None: 951 WriteFullOTAPackage(input_zip, output_zip) 952 if OPTIONS.package_key is None: 953 OPTIONS.package_key = OPTIONS.info_dict.get( 954 "default_system_dev_certificate", 955 "build/target/product/security/testkey") 956 else: 957 print "unzipping source target-files..." 958 OPTIONS.source_tmp, source_zip = common.UnzipTemp(OPTIONS.incremental_source) 959 OPTIONS.target_info_dict = OPTIONS.info_dict 960 OPTIONS.source_info_dict = common.LoadInfoDict(source_zip) 961 if OPTIONS.package_key is None: 962 OPTIONS.package_key = OPTIONS.source_info_dict.get( 963 "default_system_dev_certificate", 964 "build/target/product/security/testkey") 965 if OPTIONS.verbose: 966 print "--- source info ---" 967 common.DumpInfoDict(OPTIONS.source_info_dict) 968 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) 969 970 output_zip.close() 971 972 SignOutput(temp_zip_file.name, args[1]) 973 temp_zip_file.close() 974 975 common.Cleanup() 976 977 print "done." 978 979 980if __name__ == '__main__': 981 try: 982 common.CloseInheritedPipes() 983 main(sys.argv[1:]) 984 except common.ExternalError, e: 985 print 986 print " ERROR: %s" % (e,) 987 print 988 sys.exit(1) 989