ota_from_target_files.py revision eb0a78afc00265479c002364fa62c9e09c3f613d
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 -b (--board_config) <file> 25 Deprecated. 26 27 -k (--package_key) <key> Key to use to sign the package (default is 28 the value of default_system_dev_certificate from the input 29 target-files's META/misc_info.txt, or 30 "build/target/product/security/testkey" if that value is not 31 specified). 32 33 For incremental OTAs, the default value is based on the source 34 target-file, not the target build. 35 36 -i (--incremental_from) <file> 37 Generate an incremental OTA using the given target-files zip as 38 the starting build. 39 40 -w (--wipe_user_data) 41 Generate an OTA package that will wipe the user data partition 42 when installed. 43 44 -n (--no_prereq) 45 Omit the timestamp prereq check normally included at the top of 46 the build scripts (used for developer OTA packages which 47 legitimately need to go back and forth). 48 49 -e (--extra_script) <file> 50 Insert the contents of file at the end of the update script. 51 52 -a (--aslr_mode) <on|off> 53 Specify whether to turn on ASLR for the package (on by default). 54 55 -2 (--two_step) 56 Generate a 'two-step' OTA package, where recovery is updated 57 first, so that any changes made to the system partition are done 58 using the new recovery (new kernel, etc.). 59 60""" 61 62import sys 63 64if sys.hexversion < 0x02040000: 65 print >> sys.stderr, "Python 2.4 or newer is required." 66 sys.exit(1) 67 68import copy 69import errno 70import os 71import re 72import subprocess 73import tempfile 74import time 75import zipfile 76 77try: 78 from hashlib import sha1 as sha1 79except ImportError: 80 from sha import sha as sha1 81 82import common 83import edify_generator 84 85OPTIONS = common.OPTIONS 86OPTIONS.package_key = None 87OPTIONS.incremental_source = None 88OPTIONS.require_verbatim = set() 89OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 90OPTIONS.patch_threshold = 0.95 91OPTIONS.wipe_user_data = False 92OPTIONS.omit_prereq = False 93OPTIONS.extra_script = None 94OPTIONS.aslr_mode = True 95OPTIONS.worker_threads = 3 96OPTIONS.two_step = False 97 98def MostPopularKey(d, default): 99 """Given a dict, return the key corresponding to the largest 100 value. Returns 'default' if the dict is empty.""" 101 x = [(v, k) for (k, v) in d.iteritems()] 102 if not x: return default 103 x.sort() 104 return x[-1][1] 105 106 107def IsSymlink(info): 108 """Return true if the zipfile.ZipInfo object passed in represents a 109 symlink.""" 110 return (info.external_attr >> 16) == 0120777 111 112def IsRegular(info): 113 """Return true if the zipfile.ZipInfo object passed in represents a 114 symlink.""" 115 return (info.external_attr >> 28) == 010 116 117def ClosestFileMatch(src, tgtfiles, existing): 118 """Returns the closest file match between a source file and list 119 of potential matches. The exact filename match is preferred, 120 then the sha1 is searched for, and finally a file with the same 121 basename is evaluated. Rename support in the updater-binary is 122 required for the latter checks to be used.""" 123 124 result = tgtfiles.get("path:" + src.name) 125 if result is not None: 126 return result 127 128 if not OPTIONS.target_info_dict.get("update_rename_support", False): 129 return None 130 131 if src.size < 1000: 132 return None 133 134 result = tgtfiles.get("sha1:" + src.sha1) 135 if result is not None and existing.get(result.name) is None: 136 return result 137 result = tgtfiles.get("file:" + src.name.split("/")[-1]) 138 if result is not None and existing.get(result.name) is None: 139 return result 140 return None 141 142class Item: 143 """Items represent the metadata (user, group, mode) of files and 144 directories in the system image.""" 145 ITEMS = {} 146 def __init__(self, name, dir=False): 147 self.name = name 148 self.uid = None 149 self.gid = None 150 self.mode = None 151 self.selabel = None 152 self.capabilities = None 153 self.dir = dir 154 155 if name: 156 self.parent = Item.Get(os.path.dirname(name), dir=True) 157 self.parent.children.append(self) 158 else: 159 self.parent = None 160 if dir: 161 self.children = [] 162 163 def Dump(self, indent=0): 164 if self.uid is not None: 165 print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode) 166 else: 167 print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode) 168 if self.dir: 169 print "%s%s" % (" "*indent, self.descendants) 170 print "%s%s" % (" "*indent, self.best_subtree) 171 for i in self.children: 172 i.Dump(indent=indent+1) 173 174 @classmethod 175 def Get(cls, name, dir=False): 176 if name not in cls.ITEMS: 177 cls.ITEMS[name] = Item(name, dir=dir) 178 return cls.ITEMS[name] 179 180 @classmethod 181 def GetMetadata(cls, input_zip): 182 183 # The target_files contains a record of what the uid, 184 # gid, and mode are supposed to be. 185 output = input_zip.read("META/filesystem_config.txt") 186 187 for line in output.split("\n"): 188 if not line: continue 189 columns = line.split() 190 name, uid, gid, mode = columns[:4] 191 selabel = None 192 capabilities = None 193 194 # After the first 4 columns, there are a series of key=value 195 # pairs. Extract out the fields we care about. 196 for element in columns[4:]: 197 key, value = element.split("=") 198 if key == "selabel": 199 selabel = value 200 if key == "capabilities": 201 capabilities = value 202 203 i = cls.ITEMS.get(name, None) 204 if i is not None: 205 i.uid = int(uid) 206 i.gid = int(gid) 207 i.mode = int(mode, 8) 208 i.selabel = selabel 209 i.capabilities = capabilities 210 if i.dir: 211 i.children.sort(key=lambda i: i.name) 212 213 # set metadata for the files generated by this script. 214 i = cls.ITEMS.get("system/recovery-from-boot.p", None) 215 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0644, None, None 216 i = cls.ITEMS.get("system/etc/install-recovery.sh", None) 217 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0544, None, None 218 219 def CountChildMetadata(self): 220 """Count up the (uid, gid, mode, selabel, capabilities) tuples for 221 all children and determine the best strategy for using set_perm_recursive and 222 set_perm to correctly chown/chmod all the files to their desired 223 values. Recursively calls itself for all descendants. 224 225 Returns a dict of {(uid, gid, dmode, fmode, selabel, capabilities): count} counting up 226 all descendants of this node. (dmode or fmode may be None.) Also 227 sets the best_subtree of each directory Item to the (uid, gid, 228 dmode, fmode, selabel, capabilities) tuple that will match the most 229 descendants of that Item. 230 """ 231 232 assert self.dir 233 d = self.descendants = {(self.uid, self.gid, self.mode, None, self.selabel, self.capabilities): 1} 234 for i in self.children: 235 if i.dir: 236 for k, v in i.CountChildMetadata().iteritems(): 237 d[k] = d.get(k, 0) + v 238 else: 239 k = (i.uid, i.gid, None, i.mode, i.selabel, i.capabilities) 240 d[k] = d.get(k, 0) + 1 241 242 # Find the (uid, gid, dmode, fmode, selabel, capabilities) 243 # tuple that matches the most descendants. 244 245 # First, find the (uid, gid) pair that matches the most 246 # descendants. 247 ug = {} 248 for (uid, gid, _, _, _, _), count in d.iteritems(): 249 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 250 ug = MostPopularKey(ug, (0, 0)) 251 252 # Now find the dmode, fmode, selabel, and capabilities that match 253 # the most descendants with that (uid, gid), and choose those. 254 best_dmode = (0, 0755) 255 best_fmode = (0, 0644) 256 best_selabel = (0, None) 257 best_capabilities = (0, None) 258 for k, count in d.iteritems(): 259 if k[:2] != ug: continue 260 if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2]) 261 if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3]) 262 if k[4] is not None and count >= best_selabel[0]: best_selabel = (count, k[4]) 263 if k[5] is not None and count >= best_capabilities[0]: best_capabilities = (count, k[5]) 264 self.best_subtree = ug + (best_dmode[1], best_fmode[1], best_selabel[1], best_capabilities[1]) 265 266 return d 267 268 def SetPermissions(self, script): 269 """Append set_perm/set_perm_recursive commands to 'script' to 270 set all permissions, users, and groups for the tree of files 271 rooted at 'self'.""" 272 273 self.CountChildMetadata() 274 275 def recurse(item, current): 276 # current is the (uid, gid, dmode, fmode, selabel, capabilities) tuple that the current 277 # item (and all its children) have already been set to. We only 278 # need to issue set_perm/set_perm_recursive commands if we're 279 # supposed to be something different. 280 if item.dir: 281 if current != item.best_subtree: 282 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 283 current = item.best_subtree 284 285 if item.uid != current[0] or item.gid != current[1] or \ 286 item.mode != current[2] or item.selabel != current[4] or \ 287 item.capabilities != current[5]: 288 script.SetPermissions("/"+item.name, item.uid, item.gid, 289 item.mode, item.selabel, item.capabilities) 290 291 for i in item.children: 292 recurse(i, current) 293 else: 294 if item.uid != current[0] or item.gid != current[1] or \ 295 item.mode != current[3] or item.selabel != current[4] or \ 296 item.capabilities != current[5]: 297 script.SetPermissions("/"+item.name, item.uid, item.gid, 298 item.mode, item.selabel, item.capabilities) 299 300 recurse(self, (-1, -1, -1, -1, None, None)) 301 302 303def CopySystemFiles(input_zip, output_zip=None, 304 substitute=None): 305 """Copies files underneath system/ in the input zip to the output 306 zip. Populates the Item class with their metadata, and returns a 307 list of symlinks. output_zip may be None, in which case the copy is 308 skipped (but the other side effects still happen). substitute is an 309 optional dict of {output filename: contents} to be output instead of 310 certain input files. 311 """ 312 313 symlinks = [] 314 315 for info in input_zip.infolist(): 316 if info.filename.startswith("SYSTEM/"): 317 basefilename = info.filename[7:] 318 if IsSymlink(info): 319 symlinks.append((input_zip.read(info.filename), 320 "/system/" + basefilename)) 321 else: 322 info2 = copy.copy(info) 323 fn = info2.filename = "system/" + basefilename 324 if substitute and fn in substitute and substitute[fn] is None: 325 continue 326 if output_zip is not None: 327 if substitute and fn in substitute: 328 data = substitute[fn] 329 else: 330 data = input_zip.read(info.filename) 331 output_zip.writestr(info2, data) 332 if fn.endswith("/"): 333 Item.Get(fn[:-1], dir=True) 334 else: 335 Item.Get(fn, dir=False) 336 337 symlinks.sort() 338 return symlinks 339 340 341def SignOutput(temp_zip_name, output_zip_name): 342 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 343 pw = key_passwords[OPTIONS.package_key] 344 345 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 346 whole_file=True) 347 348 349def AppendAssertions(script, info_dict): 350 device = GetBuildProp("ro.product.device", info_dict) 351 script.AssertDevice(device) 352 353 354def MakeRecoveryPatch(input_tmp, output_zip, recovery_img, boot_img): 355 """Generate a binary patch that creates the recovery image starting 356 with the boot image. (Most of the space in these images is just the 357 kernel, which is identical for the two, so the resulting patch 358 should be efficient.) Add it to the output zip, along with a shell 359 script that is run from init.rc on first boot to actually do the 360 patching and install the new recovery image. 361 362 recovery_img and boot_img should be File objects for the 363 corresponding images. info should be the dictionary returned by 364 common.LoadInfoDict() on the input target_files. 365 366 Returns an Item for the shell script, which must be made 367 executable. 368 """ 369 370 diff_program = ["imgdiff"] 371 path = os.path.join(input_tmp, "SYSTEM", "etc", "recovery-resource.dat") 372 if os.path.exists(path): 373 diff_program.append("-b") 374 diff_program.append(path) 375 bonus_args = "-b /system/etc/recovery-resource.dat" 376 else: 377 bonus_args = "" 378 379 d = common.Difference(recovery_img, boot_img, diff_program=diff_program) 380 _, _, patch = d.ComputePatch() 381 common.ZipWriteStr(output_zip, "recovery/recovery-from-boot.p", patch) 382 Item.Get("system/recovery-from-boot.p", dir=False) 383 384 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 385 recovery_type, recovery_device = common.GetTypeAndDevice("/recovery", OPTIONS.info_dict) 386 387 sh = """#!/system/bin/sh 388if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 389 log -t recovery "Installing new recovery image" 390 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p 391else 392 log -t recovery "Recovery image already installed" 393fi 394""" % { 'boot_size': boot_img.size, 395 'boot_sha1': boot_img.sha1, 396 'recovery_size': recovery_img.size, 397 'recovery_sha1': recovery_img.sha1, 398 'boot_type': boot_type, 399 'boot_device': boot_device, 400 'recovery_type': recovery_type, 401 'recovery_device': recovery_device, 402 'bonus_args': bonus_args, 403 } 404 common.ZipWriteStr(output_zip, "recovery/etc/install-recovery.sh", sh) 405 return Item.Get("system/etc/install-recovery.sh", dir=False) 406 407 408def WriteFullOTAPackage(input_zip, output_zip): 409 # TODO: how to determine this? We don't know what version it will 410 # be installed on top of. For now, we expect the API just won't 411 # change very often. 412 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 413 414 metadata = {"post-build": GetBuildProp("ro.build.fingerprint", 415 OPTIONS.info_dict), 416 "pre-device": GetBuildProp("ro.product.device", 417 OPTIONS.info_dict), 418 "post-timestamp": GetBuildProp("ro.build.date.utc", 419 OPTIONS.info_dict), 420 } 421 422 device_specific = common.DeviceSpecificParams( 423 input_zip=input_zip, 424 input_version=OPTIONS.info_dict["recovery_api_version"], 425 output_zip=output_zip, 426 script=script, 427 input_tmp=OPTIONS.input_tmp, 428 metadata=metadata, 429 info_dict=OPTIONS.info_dict) 430 431 if not OPTIONS.omit_prereq: 432 ts = GetBuildProp("ro.build.date.utc", OPTIONS.info_dict) 433 ts_text = GetBuildProp("ro.build.date", OPTIONS.info_dict) 434 script.AssertOlderBuild(ts, ts_text) 435 436 AppendAssertions(script, OPTIONS.info_dict) 437 device_specific.FullOTA_Assertions() 438 439 # Two-step package strategy (in chronological order, which is *not* 440 # the order in which the generated script has things): 441 # 442 # if stage is not "2/3" or "3/3": 443 # write recovery image to boot partition 444 # set stage to "2/3" 445 # reboot to boot partition and restart recovery 446 # else if stage is "2/3": 447 # write recovery image to recovery partition 448 # set stage to "3/3" 449 # reboot to recovery partition and restart recovery 450 # else: 451 # (stage must be "3/3") 452 # set stage to "" 453 # do normal full package installation: 454 # wipe and install system, boot image, etc. 455 # set up system to update recovery partition on first boot 456 # complete script normally (allow recovery to mark itself finished and reboot) 457 458 recovery_img = common.GetBootableImage("recovery.img", "recovery.img", 459 OPTIONS.input_tmp, "RECOVERY") 460 if OPTIONS.two_step: 461 if not OPTIONS.info_dict.get("multistage_support", None): 462 assert False, "two-step packages not supported by this build" 463 fs = OPTIONS.info_dict["fstab"]["/misc"] 464 assert fs.fs_type.upper() == "EMMC", \ 465 "two-step packages only supported on devices with EMMC /misc partitions" 466 bcb_dev = {"bcb_dev": fs.device} 467 common.ZipWriteStr(output_zip, "recovery.img", recovery_img.data) 468 script.AppendExtra(""" 469if get_stage("%(bcb_dev)s", "stage") == "2/3" then 470""" % bcb_dev) 471 script.WriteRawImage("/recovery", "recovery.img") 472 script.AppendExtra(""" 473set_stage("%(bcb_dev)s", "3/3"); 474reboot_now("%(bcb_dev)s", "recovery"); 475else if get_stage("%(bcb_dev)s", "stage") == "3/3" then 476""" % bcb_dev) 477 478 device_specific.FullOTA_InstallBegin() 479 480 script.ShowProgress(0.5, 0) 481 482 if OPTIONS.wipe_user_data: 483 script.FormatPartition("/data") 484 485 if "selinux_fc" in OPTIONS.info_dict: 486 WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip) 487 488 script.FormatPartition("/system") 489 script.Mount("/system") 490 script.UnpackPackageDir("recovery", "/system") 491 script.UnpackPackageDir("system", "/system") 492 493 symlinks = CopySystemFiles(input_zip, output_zip) 494 script.MakeSymlinks(symlinks) 495 496 boot_img = common.GetBootableImage("boot.img", "boot.img", 497 OPTIONS.input_tmp, "BOOT") 498 MakeRecoveryPatch(OPTIONS.input_tmp, output_zip, recovery_img, boot_img) 499 500 Item.GetMetadata(input_zip) 501 Item.Get("system").SetPermissions(script) 502 503 common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict) 504 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 505 script.ShowProgress(0.2, 0) 506 507 script.ShowProgress(0.2, 10) 508 script.WriteRawImage("/boot", "boot.img") 509 510 script.ShowProgress(0.1, 0) 511 device_specific.FullOTA_InstallEnd() 512 513 if OPTIONS.extra_script is not None: 514 script.AppendExtra(OPTIONS.extra_script) 515 516 script.UnmountAll() 517 518 if OPTIONS.two_step: 519 script.AppendExtra(""" 520set_stage("%(bcb_dev)s", ""); 521""" % bcb_dev) 522 script.AppendExtra("else\n") 523 script.WriteRawImage("/boot", "recovery.img") 524 script.AppendExtra(""" 525set_stage("%(bcb_dev)s", "2/3"); 526reboot_now("%(bcb_dev)s", ""); 527endif; 528endif; 529""" % bcb_dev) 530 script.AddToZip(input_zip, output_zip) 531 WriteMetadata(metadata, output_zip) 532 533def WritePolicyConfig(file_context, output_zip): 534 f = open(file_context, 'r'); 535 basename = os.path.basename(file_context) 536 common.ZipWriteStr(output_zip, basename, f.read()) 537 538 539def WriteMetadata(metadata, output_zip): 540 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata", 541 "".join(["%s=%s\n" % kv 542 for kv in sorted(metadata.iteritems())])) 543 544def LoadSystemFiles(z): 545 """Load all the files from SYSTEM/... in a given target-files 546 ZipFile, and return a dict of {filename: File object}.""" 547 out = {} 548 for info in z.infolist(): 549 if info.filename.startswith("SYSTEM/") and not IsSymlink(info): 550 basefilename = info.filename[7:] 551 fn = "system/" + basefilename 552 data = z.read(info.filename) 553 out[fn] = common.File(fn, data) 554 return out 555 556 557def GetBuildProp(prop, info_dict): 558 """Return the fingerprint of the build of a given target-files info_dict.""" 559 try: 560 return info_dict.get("build.prop", {})[prop] 561 except KeyError: 562 raise common.ExternalError("couldn't find %s in build.prop" % (property,)) 563 564def AddToKnownPaths(filename, known_paths): 565 if filename[-1] == "/": 566 return 567 dirs = filename.split("/")[:-1] 568 while len(dirs) > 0: 569 path = "/".join(dirs) 570 if path in known_paths: 571 break; 572 known_paths.add(path) 573 dirs.pop() 574 575def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): 576 source_version = OPTIONS.source_info_dict["recovery_api_version"] 577 target_version = OPTIONS.target_info_dict["recovery_api_version"] 578 579 if source_version == 0: 580 print ("WARNING: generating edify script for a source that " 581 "can't install it.") 582 script = edify_generator.EdifyGenerator(source_version, 583 OPTIONS.target_info_dict) 584 585 metadata = {"pre-device": GetBuildProp("ro.product.device", 586 OPTIONS.source_info_dict), 587 "post-timestamp": GetBuildProp("ro.build.date.utc", 588 OPTIONS.target_info_dict), 589 } 590 591 device_specific = common.DeviceSpecificParams( 592 source_zip=source_zip, 593 source_version=source_version, 594 target_zip=target_zip, 595 target_version=target_version, 596 output_zip=output_zip, 597 script=script, 598 metadata=metadata, 599 info_dict=OPTIONS.info_dict) 600 601 print "Loading target..." 602 target_data = LoadSystemFiles(target_zip) 603 print "Loading source..." 604 source_data = LoadSystemFiles(source_zip) 605 606 verbatim_targets = [] 607 patch_list = [] 608 diffs = [] 609 renames = {} 610 known_paths = set() 611 largest_source_size = 0 612 613 matching_file_cache = {} 614 for fn, sf in source_data.items(): 615 assert fn == sf.name 616 matching_file_cache["path:" + fn] = sf 617 if fn in target_data.keys(): 618 AddToKnownPaths(fn, known_paths) 619 # Only allow eligibility for filename/sha matching 620 # if there isn't a perfect path match. 621 if target_data.get(sf.name) is None: 622 matching_file_cache["file:" + fn.split("/")[-1]] = sf 623 matching_file_cache["sha:" + sf.sha1] = sf 624 625 for fn in sorted(target_data.keys()): 626 tf = target_data[fn] 627 assert fn == tf.name 628 sf = ClosestFileMatch(tf, matching_file_cache, renames) 629 if sf is not None and sf.name != tf.name: 630 print "File has moved from " + sf.name + " to " + tf.name 631 renames[sf.name] = tf 632 633 if sf is None or fn in OPTIONS.require_verbatim: 634 # This file should be included verbatim 635 if fn in OPTIONS.prohibit_verbatim: 636 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 637 print "send", fn, "verbatim" 638 tf.AddToZip(output_zip) 639 verbatim_targets.append((fn, tf.size)) 640 if fn in target_data.keys(): 641 AddToKnownPaths(fn, known_paths) 642 elif tf.sha1 != sf.sha1: 643 # File is different; consider sending as a patch 644 diffs.append(common.Difference(tf, sf)) 645 else: 646 # Target file data identical to source (may still be renamed) 647 pass 648 649 common.ComputeDifferences(diffs) 650 651 for diff in diffs: 652 tf, sf, d = diff.GetPatch() 653 path = "/".join(tf.name.split("/")[:-1]) 654 if d is None or len(d) > tf.size * OPTIONS.patch_threshold or \ 655 path not in known_paths: 656 # patch is almost as big as the file; don't bother patching 657 # or a patch + rename cannot take place due to the target 658 # directory not existing 659 tf.AddToZip(output_zip) 660 verbatim_targets.append((tf.name, tf.size)) 661 if sf.name in renames: 662 del renames[sf.name] 663 AddToKnownPaths(tf.name, known_paths) 664 else: 665 common.ZipWriteStr(output_zip, "patch/" + sf.name + ".p", d) 666 patch_list.append((tf, sf, tf.size, common.sha1(d).hexdigest())) 667 largest_source_size = max(largest_source_size, sf.size) 668 669 source_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.source_info_dict) 670 target_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.target_info_dict) 671 metadata["pre-build"] = source_fp 672 metadata["post-build"] = target_fp 673 674 script.Mount("/system") 675 script.AssertSomeFingerprint(source_fp, target_fp) 676 677 source_boot = common.GetBootableImage( 678 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 679 OPTIONS.source_info_dict) 680 target_boot = common.GetBootableImage( 681 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 682 updating_boot = (not OPTIONS.two_step and 683 (source_boot.data != target_boot.data)) 684 685 source_recovery = common.GetBootableImage( 686 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 687 OPTIONS.source_info_dict) 688 target_recovery = common.GetBootableImage( 689 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 690 updating_recovery = (source_recovery.data != target_recovery.data) 691 692 # Here's how we divide up the progress bar: 693 # 0.1 for verifying the start state (PatchCheck calls) 694 # 0.8 for applying patches (ApplyPatch calls) 695 # 0.1 for unpacking verbatim files, symlinking, and doing the 696 # device-specific commands. 697 698 AppendAssertions(script, OPTIONS.target_info_dict) 699 device_specific.IncrementalOTA_Assertions() 700 701 # Two-step incremental package strategy (in chronological order, 702 # which is *not* the order in which the generated script has 703 # things): 704 # 705 # if stage is not "2/3" or "3/3": 706 # do verification on current system 707 # write recovery image to boot partition 708 # set stage to "2/3" 709 # reboot to boot partition and restart recovery 710 # else if stage is "2/3": 711 # write recovery image to recovery partition 712 # set stage to "3/3" 713 # reboot to recovery partition and restart recovery 714 # else: 715 # (stage must be "3/3") 716 # perform update: 717 # patch system files, etc. 718 # force full install of new boot image 719 # set up system to update recovery partition on first boot 720 # complete script normally (allow recovery to mark itself finished and reboot) 721 722 if OPTIONS.two_step: 723 if not OPTIONS.info_dict.get("multistage_support", None): 724 assert False, "two-step packages not supported by this build" 725 fs = OPTIONS.info_dict["fstab"]["/misc"] 726 assert fs.fs_type.upper() == "EMMC", \ 727 "two-step packages only supported on devices with EMMC /misc partitions" 728 bcb_dev = {"bcb_dev": fs.device} 729 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 730 script.AppendExtra(""" 731if get_stage("%(bcb_dev)s", "stage") == "2/3" then 732""" % bcb_dev) 733 script.AppendExtra("sleep(20);\n"); 734 script.WriteRawImage("/recovery", "recovery.img") 735 script.AppendExtra(""" 736set_stage("%(bcb_dev)s", "3/3"); 737reboot_now("%(bcb_dev)s", "recovery"); 738else if get_stage("%(bcb_dev)s", "stage") != "3/3" then 739""" % bcb_dev) 740 741 script.Print("Verifying current system...") 742 743 device_specific.IncrementalOTA_VerifyBegin() 744 745 script.ShowProgress(0.1, 0) 746 total_verify_size = float(sum([i[1].size for i in patch_list]) + 1) 747 if updating_boot: 748 total_verify_size += source_boot.size 749 so_far = 0 750 751 for tf, sf, size, patch_sha in patch_list: 752 if tf.name != sf.name: 753 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 754 script.PatchCheck("/"+sf.name, tf.sha1, sf.sha1) 755 so_far += sf.size 756 script.SetProgress(so_far / total_verify_size) 757 758 if updating_boot: 759 d = common.Difference(target_boot, source_boot) 760 _, _, d = d.ComputePatch() 761 print "boot target: %d source: %d diff: %d" % ( 762 target_boot.size, source_boot.size, len(d)) 763 764 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 765 766 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 767 768 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 769 (boot_type, boot_device, 770 source_boot.size, source_boot.sha1, 771 target_boot.size, target_boot.sha1)) 772 so_far += source_boot.size 773 script.SetProgress(so_far / total_verify_size) 774 775 if patch_list or updating_recovery or updating_boot: 776 script.CacheFreeSpaceCheck(largest_source_size) 777 778 device_specific.IncrementalOTA_VerifyEnd() 779 780 if OPTIONS.two_step: 781 script.WriteRawImage("/boot", "recovery.img") 782 script.AppendExtra(""" 783set_stage("%(bcb_dev)s", "2/3"); 784reboot_now("%(bcb_dev)s", ""); 785else 786""" % bcb_dev) 787 788 script.Comment("---- start making changes here ----") 789 790 device_specific.IncrementalOTA_InstallBegin() 791 792 if OPTIONS.two_step: 793 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 794 script.WriteRawImage("/boot", "boot.img") 795 print "writing full boot image (forced by two-step mode)" 796 797 if OPTIONS.wipe_user_data: 798 script.Print("Erasing user data...") 799 script.FormatPartition("/data") 800 801 script.Print("Removing unneeded files...") 802 script.DeleteFiles(["/"+i[0] for i in verbatim_targets] + 803 ["/"+i for i in sorted(source_data) 804 if i not in target_data and 805 i not in renames] + 806 ["/system/recovery.img"]) 807 808 script.ShowProgress(0.8, 0) 809 total_patch_size = float(sum([i[1].size for i in patch_list]) + 1) 810 if updating_boot: 811 total_patch_size += target_boot.size 812 so_far = 0 813 814 script.Print("Patching system files...") 815 deferred_patch_list = [] 816 for item in patch_list: 817 tf, sf, size, _ = item 818 if tf.name == "system/build.prop": 819 deferred_patch_list.append(item) 820 continue 821 if (sf.name != tf.name): 822 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 823 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p") 824 so_far += tf.size 825 script.SetProgress(so_far / total_patch_size) 826 827 if not OPTIONS.two_step: 828 if updating_boot: 829 # Produce the boot image by applying a patch to the current 830 # contents of the boot partition, and write it back to the 831 # partition. 832 script.Print("Patching boot image...") 833 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 834 % (boot_type, boot_device, 835 source_boot.size, source_boot.sha1, 836 target_boot.size, target_boot.sha1), 837 "-", 838 target_boot.size, target_boot.sha1, 839 source_boot.sha1, "patch/boot.img.p") 840 so_far += target_boot.size 841 script.SetProgress(so_far / total_patch_size) 842 print "boot image changed; including." 843 else: 844 print "boot image unchanged; skipping." 845 846 if updating_recovery: 847 # Recovery is generated as a patch using both the boot image 848 # (which contains the same linux kernel as recovery) and the file 849 # /system/etc/recovery-resource.dat (which contains all the images 850 # used in the recovery UI) as sources. This lets us minimize the 851 # size of the patch, which must be included in every OTA package. 852 # 853 # For older builds where recovery-resource.dat is not present, we 854 # use only the boot image as the source. 855 856 MakeRecoveryPatch(OPTIONS.target_tmp, output_zip, 857 target_recovery, target_boot) 858 script.DeleteFiles(["/system/recovery-from-boot.p", 859 "/system/etc/install-recovery.sh"]) 860 print "recovery image changed; including as patch from boot." 861 else: 862 print "recovery image unchanged; skipping." 863 864 script.ShowProgress(0.1, 10) 865 866 target_symlinks = CopySystemFiles(target_zip, None) 867 868 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 869 temp_script = script.MakeTemporary() 870 Item.GetMetadata(target_zip) 871 Item.Get("system").SetPermissions(temp_script) 872 873 # Note that this call will mess up the tree of Items, so make sure 874 # we're done with it. 875 source_symlinks = CopySystemFiles(source_zip, None) 876 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 877 878 # Delete all the symlinks in source that aren't in target. This 879 # needs to happen before verbatim files are unpacked, in case a 880 # symlink in the source is replaced by a real file in the target. 881 to_delete = [] 882 for dest, link in source_symlinks: 883 if link not in target_symlinks_d: 884 to_delete.append(link) 885 script.DeleteFiles(to_delete) 886 887 if verbatim_targets: 888 script.Print("Unpacking new files...") 889 script.UnpackPackageDir("system", "/system") 890 891 if updating_recovery: 892 script.Print("Unpacking new recovery...") 893 script.UnpackPackageDir("recovery", "/system") 894 895 if len(renames) > 0: 896 script.Print("Renaming files...") 897 898 for src in renames: 899 print "Renaming " + src + " to " + renames[src].name 900 script.RenameFile(src, renames[src].name) 901 902 script.Print("Symlinks and permissions...") 903 904 # Create all the symlinks that don't already exist, or point to 905 # somewhere different than what we want. Delete each symlink before 906 # creating it, since the 'symlink' command won't overwrite. 907 to_create = [] 908 for dest, link in target_symlinks: 909 if link in source_symlinks_d: 910 if dest != source_symlinks_d[link]: 911 to_create.append((dest, link)) 912 else: 913 to_create.append((dest, link)) 914 script.DeleteFiles([i[1] for i in to_create]) 915 script.MakeSymlinks(to_create) 916 917 # Now that the symlinks are created, we can set all the 918 # permissions. 919 script.AppendScript(temp_script) 920 921 # Do device-specific installation (eg, write radio image). 922 device_specific.IncrementalOTA_InstallEnd() 923 924 if OPTIONS.extra_script is not None: 925 script.AppendExtra(OPTIONS.extra_script) 926 927 # Patch the build.prop file last, so if something fails but the 928 # device can still come up, it appears to be the old build and will 929 # get set the OTA package again to retry. 930 script.Print("Patching remaining system files...") 931 for item in deferred_patch_list: 932 tf, sf, size, _ = item 933 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p") 934 script.SetPermissions("/system/build.prop", 0, 0, 0644, None, None) 935 936 if OPTIONS.two_step: 937 script.AppendExtra(""" 938set_stage("%(bcb_dev)s", ""); 939endif; 940endif; 941""" % bcb_dev) 942 943 script.AddToZip(target_zip, output_zip) 944 WriteMetadata(metadata, output_zip) 945 946 947def main(argv): 948 949 def option_handler(o, a): 950 if o in ("-b", "--board_config"): 951 pass # deprecated 952 elif o in ("-k", "--package_key"): 953 OPTIONS.package_key = a 954 elif o in ("-i", "--incremental_from"): 955 OPTIONS.incremental_source = a 956 elif o in ("-w", "--wipe_user_data"): 957 OPTIONS.wipe_user_data = True 958 elif o in ("-n", "--no_prereq"): 959 OPTIONS.omit_prereq = True 960 elif o in ("-e", "--extra_script"): 961 OPTIONS.extra_script = a 962 elif o in ("-a", "--aslr_mode"): 963 if a in ("on", "On", "true", "True", "yes", "Yes"): 964 OPTIONS.aslr_mode = True 965 else: 966 OPTIONS.aslr_mode = False 967 elif o in ("--worker_threads"): 968 OPTIONS.worker_threads = int(a) 969 elif o in ("-2", "--two_step"): 970 OPTIONS.two_step = True 971 else: 972 return False 973 return True 974 975 args = common.ParseOptions(argv, __doc__, 976 extra_opts="b:k:i:d:wne:a:2", 977 extra_long_opts=["board_config=", 978 "package_key=", 979 "incremental_from=", 980 "wipe_user_data", 981 "no_prereq", 982 "extra_script=", 983 "worker_threads=", 984 "aslr_mode=", 985 "two_step", 986 ], 987 extra_option_handler=option_handler) 988 989 if len(args) != 2: 990 common.Usage(__doc__) 991 sys.exit(1) 992 993 if OPTIONS.extra_script is not None: 994 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 995 996 print "unzipping target target-files..." 997 OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0]) 998 999 OPTIONS.target_tmp = OPTIONS.input_tmp 1000 OPTIONS.info_dict = common.LoadInfoDict(input_zip) 1001 1002 # If this image was originally labelled with SELinux contexts, make sure we 1003 # also apply the labels in our new image. During building, the "file_contexts" 1004 # is in the out/ directory tree, but for repacking from target-files.zip it's 1005 # in the root directory of the ramdisk. 1006 if "selinux_fc" in OPTIONS.info_dict: 1007 OPTIONS.info_dict["selinux_fc"] = os.path.join(OPTIONS.input_tmp, "BOOT", "RAMDISK", 1008 "file_contexts") 1009 1010 if OPTIONS.verbose: 1011 print "--- target info ---" 1012 common.DumpInfoDict(OPTIONS.info_dict) 1013 1014 # If the caller explicitly specified the device-specific extensions 1015 # path via -s/--device_specific, use that. Otherwise, use 1016 # META/releasetools.py if it is present in the target target_files. 1017 # Otherwise, take the path of the file from 'tool_extensions' in the 1018 # info dict and look for that in the local filesystem, relative to 1019 # the current directory. 1020 1021 if OPTIONS.device_specific is None: 1022 from_input = os.path.join(OPTIONS.input_tmp, "META", "releasetools.py") 1023 if os.path.exists(from_input): 1024 print "(using device-specific extensions from target_files)" 1025 OPTIONS.device_specific = from_input 1026 else: 1027 OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None) 1028 1029 if OPTIONS.device_specific is not None: 1030 OPTIONS.device_specific = os.path.abspath(OPTIONS.device_specific) 1031 1032 temp_zip_file = tempfile.NamedTemporaryFile() 1033 output_zip = zipfile.ZipFile(temp_zip_file, "w", 1034 compression=zipfile.ZIP_DEFLATED) 1035 1036 if OPTIONS.incremental_source is None: 1037 WriteFullOTAPackage(input_zip, output_zip) 1038 if OPTIONS.package_key is None: 1039 OPTIONS.package_key = OPTIONS.info_dict.get( 1040 "default_system_dev_certificate", 1041 "build/target/product/security/testkey") 1042 else: 1043 print "unzipping source target-files..." 1044 OPTIONS.source_tmp, source_zip = common.UnzipTemp(OPTIONS.incremental_source) 1045 OPTIONS.target_info_dict = OPTIONS.info_dict 1046 OPTIONS.source_info_dict = common.LoadInfoDict(source_zip) 1047 if OPTIONS.package_key is None: 1048 OPTIONS.package_key = OPTIONS.source_info_dict.get( 1049 "default_system_dev_certificate", 1050 "build/target/product/security/testkey") 1051 if OPTIONS.verbose: 1052 print "--- source info ---" 1053 common.DumpInfoDict(OPTIONS.source_info_dict) 1054 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) 1055 1056 output_zip.close() 1057 1058 SignOutput(temp_zip_file.name, args[1]) 1059 temp_zip_file.close() 1060 1061 common.Cleanup() 1062 1063 print "done." 1064 1065 1066if __name__ == '__main__': 1067 try: 1068 common.CloseInheritedPipes() 1069 main(sys.argv[1:]) 1070 except common.ExternalError, e: 1071 print 1072 print " ERROR: %s" % (e,) 1073 print 1074 sys.exit(1) 1075