ota_from_target_files.py revision df4cb0b2c1e6d1cf5973e9008bd6dc15bd590ac0
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 --board_config <file> 25 Deprecated. 26 27 -k (--package_key) <key> Key to use to sign the package (default is 28 the value of default_system_dev_certificate from the input 29 target-files's META/misc_info.txt, or 30 "build/target/product/security/testkey" if that value is not 31 specified). 32 33 For incremental OTAs, the default value is based on the source 34 target-file, not the target build. 35 36 -i (--incremental_from) <file> 37 Generate an incremental OTA using the given target-files zip as 38 the starting build. 39 40 --full_radio 41 When generating an incremental OTA, always include a full copy of 42 radio image. This option is only meaningful when -i is specified, 43 because a full radio is always included in a full OTA if applicable. 44 45 --full_bootloader 46 When generating an incremental OTA, always include a full copy of 47 bootloader image. This option is only meaningful when -i is specified, 48 because a full bootloader is always included in a full OTA if applicable. 49 50 -v (--verify) 51 Remount and verify the checksums of the files written to the 52 system and vendor (if used) partitions. Incremental builds only. 53 54 -o (--oem_settings) <file> 55 Use the file to specify the expected OEM-specific properties 56 on the OEM partition of the intended device. 57 58 --oem_no_mount 59 For devices with OEM-specific properties but without an OEM partition, 60 do not mount the OEM partition in the updater-script. This should be 61 very rarely used, since it's expected to have a dedicated OEM partition 62 for OEM-specific properties. Only meaningful when -o is specified. 63 64 -w (--wipe_user_data) 65 Generate an OTA package that will wipe the user data partition 66 when installed. 67 68 -n (--no_prereq) 69 Omit the timestamp prereq check normally included at the top of 70 the build scripts (used for developer OTA packages which 71 legitimately need to go back and forth). 72 73 -e (--extra_script) <file> 74 Insert the contents of file at the end of the update script. 75 76 -a (--aslr_mode) <on|off> 77 Specify whether to turn on ASLR for the package (on by default). 78 79 -2 (--two_step) 80 Generate a 'two-step' OTA package, where recovery is updated 81 first, so that any changes made to the system partition are done 82 using the new recovery (new kernel, etc.). 83 84 --block 85 Generate a block-based OTA if possible. Will fall back to a 86 file-based OTA if the target_files is older and doesn't support 87 block-based OTAs. 88 89 -b (--binary) <file> 90 Use the given binary as the update-binary in the output package, 91 instead of the binary in the build's target_files. Use for 92 development only. 93 94 -t (--worker_threads) <int> 95 Specifies the number of worker-threads that will be used when 96 generating patches for incremental updates (defaults to 3). 97 98 --stash_threshold <float> 99 Specifies the threshold that will be used to compute the maximum 100 allowed stash size (defaults to 0.8). 101""" 102 103import sys 104 105if sys.hexversion < 0x02070000: 106 print >> sys.stderr, "Python 2.7 or newer is required." 107 sys.exit(1) 108 109import multiprocessing 110import os 111import tempfile 112import zipfile 113 114import common 115import edify_generator 116import sparse_img 117 118OPTIONS = common.OPTIONS 119OPTIONS.package_key = None 120OPTIONS.incremental_source = None 121OPTIONS.verify = False 122OPTIONS.require_verbatim = set() 123OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 124OPTIONS.patch_threshold = 0.95 125OPTIONS.wipe_user_data = False 126OPTIONS.omit_prereq = False 127OPTIONS.extra_script = None 128OPTIONS.aslr_mode = True 129OPTIONS.worker_threads = multiprocessing.cpu_count() // 2 130if OPTIONS.worker_threads == 0: 131 OPTIONS.worker_threads = 1 132OPTIONS.two_step = False 133OPTIONS.no_signing = False 134OPTIONS.block_based = False 135OPTIONS.updater_binary = None 136OPTIONS.oem_source = None 137OPTIONS.oem_no_mount = False 138OPTIONS.fallback_to_full = True 139OPTIONS.full_radio = False 140OPTIONS.full_bootloader = False 141# Stash size cannot exceed cache_size * threshold. 142OPTIONS.cache_size = None 143OPTIONS.stash_threshold = 0.8 144 145def MostPopularKey(d, default): 146 """Given a dict, return the key corresponding to the largest 147 value. Returns 'default' if the dict is empty.""" 148 x = [(v, k) for (k, v) in d.iteritems()] 149 if not x: 150 return default 151 x.sort() 152 return x[-1][1] 153 154 155def IsSymlink(info): 156 """Return true if the zipfile.ZipInfo object passed in represents a 157 symlink.""" 158 return (info.external_attr >> 16) == 0o120777 159 160def IsRegular(info): 161 """Return true if the zipfile.ZipInfo object passed in represents a 162 symlink.""" 163 return (info.external_attr >> 28) == 0o10 164 165def ClosestFileMatch(src, tgtfiles, existing): 166 """Returns the closest file match between a source file and list 167 of potential matches. The exact filename match is preferred, 168 then the sha1 is searched for, and finally a file with the same 169 basename is evaluated. Rename support in the updater-binary is 170 required for the latter checks to be used.""" 171 172 result = tgtfiles.get("path:" + src.name) 173 if result is not None: 174 return result 175 176 if not OPTIONS.target_info_dict.get("update_rename_support", False): 177 return None 178 179 if src.size < 1000: 180 return None 181 182 result = tgtfiles.get("sha1:" + src.sha1) 183 if result is not None and existing.get(result.name) is None: 184 return result 185 result = tgtfiles.get("file:" + src.name.split("/")[-1]) 186 if result is not None and existing.get(result.name) is None: 187 return result 188 return None 189 190class ItemSet(object): 191 def __init__(self, partition, fs_config): 192 self.partition = partition 193 self.fs_config = fs_config 194 self.ITEMS = {} 195 196 def Get(self, name, is_dir=False): 197 if name not in self.ITEMS: 198 self.ITEMS[name] = Item(self, name, is_dir=is_dir) 199 return self.ITEMS[name] 200 201 def GetMetadata(self, input_zip): 202 # The target_files contains a record of what the uid, 203 # gid, and mode are supposed to be. 204 output = input_zip.read(self.fs_config) 205 206 for line in output.split("\n"): 207 if not line: 208 continue 209 columns = line.split() 210 name, uid, gid, mode = columns[:4] 211 selabel = None 212 capabilities = None 213 214 # After the first 4 columns, there are a series of key=value 215 # pairs. Extract out the fields we care about. 216 for element in columns[4:]: 217 key, value = element.split("=") 218 if key == "selabel": 219 selabel = value 220 if key == "capabilities": 221 capabilities = value 222 223 i = self.ITEMS.get(name, None) 224 if i is not None: 225 i.uid = int(uid) 226 i.gid = int(gid) 227 i.mode = int(mode, 8) 228 i.selabel = selabel 229 i.capabilities = capabilities 230 if i.is_dir: 231 i.children.sort(key=lambda i: i.name) 232 233 # set metadata for the files generated by this script. 234 i = self.ITEMS.get("system/recovery-from-boot.p", None) 235 if i: 236 i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0o644, None, None 237 i = self.ITEMS.get("system/etc/install-recovery.sh", None) 238 if i: 239 i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0o544, None, None 240 241 242class Item(object): 243 """Items represent the metadata (user, group, mode) of files and 244 directories in the system image.""" 245 def __init__(self, itemset, name, is_dir=False): 246 self.itemset = itemset 247 self.name = name 248 self.uid = None 249 self.gid = None 250 self.mode = None 251 self.selabel = None 252 self.capabilities = None 253 self.is_dir = is_dir 254 self.descendants = None 255 self.best_subtree = None 256 257 if name: 258 self.parent = itemset.Get(os.path.dirname(name), is_dir=True) 259 self.parent.children.append(self) 260 else: 261 self.parent = None 262 if self.is_dir: 263 self.children = [] 264 265 def Dump(self, indent=0): 266 if self.uid is not None: 267 print "%s%s %d %d %o" % ( 268 " " * indent, self.name, self.uid, self.gid, self.mode) 269 else: 270 print "%s%s %s %s %s" % ( 271 " " * indent, self.name, self.uid, self.gid, self.mode) 272 if self.is_dir: 273 print "%s%s" % (" "*indent, self.descendants) 274 print "%s%s" % (" "*indent, self.best_subtree) 275 for i in self.children: 276 i.Dump(indent=indent+1) 277 278 def CountChildMetadata(self): 279 """Count up the (uid, gid, mode, selabel, capabilities) tuples for 280 all children and determine the best strategy for using set_perm_recursive 281 and set_perm to correctly chown/chmod all the files to their desired 282 values. Recursively calls itself for all descendants. 283 284 Returns a dict of {(uid, gid, dmode, fmode, selabel, capabilities): count} 285 counting up all descendants of this node. (dmode or fmode may be None.) 286 Also sets the best_subtree of each directory Item to the (uid, gid, dmode, 287 fmode, selabel, capabilities) tuple that will match the most descendants of 288 that Item. 289 """ 290 291 assert self.is_dir 292 key = (self.uid, self.gid, self.mode, None, self.selabel, 293 self.capabilities) 294 self.descendants = {key: 1} 295 d = self.descendants 296 for i in self.children: 297 if i.is_dir: 298 for k, v in i.CountChildMetadata().iteritems(): 299 d[k] = d.get(k, 0) + v 300 else: 301 k = (i.uid, i.gid, None, i.mode, i.selabel, i.capabilities) 302 d[k] = d.get(k, 0) + 1 303 304 # Find the (uid, gid, dmode, fmode, selabel, capabilities) 305 # tuple that matches the most descendants. 306 307 # First, find the (uid, gid) pair that matches the most 308 # descendants. 309 ug = {} 310 for (uid, gid, _, _, _, _), count in d.iteritems(): 311 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 312 ug = MostPopularKey(ug, (0, 0)) 313 314 # Now find the dmode, fmode, selabel, and capabilities that match 315 # the most descendants with that (uid, gid), and choose those. 316 best_dmode = (0, 0o755) 317 best_fmode = (0, 0o644) 318 best_selabel = (0, None) 319 best_capabilities = (0, None) 320 for k, count in d.iteritems(): 321 if k[:2] != ug: 322 continue 323 if k[2] is not None and count >= best_dmode[0]: 324 best_dmode = (count, k[2]) 325 if k[3] is not None and count >= best_fmode[0]: 326 best_fmode = (count, k[3]) 327 if k[4] is not None and count >= best_selabel[0]: 328 best_selabel = (count, k[4]) 329 if k[5] is not None and count >= best_capabilities[0]: 330 best_capabilities = (count, k[5]) 331 self.best_subtree = ug + ( 332 best_dmode[1], best_fmode[1], best_selabel[1], best_capabilities[1]) 333 334 return d 335 336 def SetPermissions(self, script): 337 """Append set_perm/set_perm_recursive commands to 'script' to 338 set all permissions, users, and groups for the tree of files 339 rooted at 'self'.""" 340 341 self.CountChildMetadata() 342 343 def recurse(item, current): 344 # current is the (uid, gid, dmode, fmode, selabel, capabilities) tuple 345 # that the current item (and all its children) have already been set to. 346 # We only need to issue set_perm/set_perm_recursive commands if we're 347 # supposed to be something different. 348 if item.is_dir: 349 if current != item.best_subtree: 350 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 351 current = item.best_subtree 352 353 if item.uid != current[0] or item.gid != current[1] or \ 354 item.mode != current[2] or item.selabel != current[4] or \ 355 item.capabilities != current[5]: 356 script.SetPermissions("/"+item.name, item.uid, item.gid, 357 item.mode, item.selabel, item.capabilities) 358 359 for i in item.children: 360 recurse(i, current) 361 else: 362 if item.uid != current[0] or item.gid != current[1] or \ 363 item.mode != current[3] or item.selabel != current[4] or \ 364 item.capabilities != current[5]: 365 script.SetPermissions("/"+item.name, item.uid, item.gid, 366 item.mode, item.selabel, item.capabilities) 367 368 recurse(self, (-1, -1, -1, -1, None, None)) 369 370 371def CopyPartitionFiles(itemset, input_zip, output_zip=None, substitute=None): 372 """Copies files for the partition in the input zip to the output 373 zip. Populates the Item class with their metadata, and returns a 374 list of symlinks. output_zip may be None, in which case the copy is 375 skipped (but the other side effects still happen). substitute is an 376 optional dict of {output filename: contents} to be output instead of 377 certain input files. 378 """ 379 380 symlinks = [] 381 382 partition = itemset.partition 383 384 for info in input_zip.infolist(): 385 prefix = partition.upper() + "/" 386 if info.filename.startswith(prefix): 387 basefilename = info.filename[len(prefix):] 388 if IsSymlink(info): 389 symlinks.append((input_zip.read(info.filename), 390 "/" + partition + "/" + basefilename)) 391 else: 392 import copy 393 info2 = copy.copy(info) 394 fn = info2.filename = partition + "/" + basefilename 395 if substitute and fn in substitute and substitute[fn] is None: 396 continue 397 if output_zip is not None: 398 if substitute and fn in substitute: 399 data = substitute[fn] 400 else: 401 data = input_zip.read(info.filename) 402 common.ZipWriteStr(output_zip, info2, data) 403 if fn.endswith("/"): 404 itemset.Get(fn[:-1], is_dir=True) 405 else: 406 itemset.Get(fn) 407 408 symlinks.sort() 409 return symlinks 410 411 412def SignOutput(temp_zip_name, output_zip_name): 413 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 414 pw = key_passwords[OPTIONS.package_key] 415 416 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 417 whole_file=True) 418 419 420def AppendAssertions(script, info_dict, oem_dict=None): 421 oem_props = info_dict.get("oem_fingerprint_properties") 422 if oem_props is None or len(oem_props) == 0: 423 device = GetBuildProp("ro.product.device", info_dict) 424 script.AssertDevice(device) 425 else: 426 if oem_dict is None: 427 raise common.ExternalError( 428 "No OEM file provided to answer expected assertions") 429 for prop in oem_props.split(): 430 if oem_dict.get(prop) is None: 431 raise common.ExternalError( 432 "The OEM file is missing the property %s" % prop) 433 script.AssertOemProperty(prop, oem_dict.get(prop)) 434 435 436def HasRecoveryPatch(target_files_zip): 437 try: 438 target_files_zip.getinfo("SYSTEM/recovery-from-boot.p") 439 return True 440 except KeyError: 441 return False 442 443def HasVendorPartition(target_files_zip): 444 try: 445 target_files_zip.getinfo("VENDOR/") 446 return True 447 except KeyError: 448 return False 449 450def GetOemProperty(name, oem_props, oem_dict, info_dict): 451 if oem_props is not None and name in oem_props: 452 return oem_dict[name] 453 return GetBuildProp(name, info_dict) 454 455 456def CalculateFingerprint(oem_props, oem_dict, info_dict): 457 if oem_props is None: 458 return GetBuildProp("ro.build.fingerprint", info_dict) 459 return "%s/%s/%s:%s" % ( 460 GetOemProperty("ro.product.brand", oem_props, oem_dict, info_dict), 461 GetOemProperty("ro.product.name", oem_props, oem_dict, info_dict), 462 GetOemProperty("ro.product.device", oem_props, oem_dict, info_dict), 463 GetBuildProp("ro.build.thumbprint", info_dict)) 464 465 466def GetImage(which, tmpdir, info_dict): 467 # Return an image object (suitable for passing to BlockImageDiff) 468 # for the 'which' partition (most be "system" or "vendor"). If a 469 # prebuilt image and file map are found in tmpdir they are used, 470 # otherwise they are reconstructed from the individual files. 471 472 assert which in ("system", "vendor") 473 474 path = os.path.join(tmpdir, "IMAGES", which + ".img") 475 mappath = os.path.join(tmpdir, "IMAGES", which + ".map") 476 if os.path.exists(path) and os.path.exists(mappath): 477 print "using %s.img from target-files" % (which,) 478 # This is a 'new' target-files, which already has the image in it. 479 480 else: 481 print "building %s.img from target-files" % (which,) 482 483 # This is an 'old' target-files, which does not contain images 484 # already built. Build them. 485 486 mappath = tempfile.mkstemp()[1] 487 OPTIONS.tempfiles.append(mappath) 488 489 import add_img_to_target_files 490 if which == "system": 491 path = add_img_to_target_files.BuildSystem( 492 tmpdir, info_dict, block_list=mappath) 493 elif which == "vendor": 494 path = add_img_to_target_files.BuildVendor( 495 tmpdir, info_dict, block_list=mappath) 496 497 # Bug: http://b/20939131 498 # In ext4 filesystems, block 0 might be changed even being mounted 499 # R/O. We add it to clobbered_blocks so that it will be written to the 500 # target unconditionally. Note that they are still part of care_map. 501 clobbered_blocks = "0" 502 503 return sparse_img.SparseImage(path, mappath, clobbered_blocks) 504 505 506def WriteFullOTAPackage(input_zip, output_zip): 507 # TODO: how to determine this? We don't know what version it will 508 # be installed on top of. For now, we expect the API just won't 509 # change very often. Similarly for fstab, it might have changed 510 # in the target build. 511 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 512 513 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 514 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 515 oem_dict = None 516 if oem_props is not None and len(oem_props) > 0: 517 if OPTIONS.oem_source is None: 518 raise common.ExternalError("OEM source required for this build") 519 if not OPTIONS.oem_no_mount: 520 script.Mount("/oem", recovery_mount_options) 521 oem_dict = common.LoadDictionaryFromLines( 522 open(OPTIONS.oem_source).readlines()) 523 524 metadata = { 525 "post-build": CalculateFingerprint(oem_props, oem_dict, 526 OPTIONS.info_dict), 527 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 528 OPTIONS.info_dict), 529 "post-timestamp": GetBuildProp("ro.build.date.utc", OPTIONS.info_dict), 530 } 531 532 device_specific = common.DeviceSpecificParams( 533 input_zip=input_zip, 534 input_version=OPTIONS.info_dict["recovery_api_version"], 535 output_zip=output_zip, 536 script=script, 537 input_tmp=OPTIONS.input_tmp, 538 metadata=metadata, 539 info_dict=OPTIONS.info_dict) 540 541 has_recovery_patch = HasRecoveryPatch(input_zip) 542 block_based = OPTIONS.block_based and has_recovery_patch 543 544 if not OPTIONS.omit_prereq: 545 ts = GetBuildProp("ro.build.date.utc", OPTIONS.info_dict) 546 ts_text = GetBuildProp("ro.build.date", OPTIONS.info_dict) 547 script.AssertOlderBuild(ts, ts_text) 548 549 AppendAssertions(script, OPTIONS.info_dict, oem_dict) 550 device_specific.FullOTA_Assertions() 551 552 # Two-step package strategy (in chronological order, which is *not* 553 # the order in which the generated script has things): 554 # 555 # if stage is not "2/3" or "3/3": 556 # write recovery image to boot partition 557 # set stage to "2/3" 558 # reboot to boot partition and restart recovery 559 # else if stage is "2/3": 560 # write recovery image to recovery partition 561 # set stage to "3/3" 562 # reboot to recovery partition and restart recovery 563 # else: 564 # (stage must be "3/3") 565 # set stage to "" 566 # do normal full package installation: 567 # wipe and install system, boot image, etc. 568 # set up system to update recovery partition on first boot 569 # complete script normally 570 # (allow recovery to mark itself finished and reboot) 571 572 recovery_img = common.GetBootableImage("recovery.img", "recovery.img", 573 OPTIONS.input_tmp, "RECOVERY") 574 if OPTIONS.two_step: 575 if not OPTIONS.info_dict.get("multistage_support", None): 576 assert False, "two-step packages not supported by this build" 577 fs = OPTIONS.info_dict["fstab"]["/misc"] 578 assert fs.fs_type.upper() == "EMMC", \ 579 "two-step packages only supported on devices with EMMC /misc partitions" 580 bcb_dev = {"bcb_dev": fs.device} 581 common.ZipWriteStr(output_zip, "recovery.img", recovery_img.data) 582 script.AppendExtra(""" 583if get_stage("%(bcb_dev)s") == "2/3" then 584""" % bcb_dev) 585 script.WriteRawImage("/recovery", "recovery.img") 586 script.AppendExtra(""" 587set_stage("%(bcb_dev)s", "3/3"); 588reboot_now("%(bcb_dev)s", "recovery"); 589else if get_stage("%(bcb_dev)s") == "3/3" then 590""" % bcb_dev) 591 592 # Dump fingerprints 593 script.Print("Target: %s" % CalculateFingerprint( 594 oem_props, oem_dict, OPTIONS.info_dict)) 595 596 device_specific.FullOTA_InstallBegin() 597 598 system_progress = 0.75 599 600 if OPTIONS.wipe_user_data: 601 system_progress -= 0.1 602 if HasVendorPartition(input_zip): 603 system_progress -= 0.1 604 605 if "selinux_fc" in OPTIONS.info_dict: 606 WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip) 607 608 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 609 610 system_items = ItemSet("system", "META/filesystem_config.txt") 611 script.ShowProgress(system_progress, 0) 612 613 if block_based: 614 # Full OTA is done as an "incremental" against an empty source 615 # image. This has the effect of writing new data from the package 616 # to the entire partition, but lets us reuse the updater code that 617 # writes incrementals to do it. 618 system_tgt = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict) 619 system_tgt.ResetFileMap() 620 system_diff = common.BlockDifference("system", system_tgt, src=None) 621 system_diff.WriteScript(script, output_zip) 622 else: 623 script.FormatPartition("/system") 624 script.Mount("/system", recovery_mount_options) 625 if not has_recovery_patch: 626 script.UnpackPackageDir("recovery", "/system") 627 script.UnpackPackageDir("system", "/system") 628 629 symlinks = CopyPartitionFiles(system_items, input_zip, output_zip) 630 script.MakeSymlinks(symlinks) 631 632 boot_img = common.GetBootableImage("boot.img", "boot.img", 633 OPTIONS.input_tmp, "BOOT") 634 635 if not block_based: 636 def output_sink(fn, data): 637 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 638 system_items.Get("system/" + fn) 639 640 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, 641 recovery_img, boot_img) 642 643 system_items.GetMetadata(input_zip) 644 system_items.Get("system").SetPermissions(script) 645 646 if HasVendorPartition(input_zip): 647 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 648 script.ShowProgress(0.1, 0) 649 650 if block_based: 651 vendor_tgt = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict) 652 vendor_tgt.ResetFileMap() 653 vendor_diff = common.BlockDifference("vendor", vendor_tgt) 654 vendor_diff.WriteScript(script, output_zip) 655 else: 656 script.FormatPartition("/vendor") 657 script.Mount("/vendor", recovery_mount_options) 658 script.UnpackPackageDir("vendor", "/vendor") 659 660 symlinks = CopyPartitionFiles(vendor_items, input_zip, output_zip) 661 script.MakeSymlinks(symlinks) 662 663 vendor_items.GetMetadata(input_zip) 664 vendor_items.Get("vendor").SetPermissions(script) 665 666 common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict) 667 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 668 669 script.ShowProgress(0.05, 5) 670 script.WriteRawImage("/boot", "boot.img") 671 672 script.ShowProgress(0.2, 10) 673 device_specific.FullOTA_InstallEnd() 674 675 if OPTIONS.extra_script is not None: 676 script.AppendExtra(OPTIONS.extra_script) 677 678 script.UnmountAll() 679 680 if OPTIONS.wipe_user_data: 681 script.ShowProgress(0.1, 10) 682 script.FormatPartition("/data") 683 684 if OPTIONS.two_step: 685 script.AppendExtra(""" 686set_stage("%(bcb_dev)s", ""); 687""" % bcb_dev) 688 script.AppendExtra("else\n") 689 script.WriteRawImage("/boot", "recovery.img") 690 script.AppendExtra(""" 691set_stage("%(bcb_dev)s", "2/3"); 692reboot_now("%(bcb_dev)s", ""); 693endif; 694endif; 695""" % bcb_dev) 696 script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary) 697 WriteMetadata(metadata, output_zip) 698 699 700def WritePolicyConfig(file_name, output_zip): 701 common.ZipWrite(output_zip, file_name, os.path.basename(file_name)) 702 703 704def WriteMetadata(metadata, output_zip): 705 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata", 706 "".join(["%s=%s\n" % kv 707 for kv in sorted(metadata.iteritems())])) 708 709 710def LoadPartitionFiles(z, partition): 711 """Load all the files from the given partition in a given target-files 712 ZipFile, and return a dict of {filename: File object}.""" 713 out = {} 714 prefix = partition.upper() + "/" 715 for info in z.infolist(): 716 if info.filename.startswith(prefix) and not IsSymlink(info): 717 basefilename = info.filename[len(prefix):] 718 fn = partition + "/" + basefilename 719 data = z.read(info.filename) 720 out[fn] = common.File(fn, data) 721 return out 722 723 724def GetBuildProp(prop, info_dict): 725 """Return the fingerprint of the build of a given target-files info_dict.""" 726 try: 727 return info_dict.get("build.prop", {})[prop] 728 except KeyError: 729 raise common.ExternalError("couldn't find %s in build.prop" % (prop,)) 730 731 732def AddToKnownPaths(filename, known_paths): 733 if filename[-1] == "/": 734 return 735 dirs = filename.split("/")[:-1] 736 while len(dirs) > 0: 737 path = "/".join(dirs) 738 if path in known_paths: 739 break 740 known_paths.add(path) 741 dirs.pop() 742 743 744def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 745 source_version = OPTIONS.source_info_dict["recovery_api_version"] 746 target_version = OPTIONS.target_info_dict["recovery_api_version"] 747 748 if source_version == 0: 749 print ("WARNING: generating edify script for a source that " 750 "can't install it.") 751 script = edify_generator.EdifyGenerator( 752 source_version, OPTIONS.target_info_dict, 753 fstab=OPTIONS.source_info_dict["fstab"]) 754 755 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 756 recovery_mount_options = OPTIONS.source_info_dict.get( 757 "recovery_mount_options") 758 oem_dict = None 759 if oem_props is not None and len(oem_props) > 0: 760 if OPTIONS.oem_source is None: 761 raise common.ExternalError("OEM source required for this build") 762 if not OPTIONS.oem_no_mount: 763 script.Mount("/oem", recovery_mount_options) 764 oem_dict = common.LoadDictionaryFromLines( 765 open(OPTIONS.oem_source).readlines()) 766 767 metadata = { 768 "pre-device": GetBuildProp("ro.product.device", 769 OPTIONS.source_info_dict), 770 "post-timestamp": GetBuildProp("ro.build.date.utc", 771 OPTIONS.target_info_dict), 772 } 773 774 device_specific = common.DeviceSpecificParams( 775 source_zip=source_zip, 776 source_version=source_version, 777 target_zip=target_zip, 778 target_version=target_version, 779 output_zip=output_zip, 780 script=script, 781 metadata=metadata, 782 info_dict=OPTIONS.source_info_dict) 783 784 # TODO: Currently this works differently from WriteIncrementalOTAPackage(). 785 # This function doesn't consider thumbprints when writing 786 # metadata["pre/post-build"]. One possible reason is that the current 787 # devices with thumbprints are all using file-based OTAs. Long term we 788 # should factor out the common parts into a shared one to avoid further 789 # divergence. 790 source_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.source_info_dict) 791 target_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.target_info_dict) 792 metadata["pre-build"] = source_fp 793 metadata["post-build"] = target_fp 794 795 source_boot = common.GetBootableImage( 796 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 797 OPTIONS.source_info_dict) 798 target_boot = common.GetBootableImage( 799 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 800 updating_boot = (not OPTIONS.two_step and 801 (source_boot.data != target_boot.data)) 802 803 target_recovery = common.GetBootableImage( 804 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 805 806 system_src = GetImage("system", OPTIONS.source_tmp, OPTIONS.source_info_dict) 807 system_tgt = GetImage("system", OPTIONS.target_tmp, OPTIONS.target_info_dict) 808 809 blockimgdiff_version = 1 810 if OPTIONS.info_dict: 811 blockimgdiff_version = max( 812 int(i) for i in 813 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(",")) 814 815 system_diff = common.BlockDifference("system", system_tgt, system_src, 816 version=blockimgdiff_version) 817 818 if HasVendorPartition(target_zip): 819 if not HasVendorPartition(source_zip): 820 raise RuntimeError("can't generate incremental that adds /vendor") 821 vendor_src = GetImage("vendor", OPTIONS.source_tmp, 822 OPTIONS.source_info_dict) 823 vendor_tgt = GetImage("vendor", OPTIONS.target_tmp, 824 OPTIONS.target_info_dict) 825 vendor_diff = common.BlockDifference("vendor", vendor_tgt, vendor_src, 826 version=blockimgdiff_version) 827 else: 828 vendor_diff = None 829 830 oem_props = OPTIONS.target_info_dict.get("oem_fingerprint_properties") 831 recovery_mount_options = OPTIONS.source_info_dict.get( 832 "recovery_mount_options") 833 oem_dict = None 834 if oem_props is not None and len(oem_props) > 0: 835 if OPTIONS.oem_source is None: 836 raise common.ExternalError("OEM source required for this build") 837 script.Mount("/oem", recovery_mount_options) 838 oem_dict = common.LoadDictionaryFromLines( 839 open(OPTIONS.oem_source).readlines()) 840 841 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 842 device_specific.IncrementalOTA_Assertions() 843 844 # Two-step incremental package strategy (in chronological order, 845 # which is *not* the order in which the generated script has 846 # things): 847 # 848 # if stage is not "2/3" or "3/3": 849 # do verification on current system 850 # write recovery image to boot partition 851 # set stage to "2/3" 852 # reboot to boot partition and restart recovery 853 # else if stage is "2/3": 854 # write recovery image to recovery partition 855 # set stage to "3/3" 856 # reboot to recovery partition and restart recovery 857 # else: 858 # (stage must be "3/3") 859 # perform update: 860 # patch system files, etc. 861 # force full install of new boot image 862 # set up system to update recovery partition on first boot 863 # complete script normally 864 # (allow recovery to mark itself finished and reboot) 865 866 if OPTIONS.two_step: 867 if not OPTIONS.source_info_dict.get("multistage_support", None): 868 assert False, "two-step packages not supported by this build" 869 fs = OPTIONS.source_info_dict["fstab"]["/misc"] 870 assert fs.fs_type.upper() == "EMMC", \ 871 "two-step packages only supported on devices with EMMC /misc partitions" 872 bcb_dev = {"bcb_dev": fs.device} 873 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 874 script.AppendExtra(""" 875if get_stage("%(bcb_dev)s") == "2/3" then 876""" % bcb_dev) 877 script.AppendExtra("sleep(20);\n") 878 script.WriteRawImage("/recovery", "recovery.img") 879 script.AppendExtra(""" 880set_stage("%(bcb_dev)s", "3/3"); 881reboot_now("%(bcb_dev)s", "recovery"); 882else if get_stage("%(bcb_dev)s") != "3/3" then 883""" % bcb_dev) 884 885 # Dump fingerprints 886 script.Print("Source: %s" % CalculateFingerprint( 887 oem_props, oem_dict, OPTIONS.source_info_dict)) 888 script.Print("Target: %s" % CalculateFingerprint( 889 oem_props, oem_dict, OPTIONS.target_info_dict)) 890 891 script.Print("Verifying current system...") 892 893 device_specific.IncrementalOTA_VerifyBegin() 894 895 if oem_props is None: 896 # When blockimgdiff version is less than 3 (non-resumable block-based OTA), 897 # patching on a device that's already on the target build will damage the 898 # system. Because operations like move don't check the block state, they 899 # always apply the changes unconditionally. 900 if blockimgdiff_version <= 2: 901 script.AssertSomeFingerprint(source_fp) 902 else: 903 script.AssertSomeFingerprint(source_fp, target_fp) 904 else: 905 if blockimgdiff_version <= 2: 906 script.AssertSomeThumbprint( 907 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 908 else: 909 script.AssertSomeThumbprint( 910 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 911 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 912 913 if updating_boot: 914 boot_type, boot_device = common.GetTypeAndDevice( 915 "/boot", OPTIONS.source_info_dict) 916 d = common.Difference(target_boot, source_boot) 917 _, _, d = d.ComputePatch() 918 if d is None: 919 include_full_boot = True 920 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 921 else: 922 include_full_boot = False 923 924 print "boot target: %d source: %d diff: %d" % ( 925 target_boot.size, source_boot.size, len(d)) 926 927 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 928 929 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 930 (boot_type, boot_device, 931 source_boot.size, source_boot.sha1, 932 target_boot.size, target_boot.sha1)) 933 934 device_specific.IncrementalOTA_VerifyEnd() 935 936 if OPTIONS.two_step: 937 script.WriteRawImage("/boot", "recovery.img") 938 script.AppendExtra(""" 939set_stage("%(bcb_dev)s", "2/3"); 940reboot_now("%(bcb_dev)s", ""); 941else 942""" % bcb_dev) 943 944 # Verify the existing partitions. 945 system_diff.WriteVerifyScript(script) 946 if vendor_diff: 947 vendor_diff.WriteVerifyScript(script) 948 949 script.Comment("---- start making changes here ----") 950 951 device_specific.IncrementalOTA_InstallBegin() 952 953 system_diff.WriteScript(script, output_zip, 954 progress=0.8 if vendor_diff else 0.9) 955 if vendor_diff: 956 vendor_diff.WriteScript(script, output_zip, progress=0.1) 957 958 if OPTIONS.two_step: 959 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 960 script.WriteRawImage("/boot", "boot.img") 961 print "writing full boot image (forced by two-step mode)" 962 963 if not OPTIONS.two_step: 964 if updating_boot: 965 if include_full_boot: 966 print "boot image changed; including full." 967 script.Print("Installing boot image...") 968 script.WriteRawImage("/boot", "boot.img") 969 else: 970 # Produce the boot image by applying a patch to the current 971 # contents of the boot partition, and write it back to the 972 # partition. 973 print "boot image changed; including patch." 974 script.Print("Patching boot image...") 975 script.ShowProgress(0.1, 10) 976 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 977 % (boot_type, boot_device, 978 source_boot.size, source_boot.sha1, 979 target_boot.size, target_boot.sha1), 980 "-", 981 target_boot.size, target_boot.sha1, 982 source_boot.sha1, "patch/boot.img.p") 983 else: 984 print "boot image unchanged; skipping." 985 986 # Do device-specific installation (eg, write radio image). 987 device_specific.IncrementalOTA_InstallEnd() 988 989 if OPTIONS.extra_script is not None: 990 script.AppendExtra(OPTIONS.extra_script) 991 992 if OPTIONS.wipe_user_data: 993 script.Print("Erasing user data...") 994 script.FormatPartition("/data") 995 996 if OPTIONS.two_step: 997 script.AppendExtra(""" 998set_stage("%(bcb_dev)s", ""); 999endif; 1000endif; 1001""" % bcb_dev) 1002 1003 script.SetProgress(1) 1004 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 1005 WriteMetadata(metadata, output_zip) 1006 1007 1008class FileDifference(object): 1009 def __init__(self, partition, source_zip, target_zip, output_zip): 1010 self.deferred_patch_list = None 1011 print "Loading target..." 1012 self.target_data = target_data = LoadPartitionFiles(target_zip, partition) 1013 print "Loading source..." 1014 self.source_data = source_data = LoadPartitionFiles(source_zip, partition) 1015 1016 self.verbatim_targets = verbatim_targets = [] 1017 self.patch_list = patch_list = [] 1018 diffs = [] 1019 self.renames = renames = {} 1020 known_paths = set() 1021 largest_source_size = 0 1022 1023 matching_file_cache = {} 1024 for fn, sf in source_data.items(): 1025 assert fn == sf.name 1026 matching_file_cache["path:" + fn] = sf 1027 if fn in target_data.keys(): 1028 AddToKnownPaths(fn, known_paths) 1029 # Only allow eligibility for filename/sha matching 1030 # if there isn't a perfect path match. 1031 if target_data.get(sf.name) is None: 1032 matching_file_cache["file:" + fn.split("/")[-1]] = sf 1033 matching_file_cache["sha:" + sf.sha1] = sf 1034 1035 for fn in sorted(target_data.keys()): 1036 tf = target_data[fn] 1037 assert fn == tf.name 1038 sf = ClosestFileMatch(tf, matching_file_cache, renames) 1039 if sf is not None and sf.name != tf.name: 1040 print "File has moved from " + sf.name + " to " + tf.name 1041 renames[sf.name] = tf 1042 1043 if sf is None or fn in OPTIONS.require_verbatim: 1044 # This file should be included verbatim 1045 if fn in OPTIONS.prohibit_verbatim: 1046 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 1047 print "send", fn, "verbatim" 1048 tf.AddToZip(output_zip) 1049 verbatim_targets.append((fn, tf.size, tf.sha1)) 1050 if fn in target_data.keys(): 1051 AddToKnownPaths(fn, known_paths) 1052 elif tf.sha1 != sf.sha1: 1053 # File is different; consider sending as a patch 1054 diffs.append(common.Difference(tf, sf)) 1055 else: 1056 # Target file data identical to source (may still be renamed) 1057 pass 1058 1059 common.ComputeDifferences(diffs) 1060 1061 for diff in diffs: 1062 tf, sf, d = diff.GetPatch() 1063 path = "/".join(tf.name.split("/")[:-1]) 1064 if d is None or len(d) > tf.size * OPTIONS.patch_threshold or \ 1065 path not in known_paths: 1066 # patch is almost as big as the file; don't bother patching 1067 # or a patch + rename cannot take place due to the target 1068 # directory not existing 1069 tf.AddToZip(output_zip) 1070 verbatim_targets.append((tf.name, tf.size, tf.sha1)) 1071 if sf.name in renames: 1072 del renames[sf.name] 1073 AddToKnownPaths(tf.name, known_paths) 1074 else: 1075 common.ZipWriteStr(output_zip, "patch/" + sf.name + ".p", d) 1076 patch_list.append((tf, sf, tf.size, common.sha1(d).hexdigest())) 1077 largest_source_size = max(largest_source_size, sf.size) 1078 1079 self.largest_source_size = largest_source_size 1080 1081 def EmitVerification(self, script): 1082 so_far = 0 1083 for tf, sf, _, _ in self.patch_list: 1084 if tf.name != sf.name: 1085 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1086 script.PatchCheck("/"+sf.name, tf.sha1, sf.sha1) 1087 so_far += sf.size 1088 return so_far 1089 1090 def EmitExplicitTargetVerification(self, script): 1091 for fn, _, sha1 in self.verbatim_targets: 1092 if fn[-1] != "/": 1093 script.FileCheck("/"+fn, sha1) 1094 for tf, _, _, _ in self.patch_list: 1095 script.FileCheck(tf.name, tf.sha1) 1096 1097 def RemoveUnneededFiles(self, script, extras=()): 1098 script.DeleteFiles( 1099 ["/" + i[0] for i in self.verbatim_targets] + 1100 ["/" + i for i in sorted(self.source_data) 1101 if i not in self.target_data and i not in self.renames] + 1102 list(extras)) 1103 1104 def TotalPatchSize(self): 1105 return sum(i[1].size for i in self.patch_list) 1106 1107 def EmitPatches(self, script, total_patch_size, so_far): 1108 self.deferred_patch_list = deferred_patch_list = [] 1109 for item in self.patch_list: 1110 tf, sf, _, _ = item 1111 if tf.name == "system/build.prop": 1112 deferred_patch_list.append(item) 1113 continue 1114 if sf.name != tf.name: 1115 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1116 script.ApplyPatch("/" + sf.name, "-", tf.size, tf.sha1, sf.sha1, 1117 "patch/" + sf.name + ".p") 1118 so_far += tf.size 1119 script.SetProgress(so_far / total_patch_size) 1120 return so_far 1121 1122 def EmitDeferredPatches(self, script): 1123 for item in self.deferred_patch_list: 1124 tf, sf, _, _ = item 1125 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, 1126 "patch/" + sf.name + ".p") 1127 script.SetPermissions("/system/build.prop", 0, 0, 0o644, None, None) 1128 1129 def EmitRenames(self, script): 1130 if len(self.renames) > 0: 1131 script.Print("Renaming files...") 1132 for src, tgt in self.renames.iteritems(): 1133 print "Renaming " + src + " to " + tgt.name 1134 script.RenameFile(src, tgt.name) 1135 1136 1137def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): 1138 target_has_recovery_patch = HasRecoveryPatch(target_zip) 1139 source_has_recovery_patch = HasRecoveryPatch(source_zip) 1140 1141 if (OPTIONS.block_based and 1142 target_has_recovery_patch and 1143 source_has_recovery_patch): 1144 return WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip) 1145 1146 source_version = OPTIONS.source_info_dict["recovery_api_version"] 1147 target_version = OPTIONS.target_info_dict["recovery_api_version"] 1148 1149 if source_version == 0: 1150 print ("WARNING: generating edify script for a source that " 1151 "can't install it.") 1152 script = edify_generator.EdifyGenerator( 1153 source_version, OPTIONS.target_info_dict, 1154 fstab=OPTIONS.source_info_dict["fstab"]) 1155 1156 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 1157 recovery_mount_options = OPTIONS.source_info_dict.get( 1158 "recovery_mount_options") 1159 oem_dict = None 1160 if oem_props is not None and len(oem_props) > 0: 1161 if OPTIONS.oem_source is None: 1162 raise common.ExternalError("OEM source required for this build") 1163 script.Mount("/oem", recovery_mount_options) 1164 oem_dict = common.LoadDictionaryFromLines( 1165 open(OPTIONS.oem_source).readlines()) 1166 1167 metadata = { 1168 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 1169 OPTIONS.source_info_dict), 1170 "post-timestamp": GetBuildProp("ro.build.date.utc", 1171 OPTIONS.target_info_dict), 1172 } 1173 1174 device_specific = common.DeviceSpecificParams( 1175 source_zip=source_zip, 1176 source_version=source_version, 1177 target_zip=target_zip, 1178 target_version=target_version, 1179 output_zip=output_zip, 1180 script=script, 1181 metadata=metadata, 1182 info_dict=OPTIONS.source_info_dict) 1183 1184 system_diff = FileDifference("system", source_zip, target_zip, output_zip) 1185 script.Mount("/system", recovery_mount_options) 1186 if HasVendorPartition(target_zip): 1187 vendor_diff = FileDifference("vendor", source_zip, target_zip, output_zip) 1188 script.Mount("/vendor", recovery_mount_options) 1189 else: 1190 vendor_diff = None 1191 1192 target_fp = CalculateFingerprint(oem_props, oem_dict, 1193 OPTIONS.target_info_dict) 1194 source_fp = CalculateFingerprint(oem_props, oem_dict, 1195 OPTIONS.source_info_dict) 1196 1197 if oem_props is None: 1198 script.AssertSomeFingerprint(source_fp, target_fp) 1199 else: 1200 script.AssertSomeThumbprint( 1201 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 1202 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 1203 1204 metadata["pre-build"] = source_fp 1205 metadata["post-build"] = target_fp 1206 1207 source_boot = common.GetBootableImage( 1208 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 1209 OPTIONS.source_info_dict) 1210 target_boot = common.GetBootableImage( 1211 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 1212 updating_boot = (not OPTIONS.two_step and 1213 (source_boot.data != target_boot.data)) 1214 1215 source_recovery = common.GetBootableImage( 1216 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 1217 OPTIONS.source_info_dict) 1218 target_recovery = common.GetBootableImage( 1219 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 1220 updating_recovery = (source_recovery.data != target_recovery.data) 1221 1222 # Here's how we divide up the progress bar: 1223 # 0.1 for verifying the start state (PatchCheck calls) 1224 # 0.8 for applying patches (ApplyPatch calls) 1225 # 0.1 for unpacking verbatim files, symlinking, and doing the 1226 # device-specific commands. 1227 1228 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 1229 device_specific.IncrementalOTA_Assertions() 1230 1231 # Two-step incremental package strategy (in chronological order, 1232 # which is *not* the order in which the generated script has 1233 # things): 1234 # 1235 # if stage is not "2/3" or "3/3": 1236 # do verification on current system 1237 # write recovery image to boot partition 1238 # set stage to "2/3" 1239 # reboot to boot partition and restart recovery 1240 # else if stage is "2/3": 1241 # write recovery image to recovery partition 1242 # set stage to "3/3" 1243 # reboot to recovery partition and restart recovery 1244 # else: 1245 # (stage must be "3/3") 1246 # perform update: 1247 # patch system files, etc. 1248 # force full install of new boot image 1249 # set up system to update recovery partition on first boot 1250 # complete script normally 1251 # (allow recovery to mark itself finished and reboot) 1252 1253 if OPTIONS.two_step: 1254 if not OPTIONS.source_info_dict.get("multistage_support", None): 1255 assert False, "two-step packages not supported by this build" 1256 fs = OPTIONS.source_info_dict["fstab"]["/misc"] 1257 assert fs.fs_type.upper() == "EMMC", \ 1258 "two-step packages only supported on devices with EMMC /misc partitions" 1259 bcb_dev = {"bcb_dev": fs.device} 1260 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 1261 script.AppendExtra(""" 1262if get_stage("%(bcb_dev)s") == "2/3" then 1263""" % bcb_dev) 1264 script.AppendExtra("sleep(20);\n") 1265 script.WriteRawImage("/recovery", "recovery.img") 1266 script.AppendExtra(""" 1267set_stage("%(bcb_dev)s", "3/3"); 1268reboot_now("%(bcb_dev)s", "recovery"); 1269else if get_stage("%(bcb_dev)s") != "3/3" then 1270""" % bcb_dev) 1271 1272 # Dump fingerprints 1273 script.Print("Source: %s" % (source_fp,)) 1274 script.Print("Target: %s" % (target_fp,)) 1275 1276 script.Print("Verifying current system...") 1277 1278 device_specific.IncrementalOTA_VerifyBegin() 1279 1280 script.ShowProgress(0.1, 0) 1281 so_far = system_diff.EmitVerification(script) 1282 if vendor_diff: 1283 so_far += vendor_diff.EmitVerification(script) 1284 1285 if updating_boot: 1286 d = common.Difference(target_boot, source_boot) 1287 _, _, d = d.ComputePatch() 1288 print "boot target: %d source: %d diff: %d" % ( 1289 target_boot.size, source_boot.size, len(d)) 1290 1291 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 1292 1293 boot_type, boot_device = common.GetTypeAndDevice( 1294 "/boot", OPTIONS.source_info_dict) 1295 1296 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 1297 (boot_type, boot_device, 1298 source_boot.size, source_boot.sha1, 1299 target_boot.size, target_boot.sha1)) 1300 so_far += source_boot.size 1301 1302 size = [] 1303 if system_diff.patch_list: 1304 size.append(system_diff.largest_source_size) 1305 if vendor_diff: 1306 if vendor_diff.patch_list: 1307 size.append(vendor_diff.largest_source_size) 1308 if size or updating_recovery or updating_boot: 1309 script.CacheFreeSpaceCheck(max(size)) 1310 1311 device_specific.IncrementalOTA_VerifyEnd() 1312 1313 if OPTIONS.two_step: 1314 script.WriteRawImage("/boot", "recovery.img") 1315 script.AppendExtra(""" 1316set_stage("%(bcb_dev)s", "2/3"); 1317reboot_now("%(bcb_dev)s", ""); 1318else 1319""" % bcb_dev) 1320 1321 script.Comment("---- start making changes here ----") 1322 1323 device_specific.IncrementalOTA_InstallBegin() 1324 1325 if OPTIONS.two_step: 1326 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 1327 script.WriteRawImage("/boot", "boot.img") 1328 print "writing full boot image (forced by two-step mode)" 1329 1330 script.Print("Removing unneeded files...") 1331 system_diff.RemoveUnneededFiles(script, ("/system/recovery.img",)) 1332 if vendor_diff: 1333 vendor_diff.RemoveUnneededFiles(script) 1334 1335 script.ShowProgress(0.8, 0) 1336 total_patch_size = 1.0 + system_diff.TotalPatchSize() 1337 if vendor_diff: 1338 total_patch_size += vendor_diff.TotalPatchSize() 1339 if updating_boot: 1340 total_patch_size += target_boot.size 1341 1342 script.Print("Patching system files...") 1343 so_far = system_diff.EmitPatches(script, total_patch_size, 0) 1344 if vendor_diff: 1345 script.Print("Patching vendor files...") 1346 so_far = vendor_diff.EmitPatches(script, total_patch_size, so_far) 1347 1348 if not OPTIONS.two_step: 1349 if updating_boot: 1350 # Produce the boot image by applying a patch to the current 1351 # contents of the boot partition, and write it back to the 1352 # partition. 1353 script.Print("Patching boot image...") 1354 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 1355 % (boot_type, boot_device, 1356 source_boot.size, source_boot.sha1, 1357 target_boot.size, target_boot.sha1), 1358 "-", 1359 target_boot.size, target_boot.sha1, 1360 source_boot.sha1, "patch/boot.img.p") 1361 so_far += target_boot.size 1362 script.SetProgress(so_far / total_patch_size) 1363 print "boot image changed; including." 1364 else: 1365 print "boot image unchanged; skipping." 1366 1367 system_items = ItemSet("system", "META/filesystem_config.txt") 1368 if vendor_diff: 1369 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 1370 1371 if updating_recovery: 1372 # Recovery is generated as a patch using both the boot image 1373 # (which contains the same linux kernel as recovery) and the file 1374 # /system/etc/recovery-resource.dat (which contains all the images 1375 # used in the recovery UI) as sources. This lets us minimize the 1376 # size of the patch, which must be included in every OTA package. 1377 # 1378 # For older builds where recovery-resource.dat is not present, we 1379 # use only the boot image as the source. 1380 1381 if not target_has_recovery_patch: 1382 def output_sink(fn, data): 1383 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 1384 system_items.Get("system/" + fn) 1385 1386 common.MakeRecoveryPatch(OPTIONS.target_tmp, output_sink, 1387 target_recovery, target_boot) 1388 script.DeleteFiles(["/system/recovery-from-boot.p", 1389 "/system/etc/install-recovery.sh"]) 1390 print "recovery image changed; including as patch from boot." 1391 else: 1392 print "recovery image unchanged; skipping." 1393 1394 script.ShowProgress(0.1, 10) 1395 1396 target_symlinks = CopyPartitionFiles(system_items, target_zip, None) 1397 if vendor_diff: 1398 target_symlinks.extend(CopyPartitionFiles(vendor_items, target_zip, None)) 1399 1400 temp_script = script.MakeTemporary() 1401 system_items.GetMetadata(target_zip) 1402 system_items.Get("system").SetPermissions(temp_script) 1403 if vendor_diff: 1404 vendor_items.GetMetadata(target_zip) 1405 vendor_items.Get("vendor").SetPermissions(temp_script) 1406 1407 # Note that this call will mess up the trees of Items, so make sure 1408 # we're done with them. 1409 source_symlinks = CopyPartitionFiles(system_items, source_zip, None) 1410 if vendor_diff: 1411 source_symlinks.extend(CopyPartitionFiles(vendor_items, source_zip, None)) 1412 1413 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 1414 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 1415 1416 # Delete all the symlinks in source that aren't in target. This 1417 # needs to happen before verbatim files are unpacked, in case a 1418 # symlink in the source is replaced by a real file in the target. 1419 1420 # If a symlink in the source will be replaced by a regular file, we cannot 1421 # delete the symlink/file in case the package gets applied again. For such 1422 # a symlink, we prepend a sha1_check() to detect if it has been updated. 1423 # (Bug: 23646151) 1424 replaced_symlinks = dict() 1425 if system_diff: 1426 for i in system_diff.verbatim_targets: 1427 replaced_symlinks["/%s" % (i[0],)] = i[2] 1428 if vendor_diff: 1429 for i in vendor_diff.verbatim_targets: 1430 replaced_symlinks["/%s" % (i[0],)] = i[2] 1431 1432 if system_diff: 1433 for tf in system_diff.renames.values(): 1434 replaced_symlinks["/%s" % (tf.name,)] = tf.sha1 1435 if vendor_diff: 1436 for tf in vendor_diff.renames.values(): 1437 replaced_symlinks["/%s" % (tf.name,)] = tf.sha1 1438 1439 always_delete = [] 1440 may_delete = [] 1441 for dest, link in source_symlinks: 1442 if link not in target_symlinks_d: 1443 if link in replaced_symlinks: 1444 may_delete.append((link, replaced_symlinks[link])) 1445 else: 1446 always_delete.append(link) 1447 script.DeleteFiles(always_delete) 1448 script.DeleteFilesIfNotMatching(may_delete) 1449 1450 if system_diff.verbatim_targets: 1451 script.Print("Unpacking new system files...") 1452 script.UnpackPackageDir("system", "/system") 1453 if vendor_diff and vendor_diff.verbatim_targets: 1454 script.Print("Unpacking new vendor files...") 1455 script.UnpackPackageDir("vendor", "/vendor") 1456 1457 if updating_recovery and not target_has_recovery_patch: 1458 script.Print("Unpacking new recovery...") 1459 script.UnpackPackageDir("recovery", "/system") 1460 1461 system_diff.EmitRenames(script) 1462 if vendor_diff: 1463 vendor_diff.EmitRenames(script) 1464 1465 script.Print("Symlinks and permissions...") 1466 1467 # Create all the symlinks that don't already exist, or point to 1468 # somewhere different than what we want. Delete each symlink before 1469 # creating it, since the 'symlink' command won't overwrite. 1470 to_create = [] 1471 for dest, link in target_symlinks: 1472 if link in source_symlinks_d: 1473 if dest != source_symlinks_d[link]: 1474 to_create.append((dest, link)) 1475 else: 1476 to_create.append((dest, link)) 1477 script.DeleteFiles([i[1] for i in to_create]) 1478 script.MakeSymlinks(to_create) 1479 1480 # Now that the symlinks are created, we can set all the 1481 # permissions. 1482 script.AppendScript(temp_script) 1483 1484 # Do device-specific installation (eg, write radio image). 1485 device_specific.IncrementalOTA_InstallEnd() 1486 1487 if OPTIONS.extra_script is not None: 1488 script.AppendExtra(OPTIONS.extra_script) 1489 1490 # Patch the build.prop file last, so if something fails but the 1491 # device can still come up, it appears to be the old build and will 1492 # get set the OTA package again to retry. 1493 script.Print("Patching remaining system files...") 1494 system_diff.EmitDeferredPatches(script) 1495 1496 if OPTIONS.wipe_user_data: 1497 script.Print("Erasing user data...") 1498 script.FormatPartition("/data") 1499 1500 if OPTIONS.two_step: 1501 script.AppendExtra(""" 1502set_stage("%(bcb_dev)s", ""); 1503endif; 1504endif; 1505""" % bcb_dev) 1506 1507 if OPTIONS.verify and system_diff: 1508 script.Print("Remounting and verifying system partition files...") 1509 script.Unmount("/system") 1510 script.Mount("/system") 1511 system_diff.EmitExplicitTargetVerification(script) 1512 1513 if OPTIONS.verify and vendor_diff: 1514 script.Print("Remounting and verifying vendor partition files...") 1515 script.Unmount("/vendor") 1516 script.Mount("/vendor") 1517 vendor_diff.EmitExplicitTargetVerification(script) 1518 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 1519 1520 WriteMetadata(metadata, output_zip) 1521 1522 1523def main(argv): 1524 1525 def option_handler(o, a): 1526 if o == "--board_config": 1527 pass # deprecated 1528 elif o in ("-k", "--package_key"): 1529 OPTIONS.package_key = a 1530 elif o in ("-i", "--incremental_from"): 1531 OPTIONS.incremental_source = a 1532 elif o == "--full_radio": 1533 OPTIONS.full_radio = True 1534 elif o == "--full_bootloader": 1535 OPTIONS.full_bootloader = True 1536 elif o in ("-w", "--wipe_user_data"): 1537 OPTIONS.wipe_user_data = True 1538 elif o in ("-n", "--no_prereq"): 1539 OPTIONS.omit_prereq = True 1540 elif o in ("-o", "--oem_settings"): 1541 OPTIONS.oem_source = a 1542 elif o == "--oem_no_mount": 1543 OPTIONS.oem_no_mount = True 1544 elif o in ("-e", "--extra_script"): 1545 OPTIONS.extra_script = a 1546 elif o in ("-a", "--aslr_mode"): 1547 if a in ("on", "On", "true", "True", "yes", "Yes"): 1548 OPTIONS.aslr_mode = True 1549 else: 1550 OPTIONS.aslr_mode = False 1551 elif o in ("-t", "--worker_threads"): 1552 if a.isdigit(): 1553 OPTIONS.worker_threads = int(a) 1554 else: 1555 raise ValueError("Cannot parse value %r for option %r - only " 1556 "integers are allowed." % (a, o)) 1557 elif o in ("-2", "--two_step"): 1558 OPTIONS.two_step = True 1559 elif o == "--no_signing": 1560 OPTIONS.no_signing = True 1561 elif o == "--verify": 1562 OPTIONS.verify = True 1563 elif o == "--block": 1564 OPTIONS.block_based = True 1565 elif o in ("-b", "--binary"): 1566 OPTIONS.updater_binary = a 1567 elif o in ("--no_fallback_to_full",): 1568 OPTIONS.fallback_to_full = False 1569 elif o == "--stash_threshold": 1570 try: 1571 OPTIONS.stash_threshold = float(a) 1572 except ValueError: 1573 raise ValueError("Cannot parse value %r for option %r - expecting " 1574 "a float" % (a, o)) 1575 else: 1576 return False 1577 return True 1578 1579 args = common.ParseOptions(argv, __doc__, 1580 extra_opts="b:k:i:d:wne:t:a:2o:", 1581 extra_long_opts=[ 1582 "board_config=", 1583 "package_key=", 1584 "incremental_from=", 1585 "full_radio", 1586 "full_bootloader", 1587 "wipe_user_data", 1588 "no_prereq", 1589 "extra_script=", 1590 "worker_threads=", 1591 "aslr_mode=", 1592 "two_step", 1593 "no_signing", 1594 "block", 1595 "binary=", 1596 "oem_settings=", 1597 "oem_no_mount", 1598 "verify", 1599 "no_fallback_to_full", 1600 "stash_threshold=", 1601 ], extra_option_handler=option_handler) 1602 1603 if len(args) != 2: 1604 common.Usage(__doc__) 1605 sys.exit(1) 1606 1607 if OPTIONS.extra_script is not None: 1608 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 1609 1610 print "unzipping target target-files..." 1611 OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0]) 1612 1613 OPTIONS.target_tmp = OPTIONS.input_tmp 1614 OPTIONS.info_dict = common.LoadInfoDict(input_zip) 1615 1616 # If this image was originally labelled with SELinux contexts, make sure we 1617 # also apply the labels in our new image. During building, the "file_contexts" 1618 # is in the out/ directory tree, but for repacking from target-files.zip it's 1619 # in the root directory of the ramdisk. 1620 if "selinux_fc" in OPTIONS.info_dict: 1621 OPTIONS.info_dict["selinux_fc"] = os.path.join( 1622 OPTIONS.input_tmp, "BOOT", "RAMDISK", "file_contexts") 1623 1624 if OPTIONS.verbose: 1625 print "--- target info ---" 1626 common.DumpInfoDict(OPTIONS.info_dict) 1627 1628 # If the caller explicitly specified the device-specific extensions 1629 # path via -s/--device_specific, use that. Otherwise, use 1630 # META/releasetools.py if it is present in the target target_files. 1631 # Otherwise, take the path of the file from 'tool_extensions' in the 1632 # info dict and look for that in the local filesystem, relative to 1633 # the current directory. 1634 1635 if OPTIONS.device_specific is None: 1636 from_input = os.path.join(OPTIONS.input_tmp, "META", "releasetools.py") 1637 if os.path.exists(from_input): 1638 print "(using device-specific extensions from target_files)" 1639 OPTIONS.device_specific = from_input 1640 else: 1641 OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None) 1642 1643 if OPTIONS.device_specific is not None: 1644 OPTIONS.device_specific = os.path.abspath(OPTIONS.device_specific) 1645 1646 while True: 1647 1648 if OPTIONS.no_signing: 1649 if os.path.exists(args[1]): 1650 os.unlink(args[1]) 1651 output_zip = zipfile.ZipFile(args[1], "w", 1652 compression=zipfile.ZIP_DEFLATED) 1653 else: 1654 temp_zip_file = tempfile.NamedTemporaryFile() 1655 output_zip = zipfile.ZipFile(temp_zip_file, "w", 1656 compression=zipfile.ZIP_DEFLATED) 1657 1658 cache_size = OPTIONS.info_dict.get("cache_size", None) 1659 if cache_size is None: 1660 raise RuntimeError("can't determine the cache partition size") 1661 OPTIONS.cache_size = cache_size 1662 1663 if OPTIONS.incremental_source is None: 1664 WriteFullOTAPackage(input_zip, output_zip) 1665 if OPTIONS.package_key is None: 1666 OPTIONS.package_key = OPTIONS.info_dict.get( 1667 "default_system_dev_certificate", 1668 "build/target/product/security/testkey") 1669 common.ZipClose(output_zip) 1670 break 1671 1672 else: 1673 print "unzipping source target-files..." 1674 OPTIONS.source_tmp, source_zip = common.UnzipTemp( 1675 OPTIONS.incremental_source) 1676 OPTIONS.target_info_dict = OPTIONS.info_dict 1677 OPTIONS.source_info_dict = common.LoadInfoDict(source_zip) 1678 if "selinux_fc" in OPTIONS.source_info_dict: 1679 OPTIONS.source_info_dict["selinux_fc"] = os.path.join( 1680 OPTIONS.source_tmp, "BOOT", "RAMDISK", "file_contexts") 1681 if OPTIONS.package_key is None: 1682 OPTIONS.package_key = OPTIONS.source_info_dict.get( 1683 "default_system_dev_certificate", 1684 "build/target/product/security/testkey") 1685 if OPTIONS.verbose: 1686 print "--- source info ---" 1687 common.DumpInfoDict(OPTIONS.source_info_dict) 1688 try: 1689 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) 1690 common.ZipClose(output_zip) 1691 break 1692 except ValueError: 1693 if not OPTIONS.fallback_to_full: 1694 raise 1695 print "--- failed to build incremental; falling back to full ---" 1696 OPTIONS.incremental_source = None 1697 common.ZipClose(output_zip) 1698 1699 if not OPTIONS.no_signing: 1700 SignOutput(temp_zip_file.name, args[1]) 1701 temp_zip_file.close() 1702 1703 print "done." 1704 1705 1706if __name__ == '__main__': 1707 try: 1708 common.CloseInheritedPipes() 1709 main(sys.argv[1:]) 1710 except common.ExternalError as e: 1711 print 1712 print " ERROR: %s" % (e,) 1713 print 1714 sys.exit(1) 1715 finally: 1716 common.Cleanup() 1717