ota_from_target_files.py revision 177c610e7ffcbfe4dc0e178feb9d235f1f4be047
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 --board_config <file> 25 Deprecated. 26 27 -k (--package_key) <key> Key to use to sign the package (default is 28 the value of default_system_dev_certificate from the input 29 target-files's META/misc_info.txt, or 30 "build/target/product/security/testkey" if that value is not 31 specified). 32 33 For incremental OTAs, the default value is based on the source 34 target-file, not the target build. 35 36 -i (--incremental_from) <file> 37 Generate an incremental OTA using the given target-files zip as 38 the starting build. 39 40 --full_radio 41 When generating an incremental OTA, always include a full copy of 42 radio image. This option is only meaningful when -i is specified, 43 because a full radio is always included in a full OTA if applicable. 44 45 --full_bootloader 46 When generating an incremental OTA, always include a full copy of 47 bootloader image. This option is only meaningful when -i is specified, 48 because a full bootloader is always included in a full OTA if applicable. 49 50 -v (--verify) 51 Remount and verify the checksums of the files written to the 52 system and vendor (if used) partitions. Incremental builds only. 53 54 -o (--oem_settings) <file> 55 Use the file to specify the expected OEM-specific properties 56 on the OEM partition of the intended device. 57 58 -w (--wipe_user_data) 59 Generate an OTA package that will wipe the user data partition 60 when installed. 61 62 -n (--no_prereq) 63 Omit the timestamp prereq check normally included at the top of 64 the build scripts (used for developer OTA packages which 65 legitimately need to go back and forth). 66 67 --downgrade 68 Intentionally generate an incremental OTA that updates from a newer 69 build to an older one (based on timestamp comparison). "post-timestamp" 70 will be replaced by "ota-downgrade=yes" in the metadata file. A data 71 wipe will always be enforced, so "ota-wipe=yes" will also be included in 72 the metadata file. 73 74 -e (--extra_script) <file> 75 Insert the contents of file at the end of the update script. 76 77 -a (--aslr_mode) <on|off> 78 Specify whether to turn on ASLR for the package (on by default). 79 80 -2 (--two_step) 81 Generate a 'two-step' OTA package, where recovery is updated 82 first, so that any changes made to the system partition are done 83 using the new recovery (new kernel, etc.). 84 85 --block 86 Generate a block-based OTA if possible. Will fall back to a 87 file-based OTA if the target_files is older and doesn't support 88 block-based OTAs. 89 90 -b (--binary) <file> 91 Use the given binary as the update-binary in the output package, 92 instead of the binary in the build's target_files. Use for 93 development only. 94 95 -t (--worker_threads) <int> 96 Specifies the number of worker-threads that will be used when 97 generating patches for incremental updates (defaults to 3). 98 99 --stash_threshold <float> 100 Specifies the threshold that will be used to compute the maximum 101 allowed stash size (defaults to 0.8). 102""" 103 104import sys 105 106if sys.hexversion < 0x02070000: 107 print >> sys.stderr, "Python 2.7 or newer is required." 108 sys.exit(1) 109 110import multiprocessing 111import os 112import tempfile 113import zipfile 114 115import common 116import edify_generator 117import sparse_img 118 119OPTIONS = common.OPTIONS 120OPTIONS.package_key = None 121OPTIONS.incremental_source = None 122OPTIONS.verify = False 123OPTIONS.require_verbatim = set() 124OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 125OPTIONS.patch_threshold = 0.95 126OPTIONS.wipe_user_data = False 127OPTIONS.omit_prereq = False 128OPTIONS.downgrade = False 129OPTIONS.extra_script = None 130OPTIONS.aslr_mode = True 131OPTIONS.worker_threads = multiprocessing.cpu_count() // 2 132if OPTIONS.worker_threads == 0: 133 OPTIONS.worker_threads = 1 134OPTIONS.two_step = False 135OPTIONS.no_signing = False 136OPTIONS.block_based = False 137OPTIONS.updater_binary = None 138OPTIONS.oem_source = None 139OPTIONS.fallback_to_full = True 140OPTIONS.full_radio = False 141OPTIONS.full_bootloader = False 142# Stash size cannot exceed cache_size * threshold. 143OPTIONS.cache_size = None 144OPTIONS.stash_threshold = 0.8 145 146def MostPopularKey(d, default): 147 """Given a dict, return the key corresponding to the largest 148 value. Returns 'default' if the dict is empty.""" 149 x = [(v, k) for (k, v) in d.iteritems()] 150 if not x: 151 return default 152 x.sort() 153 return x[-1][1] 154 155 156def IsSymlink(info): 157 """Return true if the zipfile.ZipInfo object passed in represents a 158 symlink.""" 159 return (info.external_attr >> 16) == 0o120777 160 161def IsRegular(info): 162 """Return true if the zipfile.ZipInfo object passed in represents a 163 symlink.""" 164 return (info.external_attr >> 28) == 0o10 165 166def ClosestFileMatch(src, tgtfiles, existing): 167 """Returns the closest file match between a source file and list 168 of potential matches. The exact filename match is preferred, 169 then the sha1 is searched for, and finally a file with the same 170 basename is evaluated. Rename support in the updater-binary is 171 required for the latter checks to be used.""" 172 173 result = tgtfiles.get("path:" + src.name) 174 if result is not None: 175 return result 176 177 if not OPTIONS.target_info_dict.get("update_rename_support", False): 178 return None 179 180 if src.size < 1000: 181 return None 182 183 result = tgtfiles.get("sha1:" + src.sha1) 184 if result is not None and existing.get(result.name) is None: 185 return result 186 result = tgtfiles.get("file:" + src.name.split("/")[-1]) 187 if result is not None and existing.get(result.name) is None: 188 return result 189 return None 190 191class ItemSet(object): 192 def __init__(self, partition, fs_config): 193 self.partition = partition 194 self.fs_config = fs_config 195 self.ITEMS = {} 196 197 def Get(self, name, is_dir=False): 198 if name not in self.ITEMS: 199 self.ITEMS[name] = Item(self, name, is_dir=is_dir) 200 return self.ITEMS[name] 201 202 def GetMetadata(self, input_zip): 203 # The target_files contains a record of what the uid, 204 # gid, and mode are supposed to be. 205 output = input_zip.read(self.fs_config) 206 207 for line in output.split("\n"): 208 if not line: 209 continue 210 columns = line.split() 211 name, uid, gid, mode = columns[:4] 212 selabel = None 213 capabilities = None 214 215 # After the first 4 columns, there are a series of key=value 216 # pairs. Extract out the fields we care about. 217 for element in columns[4:]: 218 key, value = element.split("=") 219 if key == "selabel": 220 selabel = value 221 if key == "capabilities": 222 capabilities = value 223 224 i = self.ITEMS.get(name, None) 225 if i is not None: 226 i.uid = int(uid) 227 i.gid = int(gid) 228 i.mode = int(mode, 8) 229 i.selabel = selabel 230 i.capabilities = capabilities 231 if i.is_dir: 232 i.children.sort(key=lambda i: i.name) 233 234 # set metadata for the files generated by this script. 235 i = self.ITEMS.get("system/recovery-from-boot.p", None) 236 if i: 237 i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0o644, None, None 238 i = self.ITEMS.get("system/etc/install-recovery.sh", None) 239 if i: 240 i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0o544, None, None 241 242 243class Item(object): 244 """Items represent the metadata (user, group, mode) of files and 245 directories in the system image.""" 246 def __init__(self, itemset, name, is_dir=False): 247 self.itemset = itemset 248 self.name = name 249 self.uid = None 250 self.gid = None 251 self.mode = None 252 self.selabel = None 253 self.capabilities = None 254 self.is_dir = is_dir 255 self.descendants = None 256 self.best_subtree = None 257 258 if name: 259 self.parent = itemset.Get(os.path.dirname(name), is_dir=True) 260 self.parent.children.append(self) 261 else: 262 self.parent = None 263 if self.is_dir: 264 self.children = [] 265 266 def Dump(self, indent=0): 267 if self.uid is not None: 268 print "%s%s %d %d %o" % ( 269 " " * indent, self.name, self.uid, self.gid, self.mode) 270 else: 271 print "%s%s %s %s %s" % ( 272 " " * indent, self.name, self.uid, self.gid, self.mode) 273 if self.is_dir: 274 print "%s%s" % (" "*indent, self.descendants) 275 print "%s%s" % (" "*indent, self.best_subtree) 276 for i in self.children: 277 i.Dump(indent=indent+1) 278 279 def CountChildMetadata(self): 280 """Count up the (uid, gid, mode, selabel, capabilities) tuples for 281 all children and determine the best strategy for using set_perm_recursive 282 and set_perm to correctly chown/chmod all the files to their desired 283 values. Recursively calls itself for all descendants. 284 285 Returns a dict of {(uid, gid, dmode, fmode, selabel, capabilities): count} 286 counting up all descendants of this node. (dmode or fmode may be None.) 287 Also sets the best_subtree of each directory Item to the (uid, gid, dmode, 288 fmode, selabel, capabilities) tuple that will match the most descendants of 289 that Item. 290 """ 291 292 assert self.is_dir 293 key = (self.uid, self.gid, self.mode, None, self.selabel, 294 self.capabilities) 295 self.descendants = {key: 1} 296 d = self.descendants 297 for i in self.children: 298 if i.is_dir: 299 for k, v in i.CountChildMetadata().iteritems(): 300 d[k] = d.get(k, 0) + v 301 else: 302 k = (i.uid, i.gid, None, i.mode, i.selabel, i.capabilities) 303 d[k] = d.get(k, 0) + 1 304 305 # Find the (uid, gid, dmode, fmode, selabel, capabilities) 306 # tuple that matches the most descendants. 307 308 # First, find the (uid, gid) pair that matches the most 309 # descendants. 310 ug = {} 311 for (uid, gid, _, _, _, _), count in d.iteritems(): 312 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 313 ug = MostPopularKey(ug, (0, 0)) 314 315 # Now find the dmode, fmode, selabel, and capabilities that match 316 # the most descendants with that (uid, gid), and choose those. 317 best_dmode = (0, 0o755) 318 best_fmode = (0, 0o644) 319 best_selabel = (0, None) 320 best_capabilities = (0, None) 321 for k, count in d.iteritems(): 322 if k[:2] != ug: 323 continue 324 if k[2] is not None and count >= best_dmode[0]: 325 best_dmode = (count, k[2]) 326 if k[3] is not None and count >= best_fmode[0]: 327 best_fmode = (count, k[3]) 328 if k[4] is not None and count >= best_selabel[0]: 329 best_selabel = (count, k[4]) 330 if k[5] is not None and count >= best_capabilities[0]: 331 best_capabilities = (count, k[5]) 332 self.best_subtree = ug + ( 333 best_dmode[1], best_fmode[1], best_selabel[1], best_capabilities[1]) 334 335 return d 336 337 def SetPermissions(self, script): 338 """Append set_perm/set_perm_recursive commands to 'script' to 339 set all permissions, users, and groups for the tree of files 340 rooted at 'self'.""" 341 342 self.CountChildMetadata() 343 344 def recurse(item, current): 345 # current is the (uid, gid, dmode, fmode, selabel, capabilities) tuple 346 # that the current item (and all its children) have already been set to. 347 # We only need to issue set_perm/set_perm_recursive commands if we're 348 # supposed to be something different. 349 if item.is_dir: 350 if current != item.best_subtree: 351 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 352 current = item.best_subtree 353 354 if item.uid != current[0] or item.gid != current[1] or \ 355 item.mode != current[2] or item.selabel != current[4] or \ 356 item.capabilities != current[5]: 357 script.SetPermissions("/"+item.name, item.uid, item.gid, 358 item.mode, item.selabel, item.capabilities) 359 360 for i in item.children: 361 recurse(i, current) 362 else: 363 if item.uid != current[0] or item.gid != current[1] or \ 364 item.mode != current[3] or item.selabel != current[4] or \ 365 item.capabilities != current[5]: 366 script.SetPermissions("/"+item.name, item.uid, item.gid, 367 item.mode, item.selabel, item.capabilities) 368 369 recurse(self, (-1, -1, -1, -1, None, None)) 370 371 372def CopyPartitionFiles(itemset, input_zip, output_zip=None, substitute=None): 373 """Copies files for the partition in the input zip to the output 374 zip. Populates the Item class with their metadata, and returns a 375 list of symlinks. output_zip may be None, in which case the copy is 376 skipped (but the other side effects still happen). substitute is an 377 optional dict of {output filename: contents} to be output instead of 378 certain input files. 379 """ 380 381 symlinks = [] 382 383 partition = itemset.partition 384 385 for info in input_zip.infolist(): 386 prefix = partition.upper() + "/" 387 if info.filename.startswith(prefix): 388 basefilename = info.filename[len(prefix):] 389 if IsSymlink(info): 390 symlinks.append((input_zip.read(info.filename), 391 "/" + partition + "/" + basefilename)) 392 else: 393 import copy 394 info2 = copy.copy(info) 395 fn = info2.filename = partition + "/" + basefilename 396 if substitute and fn in substitute and substitute[fn] is None: 397 continue 398 if output_zip is not None: 399 if substitute and fn in substitute: 400 data = substitute[fn] 401 else: 402 data = input_zip.read(info.filename) 403 common.ZipWriteStr(output_zip, info2, data) 404 if fn.endswith("/"): 405 itemset.Get(fn[:-1], is_dir=True) 406 else: 407 itemset.Get(fn) 408 409 symlinks.sort() 410 return symlinks 411 412 413def SignOutput(temp_zip_name, output_zip_name): 414 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 415 pw = key_passwords[OPTIONS.package_key] 416 417 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 418 whole_file=True) 419 420 421def AppendAssertions(script, info_dict, oem_dict=None): 422 oem_props = info_dict.get("oem_fingerprint_properties") 423 if oem_props is None or len(oem_props) == 0: 424 device = GetBuildProp("ro.product.device", info_dict) 425 script.AssertDevice(device) 426 else: 427 if oem_dict is None: 428 raise common.ExternalError( 429 "No OEM file provided to answer expected assertions") 430 for prop in oem_props.split(): 431 if oem_dict.get(prop) is None: 432 raise common.ExternalError( 433 "The OEM file is missing the property %s" % prop) 434 script.AssertOemProperty(prop, oem_dict.get(prop)) 435 436 437def HasRecoveryPatch(target_files_zip): 438 try: 439 target_files_zip.getinfo("SYSTEM/recovery-from-boot.p") 440 return True 441 except KeyError: 442 return False 443 444def HasVendorPartition(target_files_zip): 445 try: 446 target_files_zip.getinfo("VENDOR/") 447 return True 448 except KeyError: 449 return False 450 451def GetOemProperty(name, oem_props, oem_dict, info_dict): 452 if oem_props is not None and name in oem_props: 453 return oem_dict[name] 454 return GetBuildProp(name, info_dict) 455 456 457def CalculateFingerprint(oem_props, oem_dict, info_dict): 458 if oem_props is None: 459 return GetBuildProp("ro.build.fingerprint", info_dict) 460 return "%s/%s/%s:%s" % ( 461 GetOemProperty("ro.product.brand", oem_props, oem_dict, info_dict), 462 GetOemProperty("ro.product.name", oem_props, oem_dict, info_dict), 463 GetOemProperty("ro.product.device", oem_props, oem_dict, info_dict), 464 GetBuildProp("ro.build.thumbprint", info_dict)) 465 466 467def GetImage(which, tmpdir, info_dict): 468 # Return an image object (suitable for passing to BlockImageDiff) 469 # for the 'which' partition (most be "system" or "vendor"). If a 470 # prebuilt image and file map are found in tmpdir they are used, 471 # otherwise they are reconstructed from the individual files. 472 473 assert which in ("system", "vendor") 474 475 path = os.path.join(tmpdir, "IMAGES", which + ".img") 476 mappath = os.path.join(tmpdir, "IMAGES", which + ".map") 477 if os.path.exists(path) and os.path.exists(mappath): 478 print "using %s.img from target-files" % (which,) 479 # This is a 'new' target-files, which already has the image in it. 480 481 else: 482 print "building %s.img from target-files" % (which,) 483 484 # This is an 'old' target-files, which does not contain images 485 # already built. Build them. 486 487 mappath = tempfile.mkstemp()[1] 488 OPTIONS.tempfiles.append(mappath) 489 490 import add_img_to_target_files 491 if which == "system": 492 path = add_img_to_target_files.BuildSystem( 493 tmpdir, info_dict, block_list=mappath) 494 elif which == "vendor": 495 path = add_img_to_target_files.BuildVendor( 496 tmpdir, info_dict, block_list=mappath) 497 498 # Bug: http://b/20939131 499 # In ext4 filesystems, block 0 might be changed even being mounted 500 # R/O. We add it to clobbered_blocks so that it will be written to the 501 # target unconditionally. Note that they are still part of care_map. 502 clobbered_blocks = "0" 503 504 return sparse_img.SparseImage(path, mappath, clobbered_blocks) 505 506 507def WriteFullOTAPackage(input_zip, output_zip): 508 # TODO: how to determine this? We don't know what version it will 509 # be installed on top of. For now, we expect the API just won't 510 # change very often. Similarly for fstab, it might have changed 511 # in the target build. 512 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 513 514 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 515 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 516 oem_dict = None 517 if oem_props is not None and len(oem_props) > 0: 518 if OPTIONS.oem_source is None: 519 raise common.ExternalError("OEM source required for this build") 520 script.Mount("/oem", recovery_mount_options) 521 oem_dict = common.LoadDictionaryFromLines( 522 open(OPTIONS.oem_source).readlines()) 523 524 metadata = { 525 "post-build": CalculateFingerprint(oem_props, oem_dict, 526 OPTIONS.info_dict), 527 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 528 OPTIONS.info_dict), 529 "post-timestamp": GetBuildProp("ro.build.date.utc", OPTIONS.info_dict), 530 } 531 532 device_specific = common.DeviceSpecificParams( 533 input_zip=input_zip, 534 input_version=OPTIONS.info_dict["recovery_api_version"], 535 output_zip=output_zip, 536 script=script, 537 input_tmp=OPTIONS.input_tmp, 538 metadata=metadata, 539 info_dict=OPTIONS.info_dict) 540 541 has_recovery_patch = HasRecoveryPatch(input_zip) 542 block_based = OPTIONS.block_based and has_recovery_patch 543 544 if not OPTIONS.omit_prereq: 545 ts = GetBuildProp("ro.build.date.utc", OPTIONS.info_dict) 546 ts_text = GetBuildProp("ro.build.date", OPTIONS.info_dict) 547 script.AssertOlderBuild(ts, ts_text) 548 549 AppendAssertions(script, OPTIONS.info_dict, oem_dict) 550 device_specific.FullOTA_Assertions() 551 552 # Two-step package strategy (in chronological order, which is *not* 553 # the order in which the generated script has things): 554 # 555 # if stage is not "2/3" or "3/3": 556 # write recovery image to boot partition 557 # set stage to "2/3" 558 # reboot to boot partition and restart recovery 559 # else if stage is "2/3": 560 # write recovery image to recovery partition 561 # set stage to "3/3" 562 # reboot to recovery partition and restart recovery 563 # else: 564 # (stage must be "3/3") 565 # set stage to "" 566 # do normal full package installation: 567 # wipe and install system, boot image, etc. 568 # set up system to update recovery partition on first boot 569 # complete script normally 570 # (allow recovery to mark itself finished and reboot) 571 572 recovery_img = common.GetBootableImage("recovery.img", "recovery.img", 573 OPTIONS.input_tmp, "RECOVERY") 574 if OPTIONS.two_step: 575 if not OPTIONS.info_dict.get("multistage_support", None): 576 assert False, "two-step packages not supported by this build" 577 fs = OPTIONS.info_dict["fstab"]["/misc"] 578 assert fs.fs_type.upper() == "EMMC", \ 579 "two-step packages only supported on devices with EMMC /misc partitions" 580 bcb_dev = {"bcb_dev": fs.device} 581 common.ZipWriteStr(output_zip, "recovery.img", recovery_img.data) 582 script.AppendExtra(""" 583if get_stage("%(bcb_dev)s") == "2/3" then 584""" % bcb_dev) 585 script.WriteRawImage("/recovery", "recovery.img") 586 script.AppendExtra(""" 587set_stage("%(bcb_dev)s", "3/3"); 588reboot_now("%(bcb_dev)s", "recovery"); 589else if get_stage("%(bcb_dev)s") == "3/3" then 590""" % bcb_dev) 591 592 # Dump fingerprints 593 script.Print("Target: %s" % CalculateFingerprint( 594 oem_props, oem_dict, OPTIONS.info_dict)) 595 596 device_specific.FullOTA_InstallBegin() 597 598 system_progress = 0.75 599 600 if OPTIONS.wipe_user_data: 601 system_progress -= 0.1 602 if HasVendorPartition(input_zip): 603 system_progress -= 0.1 604 605 if "selinux_fc" in OPTIONS.info_dict: 606 WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip) 607 608 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 609 610 system_items = ItemSet("system", "META/filesystem_config.txt") 611 script.ShowProgress(system_progress, 0) 612 613 if block_based: 614 # Full OTA is done as an "incremental" against an empty source 615 # image. This has the effect of writing new data from the package 616 # to the entire partition, but lets us reuse the updater code that 617 # writes incrementals to do it. 618 system_tgt = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict) 619 system_tgt.ResetFileMap() 620 system_diff = common.BlockDifference("system", system_tgt, src=None) 621 system_diff.WriteScript(script, output_zip) 622 else: 623 script.FormatPartition("/system") 624 script.Mount("/system", recovery_mount_options) 625 if not has_recovery_patch: 626 script.UnpackPackageDir("recovery", "/system") 627 script.UnpackPackageDir("system", "/system") 628 629 symlinks = CopyPartitionFiles(system_items, input_zip, output_zip) 630 script.MakeSymlinks(symlinks) 631 632 boot_img = common.GetBootableImage("boot.img", "boot.img", 633 OPTIONS.input_tmp, "BOOT") 634 635 if not block_based: 636 def output_sink(fn, data): 637 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 638 system_items.Get("system/" + fn) 639 640 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, 641 recovery_img, boot_img) 642 643 system_items.GetMetadata(input_zip) 644 system_items.Get("system").SetPermissions(script) 645 646 if HasVendorPartition(input_zip): 647 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 648 script.ShowProgress(0.1, 0) 649 650 if block_based: 651 vendor_tgt = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict) 652 vendor_tgt.ResetFileMap() 653 vendor_diff = common.BlockDifference("vendor", vendor_tgt) 654 vendor_diff.WriteScript(script, output_zip) 655 else: 656 script.FormatPartition("/vendor") 657 script.Mount("/vendor", recovery_mount_options) 658 script.UnpackPackageDir("vendor", "/vendor") 659 660 symlinks = CopyPartitionFiles(vendor_items, input_zip, output_zip) 661 script.MakeSymlinks(symlinks) 662 663 vendor_items.GetMetadata(input_zip) 664 vendor_items.Get("vendor").SetPermissions(script) 665 666 common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict) 667 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 668 669 script.ShowProgress(0.05, 5) 670 script.WriteRawImage("/boot", "boot.img") 671 672 script.ShowProgress(0.2, 10) 673 device_specific.FullOTA_InstallEnd() 674 675 if OPTIONS.extra_script is not None: 676 script.AppendExtra(OPTIONS.extra_script) 677 678 script.UnmountAll() 679 680 if OPTIONS.wipe_user_data: 681 script.ShowProgress(0.1, 10) 682 script.FormatPartition("/data") 683 684 if OPTIONS.two_step: 685 script.AppendExtra(""" 686set_stage("%(bcb_dev)s", ""); 687""" % bcb_dev) 688 script.AppendExtra("else\n") 689 script.WriteRawImage("/boot", "recovery.img") 690 script.AppendExtra(""" 691set_stage("%(bcb_dev)s", "2/3"); 692reboot_now("%(bcb_dev)s", ""); 693endif; 694endif; 695""" % bcb_dev) 696 697 script.SetProgress(1) 698 script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary) 699 WriteMetadata(metadata, output_zip) 700 701 702def WritePolicyConfig(file_name, output_zip): 703 common.ZipWrite(output_zip, file_name, os.path.basename(file_name)) 704 705 706def WriteMetadata(metadata, output_zip): 707 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata", 708 "".join(["%s=%s\n" % kv 709 for kv in sorted(metadata.iteritems())])) 710 711 712def LoadPartitionFiles(z, partition): 713 """Load all the files from the given partition in a given target-files 714 ZipFile, and return a dict of {filename: File object}.""" 715 out = {} 716 prefix = partition.upper() + "/" 717 for info in z.infolist(): 718 if info.filename.startswith(prefix) and not IsSymlink(info): 719 basefilename = info.filename[len(prefix):] 720 fn = partition + "/" + basefilename 721 data = z.read(info.filename) 722 out[fn] = common.File(fn, data) 723 return out 724 725 726def GetBuildProp(prop, info_dict): 727 """Return the fingerprint of the build of a given target-files info_dict.""" 728 try: 729 return info_dict.get("build.prop", {})[prop] 730 except KeyError: 731 raise common.ExternalError("couldn't find %s in build.prop" % (prop,)) 732 733 734def AddToKnownPaths(filename, known_paths): 735 if filename[-1] == "/": 736 return 737 dirs = filename.split("/")[:-1] 738 while len(dirs) > 0: 739 path = "/".join(dirs) 740 if path in known_paths: 741 break 742 known_paths.add(path) 743 dirs.pop() 744 745 746def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 747 source_version = OPTIONS.source_info_dict["recovery_api_version"] 748 target_version = OPTIONS.target_info_dict["recovery_api_version"] 749 750 if source_version == 0: 751 print ("WARNING: generating edify script for a source that " 752 "can't install it.") 753 script = edify_generator.EdifyGenerator( 754 source_version, OPTIONS.target_info_dict, 755 fstab=OPTIONS.source_info_dict["fstab"]) 756 757 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 758 recovery_mount_options = OPTIONS.source_info_dict.get( 759 "recovery_mount_options") 760 oem_dict = None 761 if oem_props is not None and len(oem_props) > 0: 762 if OPTIONS.oem_source is None: 763 raise common.ExternalError("OEM source required for this build") 764 script.Mount("/oem", recovery_mount_options) 765 oem_dict = common.LoadDictionaryFromLines( 766 open(OPTIONS.oem_source).readlines()) 767 768 metadata = { 769 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 770 OPTIONS.source_info_dict), 771 "ota-type": "BLOCK", 772 } 773 774 post_timestamp = GetBuildProp("ro.build.date.utc", OPTIONS.target_info_dict) 775 pre_timestamp = GetBuildProp("ro.build.date.utc", OPTIONS.source_info_dict) 776 is_downgrade = long(post_timestamp) < long(pre_timestamp) 777 778 if OPTIONS.downgrade: 779 metadata["ota-downgrade"] = "yes" 780 if not is_downgrade: 781 raise RuntimeError("--downgrade specified but no downgrade detected: " 782 "pre: %s, post: %s" % (pre_timestamp, post_timestamp)) 783 else: 784 if is_downgrade: 785 # Non-fatal here to allow generating such a package which may require 786 # manual work to adjust the post-timestamp. A legit use case is that we 787 # cut a new build C (after having A and B), but want to enfore the 788 # update path of A -> C -> B. Specifying --downgrade may not help since 789 # that would enforce a data wipe for C -> B update. 790 print("\nWARNING: downgrade detected: pre: %s, post: %s.\n" 791 "The package may not be deployed properly. " 792 "Try --downgrade?\n" % (pre_timestamp, post_timestamp)) 793 metadata["post-timestamp"] = post_timestamp 794 795 device_specific = common.DeviceSpecificParams( 796 source_zip=source_zip, 797 source_version=source_version, 798 target_zip=target_zip, 799 target_version=target_version, 800 output_zip=output_zip, 801 script=script, 802 metadata=metadata, 803 info_dict=OPTIONS.source_info_dict) 804 805 target_fp = CalculateFingerprint(oem_props, oem_dict, 806 OPTIONS.target_info_dict) 807 source_fp = CalculateFingerprint(oem_props, oem_dict, 808 OPTIONS.source_info_dict) 809 metadata["pre-build"] = source_fp 810 metadata["post-build"] = target_fp 811 812 source_boot = common.GetBootableImage( 813 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 814 OPTIONS.source_info_dict) 815 target_boot = common.GetBootableImage( 816 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 817 updating_boot = (not OPTIONS.two_step and 818 (source_boot.data != target_boot.data)) 819 820 target_recovery = common.GetBootableImage( 821 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 822 823 system_src = GetImage("system", OPTIONS.source_tmp, OPTIONS.source_info_dict) 824 system_tgt = GetImage("system", OPTIONS.target_tmp, OPTIONS.target_info_dict) 825 826 blockimgdiff_version = 1 827 if OPTIONS.info_dict: 828 blockimgdiff_version = max( 829 int(i) for i in 830 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(",")) 831 832 system_diff = common.BlockDifference("system", system_tgt, system_src, 833 version=blockimgdiff_version) 834 835 if HasVendorPartition(target_zip): 836 if not HasVendorPartition(source_zip): 837 raise RuntimeError("can't generate incremental that adds /vendor") 838 vendor_src = GetImage("vendor", OPTIONS.source_tmp, 839 OPTIONS.source_info_dict) 840 vendor_tgt = GetImage("vendor", OPTIONS.target_tmp, 841 OPTIONS.target_info_dict) 842 vendor_diff = common.BlockDifference("vendor", vendor_tgt, vendor_src, 843 version=blockimgdiff_version) 844 else: 845 vendor_diff = None 846 847 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 848 device_specific.IncrementalOTA_Assertions() 849 850 # Two-step incremental package strategy (in chronological order, 851 # which is *not* the order in which the generated script has 852 # things): 853 # 854 # if stage is not "2/3" or "3/3": 855 # do verification on current system 856 # write recovery image to boot partition 857 # set stage to "2/3" 858 # reboot to boot partition and restart recovery 859 # else if stage is "2/3": 860 # write recovery image to recovery partition 861 # set stage to "3/3" 862 # reboot to recovery partition and restart recovery 863 # else: 864 # (stage must be "3/3") 865 # perform update: 866 # patch system files, etc. 867 # force full install of new boot image 868 # set up system to update recovery partition on first boot 869 # complete script normally 870 # (allow recovery to mark itself finished and reboot) 871 872 if OPTIONS.two_step: 873 if not OPTIONS.source_info_dict.get("multistage_support", None): 874 assert False, "two-step packages not supported by this build" 875 fs = OPTIONS.source_info_dict["fstab"]["/misc"] 876 assert fs.fs_type.upper() == "EMMC", \ 877 "two-step packages only supported on devices with EMMC /misc partitions" 878 bcb_dev = {"bcb_dev": fs.device} 879 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 880 script.AppendExtra(""" 881if get_stage("%(bcb_dev)s") == "2/3" then 882""" % bcb_dev) 883 script.AppendExtra("sleep(20);\n") 884 script.WriteRawImage("/recovery", "recovery.img") 885 script.AppendExtra(""" 886set_stage("%(bcb_dev)s", "3/3"); 887reboot_now("%(bcb_dev)s", "recovery"); 888else if get_stage("%(bcb_dev)s") != "3/3" then 889""" % bcb_dev) 890 891 # Dump fingerprints 892 script.Print("Source: %s" % CalculateFingerprint( 893 oem_props, oem_dict, OPTIONS.source_info_dict)) 894 script.Print("Target: %s" % CalculateFingerprint( 895 oem_props, oem_dict, OPTIONS.target_info_dict)) 896 897 script.Print("Verifying current system...") 898 899 device_specific.IncrementalOTA_VerifyBegin() 900 901 if oem_props is None: 902 # When blockimgdiff version is less than 3 (non-resumable block-based OTA), 903 # patching on a device that's already on the target build will damage the 904 # system. Because operations like move don't check the block state, they 905 # always apply the changes unconditionally. 906 if blockimgdiff_version <= 2: 907 script.AssertSomeFingerprint(source_fp) 908 else: 909 script.AssertSomeFingerprint(source_fp, target_fp) 910 else: 911 if blockimgdiff_version <= 2: 912 script.AssertSomeThumbprint( 913 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 914 else: 915 script.AssertSomeThumbprint( 916 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 917 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 918 919 if updating_boot: 920 boot_type, boot_device = common.GetTypeAndDevice( 921 "/boot", OPTIONS.source_info_dict) 922 d = common.Difference(target_boot, source_boot) 923 _, _, d = d.ComputePatch() 924 if d is None: 925 include_full_boot = True 926 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 927 else: 928 include_full_boot = False 929 930 print "boot target: %d source: %d diff: %d" % ( 931 target_boot.size, source_boot.size, len(d)) 932 933 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 934 935 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 936 (boot_type, boot_device, 937 source_boot.size, source_boot.sha1, 938 target_boot.size, target_boot.sha1)) 939 940 device_specific.IncrementalOTA_VerifyEnd() 941 942 if OPTIONS.two_step: 943 script.WriteRawImage("/boot", "recovery.img") 944 script.AppendExtra(""" 945set_stage("%(bcb_dev)s", "2/3"); 946reboot_now("%(bcb_dev)s", ""); 947else 948""" % bcb_dev) 949 950 # Verify the existing partitions. 951 system_diff.WriteVerifyScript(script) 952 if vendor_diff: 953 vendor_diff.WriteVerifyScript(script) 954 955 script.Comment("---- start making changes here ----") 956 957 device_specific.IncrementalOTA_InstallBegin() 958 959 system_diff.WriteScript(script, output_zip, 960 progress=0.8 if vendor_diff else 0.9) 961 if vendor_diff: 962 vendor_diff.WriteScript(script, output_zip, progress=0.1) 963 964 if OPTIONS.two_step: 965 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 966 script.WriteRawImage("/boot", "boot.img") 967 print "writing full boot image (forced by two-step mode)" 968 969 if not OPTIONS.two_step: 970 if updating_boot: 971 if include_full_boot: 972 print "boot image changed; including full." 973 script.Print("Installing boot image...") 974 script.WriteRawImage("/boot", "boot.img") 975 else: 976 # Produce the boot image by applying a patch to the current 977 # contents of the boot partition, and write it back to the 978 # partition. 979 print "boot image changed; including patch." 980 script.Print("Patching boot image...") 981 script.ShowProgress(0.1, 10) 982 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 983 % (boot_type, boot_device, 984 source_boot.size, source_boot.sha1, 985 target_boot.size, target_boot.sha1), 986 "-", 987 target_boot.size, target_boot.sha1, 988 source_boot.sha1, "patch/boot.img.p") 989 else: 990 print "boot image unchanged; skipping." 991 992 # Do device-specific installation (eg, write radio image). 993 device_specific.IncrementalOTA_InstallEnd() 994 995 if OPTIONS.extra_script is not None: 996 script.AppendExtra(OPTIONS.extra_script) 997 998 if OPTIONS.wipe_user_data: 999 script.Print("Erasing user data...") 1000 script.FormatPartition("/data") 1001 metadata["ota-wipe"] = "yes" 1002 1003 if OPTIONS.two_step: 1004 script.AppendExtra(""" 1005set_stage("%(bcb_dev)s", ""); 1006endif; 1007endif; 1008""" % bcb_dev) 1009 1010 script.SetProgress(1) 1011 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 1012 WriteMetadata(metadata, output_zip) 1013 1014 1015class FileDifference(object): 1016 def __init__(self, partition, source_zip, target_zip, output_zip): 1017 self.deferred_patch_list = None 1018 print "Loading target..." 1019 self.target_data = target_data = LoadPartitionFiles(target_zip, partition) 1020 print "Loading source..." 1021 self.source_data = source_data = LoadPartitionFiles(source_zip, partition) 1022 1023 self.verbatim_targets = verbatim_targets = [] 1024 self.patch_list = patch_list = [] 1025 diffs = [] 1026 self.renames = renames = {} 1027 known_paths = set() 1028 largest_source_size = 0 1029 1030 matching_file_cache = {} 1031 for fn, sf in source_data.items(): 1032 assert fn == sf.name 1033 matching_file_cache["path:" + fn] = sf 1034 if fn in target_data.keys(): 1035 AddToKnownPaths(fn, known_paths) 1036 # Only allow eligibility for filename/sha matching 1037 # if there isn't a perfect path match. 1038 if target_data.get(sf.name) is None: 1039 matching_file_cache["file:" + fn.split("/")[-1]] = sf 1040 matching_file_cache["sha:" + sf.sha1] = sf 1041 1042 for fn in sorted(target_data.keys()): 1043 tf = target_data[fn] 1044 assert fn == tf.name 1045 sf = ClosestFileMatch(tf, matching_file_cache, renames) 1046 if sf is not None and sf.name != tf.name: 1047 print "File has moved from " + sf.name + " to " + tf.name 1048 renames[sf.name] = tf 1049 1050 if sf is None or fn in OPTIONS.require_verbatim: 1051 # This file should be included verbatim 1052 if fn in OPTIONS.prohibit_verbatim: 1053 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 1054 print "send", fn, "verbatim" 1055 tf.AddToZip(output_zip) 1056 verbatim_targets.append((fn, tf.size, tf.sha1)) 1057 if fn in target_data.keys(): 1058 AddToKnownPaths(fn, known_paths) 1059 elif tf.sha1 != sf.sha1: 1060 # File is different; consider sending as a patch 1061 diffs.append(common.Difference(tf, sf)) 1062 else: 1063 # Target file data identical to source (may still be renamed) 1064 pass 1065 1066 common.ComputeDifferences(diffs) 1067 1068 for diff in diffs: 1069 tf, sf, d = diff.GetPatch() 1070 path = "/".join(tf.name.split("/")[:-1]) 1071 if d is None or len(d) > tf.size * OPTIONS.patch_threshold or \ 1072 path not in known_paths: 1073 # patch is almost as big as the file; don't bother patching 1074 # or a patch + rename cannot take place due to the target 1075 # directory not existing 1076 tf.AddToZip(output_zip) 1077 verbatim_targets.append((tf.name, tf.size, tf.sha1)) 1078 if sf.name in renames: 1079 del renames[sf.name] 1080 AddToKnownPaths(tf.name, known_paths) 1081 else: 1082 common.ZipWriteStr(output_zip, "patch/" + sf.name + ".p", d) 1083 patch_list.append((tf, sf, tf.size, common.sha1(d).hexdigest())) 1084 largest_source_size = max(largest_source_size, sf.size) 1085 1086 self.largest_source_size = largest_source_size 1087 1088 def EmitVerification(self, script): 1089 so_far = 0 1090 for tf, sf, _, _ in self.patch_list: 1091 if tf.name != sf.name: 1092 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1093 script.PatchCheck("/"+sf.name, tf.sha1, sf.sha1) 1094 so_far += sf.size 1095 return so_far 1096 1097 def EmitExplicitTargetVerification(self, script): 1098 for fn, _, sha1 in self.verbatim_targets: 1099 if fn[-1] != "/": 1100 script.FileCheck("/"+fn, sha1) 1101 for tf, _, _, _ in self.patch_list: 1102 script.FileCheck(tf.name, tf.sha1) 1103 1104 def RemoveUnneededFiles(self, script, extras=()): 1105 script.DeleteFiles( 1106 ["/" + i[0] for i in self.verbatim_targets] + 1107 ["/" + i for i in sorted(self.source_data) 1108 if i not in self.target_data and i not in self.renames] + 1109 list(extras)) 1110 1111 def TotalPatchSize(self): 1112 return sum(i[1].size for i in self.patch_list) 1113 1114 def EmitPatches(self, script, total_patch_size, so_far): 1115 self.deferred_patch_list = deferred_patch_list = [] 1116 for item in self.patch_list: 1117 tf, sf, _, _ = item 1118 if tf.name == "system/build.prop": 1119 deferred_patch_list.append(item) 1120 continue 1121 if sf.name != tf.name: 1122 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1123 script.ApplyPatch("/" + sf.name, "-", tf.size, tf.sha1, sf.sha1, 1124 "patch/" + sf.name + ".p") 1125 so_far += tf.size 1126 script.SetProgress(so_far / total_patch_size) 1127 return so_far 1128 1129 def EmitDeferredPatches(self, script): 1130 for item in self.deferred_patch_list: 1131 tf, sf, _, _ = item 1132 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, 1133 "patch/" + sf.name + ".p") 1134 script.SetPermissions("/system/build.prop", 0, 0, 0o644, None, None) 1135 1136 def EmitRenames(self, script): 1137 if len(self.renames) > 0: 1138 script.Print("Renaming files...") 1139 for src, tgt in self.renames.iteritems(): 1140 print "Renaming " + src + " to " + tgt.name 1141 script.RenameFile(src, tgt.name) 1142 1143 1144def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): 1145 target_has_recovery_patch = HasRecoveryPatch(target_zip) 1146 source_has_recovery_patch = HasRecoveryPatch(source_zip) 1147 1148 if (OPTIONS.block_based and 1149 target_has_recovery_patch and 1150 source_has_recovery_patch): 1151 return WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip) 1152 1153 source_version = OPTIONS.source_info_dict["recovery_api_version"] 1154 target_version = OPTIONS.target_info_dict["recovery_api_version"] 1155 1156 if source_version == 0: 1157 print ("WARNING: generating edify script for a source that " 1158 "can't install it.") 1159 script = edify_generator.EdifyGenerator( 1160 source_version, OPTIONS.target_info_dict, 1161 fstab=OPTIONS.source_info_dict["fstab"]) 1162 1163 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 1164 recovery_mount_options = OPTIONS.source_info_dict.get( 1165 "recovery_mount_options") 1166 oem_dict = None 1167 if oem_props is not None and len(oem_props) > 0: 1168 if OPTIONS.oem_source is None: 1169 raise common.ExternalError("OEM source required for this build") 1170 script.Mount("/oem", recovery_mount_options) 1171 oem_dict = common.LoadDictionaryFromLines( 1172 open(OPTIONS.oem_source).readlines()) 1173 1174 metadata = { 1175 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 1176 OPTIONS.source_info_dict), 1177 "ota-type": "FILE", 1178 } 1179 1180 post_timestamp = GetBuildProp("ro.build.date.utc", OPTIONS.target_info_dict) 1181 pre_timestamp = GetBuildProp("ro.build.date.utc", OPTIONS.source_info_dict) 1182 is_downgrade = long(post_timestamp) < long(pre_timestamp) 1183 1184 if OPTIONS.downgrade: 1185 metadata["ota-downgrade"] = "yes" 1186 if not is_downgrade: 1187 raise RuntimeError("--downgrade specified but no downgrade detected: " 1188 "pre: %s, post: %s" % (pre_timestamp, post_timestamp)) 1189 else: 1190 if is_downgrade: 1191 # Non-fatal here to allow generating such a package which may require 1192 # manual work to adjust the post-timestamp. A legit use case is that we 1193 # cut a new build C (after having A and B), but want to enfore the 1194 # update path of A -> C -> B. Specifying --downgrade may not help since 1195 # that would enforce a data wipe for C -> B update. 1196 print("\nWARNING: downgrade detected: pre: %s, post: %s.\n" 1197 "The package may not be deployed properly. " 1198 "Try --downgrade?\n" % (pre_timestamp, post_timestamp)) 1199 metadata["post-timestamp"] = post_timestamp 1200 1201 device_specific = common.DeviceSpecificParams( 1202 source_zip=source_zip, 1203 source_version=source_version, 1204 target_zip=target_zip, 1205 target_version=target_version, 1206 output_zip=output_zip, 1207 script=script, 1208 metadata=metadata, 1209 info_dict=OPTIONS.source_info_dict) 1210 1211 system_diff = FileDifference("system", source_zip, target_zip, output_zip) 1212 script.Mount("/system", recovery_mount_options) 1213 if HasVendorPartition(target_zip): 1214 vendor_diff = FileDifference("vendor", source_zip, target_zip, output_zip) 1215 script.Mount("/vendor", recovery_mount_options) 1216 else: 1217 vendor_diff = None 1218 1219 target_fp = CalculateFingerprint(oem_props, oem_dict, 1220 OPTIONS.target_info_dict) 1221 source_fp = CalculateFingerprint(oem_props, oem_dict, 1222 OPTIONS.source_info_dict) 1223 1224 if oem_props is None: 1225 script.AssertSomeFingerprint(source_fp, target_fp) 1226 else: 1227 script.AssertSomeThumbprint( 1228 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 1229 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 1230 1231 metadata["pre-build"] = source_fp 1232 metadata["post-build"] = target_fp 1233 1234 source_boot = common.GetBootableImage( 1235 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 1236 OPTIONS.source_info_dict) 1237 target_boot = common.GetBootableImage( 1238 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 1239 updating_boot = (not OPTIONS.two_step and 1240 (source_boot.data != target_boot.data)) 1241 1242 source_recovery = common.GetBootableImage( 1243 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 1244 OPTIONS.source_info_dict) 1245 target_recovery = common.GetBootableImage( 1246 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 1247 updating_recovery = (source_recovery.data != target_recovery.data) 1248 1249 # Here's how we divide up the progress bar: 1250 # 0.1 for verifying the start state (PatchCheck calls) 1251 # 0.8 for applying patches (ApplyPatch calls) 1252 # 0.1 for unpacking verbatim files, symlinking, and doing the 1253 # device-specific commands. 1254 1255 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 1256 device_specific.IncrementalOTA_Assertions() 1257 1258 # Two-step incremental package strategy (in chronological order, 1259 # which is *not* the order in which the generated script has 1260 # things): 1261 # 1262 # if stage is not "2/3" or "3/3": 1263 # do verification on current system 1264 # write recovery image to boot partition 1265 # set stage to "2/3" 1266 # reboot to boot partition and restart recovery 1267 # else if stage is "2/3": 1268 # write recovery image to recovery partition 1269 # set stage to "3/3" 1270 # reboot to recovery partition and restart recovery 1271 # else: 1272 # (stage must be "3/3") 1273 # perform update: 1274 # patch system files, etc. 1275 # force full install of new boot image 1276 # set up system to update recovery partition on first boot 1277 # complete script normally 1278 # (allow recovery to mark itself finished and reboot) 1279 1280 if OPTIONS.two_step: 1281 if not OPTIONS.source_info_dict.get("multistage_support", None): 1282 assert False, "two-step packages not supported by this build" 1283 fs = OPTIONS.source_info_dict["fstab"]["/misc"] 1284 assert fs.fs_type.upper() == "EMMC", \ 1285 "two-step packages only supported on devices with EMMC /misc partitions" 1286 bcb_dev = {"bcb_dev": fs.device} 1287 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 1288 script.AppendExtra(""" 1289if get_stage("%(bcb_dev)s") == "2/3" then 1290""" % bcb_dev) 1291 script.AppendExtra("sleep(20);\n") 1292 script.WriteRawImage("/recovery", "recovery.img") 1293 script.AppendExtra(""" 1294set_stage("%(bcb_dev)s", "3/3"); 1295reboot_now("%(bcb_dev)s", "recovery"); 1296else if get_stage("%(bcb_dev)s") != "3/3" then 1297""" % bcb_dev) 1298 1299 # Dump fingerprints 1300 script.Print("Source: %s" % (source_fp,)) 1301 script.Print("Target: %s" % (target_fp,)) 1302 1303 script.Print("Verifying current system...") 1304 1305 device_specific.IncrementalOTA_VerifyBegin() 1306 1307 script.ShowProgress(0.1, 0) 1308 so_far = system_diff.EmitVerification(script) 1309 if vendor_diff: 1310 so_far += vendor_diff.EmitVerification(script) 1311 1312 if updating_boot: 1313 d = common.Difference(target_boot, source_boot) 1314 _, _, d = d.ComputePatch() 1315 print "boot target: %d source: %d diff: %d" % ( 1316 target_boot.size, source_boot.size, len(d)) 1317 1318 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 1319 1320 boot_type, boot_device = common.GetTypeAndDevice( 1321 "/boot", OPTIONS.source_info_dict) 1322 1323 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 1324 (boot_type, boot_device, 1325 source_boot.size, source_boot.sha1, 1326 target_boot.size, target_boot.sha1)) 1327 so_far += source_boot.size 1328 1329 size = [] 1330 if system_diff.patch_list: 1331 size.append(system_diff.largest_source_size) 1332 if vendor_diff: 1333 if vendor_diff.patch_list: 1334 size.append(vendor_diff.largest_source_size) 1335 if size or updating_recovery or updating_boot: 1336 script.CacheFreeSpaceCheck(max(size)) 1337 1338 device_specific.IncrementalOTA_VerifyEnd() 1339 1340 if OPTIONS.two_step: 1341 script.WriteRawImage("/boot", "recovery.img") 1342 script.AppendExtra(""" 1343set_stage("%(bcb_dev)s", "2/3"); 1344reboot_now("%(bcb_dev)s", ""); 1345else 1346""" % bcb_dev) 1347 1348 script.Comment("---- start making changes here ----") 1349 1350 device_specific.IncrementalOTA_InstallBegin() 1351 1352 if OPTIONS.two_step: 1353 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 1354 script.WriteRawImage("/boot", "boot.img") 1355 print "writing full boot image (forced by two-step mode)" 1356 1357 script.Print("Removing unneeded files...") 1358 system_diff.RemoveUnneededFiles(script, ("/system/recovery.img",)) 1359 if vendor_diff: 1360 vendor_diff.RemoveUnneededFiles(script) 1361 1362 script.ShowProgress(0.8, 0) 1363 total_patch_size = 1.0 + system_diff.TotalPatchSize() 1364 if vendor_diff: 1365 total_patch_size += vendor_diff.TotalPatchSize() 1366 if updating_boot: 1367 total_patch_size += target_boot.size 1368 1369 script.Print("Patching system files...") 1370 so_far = system_diff.EmitPatches(script, total_patch_size, 0) 1371 if vendor_diff: 1372 script.Print("Patching vendor files...") 1373 so_far = vendor_diff.EmitPatches(script, total_patch_size, so_far) 1374 1375 if not OPTIONS.two_step: 1376 if updating_boot: 1377 # Produce the boot image by applying a patch to the current 1378 # contents of the boot partition, and write it back to the 1379 # partition. 1380 script.Print("Patching boot image...") 1381 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 1382 % (boot_type, boot_device, 1383 source_boot.size, source_boot.sha1, 1384 target_boot.size, target_boot.sha1), 1385 "-", 1386 target_boot.size, target_boot.sha1, 1387 source_boot.sha1, "patch/boot.img.p") 1388 so_far += target_boot.size 1389 script.SetProgress(so_far / total_patch_size) 1390 print "boot image changed; including." 1391 else: 1392 print "boot image unchanged; skipping." 1393 1394 system_items = ItemSet("system", "META/filesystem_config.txt") 1395 if vendor_diff: 1396 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 1397 1398 if updating_recovery: 1399 # Recovery is generated as a patch using both the boot image 1400 # (which contains the same linux kernel as recovery) and the file 1401 # /system/etc/recovery-resource.dat (which contains all the images 1402 # used in the recovery UI) as sources. This lets us minimize the 1403 # size of the patch, which must be included in every OTA package. 1404 # 1405 # For older builds where recovery-resource.dat is not present, we 1406 # use only the boot image as the source. 1407 1408 if not target_has_recovery_patch: 1409 def output_sink(fn, data): 1410 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 1411 system_items.Get("system/" + fn) 1412 1413 common.MakeRecoveryPatch(OPTIONS.target_tmp, output_sink, 1414 target_recovery, target_boot) 1415 script.DeleteFiles(["/system/recovery-from-boot.p", 1416 "/system/etc/install-recovery.sh"]) 1417 print "recovery image changed; including as patch from boot." 1418 else: 1419 print "recovery image unchanged; skipping." 1420 1421 script.ShowProgress(0.1, 10) 1422 1423 target_symlinks = CopyPartitionFiles(system_items, target_zip, None) 1424 if vendor_diff: 1425 target_symlinks.extend(CopyPartitionFiles(vendor_items, target_zip, None)) 1426 1427 temp_script = script.MakeTemporary() 1428 system_items.GetMetadata(target_zip) 1429 system_items.Get("system").SetPermissions(temp_script) 1430 if vendor_diff: 1431 vendor_items.GetMetadata(target_zip) 1432 vendor_items.Get("vendor").SetPermissions(temp_script) 1433 1434 # Note that this call will mess up the trees of Items, so make sure 1435 # we're done with them. 1436 source_symlinks = CopyPartitionFiles(system_items, source_zip, None) 1437 if vendor_diff: 1438 source_symlinks.extend(CopyPartitionFiles(vendor_items, source_zip, None)) 1439 1440 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 1441 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 1442 1443 # Delete all the symlinks in source that aren't in target. This 1444 # needs to happen before verbatim files are unpacked, in case a 1445 # symlink in the source is replaced by a real file in the target. 1446 1447 # If a symlink in the source will be replaced by a regular file, we cannot 1448 # delete the symlink/file in case the package gets applied again. For such 1449 # a symlink, we prepend a sha1_check() to detect if it has been updated. 1450 # (Bug: 23646151) 1451 replaced_symlinks = dict() 1452 if system_diff: 1453 for i in system_diff.verbatim_targets: 1454 replaced_symlinks["/%s" % (i[0],)] = i[2] 1455 if vendor_diff: 1456 for i in vendor_diff.verbatim_targets: 1457 replaced_symlinks["/%s" % (i[0],)] = i[2] 1458 1459 if system_diff: 1460 for tf in system_diff.renames.values(): 1461 replaced_symlinks["/%s" % (tf.name,)] = tf.sha1 1462 if vendor_diff: 1463 for tf in vendor_diff.renames.values(): 1464 replaced_symlinks["/%s" % (tf.name,)] = tf.sha1 1465 1466 always_delete = [] 1467 may_delete = [] 1468 for dest, link in source_symlinks: 1469 if link not in target_symlinks_d: 1470 if link in replaced_symlinks: 1471 may_delete.append((link, replaced_symlinks[link])) 1472 else: 1473 always_delete.append(link) 1474 script.DeleteFiles(always_delete) 1475 script.DeleteFilesIfNotMatching(may_delete) 1476 1477 if system_diff.verbatim_targets: 1478 script.Print("Unpacking new system files...") 1479 script.UnpackPackageDir("system", "/system") 1480 if vendor_diff and vendor_diff.verbatim_targets: 1481 script.Print("Unpacking new vendor files...") 1482 script.UnpackPackageDir("vendor", "/vendor") 1483 1484 if updating_recovery and not target_has_recovery_patch: 1485 script.Print("Unpacking new recovery...") 1486 script.UnpackPackageDir("recovery", "/system") 1487 1488 system_diff.EmitRenames(script) 1489 if vendor_diff: 1490 vendor_diff.EmitRenames(script) 1491 1492 script.Print("Symlinks and permissions...") 1493 1494 # Create all the symlinks that don't already exist, or point to 1495 # somewhere different than what we want. Delete each symlink before 1496 # creating it, since the 'symlink' command won't overwrite. 1497 to_create = [] 1498 for dest, link in target_symlinks: 1499 if link in source_symlinks_d: 1500 if dest != source_symlinks_d[link]: 1501 to_create.append((dest, link)) 1502 else: 1503 to_create.append((dest, link)) 1504 script.DeleteFiles([i[1] for i in to_create]) 1505 script.MakeSymlinks(to_create) 1506 1507 # Now that the symlinks are created, we can set all the 1508 # permissions. 1509 script.AppendScript(temp_script) 1510 1511 # Do device-specific installation (eg, write radio image). 1512 device_specific.IncrementalOTA_InstallEnd() 1513 1514 if OPTIONS.extra_script is not None: 1515 script.AppendExtra(OPTIONS.extra_script) 1516 1517 # Patch the build.prop file last, so if something fails but the 1518 # device can still come up, it appears to be the old build and will 1519 # get set the OTA package again to retry. 1520 script.Print("Patching remaining system files...") 1521 system_diff.EmitDeferredPatches(script) 1522 1523 if OPTIONS.wipe_user_data: 1524 script.Print("Erasing user data...") 1525 script.FormatPartition("/data") 1526 metadata["ota-wipe"] = "yes" 1527 1528 if OPTIONS.two_step: 1529 script.AppendExtra(""" 1530set_stage("%(bcb_dev)s", ""); 1531endif; 1532endif; 1533""" % bcb_dev) 1534 1535 if OPTIONS.verify and system_diff: 1536 script.Print("Remounting and verifying system partition files...") 1537 script.Unmount("/system") 1538 script.Mount("/system") 1539 system_diff.EmitExplicitTargetVerification(script) 1540 1541 if OPTIONS.verify and vendor_diff: 1542 script.Print("Remounting and verifying vendor partition files...") 1543 script.Unmount("/vendor") 1544 script.Mount("/vendor") 1545 vendor_diff.EmitExplicitTargetVerification(script) 1546 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 1547 1548 WriteMetadata(metadata, output_zip) 1549 1550 1551def main(argv): 1552 1553 def option_handler(o, a): 1554 if o == "--board_config": 1555 pass # deprecated 1556 elif o in ("-k", "--package_key"): 1557 OPTIONS.package_key = a 1558 elif o in ("-i", "--incremental_from"): 1559 OPTIONS.incremental_source = a 1560 elif o == "--full_radio": 1561 OPTIONS.full_radio = True 1562 elif o == "--full_bootloader": 1563 OPTIONS.full_bootloader = True 1564 elif o in ("-w", "--wipe_user_data"): 1565 OPTIONS.wipe_user_data = True 1566 elif o in ("-n", "--no_prereq"): 1567 OPTIONS.omit_prereq = True 1568 elif o == "--downgrade": 1569 OPTIONS.downgrade = True 1570 OPTIONS.wipe_user_data = True 1571 elif o in ("-o", "--oem_settings"): 1572 OPTIONS.oem_source = a 1573 elif o in ("-e", "--extra_script"): 1574 OPTIONS.extra_script = a 1575 elif o in ("-a", "--aslr_mode"): 1576 if a in ("on", "On", "true", "True", "yes", "Yes"): 1577 OPTIONS.aslr_mode = True 1578 else: 1579 OPTIONS.aslr_mode = False 1580 elif o in ("-t", "--worker_threads"): 1581 if a.isdigit(): 1582 OPTIONS.worker_threads = int(a) 1583 else: 1584 raise ValueError("Cannot parse value %r for option %r - only " 1585 "integers are allowed." % (a, o)) 1586 elif o in ("-2", "--two_step"): 1587 OPTIONS.two_step = True 1588 elif o == "--no_signing": 1589 OPTIONS.no_signing = True 1590 elif o == "--verify": 1591 OPTIONS.verify = True 1592 elif o == "--block": 1593 OPTIONS.block_based = True 1594 elif o in ("-b", "--binary"): 1595 OPTIONS.updater_binary = a 1596 elif o in ("--no_fallback_to_full",): 1597 OPTIONS.fallback_to_full = False 1598 elif o == "--stash_threshold": 1599 try: 1600 OPTIONS.stash_threshold = float(a) 1601 except ValueError: 1602 raise ValueError("Cannot parse value %r for option %r - expecting " 1603 "a float" % (a, o)) 1604 else: 1605 return False 1606 return True 1607 1608 args = common.ParseOptions(argv, __doc__, 1609 extra_opts="b:k:i:d:wne:t:a:2o:", 1610 extra_long_opts=[ 1611 "board_config=", 1612 "package_key=", 1613 "incremental_from=", 1614 "full_radio", 1615 "full_bootloader", 1616 "wipe_user_data", 1617 "no_prereq", 1618 "downgrade", 1619 "extra_script=", 1620 "worker_threads=", 1621 "aslr_mode=", 1622 "two_step", 1623 "no_signing", 1624 "block", 1625 "binary=", 1626 "oem_settings=", 1627 "verify", 1628 "no_fallback_to_full", 1629 "stash_threshold=", 1630 ], extra_option_handler=option_handler) 1631 1632 if len(args) != 2: 1633 common.Usage(__doc__) 1634 sys.exit(1) 1635 1636 if OPTIONS.downgrade: 1637 # Sanity check to enforce a data wipe. 1638 if not OPTIONS.wipe_user_data: 1639 raise ValueError("Cannot downgrade without a data wipe") 1640 1641 # We should only allow downgrading incrementals (as opposed to full). 1642 # Otherwise the device may go back from arbitrary build with this full 1643 # OTA package. 1644 if OPTIONS.incremental_source is None: 1645 raise ValueError("Cannot generate downgradable full OTAs - consider" 1646 "using --omit_prereq?") 1647 1648 if OPTIONS.extra_script is not None: 1649 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 1650 1651 print "unzipping target target-files..." 1652 OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0]) 1653 1654 OPTIONS.target_tmp = OPTIONS.input_tmp 1655 OPTIONS.info_dict = common.LoadInfoDict(input_zip) 1656 1657 # If this image was originally labelled with SELinux contexts, make sure we 1658 # also apply the labels in our new image. During building, the "file_contexts" 1659 # is in the out/ directory tree, but for repacking from target-files.zip it's 1660 # in the root directory of the ramdisk. 1661 if "selinux_fc" in OPTIONS.info_dict: 1662 OPTIONS.info_dict["selinux_fc"] = os.path.join( 1663 OPTIONS.input_tmp, "BOOT", "RAMDISK", "file_contexts") 1664 1665 if OPTIONS.verbose: 1666 print "--- target info ---" 1667 common.DumpInfoDict(OPTIONS.info_dict) 1668 1669 # If the caller explicitly specified the device-specific extensions 1670 # path via -s/--device_specific, use that. Otherwise, use 1671 # META/releasetools.py if it is present in the target target_files. 1672 # Otherwise, take the path of the file from 'tool_extensions' in the 1673 # info dict and look for that in the local filesystem, relative to 1674 # the current directory. 1675 1676 if OPTIONS.device_specific is None: 1677 from_input = os.path.join(OPTIONS.input_tmp, "META", "releasetools.py") 1678 if os.path.exists(from_input): 1679 print "(using device-specific extensions from target_files)" 1680 OPTIONS.device_specific = from_input 1681 else: 1682 OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None) 1683 1684 if OPTIONS.device_specific is not None: 1685 OPTIONS.device_specific = os.path.abspath(OPTIONS.device_specific) 1686 1687 while True: 1688 1689 if OPTIONS.no_signing: 1690 if os.path.exists(args[1]): 1691 os.unlink(args[1]) 1692 output_zip = zipfile.ZipFile(args[1], "w", 1693 compression=zipfile.ZIP_DEFLATED) 1694 else: 1695 temp_zip_file = tempfile.NamedTemporaryFile() 1696 output_zip = zipfile.ZipFile(temp_zip_file, "w", 1697 compression=zipfile.ZIP_DEFLATED) 1698 1699 cache_size = OPTIONS.info_dict.get("cache_size", None) 1700 if cache_size is None: 1701 raise RuntimeError("can't determine the cache partition size") 1702 OPTIONS.cache_size = cache_size 1703 1704 if OPTIONS.incremental_source is None: 1705 WriteFullOTAPackage(input_zip, output_zip) 1706 if OPTIONS.package_key is None: 1707 OPTIONS.package_key = OPTIONS.info_dict.get( 1708 "default_system_dev_certificate", 1709 "build/target/product/security/testkey") 1710 common.ZipClose(output_zip) 1711 break 1712 1713 else: 1714 print "unzipping source target-files..." 1715 OPTIONS.source_tmp, source_zip = common.UnzipTemp( 1716 OPTIONS.incremental_source) 1717 OPTIONS.target_info_dict = OPTIONS.info_dict 1718 OPTIONS.source_info_dict = common.LoadInfoDict(source_zip) 1719 if "selinux_fc" in OPTIONS.source_info_dict: 1720 OPTIONS.source_info_dict["selinux_fc"] = os.path.join( 1721 OPTIONS.source_tmp, "BOOT", "RAMDISK", "file_contexts") 1722 if OPTIONS.package_key is None: 1723 OPTIONS.package_key = OPTIONS.source_info_dict.get( 1724 "default_system_dev_certificate", 1725 "build/target/product/security/testkey") 1726 if OPTIONS.verbose: 1727 print "--- source info ---" 1728 common.DumpInfoDict(OPTIONS.source_info_dict) 1729 try: 1730 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) 1731 common.ZipClose(output_zip) 1732 break 1733 except ValueError: 1734 if not OPTIONS.fallback_to_full: 1735 raise 1736 print "--- failed to build incremental; falling back to full ---" 1737 OPTIONS.incremental_source = None 1738 common.ZipClose(output_zip) 1739 1740 if not OPTIONS.no_signing: 1741 SignOutput(temp_zip_file.name, args[1]) 1742 temp_zip_file.close() 1743 1744 print "done." 1745 1746 1747if __name__ == '__main__': 1748 try: 1749 common.CloseInheritedPipes() 1750 main(sys.argv[1:]) 1751 except common.ExternalError as e: 1752 print 1753 print " ERROR: %s" % (e,) 1754 print 1755 sys.exit(1) 1756 finally: 1757 common.Cleanup() 1758