ota_from_target_files.py revision 40ef35b9e916163bc408d6d9945f92f012885d33
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 --board_config <file> 25 Deprecated. 26 27 -k (--package_key) <key> Key to use to sign the package (default is 28 the value of default_system_dev_certificate from the input 29 target-files's META/misc_info.txt, or 30 "build/target/product/security/testkey" if that value is not 31 specified). 32 33 For incremental OTAs, the default value is based on the source 34 target-file, not the target build. 35 36 -i (--incremental_from) <file> 37 Generate an incremental OTA using the given target-files zip as 38 the starting build. 39 40 --full_radio 41 When generating an incremental OTA, always include a full copy of 42 radio image. This option is only meaningful when -i is specified, 43 because a full radio is always included in a full OTA if applicable. 44 45 --full_bootloader 46 Similar to --full_radio. When generating an incremental OTA, always 47 include a full copy of bootloader image. 48 49 -v (--verify) 50 Remount and verify the checksums of the files written to the 51 system and vendor (if used) partitions. Incremental builds only. 52 53 -o (--oem_settings) <file> 54 Use the file to specify the expected OEM-specific properties 55 on the OEM partition of the intended device. 56 57 -w (--wipe_user_data) 58 Generate an OTA package that will wipe the user data partition 59 when installed. 60 61 -n (--no_prereq) 62 Omit the timestamp prereq check normally included at the top of 63 the build scripts (used for developer OTA packages which 64 legitimately need to go back and forth). 65 66 -e (--extra_script) <file> 67 Insert the contents of file at the end of the update script. 68 69 -a (--aslr_mode) <on|off> 70 Specify whether to turn on ASLR for the package (on by default). 71 72 -2 (--two_step) 73 Generate a 'two-step' OTA package, where recovery is updated 74 first, so that any changes made to the system partition are done 75 using the new recovery (new kernel, etc.). 76 77 --block 78 Generate a block-based OTA if possible. Will fall back to a 79 file-based OTA if the target_files is older and doesn't support 80 block-based OTAs. 81 82 -b (--binary) <file> 83 Use the given binary as the update-binary in the output package, 84 instead of the binary in the build's target_files. Use for 85 development only. 86 87 -t (--worker_threads) <int> 88 Specifies the number of worker-threads that will be used when 89 generating patches for incremental updates (defaults to 3). 90 91 --stash_threshold <float> 92 Specifies the threshold that will be used to compute the maximum 93 allowed stash size (defaults to 0.8). 94 95 --gen_verify 96 Generate an OTA package that verifies the partitions. 97 98 --log_diff <file> 99 Generate a log file that shows the differences in the source and target 100 builds for an incremental package. This option is only meaningful when 101 -i is specified. 102""" 103 104import sys 105 106if sys.hexversion < 0x02070000: 107 print >> sys.stderr, "Python 2.7 or newer is required." 108 sys.exit(1) 109 110import multiprocessing 111import os 112import tempfile 113import zipfile 114 115import common 116import edify_generator 117import sparse_img 118 119OPTIONS = common.OPTIONS 120OPTIONS.package_key = None 121OPTIONS.incremental_source = None 122OPTIONS.verify = False 123OPTIONS.require_verbatim = set() 124OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 125OPTIONS.patch_threshold = 0.95 126OPTIONS.wipe_user_data = False 127OPTIONS.omit_prereq = False 128OPTIONS.extra_script = None 129OPTIONS.aslr_mode = True 130OPTIONS.worker_threads = multiprocessing.cpu_count() // 2 131if OPTIONS.worker_threads == 0: 132 OPTIONS.worker_threads = 1 133OPTIONS.two_step = False 134OPTIONS.no_signing = False 135OPTIONS.block_based = False 136OPTIONS.updater_binary = None 137OPTIONS.oem_source = None 138OPTIONS.fallback_to_full = True 139OPTIONS.full_radio = False 140OPTIONS.full_bootloader = False 141# Stash size cannot exceed cache_size * threshold. 142OPTIONS.cache_size = None 143OPTIONS.stash_threshold = 0.8 144OPTIONS.gen_verify = False 145OPTIONS.log_diff = None 146 147def MostPopularKey(d, default): 148 """Given a dict, return the key corresponding to the largest 149 value. Returns 'default' if the dict is empty.""" 150 x = [(v, k) for (k, v) in d.iteritems()] 151 if not x: 152 return default 153 x.sort() 154 return x[-1][1] 155 156 157def IsSymlink(info): 158 """Return true if the zipfile.ZipInfo object passed in represents a 159 symlink.""" 160 return (info.external_attr >> 16) & 0o770000 == 0o120000 161 162def IsRegular(info): 163 """Return true if the zipfile.ZipInfo object passed in represents a 164 regular file.""" 165 return (info.external_attr >> 16) & 0o770000 == 0o100000 166 167def ClosestFileMatch(src, tgtfiles, existing): 168 """Returns the closest file match between a source file and list 169 of potential matches. The exact filename match is preferred, 170 then the sha1 is searched for, and finally a file with the same 171 basename is evaluated. Rename support in the updater-binary is 172 required for the latter checks to be used.""" 173 174 result = tgtfiles.get("path:" + src.name) 175 if result is not None: 176 return result 177 178 if not OPTIONS.target_info_dict.get("update_rename_support", False): 179 return None 180 181 if src.size < 1000: 182 return None 183 184 result = tgtfiles.get("sha1:" + src.sha1) 185 if result is not None and existing.get(result.name) is None: 186 return result 187 result = tgtfiles.get("file:" + src.name.split("/")[-1]) 188 if result is not None and existing.get(result.name) is None: 189 return result 190 return None 191 192class ItemSet(object): 193 def __init__(self, partition, fs_config): 194 self.partition = partition 195 self.fs_config = fs_config 196 self.ITEMS = {} 197 198 def Get(self, name, is_dir=False): 199 if name not in self.ITEMS: 200 self.ITEMS[name] = Item(self, name, is_dir=is_dir) 201 return self.ITEMS[name] 202 203 def GetMetadata(self, input_zip): 204 # The target_files contains a record of what the uid, 205 # gid, and mode are supposed to be. 206 output = input_zip.read(self.fs_config) 207 208 for line in output.split("\n"): 209 if not line: 210 continue 211 columns = line.split() 212 name, uid, gid, mode = columns[:4] 213 selabel = None 214 capabilities = None 215 216 # After the first 4 columns, there are a series of key=value 217 # pairs. Extract out the fields we care about. 218 for element in columns[4:]: 219 key, value = element.split("=") 220 if key == "selabel": 221 selabel = value 222 if key == "capabilities": 223 capabilities = value 224 225 i = self.ITEMS.get(name, None) 226 if i is not None: 227 i.uid = int(uid) 228 i.gid = int(gid) 229 i.mode = int(mode, 8) 230 i.selabel = selabel 231 i.capabilities = capabilities 232 if i.is_dir: 233 i.children.sort(key=lambda i: i.name) 234 235 # Set metadata for the files generated by this script. For full recovery 236 # image at system/etc/recovery.img, it will be taken care by fs_config. 237 i = self.ITEMS.get("system/recovery-from-boot.p", None) 238 if i: 239 i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0o644, None, None 240 i = self.ITEMS.get("system/etc/install-recovery.sh", None) 241 if i: 242 i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0o544, None, None 243 244 245class Item(object): 246 """Items represent the metadata (user, group, mode) of files and 247 directories in the system image.""" 248 def __init__(self, itemset, name, is_dir=False): 249 self.itemset = itemset 250 self.name = name 251 self.uid = None 252 self.gid = None 253 self.mode = None 254 self.selabel = None 255 self.capabilities = None 256 self.is_dir = is_dir 257 self.descendants = None 258 self.best_subtree = None 259 260 if name: 261 self.parent = itemset.Get(os.path.dirname(name), is_dir=True) 262 self.parent.children.append(self) 263 else: 264 self.parent = None 265 if self.is_dir: 266 self.children = [] 267 268 def Dump(self, indent=0): 269 if self.uid is not None: 270 print "%s%s %d %d %o" % ( 271 " " * indent, self.name, self.uid, self.gid, self.mode) 272 else: 273 print "%s%s %s %s %s" % ( 274 " " * indent, self.name, self.uid, self.gid, self.mode) 275 if self.is_dir: 276 print "%s%s" % (" "*indent, self.descendants) 277 print "%s%s" % (" "*indent, self.best_subtree) 278 for i in self.children: 279 i.Dump(indent=indent+1) 280 281 def CountChildMetadata(self): 282 """Count up the (uid, gid, mode, selabel, capabilities) tuples for 283 all children and determine the best strategy for using set_perm_recursive 284 and set_perm to correctly chown/chmod all the files to their desired 285 values. Recursively calls itself for all descendants. 286 287 Returns a dict of {(uid, gid, dmode, fmode, selabel, capabilities): count} 288 counting up all descendants of this node. (dmode or fmode may be None.) 289 Also sets the best_subtree of each directory Item to the (uid, gid, dmode, 290 fmode, selabel, capabilities) tuple that will match the most descendants of 291 that Item. 292 """ 293 294 assert self.is_dir 295 key = (self.uid, self.gid, self.mode, None, self.selabel, 296 self.capabilities) 297 self.descendants = {key: 1} 298 d = self.descendants 299 for i in self.children: 300 if i.is_dir: 301 for k, v in i.CountChildMetadata().iteritems(): 302 d[k] = d.get(k, 0) + v 303 else: 304 k = (i.uid, i.gid, None, i.mode, i.selabel, i.capabilities) 305 d[k] = d.get(k, 0) + 1 306 307 # Find the (uid, gid, dmode, fmode, selabel, capabilities) 308 # tuple that matches the most descendants. 309 310 # First, find the (uid, gid) pair that matches the most 311 # descendants. 312 ug = {} 313 for (uid, gid, _, _, _, _), count in d.iteritems(): 314 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 315 ug = MostPopularKey(ug, (0, 0)) 316 317 # Now find the dmode, fmode, selabel, and capabilities that match 318 # the most descendants with that (uid, gid), and choose those. 319 best_dmode = (0, 0o755) 320 best_fmode = (0, 0o644) 321 best_selabel = (0, None) 322 best_capabilities = (0, None) 323 for k, count in d.iteritems(): 324 if k[:2] != ug: 325 continue 326 if k[2] is not None and count >= best_dmode[0]: 327 best_dmode = (count, k[2]) 328 if k[3] is not None and count >= best_fmode[0]: 329 best_fmode = (count, k[3]) 330 if k[4] is not None and count >= best_selabel[0]: 331 best_selabel = (count, k[4]) 332 if k[5] is not None and count >= best_capabilities[0]: 333 best_capabilities = (count, k[5]) 334 self.best_subtree = ug + ( 335 best_dmode[1], best_fmode[1], best_selabel[1], best_capabilities[1]) 336 337 return d 338 339 def SetPermissions(self, script): 340 """Append set_perm/set_perm_recursive commands to 'script' to 341 set all permissions, users, and groups for the tree of files 342 rooted at 'self'.""" 343 344 self.CountChildMetadata() 345 346 def recurse(item, current): 347 # current is the (uid, gid, dmode, fmode, selabel, capabilities) tuple 348 # that the current item (and all its children) have already been set to. 349 # We only need to issue set_perm/set_perm_recursive commands if we're 350 # supposed to be something different. 351 if item.is_dir: 352 if current != item.best_subtree: 353 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 354 current = item.best_subtree 355 356 if item.uid != current[0] or item.gid != current[1] or \ 357 item.mode != current[2] or item.selabel != current[4] or \ 358 item.capabilities != current[5]: 359 script.SetPermissions("/"+item.name, item.uid, item.gid, 360 item.mode, item.selabel, item.capabilities) 361 362 for i in item.children: 363 recurse(i, current) 364 else: 365 if item.uid != current[0] or item.gid != current[1] or \ 366 item.mode != current[3] or item.selabel != current[4] or \ 367 item.capabilities != current[5]: 368 script.SetPermissions("/"+item.name, item.uid, item.gid, 369 item.mode, item.selabel, item.capabilities) 370 371 recurse(self, (-1, -1, -1, -1, None, None)) 372 373 374def CopyPartitionFiles(itemset, input_zip, output_zip=None, substitute=None): 375 """Copies files for the partition in the input zip to the output 376 zip. Populates the Item class with their metadata, and returns a 377 list of symlinks. output_zip may be None, in which case the copy is 378 skipped (but the other side effects still happen). substitute is an 379 optional dict of {output filename: contents} to be output instead of 380 certain input files. 381 """ 382 383 symlinks = [] 384 385 partition = itemset.partition 386 387 for info in input_zip.infolist(): 388 prefix = partition.upper() + "/" 389 if info.filename.startswith(prefix): 390 basefilename = info.filename[len(prefix):] 391 if IsSymlink(info): 392 symlinks.append((input_zip.read(info.filename), 393 "/" + partition + "/" + basefilename)) 394 else: 395 import copy 396 info2 = copy.copy(info) 397 fn = info2.filename = partition + "/" + basefilename 398 if substitute and fn in substitute and substitute[fn] is None: 399 continue 400 if output_zip is not None: 401 if substitute and fn in substitute: 402 data = substitute[fn] 403 else: 404 data = input_zip.read(info.filename) 405 common.ZipWriteStr(output_zip, info2, data) 406 if fn.endswith("/"): 407 itemset.Get(fn[:-1], is_dir=True) 408 else: 409 itemset.Get(fn) 410 411 symlinks.sort() 412 return symlinks 413 414 415def SignOutput(temp_zip_name, output_zip_name): 416 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 417 pw = key_passwords[OPTIONS.package_key] 418 419 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 420 whole_file=True) 421 422 423def AppendAssertions(script, info_dict, oem_dict=None): 424 oem_props = info_dict.get("oem_fingerprint_properties") 425 if oem_props is None or len(oem_props) == 0: 426 device = GetBuildProp("ro.product.device", info_dict) 427 script.AssertDevice(device) 428 else: 429 if oem_dict is None: 430 raise common.ExternalError( 431 "No OEM file provided to answer expected assertions") 432 for prop in oem_props.split(): 433 if oem_dict.get(prop) is None: 434 raise common.ExternalError( 435 "The OEM file is missing the property %s" % prop) 436 script.AssertOemProperty(prop, oem_dict.get(prop)) 437 438 439def HasRecoveryPatch(target_files_zip): 440 namelist = [name for name in target_files_zip.namelist()] 441 return ("SYSTEM/recovery-from-boot.p" in namelist or 442 "SYSTEM/etc/recovery.img" in namelist) 443 444def HasVendorPartition(target_files_zip): 445 try: 446 target_files_zip.getinfo("VENDOR/") 447 return True 448 except KeyError: 449 return False 450 451def GetOemProperty(name, oem_props, oem_dict, info_dict): 452 if oem_props is not None and name in oem_props: 453 return oem_dict[name] 454 return GetBuildProp(name, info_dict) 455 456 457def CalculateFingerprint(oem_props, oem_dict, info_dict): 458 if oem_props is None: 459 return GetBuildProp("ro.build.fingerprint", info_dict) 460 return "%s/%s/%s:%s" % ( 461 GetOemProperty("ro.product.brand", oem_props, oem_dict, info_dict), 462 GetOemProperty("ro.product.name", oem_props, oem_dict, info_dict), 463 GetOemProperty("ro.product.device", oem_props, oem_dict, info_dict), 464 GetBuildProp("ro.build.thumbprint", info_dict)) 465 466 467def GetImage(which, tmpdir, info_dict): 468 # Return an image object (suitable for passing to BlockImageDiff) 469 # for the 'which' partition (most be "system" or "vendor"). If a 470 # prebuilt image and file map are found in tmpdir they are used, 471 # otherwise they are reconstructed from the individual files. 472 473 assert which in ("system", "vendor") 474 475 path = os.path.join(tmpdir, "IMAGES", which + ".img") 476 mappath = os.path.join(tmpdir, "IMAGES", which + ".map") 477 if os.path.exists(path) and os.path.exists(mappath): 478 print "using %s.img from target-files" % (which,) 479 # This is a 'new' target-files, which already has the image in it. 480 481 else: 482 print "building %s.img from target-files" % (which,) 483 484 # This is an 'old' target-files, which does not contain images 485 # already built. Build them. 486 487 mappath = tempfile.mkstemp()[1] 488 OPTIONS.tempfiles.append(mappath) 489 490 import add_img_to_target_files 491 if which == "system": 492 path = add_img_to_target_files.BuildSystem( 493 tmpdir, info_dict, block_list=mappath) 494 elif which == "vendor": 495 path = add_img_to_target_files.BuildVendor( 496 tmpdir, info_dict, block_list=mappath) 497 498 # Bug: http://b/20939131 499 # In ext4 filesystems, block 0 might be changed even being mounted 500 # R/O. We add it to clobbered_blocks so that it will be written to the 501 # target unconditionally. Note that they are still part of care_map. 502 clobbered_blocks = "0" 503 504 return sparse_img.SparseImage(path, mappath, clobbered_blocks) 505 506 507def WriteFullOTAPackage(input_zip, output_zip): 508 # TODO: how to determine this? We don't know what version it will 509 # be installed on top of. For now, we expect the API just won't 510 # change very often. Similarly for fstab, it might have changed 511 # in the target build. 512 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 513 514 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 515 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 516 oem_dict = None 517 if oem_props is not None and len(oem_props) > 0: 518 if OPTIONS.oem_source is None: 519 raise common.ExternalError("OEM source required for this build") 520 script.Mount("/oem", recovery_mount_options) 521 oem_dict = common.LoadDictionaryFromLines( 522 open(OPTIONS.oem_source).readlines()) 523 524 metadata = { 525 "post-build": CalculateFingerprint(oem_props, oem_dict, 526 OPTIONS.info_dict), 527 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 528 OPTIONS.info_dict), 529 "post-timestamp": GetBuildProp("ro.build.date.utc", OPTIONS.info_dict), 530 } 531 532 device_specific = common.DeviceSpecificParams( 533 input_zip=input_zip, 534 input_version=OPTIONS.info_dict["recovery_api_version"], 535 output_zip=output_zip, 536 script=script, 537 input_tmp=OPTIONS.input_tmp, 538 metadata=metadata, 539 info_dict=OPTIONS.info_dict) 540 541 has_recovery_patch = HasRecoveryPatch(input_zip) 542 block_based = OPTIONS.block_based and has_recovery_patch 543 544 if not OPTIONS.omit_prereq: 545 ts = GetBuildProp("ro.build.date.utc", OPTIONS.info_dict) 546 ts_text = GetBuildProp("ro.build.date", OPTIONS.info_dict) 547 script.AssertOlderBuild(ts, ts_text) 548 549 AppendAssertions(script, OPTIONS.info_dict, oem_dict) 550 device_specific.FullOTA_Assertions() 551 552 # Two-step package strategy (in chronological order, which is *not* 553 # the order in which the generated script has things): 554 # 555 # if stage is not "2/3" or "3/3": 556 # write recovery image to boot partition 557 # set stage to "2/3" 558 # reboot to boot partition and restart recovery 559 # else if stage is "2/3": 560 # write recovery image to recovery partition 561 # set stage to "3/3" 562 # reboot to recovery partition and restart recovery 563 # else: 564 # (stage must be "3/3") 565 # set stage to "" 566 # do normal full package installation: 567 # wipe and install system, boot image, etc. 568 # set up system to update recovery partition on first boot 569 # complete script normally 570 # (allow recovery to mark itself finished and reboot) 571 572 recovery_img = common.GetBootableImage("recovery.img", "recovery.img", 573 OPTIONS.input_tmp, "RECOVERY") 574 if OPTIONS.two_step: 575 if not OPTIONS.info_dict.get("multistage_support", None): 576 assert False, "two-step packages not supported by this build" 577 fs = OPTIONS.info_dict["fstab"]["/misc"] 578 assert fs.fs_type.upper() == "EMMC", \ 579 "two-step packages only supported on devices with EMMC /misc partitions" 580 bcb_dev = {"bcb_dev": fs.device} 581 common.ZipWriteStr(output_zip, "recovery.img", recovery_img.data) 582 script.AppendExtra(""" 583if get_stage("%(bcb_dev)s") == "2/3" then 584""" % bcb_dev) 585 script.WriteRawImage("/recovery", "recovery.img") 586 script.AppendExtra(""" 587set_stage("%(bcb_dev)s", "3/3"); 588reboot_now("%(bcb_dev)s", "recovery"); 589else if get_stage("%(bcb_dev)s") == "3/3" then 590""" % bcb_dev) 591 592 # Dump fingerprints 593 script.Print("Target: %s" % CalculateFingerprint( 594 oem_props, oem_dict, OPTIONS.info_dict)) 595 596 device_specific.FullOTA_InstallBegin() 597 598 system_progress = 0.75 599 600 if OPTIONS.wipe_user_data: 601 system_progress -= 0.1 602 if HasVendorPartition(input_zip): 603 system_progress -= 0.1 604 605 # Place a copy of file_contexts.bin into the OTA package which will be used 606 # by the recovery program. 607 if "selinux_fc" in OPTIONS.info_dict: 608 WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip) 609 610 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 611 612 system_items = ItemSet("system", "META/filesystem_config.txt") 613 script.ShowProgress(system_progress, 0) 614 615 if block_based: 616 # Full OTA is done as an "incremental" against an empty source 617 # image. This has the effect of writing new data from the package 618 # to the entire partition, but lets us reuse the updater code that 619 # writes incrementals to do it. 620 system_tgt = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict) 621 system_tgt.ResetFileMap() 622 system_diff = common.BlockDifference("system", system_tgt, src=None) 623 system_diff.WriteScript(script, output_zip) 624 else: 625 script.FormatPartition("/system") 626 script.Mount("/system", recovery_mount_options) 627 if not has_recovery_patch: 628 script.UnpackPackageDir("recovery", "/system") 629 script.UnpackPackageDir("system", "/system") 630 631 symlinks = CopyPartitionFiles(system_items, input_zip, output_zip) 632 script.MakeSymlinks(symlinks) 633 634 boot_img = common.GetBootableImage( 635 "boot.img", "boot.img", OPTIONS.input_tmp, "BOOT") 636 637 if not block_based: 638 def output_sink(fn, data): 639 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 640 system_items.Get("system/" + fn) 641 642 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, 643 recovery_img, boot_img) 644 645 system_items.GetMetadata(input_zip) 646 system_items.Get("system").SetPermissions(script) 647 648 if HasVendorPartition(input_zip): 649 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 650 script.ShowProgress(0.1, 0) 651 652 if block_based: 653 vendor_tgt = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict) 654 vendor_tgt.ResetFileMap() 655 vendor_diff = common.BlockDifference("vendor", vendor_tgt) 656 vendor_diff.WriteScript(script, output_zip) 657 else: 658 script.FormatPartition("/vendor") 659 script.Mount("/vendor", recovery_mount_options) 660 script.UnpackPackageDir("vendor", "/vendor") 661 662 symlinks = CopyPartitionFiles(vendor_items, input_zip, output_zip) 663 script.MakeSymlinks(symlinks) 664 665 vendor_items.GetMetadata(input_zip) 666 vendor_items.Get("vendor").SetPermissions(script) 667 668 common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict) 669 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 670 671 script.ShowProgress(0.05, 5) 672 script.WriteRawImage("/boot", "boot.img") 673 674 script.ShowProgress(0.2, 10) 675 device_specific.FullOTA_InstallEnd() 676 677 if OPTIONS.extra_script is not None: 678 script.AppendExtra(OPTIONS.extra_script) 679 680 script.UnmountAll() 681 682 if OPTIONS.wipe_user_data: 683 script.ShowProgress(0.1, 10) 684 script.FormatPartition("/data") 685 686 if OPTIONS.two_step: 687 script.AppendExtra(""" 688set_stage("%(bcb_dev)s", ""); 689""" % bcb_dev) 690 script.AppendExtra("else\n") 691 script.WriteRawImage("/boot", "recovery.img") 692 script.AppendExtra(""" 693set_stage("%(bcb_dev)s", "2/3"); 694reboot_now("%(bcb_dev)s", ""); 695endif; 696endif; 697""" % bcb_dev) 698 script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary) 699 WriteMetadata(metadata, output_zip) 700 701 702def WritePolicyConfig(file_name, output_zip): 703 common.ZipWrite(output_zip, file_name, os.path.basename(file_name)) 704 705 706def WriteMetadata(metadata, output_zip): 707 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata", 708 "".join(["%s=%s\n" % kv 709 for kv in sorted(metadata.iteritems())])) 710 711 712def LoadPartitionFiles(z, partition): 713 """Load all the files from the given partition in a given target-files 714 ZipFile, and return a dict of {filename: File object}.""" 715 out = {} 716 prefix = partition.upper() + "/" 717 for info in z.infolist(): 718 if info.filename.startswith(prefix) and not IsSymlink(info): 719 basefilename = info.filename[len(prefix):] 720 fn = partition + "/" + basefilename 721 data = z.read(info.filename) 722 out[fn] = common.File(fn, data) 723 return out 724 725 726def GetBuildProp(prop, info_dict): 727 """Return the fingerprint of the build of a given target-files info_dict.""" 728 try: 729 return info_dict.get("build.prop", {})[prop] 730 except KeyError: 731 raise common.ExternalError("couldn't find %s in build.prop" % (prop,)) 732 733 734def AddToKnownPaths(filename, known_paths): 735 if filename[-1] == "/": 736 return 737 dirs = filename.split("/")[:-1] 738 while len(dirs) > 0: 739 path = "/".join(dirs) 740 if path in known_paths: 741 break 742 known_paths.add(path) 743 dirs.pop() 744 745 746def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 747 # TODO(tbao): We should factor out the common parts between 748 # WriteBlockIncrementalOTAPackage() and WriteIncrementalOTAPackage(). 749 source_version = OPTIONS.source_info_dict["recovery_api_version"] 750 target_version = OPTIONS.target_info_dict["recovery_api_version"] 751 752 if source_version == 0: 753 print ("WARNING: generating edify script for a source that " 754 "can't install it.") 755 script = edify_generator.EdifyGenerator( 756 source_version, OPTIONS.target_info_dict, 757 fstab=OPTIONS.source_info_dict["fstab"]) 758 759 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 760 recovery_mount_options = OPTIONS.source_info_dict.get( 761 "recovery_mount_options") 762 oem_dict = None 763 if oem_props is not None and len(oem_props) > 0: 764 if OPTIONS.oem_source is None: 765 raise common.ExternalError("OEM source required for this build") 766 script.Mount("/oem", recovery_mount_options) 767 oem_dict = common.LoadDictionaryFromLines( 768 open(OPTIONS.oem_source).readlines()) 769 770 metadata = { 771 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 772 OPTIONS.source_info_dict), 773 "post-timestamp": GetBuildProp("ro.build.date.utc", 774 OPTIONS.target_info_dict), 775 } 776 777 device_specific = common.DeviceSpecificParams( 778 source_zip=source_zip, 779 source_version=source_version, 780 target_zip=target_zip, 781 target_version=target_version, 782 output_zip=output_zip, 783 script=script, 784 metadata=metadata, 785 info_dict=OPTIONS.source_info_dict) 786 787 source_fp = CalculateFingerprint(oem_props, oem_dict, 788 OPTIONS.source_info_dict) 789 target_fp = CalculateFingerprint(oem_props, oem_dict, 790 OPTIONS.target_info_dict) 791 metadata["pre-build"] = source_fp 792 metadata["post-build"] = target_fp 793 794 source_boot = common.GetBootableImage( 795 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 796 OPTIONS.source_info_dict) 797 target_boot = common.GetBootableImage( 798 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 799 updating_boot = (not OPTIONS.two_step and 800 (source_boot.data != target_boot.data)) 801 802 target_recovery = common.GetBootableImage( 803 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 804 805 system_src = GetImage("system", OPTIONS.source_tmp, OPTIONS.source_info_dict) 806 system_tgt = GetImage("system", OPTIONS.target_tmp, OPTIONS.target_info_dict) 807 808 blockimgdiff_version = 1 809 if OPTIONS.info_dict: 810 blockimgdiff_version = max( 811 int(i) for i in 812 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(",")) 813 814 system_diff = common.BlockDifference("system", system_tgt, system_src, 815 version=blockimgdiff_version) 816 817 if HasVendorPartition(target_zip): 818 if not HasVendorPartition(source_zip): 819 raise RuntimeError("can't generate incremental that adds /vendor") 820 vendor_src = GetImage("vendor", OPTIONS.source_tmp, 821 OPTIONS.source_info_dict) 822 vendor_tgt = GetImage("vendor", OPTIONS.target_tmp, 823 OPTIONS.target_info_dict) 824 vendor_diff = common.BlockDifference("vendor", vendor_tgt, vendor_src, 825 version=blockimgdiff_version) 826 else: 827 vendor_diff = None 828 829 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 830 device_specific.IncrementalOTA_Assertions() 831 832 # Two-step incremental package strategy (in chronological order, 833 # which is *not* the order in which the generated script has 834 # things): 835 # 836 # if stage is not "2/3" or "3/3": 837 # do verification on current system 838 # write recovery image to boot partition 839 # set stage to "2/3" 840 # reboot to boot partition and restart recovery 841 # else if stage is "2/3": 842 # write recovery image to recovery partition 843 # set stage to "3/3" 844 # reboot to recovery partition and restart recovery 845 # else: 846 # (stage must be "3/3") 847 # perform update: 848 # patch system files, etc. 849 # force full install of new boot image 850 # set up system to update recovery partition on first boot 851 # complete script normally 852 # (allow recovery to mark itself finished and reboot) 853 854 if OPTIONS.two_step: 855 if not OPTIONS.source_info_dict.get("multistage_support", None): 856 assert False, "two-step packages not supported by this build" 857 fs = OPTIONS.source_info_dict["fstab"]["/misc"] 858 assert fs.fs_type.upper() == "EMMC", \ 859 "two-step packages only supported on devices with EMMC /misc partitions" 860 bcb_dev = {"bcb_dev": fs.device} 861 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 862 script.AppendExtra(""" 863if get_stage("%(bcb_dev)s") == "2/3" then 864""" % bcb_dev) 865 script.AppendExtra("sleep(20);\n") 866 script.WriteRawImage("/recovery", "recovery.img") 867 script.AppendExtra(""" 868set_stage("%(bcb_dev)s", "3/3"); 869reboot_now("%(bcb_dev)s", "recovery"); 870else if get_stage("%(bcb_dev)s") != "3/3" then 871""" % bcb_dev) 872 873 # Dump fingerprints 874 script.Print("Source: %s" % CalculateFingerprint( 875 oem_props, oem_dict, OPTIONS.source_info_dict)) 876 script.Print("Target: %s" % CalculateFingerprint( 877 oem_props, oem_dict, OPTIONS.target_info_dict)) 878 879 script.Print("Verifying current system...") 880 881 device_specific.IncrementalOTA_VerifyBegin() 882 883 if oem_props is None: 884 # When blockimgdiff version is less than 3 (non-resumable block-based OTA), 885 # patching on a device that's already on the target build will damage the 886 # system. Because operations like move don't check the block state, they 887 # always apply the changes unconditionally. 888 if blockimgdiff_version <= 2: 889 script.AssertSomeFingerprint(source_fp) 890 else: 891 script.AssertSomeFingerprint(source_fp, target_fp) 892 else: 893 if blockimgdiff_version <= 2: 894 script.AssertSomeThumbprint( 895 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 896 else: 897 script.AssertSomeThumbprint( 898 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 899 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 900 901 if updating_boot: 902 boot_type, boot_device = common.GetTypeAndDevice( 903 "/boot", OPTIONS.source_info_dict) 904 d = common.Difference(target_boot, source_boot) 905 _, _, d = d.ComputePatch() 906 if d is None: 907 include_full_boot = True 908 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 909 else: 910 include_full_boot = False 911 912 print "boot target: %d source: %d diff: %d" % ( 913 target_boot.size, source_boot.size, len(d)) 914 915 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 916 917 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 918 (boot_type, boot_device, 919 source_boot.size, source_boot.sha1, 920 target_boot.size, target_boot.sha1)) 921 922 device_specific.IncrementalOTA_VerifyEnd() 923 924 if OPTIONS.two_step: 925 script.WriteRawImage("/boot", "recovery.img") 926 script.AppendExtra(""" 927set_stage("%(bcb_dev)s", "2/3"); 928reboot_now("%(bcb_dev)s", ""); 929else 930""" % bcb_dev) 931 932 # Verify the existing partitions. 933 system_diff.WriteVerifyScript(script) 934 if vendor_diff: 935 vendor_diff.WriteVerifyScript(script) 936 937 script.Comment("---- start making changes here ----") 938 939 device_specific.IncrementalOTA_InstallBegin() 940 941 system_diff.WriteScript(script, output_zip, 942 progress=0.8 if vendor_diff else 0.9) 943 944 if vendor_diff: 945 vendor_diff.WriteScript(script, output_zip, progress=0.1) 946 947 if OPTIONS.two_step: 948 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 949 script.WriteRawImage("/boot", "boot.img") 950 print "writing full boot image (forced by two-step mode)" 951 952 if not OPTIONS.two_step: 953 if updating_boot: 954 if include_full_boot: 955 print "boot image changed; including full." 956 script.Print("Installing boot image...") 957 script.WriteRawImage("/boot", "boot.img") 958 else: 959 # Produce the boot image by applying a patch to the current 960 # contents of the boot partition, and write it back to the 961 # partition. 962 print "boot image changed; including patch." 963 script.Print("Patching boot image...") 964 script.ShowProgress(0.1, 10) 965 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 966 % (boot_type, boot_device, 967 source_boot.size, source_boot.sha1, 968 target_boot.size, target_boot.sha1), 969 "-", 970 target_boot.size, target_boot.sha1, 971 source_boot.sha1, "patch/boot.img.p") 972 else: 973 print "boot image unchanged; skipping." 974 975 # Do device-specific installation (eg, write radio image). 976 device_specific.IncrementalOTA_InstallEnd() 977 978 if OPTIONS.extra_script is not None: 979 script.AppendExtra(OPTIONS.extra_script) 980 981 if OPTIONS.wipe_user_data: 982 script.Print("Erasing user data...") 983 script.FormatPartition("/data") 984 985 if OPTIONS.two_step: 986 script.AppendExtra(""" 987set_stage("%(bcb_dev)s", ""); 988endif; 989endif; 990""" % bcb_dev) 991 992 script.SetProgress(1) 993 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 994 WriteMetadata(metadata, output_zip) 995 996 997def WriteVerifyPackage(input_zip, output_zip): 998 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 999 1000 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 1001 recovery_mount_options = OPTIONS.info_dict.get( 1002 "recovery_mount_options") 1003 oem_dict = None 1004 if oem_props is not None and len(oem_props) > 0: 1005 if OPTIONS.oem_source is None: 1006 raise common.ExternalError("OEM source required for this build") 1007 script.Mount("/oem", recovery_mount_options) 1008 oem_dict = common.LoadDictionaryFromLines( 1009 open(OPTIONS.oem_source).readlines()) 1010 1011 target_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.info_dict) 1012 metadata = { 1013 "post-build": target_fp, 1014 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 1015 OPTIONS.info_dict), 1016 "post-timestamp": GetBuildProp("ro.build.date.utc", OPTIONS.info_dict), 1017 } 1018 1019 device_specific = common.DeviceSpecificParams( 1020 input_zip=input_zip, 1021 input_version=OPTIONS.info_dict["recovery_api_version"], 1022 output_zip=output_zip, 1023 script=script, 1024 input_tmp=OPTIONS.input_tmp, 1025 metadata=metadata, 1026 info_dict=OPTIONS.info_dict) 1027 1028 AppendAssertions(script, OPTIONS.info_dict, oem_dict) 1029 1030 script.Print("Verifying device images against %s..." % target_fp) 1031 script.AppendExtra("") 1032 1033 script.Print("Verifying boot...") 1034 boot_img = common.GetBootableImage( 1035 "boot.img", "boot.img", OPTIONS.input_tmp, "BOOT") 1036 boot_type, boot_device = common.GetTypeAndDevice( 1037 "/boot", OPTIONS.info_dict) 1038 script.Verify("%s:%s:%d:%s" % ( 1039 boot_type, boot_device, boot_img.size, boot_img.sha1)) 1040 script.AppendExtra("") 1041 1042 script.Print("Verifying recovery...") 1043 recovery_img = common.GetBootableImage( 1044 "recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY") 1045 recovery_type, recovery_device = common.GetTypeAndDevice( 1046 "/recovery", OPTIONS.info_dict) 1047 script.Verify("%s:%s:%d:%s" % ( 1048 recovery_type, recovery_device, recovery_img.size, recovery_img.sha1)) 1049 script.AppendExtra("") 1050 1051 system_tgt = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict) 1052 system_tgt.ResetFileMap() 1053 system_diff = common.BlockDifference("system", system_tgt, src=None) 1054 system_diff.WriteStrictVerifyScript(script) 1055 1056 if HasVendorPartition(input_zip): 1057 vendor_tgt = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict) 1058 vendor_tgt.ResetFileMap() 1059 vendor_diff = common.BlockDifference("vendor", vendor_tgt, src=None) 1060 vendor_diff.WriteStrictVerifyScript(script) 1061 1062 # Device specific partitions, such as radio, bootloader and etc. 1063 device_specific.VerifyOTA_Assertions() 1064 1065 script.SetProgress(1.0) 1066 script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary) 1067 WriteMetadata(metadata, output_zip) 1068 1069 1070class FileDifference(object): 1071 def __init__(self, partition, source_zip, target_zip, output_zip): 1072 self.deferred_patch_list = None 1073 print "Loading target..." 1074 self.target_data = target_data = LoadPartitionFiles(target_zip, partition) 1075 print "Loading source..." 1076 self.source_data = source_data = LoadPartitionFiles(source_zip, partition) 1077 1078 self.verbatim_targets = verbatim_targets = [] 1079 self.patch_list = patch_list = [] 1080 diffs = [] 1081 self.renames = renames = {} 1082 known_paths = set() 1083 largest_source_size = 0 1084 1085 matching_file_cache = {} 1086 for fn, sf in source_data.items(): 1087 assert fn == sf.name 1088 matching_file_cache["path:" + fn] = sf 1089 if fn in target_data.keys(): 1090 AddToKnownPaths(fn, known_paths) 1091 # Only allow eligibility for filename/sha matching 1092 # if there isn't a perfect path match. 1093 if target_data.get(sf.name) is None: 1094 matching_file_cache["file:" + fn.split("/")[-1]] = sf 1095 matching_file_cache["sha:" + sf.sha1] = sf 1096 1097 for fn in sorted(target_data.keys()): 1098 tf = target_data[fn] 1099 assert fn == tf.name 1100 sf = ClosestFileMatch(tf, matching_file_cache, renames) 1101 if sf is not None and sf.name != tf.name: 1102 print "File has moved from " + sf.name + " to " + tf.name 1103 renames[sf.name] = tf 1104 1105 if sf is None or fn in OPTIONS.require_verbatim: 1106 # This file should be included verbatim 1107 if fn in OPTIONS.prohibit_verbatim: 1108 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 1109 print "send", fn, "verbatim" 1110 tf.AddToZip(output_zip) 1111 verbatim_targets.append((fn, tf.size, tf.sha1)) 1112 if fn in target_data.keys(): 1113 AddToKnownPaths(fn, known_paths) 1114 elif tf.sha1 != sf.sha1: 1115 # File is different; consider sending as a patch 1116 diffs.append(common.Difference(tf, sf)) 1117 else: 1118 # Target file data identical to source (may still be renamed) 1119 pass 1120 1121 common.ComputeDifferences(diffs) 1122 1123 for diff in diffs: 1124 tf, sf, d = diff.GetPatch() 1125 path = "/".join(tf.name.split("/")[:-1]) 1126 if d is None or len(d) > tf.size * OPTIONS.patch_threshold or \ 1127 path not in known_paths: 1128 # patch is almost as big as the file; don't bother patching 1129 # or a patch + rename cannot take place due to the target 1130 # directory not existing 1131 tf.AddToZip(output_zip) 1132 verbatim_targets.append((tf.name, tf.size, tf.sha1)) 1133 if sf.name in renames: 1134 del renames[sf.name] 1135 AddToKnownPaths(tf.name, known_paths) 1136 else: 1137 common.ZipWriteStr(output_zip, "patch/" + sf.name + ".p", d) 1138 patch_list.append((tf, sf, tf.size, common.sha1(d).hexdigest())) 1139 largest_source_size = max(largest_source_size, sf.size) 1140 1141 self.largest_source_size = largest_source_size 1142 1143 def EmitVerification(self, script): 1144 so_far = 0 1145 for tf, sf, _, _ in self.patch_list: 1146 if tf.name != sf.name: 1147 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1148 script.PatchCheck("/"+sf.name, tf.sha1, sf.sha1) 1149 so_far += sf.size 1150 return so_far 1151 1152 def EmitExplicitTargetVerification(self, script): 1153 for fn, _, sha1 in self.verbatim_targets: 1154 if fn[-1] != "/": 1155 script.FileCheck("/"+fn, sha1) 1156 for tf, _, _, _ in self.patch_list: 1157 script.FileCheck(tf.name, tf.sha1) 1158 1159 def RemoveUnneededFiles(self, script, extras=()): 1160 file_list = ["/" + i[0] for i in self.verbatim_targets] 1161 file_list += ["/" + i for i in self.source_data 1162 if i not in self.target_data and i not in self.renames] 1163 file_list += list(extras) 1164 # Sort the list in descending order, which removes all the files first 1165 # before attempting to remove the folder. (Bug: 22960996) 1166 script.DeleteFiles(sorted(file_list, reverse=True)) 1167 1168 def TotalPatchSize(self): 1169 return sum(i[1].size for i in self.patch_list) 1170 1171 def EmitPatches(self, script, total_patch_size, so_far): 1172 self.deferred_patch_list = deferred_patch_list = [] 1173 for item in self.patch_list: 1174 tf, sf, _, _ = item 1175 if tf.name == "system/build.prop": 1176 deferred_patch_list.append(item) 1177 continue 1178 if sf.name != tf.name: 1179 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1180 script.ApplyPatch("/" + sf.name, "-", tf.size, tf.sha1, sf.sha1, 1181 "patch/" + sf.name + ".p") 1182 so_far += tf.size 1183 script.SetProgress(so_far / total_patch_size) 1184 return so_far 1185 1186 def EmitDeferredPatches(self, script): 1187 for item in self.deferred_patch_list: 1188 tf, sf, _, _ = item 1189 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, 1190 "patch/" + sf.name + ".p") 1191 script.SetPermissions("/system/build.prop", 0, 0, 0o644, None, None) 1192 1193 def EmitRenames(self, script): 1194 if len(self.renames) > 0: 1195 script.Print("Renaming files...") 1196 for src, tgt in self.renames.iteritems(): 1197 print "Renaming " + src + " to " + tgt.name 1198 script.RenameFile(src, tgt.name) 1199 1200 1201def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): 1202 target_has_recovery_patch = HasRecoveryPatch(target_zip) 1203 source_has_recovery_patch = HasRecoveryPatch(source_zip) 1204 1205 if (OPTIONS.block_based and 1206 target_has_recovery_patch and 1207 source_has_recovery_patch): 1208 return WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip) 1209 1210 source_version = OPTIONS.source_info_dict["recovery_api_version"] 1211 target_version = OPTIONS.target_info_dict["recovery_api_version"] 1212 1213 if source_version == 0: 1214 print ("WARNING: generating edify script for a source that " 1215 "can't install it.") 1216 script = edify_generator.EdifyGenerator( 1217 source_version, OPTIONS.target_info_dict, 1218 fstab=OPTIONS.source_info_dict["fstab"]) 1219 1220 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 1221 recovery_mount_options = OPTIONS.source_info_dict.get( 1222 "recovery_mount_options") 1223 oem_dict = None 1224 if oem_props is not None and len(oem_props) > 0: 1225 if OPTIONS.oem_source is None: 1226 raise common.ExternalError("OEM source required for this build") 1227 script.Mount("/oem", recovery_mount_options) 1228 oem_dict = common.LoadDictionaryFromLines( 1229 open(OPTIONS.oem_source).readlines()) 1230 1231 metadata = { 1232 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 1233 OPTIONS.source_info_dict), 1234 "post-timestamp": GetBuildProp("ro.build.date.utc", 1235 OPTIONS.target_info_dict), 1236 } 1237 1238 device_specific = common.DeviceSpecificParams( 1239 source_zip=source_zip, 1240 source_version=source_version, 1241 target_zip=target_zip, 1242 target_version=target_version, 1243 output_zip=output_zip, 1244 script=script, 1245 metadata=metadata, 1246 info_dict=OPTIONS.source_info_dict) 1247 1248 system_diff = FileDifference("system", source_zip, target_zip, output_zip) 1249 script.Mount("/system", recovery_mount_options) 1250 if HasVendorPartition(target_zip): 1251 vendor_diff = FileDifference("vendor", source_zip, target_zip, output_zip) 1252 script.Mount("/vendor", recovery_mount_options) 1253 else: 1254 vendor_diff = None 1255 1256 target_fp = CalculateFingerprint(oem_props, oem_dict, 1257 OPTIONS.target_info_dict) 1258 source_fp = CalculateFingerprint(oem_props, oem_dict, 1259 OPTIONS.source_info_dict) 1260 1261 if oem_props is None: 1262 script.AssertSomeFingerprint(source_fp, target_fp) 1263 else: 1264 script.AssertSomeThumbprint( 1265 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 1266 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 1267 1268 metadata["pre-build"] = source_fp 1269 metadata["post-build"] = target_fp 1270 1271 source_boot = common.GetBootableImage( 1272 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 1273 OPTIONS.source_info_dict) 1274 target_boot = common.GetBootableImage( 1275 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 1276 updating_boot = (not OPTIONS.two_step and 1277 (source_boot.data != target_boot.data)) 1278 1279 source_recovery = common.GetBootableImage( 1280 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 1281 OPTIONS.source_info_dict) 1282 target_recovery = common.GetBootableImage( 1283 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 1284 updating_recovery = (source_recovery.data != target_recovery.data) 1285 1286 # Here's how we divide up the progress bar: 1287 # 0.1 for verifying the start state (PatchCheck calls) 1288 # 0.8 for applying patches (ApplyPatch calls) 1289 # 0.1 for unpacking verbatim files, symlinking, and doing the 1290 # device-specific commands. 1291 1292 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 1293 device_specific.IncrementalOTA_Assertions() 1294 1295 # Two-step incremental package strategy (in chronological order, 1296 # which is *not* the order in which the generated script has 1297 # things): 1298 # 1299 # if stage is not "2/3" or "3/3": 1300 # do verification on current system 1301 # write recovery image to boot partition 1302 # set stage to "2/3" 1303 # reboot to boot partition and restart recovery 1304 # else if stage is "2/3": 1305 # write recovery image to recovery partition 1306 # set stage to "3/3" 1307 # reboot to recovery partition and restart recovery 1308 # else: 1309 # (stage must be "3/3") 1310 # perform update: 1311 # patch system files, etc. 1312 # force full install of new boot image 1313 # set up system to update recovery partition on first boot 1314 # complete script normally 1315 # (allow recovery to mark itself finished and reboot) 1316 1317 if OPTIONS.two_step: 1318 if not OPTIONS.source_info_dict.get("multistage_support", None): 1319 assert False, "two-step packages not supported by this build" 1320 fs = OPTIONS.source_info_dict["fstab"]["/misc"] 1321 assert fs.fs_type.upper() == "EMMC", \ 1322 "two-step packages only supported on devices with EMMC /misc partitions" 1323 bcb_dev = {"bcb_dev": fs.device} 1324 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 1325 script.AppendExtra(""" 1326if get_stage("%(bcb_dev)s") == "2/3" then 1327""" % bcb_dev) 1328 script.AppendExtra("sleep(20);\n") 1329 script.WriteRawImage("/recovery", "recovery.img") 1330 script.AppendExtra(""" 1331set_stage("%(bcb_dev)s", "3/3"); 1332reboot_now("%(bcb_dev)s", "recovery"); 1333else if get_stage("%(bcb_dev)s") != "3/3" then 1334""" % bcb_dev) 1335 1336 # Dump fingerprints 1337 script.Print("Source: %s" % (source_fp,)) 1338 script.Print("Target: %s" % (target_fp,)) 1339 1340 script.Print("Verifying current system...") 1341 1342 device_specific.IncrementalOTA_VerifyBegin() 1343 1344 script.ShowProgress(0.1, 0) 1345 so_far = system_diff.EmitVerification(script) 1346 if vendor_diff: 1347 so_far += vendor_diff.EmitVerification(script) 1348 1349 if updating_boot: 1350 d = common.Difference(target_boot, source_boot) 1351 _, _, d = d.ComputePatch() 1352 print "boot target: %d source: %d diff: %d" % ( 1353 target_boot.size, source_boot.size, len(d)) 1354 1355 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 1356 1357 boot_type, boot_device = common.GetTypeAndDevice( 1358 "/boot", OPTIONS.source_info_dict) 1359 1360 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 1361 (boot_type, boot_device, 1362 source_boot.size, source_boot.sha1, 1363 target_boot.size, target_boot.sha1)) 1364 so_far += source_boot.size 1365 1366 size = [] 1367 if system_diff.patch_list: 1368 size.append(system_diff.largest_source_size) 1369 if vendor_diff: 1370 if vendor_diff.patch_list: 1371 size.append(vendor_diff.largest_source_size) 1372 if size or updating_recovery or updating_boot: 1373 script.CacheFreeSpaceCheck(max(size)) 1374 1375 device_specific.IncrementalOTA_VerifyEnd() 1376 1377 if OPTIONS.two_step: 1378 script.WriteRawImage("/boot", "recovery.img") 1379 script.AppendExtra(""" 1380set_stage("%(bcb_dev)s", "2/3"); 1381reboot_now("%(bcb_dev)s", ""); 1382else 1383""" % bcb_dev) 1384 1385 script.Comment("---- start making changes here ----") 1386 1387 device_specific.IncrementalOTA_InstallBegin() 1388 1389 if OPTIONS.two_step: 1390 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 1391 script.WriteRawImage("/boot", "boot.img") 1392 print "writing full boot image (forced by two-step mode)" 1393 1394 script.Print("Removing unneeded files...") 1395 system_diff.RemoveUnneededFiles(script, ("/system/recovery.img",)) 1396 if vendor_diff: 1397 vendor_diff.RemoveUnneededFiles(script) 1398 1399 script.ShowProgress(0.8, 0) 1400 total_patch_size = 1.0 + system_diff.TotalPatchSize() 1401 if vendor_diff: 1402 total_patch_size += vendor_diff.TotalPatchSize() 1403 if updating_boot: 1404 total_patch_size += target_boot.size 1405 1406 script.Print("Patching system files...") 1407 so_far = system_diff.EmitPatches(script, total_patch_size, 0) 1408 if vendor_diff: 1409 script.Print("Patching vendor files...") 1410 so_far = vendor_diff.EmitPatches(script, total_patch_size, so_far) 1411 1412 if not OPTIONS.two_step: 1413 if updating_boot: 1414 # Produce the boot image by applying a patch to the current 1415 # contents of the boot partition, and write it back to the 1416 # partition. 1417 script.Print("Patching boot image...") 1418 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 1419 % (boot_type, boot_device, 1420 source_boot.size, source_boot.sha1, 1421 target_boot.size, target_boot.sha1), 1422 "-", 1423 target_boot.size, target_boot.sha1, 1424 source_boot.sha1, "patch/boot.img.p") 1425 so_far += target_boot.size 1426 script.SetProgress(so_far / total_patch_size) 1427 print "boot image changed; including." 1428 else: 1429 print "boot image unchanged; skipping." 1430 1431 system_items = ItemSet("system", "META/filesystem_config.txt") 1432 if vendor_diff: 1433 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 1434 1435 if updating_recovery: 1436 # Recovery is generated as a patch using both the boot image 1437 # (which contains the same linux kernel as recovery) and the file 1438 # /system/etc/recovery-resource.dat (which contains all the images 1439 # used in the recovery UI) as sources. This lets us minimize the 1440 # size of the patch, which must be included in every OTA package. 1441 # 1442 # For older builds where recovery-resource.dat is not present, we 1443 # use only the boot image as the source. 1444 1445 if not target_has_recovery_patch: 1446 def output_sink(fn, data): 1447 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 1448 system_items.Get("system/" + fn) 1449 1450 common.MakeRecoveryPatch(OPTIONS.target_tmp, output_sink, 1451 target_recovery, target_boot) 1452 script.DeleteFiles(["/system/recovery-from-boot.p", 1453 "/system/etc/recovery.img", 1454 "/system/etc/install-recovery.sh"]) 1455 print "recovery image changed; including as patch from boot." 1456 else: 1457 print "recovery image unchanged; skipping." 1458 1459 script.ShowProgress(0.1, 10) 1460 1461 target_symlinks = CopyPartitionFiles(system_items, target_zip, None) 1462 if vendor_diff: 1463 target_symlinks.extend(CopyPartitionFiles(vendor_items, target_zip, None)) 1464 1465 temp_script = script.MakeTemporary() 1466 system_items.GetMetadata(target_zip) 1467 system_items.Get("system").SetPermissions(temp_script) 1468 if vendor_diff: 1469 vendor_items.GetMetadata(target_zip) 1470 vendor_items.Get("vendor").SetPermissions(temp_script) 1471 1472 # Note that this call will mess up the trees of Items, so make sure 1473 # we're done with them. 1474 source_symlinks = CopyPartitionFiles(system_items, source_zip, None) 1475 if vendor_diff: 1476 source_symlinks.extend(CopyPartitionFiles(vendor_items, source_zip, None)) 1477 1478 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 1479 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 1480 1481 # Delete all the symlinks in source that aren't in target. This 1482 # needs to happen before verbatim files are unpacked, in case a 1483 # symlink in the source is replaced by a real file in the target. 1484 1485 # If a symlink in the source will be replaced by a regular file, we cannot 1486 # delete the symlink/file in case the package gets applied again. For such 1487 # a symlink, we prepend a sha1_check() to detect if it has been updated. 1488 # (Bug: 23646151) 1489 replaced_symlinks = dict() 1490 if system_diff: 1491 for i in system_diff.verbatim_targets: 1492 replaced_symlinks["/%s" % (i[0],)] = i[2] 1493 if vendor_diff: 1494 for i in vendor_diff.verbatim_targets: 1495 replaced_symlinks["/%s" % (i[0],)] = i[2] 1496 1497 if system_diff: 1498 for tf in system_diff.renames.values(): 1499 replaced_symlinks["/%s" % (tf.name,)] = tf.sha1 1500 if vendor_diff: 1501 for tf in vendor_diff.renames.values(): 1502 replaced_symlinks["/%s" % (tf.name,)] = tf.sha1 1503 1504 always_delete = [] 1505 may_delete = [] 1506 for dest, link in source_symlinks: 1507 if link not in target_symlinks_d: 1508 if link in replaced_symlinks: 1509 may_delete.append((link, replaced_symlinks[link])) 1510 else: 1511 always_delete.append(link) 1512 script.DeleteFiles(always_delete) 1513 script.DeleteFilesIfNotMatching(may_delete) 1514 1515 if system_diff.verbatim_targets: 1516 script.Print("Unpacking new system files...") 1517 script.UnpackPackageDir("system", "/system") 1518 if vendor_diff and vendor_diff.verbatim_targets: 1519 script.Print("Unpacking new vendor files...") 1520 script.UnpackPackageDir("vendor", "/vendor") 1521 1522 if updating_recovery and not target_has_recovery_patch: 1523 script.Print("Unpacking new recovery...") 1524 script.UnpackPackageDir("recovery", "/system") 1525 1526 system_diff.EmitRenames(script) 1527 if vendor_diff: 1528 vendor_diff.EmitRenames(script) 1529 1530 script.Print("Symlinks and permissions...") 1531 1532 # Create all the symlinks that don't already exist, or point to 1533 # somewhere different than what we want. Delete each symlink before 1534 # creating it, since the 'symlink' command won't overwrite. 1535 to_create = [] 1536 for dest, link in target_symlinks: 1537 if link in source_symlinks_d: 1538 if dest != source_symlinks_d[link]: 1539 to_create.append((dest, link)) 1540 else: 1541 to_create.append((dest, link)) 1542 script.DeleteFiles([i[1] for i in to_create]) 1543 script.MakeSymlinks(to_create) 1544 1545 # Now that the symlinks are created, we can set all the 1546 # permissions. 1547 script.AppendScript(temp_script) 1548 1549 # Do device-specific installation (eg, write radio image). 1550 device_specific.IncrementalOTA_InstallEnd() 1551 1552 if OPTIONS.extra_script is not None: 1553 script.AppendExtra(OPTIONS.extra_script) 1554 1555 # Patch the build.prop file last, so if something fails but the 1556 # device can still come up, it appears to be the old build and will 1557 # get set the OTA package again to retry. 1558 script.Print("Patching remaining system files...") 1559 system_diff.EmitDeferredPatches(script) 1560 1561 if OPTIONS.wipe_user_data: 1562 script.Print("Erasing user data...") 1563 script.FormatPartition("/data") 1564 1565 if OPTIONS.two_step: 1566 script.AppendExtra(""" 1567set_stage("%(bcb_dev)s", ""); 1568endif; 1569endif; 1570""" % bcb_dev) 1571 1572 if OPTIONS.verify and system_diff: 1573 script.Print("Remounting and verifying system partition files...") 1574 script.Unmount("/system") 1575 script.Mount("/system", recovery_mount_options) 1576 system_diff.EmitExplicitTargetVerification(script) 1577 1578 if OPTIONS.verify and vendor_diff: 1579 script.Print("Remounting and verifying vendor partition files...") 1580 script.Unmount("/vendor") 1581 script.Mount("/vendor", recovery_mount_options) 1582 vendor_diff.EmitExplicitTargetVerification(script) 1583 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 1584 1585 WriteMetadata(metadata, output_zip) 1586 1587 1588def main(argv): 1589 1590 def option_handler(o, a): 1591 if o == "--board_config": 1592 pass # deprecated 1593 elif o in ("-k", "--package_key"): 1594 OPTIONS.package_key = a 1595 elif o in ("-i", "--incremental_from"): 1596 OPTIONS.incremental_source = a 1597 elif o == "--full_radio": 1598 OPTIONS.full_radio = True 1599 elif o == "--full_bootloader": 1600 OPTIONS.full_bootloader = True 1601 elif o in ("-w", "--wipe_user_data"): 1602 OPTIONS.wipe_user_data = True 1603 elif o in ("-n", "--no_prereq"): 1604 OPTIONS.omit_prereq = True 1605 elif o in ("-o", "--oem_settings"): 1606 OPTIONS.oem_source = a 1607 elif o in ("-e", "--extra_script"): 1608 OPTIONS.extra_script = a 1609 elif o in ("-a", "--aslr_mode"): 1610 if a in ("on", "On", "true", "True", "yes", "Yes"): 1611 OPTIONS.aslr_mode = True 1612 else: 1613 OPTIONS.aslr_mode = False 1614 elif o in ("-t", "--worker_threads"): 1615 if a.isdigit(): 1616 OPTIONS.worker_threads = int(a) 1617 else: 1618 raise ValueError("Cannot parse value %r for option %r - only " 1619 "integers are allowed." % (a, o)) 1620 elif o in ("-2", "--two_step"): 1621 OPTIONS.two_step = True 1622 elif o == "--no_signing": 1623 OPTIONS.no_signing = True 1624 elif o == "--verify": 1625 OPTIONS.verify = True 1626 elif o == "--block": 1627 OPTIONS.block_based = True 1628 elif o in ("-b", "--binary"): 1629 OPTIONS.updater_binary = a 1630 elif o in ("--no_fallback_to_full",): 1631 OPTIONS.fallback_to_full = False 1632 elif o == "--stash_threshold": 1633 try: 1634 OPTIONS.stash_threshold = float(a) 1635 except ValueError: 1636 raise ValueError("Cannot parse value %r for option %r - expecting " 1637 "a float" % (a, o)) 1638 elif o == "--gen_verify": 1639 OPTIONS.gen_verify = True 1640 elif o == "--log_diff": 1641 OPTIONS.log_diff = a 1642 else: 1643 return False 1644 return True 1645 1646 args = common.ParseOptions(argv, __doc__, 1647 extra_opts="b:k:i:d:wne:t:a:2o:", 1648 extra_long_opts=[ 1649 "board_config=", 1650 "package_key=", 1651 "incremental_from=", 1652 "full_radio", 1653 "full_bootloader", 1654 "wipe_user_data", 1655 "no_prereq", 1656 "extra_script=", 1657 "worker_threads=", 1658 "aslr_mode=", 1659 "two_step", 1660 "no_signing", 1661 "block", 1662 "binary=", 1663 "oem_settings=", 1664 "verify", 1665 "no_fallback_to_full", 1666 "stash_threshold=", 1667 "gen_verify", 1668 "log_diff=", 1669 ], extra_option_handler=option_handler) 1670 1671 if len(args) != 2: 1672 common.Usage(__doc__) 1673 sys.exit(1) 1674 1675 if OPTIONS.extra_script is not None: 1676 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 1677 1678 print "unzipping target target-files..." 1679 OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0]) 1680 1681 OPTIONS.target_tmp = OPTIONS.input_tmp 1682 OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.target_tmp) 1683 1684 if OPTIONS.verbose: 1685 print "--- target info ---" 1686 common.DumpInfoDict(OPTIONS.info_dict) 1687 1688 # If the caller explicitly specified the device-specific extensions 1689 # path via -s/--device_specific, use that. Otherwise, use 1690 # META/releasetools.py if it is present in the target target_files. 1691 # Otherwise, take the path of the file from 'tool_extensions' in the 1692 # info dict and look for that in the local filesystem, relative to 1693 # the current directory. 1694 1695 if OPTIONS.device_specific is None: 1696 from_input = os.path.join(OPTIONS.input_tmp, "META", "releasetools.py") 1697 if os.path.exists(from_input): 1698 print "(using device-specific extensions from target_files)" 1699 OPTIONS.device_specific = from_input 1700 else: 1701 OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None) 1702 1703 if OPTIONS.device_specific is not None: 1704 OPTIONS.device_specific = os.path.abspath(OPTIONS.device_specific) 1705 1706 ab_update = OPTIONS.info_dict.get("ab_update") == "true" 1707 1708 if OPTIONS.info_dict.get("no_recovery") == "true" and not ab_update: 1709 raise common.ExternalError( 1710 "--- target build has specified no recovery ---") 1711 1712 # Use the default key to sign the package if not specified with package_key. 1713 if not OPTIONS.no_signing: 1714 if OPTIONS.package_key is None: 1715 OPTIONS.package_key = OPTIONS.info_dict.get( 1716 "default_system_dev_certificate", 1717 "build/target/product/security/testkey") 1718 1719 # Set up the output zip. Create a temporary zip file if signing is needed. 1720 if OPTIONS.no_signing: 1721 if os.path.exists(args[1]): 1722 os.unlink(args[1]) 1723 output_zip = zipfile.ZipFile(args[1], "w", 1724 compression=zipfile.ZIP_DEFLATED) 1725 else: 1726 temp_zip_file = tempfile.NamedTemporaryFile() 1727 output_zip = zipfile.ZipFile(temp_zip_file, "w", 1728 compression=zipfile.ZIP_DEFLATED) 1729 1730 # Non A/B OTAs rely on /cache partition to store temporary files. 1731 cache_size = OPTIONS.info_dict.get("cache_size", None) 1732 if cache_size is None and not ab_update: 1733 print "--- can't determine the cache partition size ---" 1734 OPTIONS.cache_size = cache_size 1735 1736 # Generate a verify package. 1737 if OPTIONS.gen_verify: 1738 WriteVerifyPackage(input_zip, output_zip) 1739 1740 # Generate a full OTA. 1741 elif OPTIONS.incremental_source is None: 1742 if ab_update: 1743 # TODO: Pending for b/25715402. 1744 pass 1745 else: 1746 WriteFullOTAPackage(input_zip, output_zip) 1747 1748 # Generate an incremental OTA. It will fall back to generate a full OTA on 1749 # failure unless no_fallback_to_full is specified. 1750 else: 1751 print "unzipping source target-files..." 1752 OPTIONS.source_tmp, source_zip = common.UnzipTemp( 1753 OPTIONS.incremental_source) 1754 OPTIONS.target_info_dict = OPTIONS.info_dict 1755 OPTIONS.source_info_dict = common.LoadInfoDict(source_zip, 1756 OPTIONS.source_tmp) 1757 if OPTIONS.verbose: 1758 print "--- source info ---" 1759 common.DumpInfoDict(OPTIONS.source_info_dict) 1760 try: 1761 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) 1762 if OPTIONS.log_diff: 1763 out_file = open(OPTIONS.log_diff, 'w') 1764 import target_files_diff 1765 target_files_diff.recursiveDiff('', 1766 OPTIONS.source_tmp, 1767 OPTIONS.input_tmp, 1768 out_file) 1769 out_file.close() 1770 except ValueError: 1771 if not OPTIONS.fallback_to_full: 1772 raise 1773 print "--- failed to build incremental; falling back to full ---" 1774 OPTIONS.incremental_source = None 1775 WriteFullOTAPackage(input_zip, output_zip) 1776 1777 common.ZipClose(output_zip) 1778 1779 # Sign the generated zip package unless no_signing is specified. 1780 if not OPTIONS.no_signing: 1781 SignOutput(temp_zip_file.name, args[1]) 1782 temp_zip_file.close() 1783 1784 print "done." 1785 1786 1787if __name__ == '__main__': 1788 try: 1789 common.CloseInheritedPipes() 1790 main(sys.argv[1:]) 1791 except common.ExternalError as e: 1792 print 1793 print " ERROR: %s" % (e,) 1794 print 1795 sys.exit(1) 1796 finally: 1797 common.Cleanup() 1798