ota_from_target_files.py revision 9bc6bb23b5bf64635275041fecace64635f41fba
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 --board_config <file> 25 Deprecated. 26 27 -k (--package_key) <key> Key to use to sign the package (default is 28 the value of default_system_dev_certificate from the input 29 target-files's META/misc_info.txt, or 30 "build/target/product/security/testkey" if that value is not 31 specified). 32 33 For incremental OTAs, the default value is based on the source 34 target-file, not the target build. 35 36 -i (--incremental_from) <file> 37 Generate an incremental OTA using the given target-files zip as 38 the starting build. 39 40 --full_radio 41 When generating an incremental OTA, always include a full copy of 42 radio image. This option is only meaningful when -i is specified, 43 because a full radio is always included in a full OTA if applicable. 44 45 --full_bootloader 46 Similar to --full_radio. When generating an incremental OTA, always 47 include a full copy of bootloader image. 48 49 -v (--verify) 50 Remount and verify the checksums of the files written to the 51 system and vendor (if used) partitions. Incremental builds only. 52 53 -o (--oem_settings) <file> 54 Use the file to specify the expected OEM-specific properties 55 on the OEM partition of the intended device. 56 57 -w (--wipe_user_data) 58 Generate an OTA package that will wipe the user data partition 59 when installed. 60 61 -n (--no_prereq) 62 Omit the timestamp prereq check normally included at the top of 63 the build scripts (used for developer OTA packages which 64 legitimately need to go back and forth). 65 66 -e (--extra_script) <file> 67 Insert the contents of file at the end of the update script. 68 69 -a (--aslr_mode) <on|off> 70 Specify whether to turn on ASLR for the package (on by default). 71 72 -2 (--two_step) 73 Generate a 'two-step' OTA package, where recovery is updated 74 first, so that any changes made to the system partition are done 75 using the new recovery (new kernel, etc.). 76 77 --block 78 Generate a block-based OTA if possible. Will fall back to a 79 file-based OTA if the target_files is older and doesn't support 80 block-based OTAs. 81 82 -b (--binary) <file> 83 Use the given binary as the update-binary in the output package, 84 instead of the binary in the build's target_files. Use for 85 development only. 86 87 -t (--worker_threads) <int> 88 Specifies the number of worker-threads that will be used when 89 generating patches for incremental updates (defaults to 3). 90 91 --stash_threshold <float> 92 Specifies the threshold that will be used to compute the maximum 93 allowed stash size (defaults to 0.8). 94 95 --gen_verify 96 Generate an OTA package that verifies the partitions. 97""" 98 99import sys 100 101if sys.hexversion < 0x02070000: 102 print >> sys.stderr, "Python 2.7 or newer is required." 103 sys.exit(1) 104 105import multiprocessing 106import os 107import tempfile 108import zipfile 109 110import common 111import edify_generator 112import sparse_img 113 114OPTIONS = common.OPTIONS 115OPTIONS.package_key = None 116OPTIONS.incremental_source = None 117OPTIONS.verify = False 118OPTIONS.require_verbatim = set() 119OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 120OPTIONS.patch_threshold = 0.95 121OPTIONS.wipe_user_data = False 122OPTIONS.omit_prereq = False 123OPTIONS.extra_script = None 124OPTIONS.aslr_mode = True 125OPTIONS.worker_threads = multiprocessing.cpu_count() // 2 126if OPTIONS.worker_threads == 0: 127 OPTIONS.worker_threads = 1 128OPTIONS.two_step = False 129OPTIONS.no_signing = False 130OPTIONS.block_based = False 131OPTIONS.updater_binary = None 132OPTIONS.oem_source = None 133OPTIONS.fallback_to_full = True 134OPTIONS.full_radio = False 135OPTIONS.full_bootloader = False 136# Stash size cannot exceed cache_size * threshold. 137OPTIONS.cache_size = None 138OPTIONS.stash_threshold = 0.8 139OPTIONS.gen_verify = False 140 141def MostPopularKey(d, default): 142 """Given a dict, return the key corresponding to the largest 143 value. Returns 'default' if the dict is empty.""" 144 x = [(v, k) for (k, v) in d.iteritems()] 145 if not x: 146 return default 147 x.sort() 148 return x[-1][1] 149 150 151def IsSymlink(info): 152 """Return true if the zipfile.ZipInfo object passed in represents a 153 symlink.""" 154 return (info.external_attr >> 16) & 0o770000 == 0o120000 155 156def IsRegular(info): 157 """Return true if the zipfile.ZipInfo object passed in represents a 158 regular file.""" 159 return (info.external_attr >> 16) & 0o770000 == 0o100000 160 161def ClosestFileMatch(src, tgtfiles, existing): 162 """Returns the closest file match between a source file and list 163 of potential matches. The exact filename match is preferred, 164 then the sha1 is searched for, and finally a file with the same 165 basename is evaluated. Rename support in the updater-binary is 166 required for the latter checks to be used.""" 167 168 result = tgtfiles.get("path:" + src.name) 169 if result is not None: 170 return result 171 172 if not OPTIONS.target_info_dict.get("update_rename_support", False): 173 return None 174 175 if src.size < 1000: 176 return None 177 178 result = tgtfiles.get("sha1:" + src.sha1) 179 if result is not None and existing.get(result.name) is None: 180 return result 181 result = tgtfiles.get("file:" + src.name.split("/")[-1]) 182 if result is not None and existing.get(result.name) is None: 183 return result 184 return None 185 186class ItemSet(object): 187 def __init__(self, partition, fs_config): 188 self.partition = partition 189 self.fs_config = fs_config 190 self.ITEMS = {} 191 192 def Get(self, name, is_dir=False): 193 if name not in self.ITEMS: 194 self.ITEMS[name] = Item(self, name, is_dir=is_dir) 195 return self.ITEMS[name] 196 197 def GetMetadata(self, input_zip): 198 # The target_files contains a record of what the uid, 199 # gid, and mode are supposed to be. 200 output = input_zip.read(self.fs_config) 201 202 for line in output.split("\n"): 203 if not line: 204 continue 205 columns = line.split() 206 name, uid, gid, mode = columns[:4] 207 selabel = None 208 capabilities = None 209 210 # After the first 4 columns, there are a series of key=value 211 # pairs. Extract out the fields we care about. 212 for element in columns[4:]: 213 key, value = element.split("=") 214 if key == "selabel": 215 selabel = value 216 if key == "capabilities": 217 capabilities = value 218 219 i = self.ITEMS.get(name, None) 220 if i is not None: 221 i.uid = int(uid) 222 i.gid = int(gid) 223 i.mode = int(mode, 8) 224 i.selabel = selabel 225 i.capabilities = capabilities 226 if i.is_dir: 227 i.children.sort(key=lambda i: i.name) 228 229 # Set metadata for the files generated by this script. For full recovery 230 # image at system/etc/recovery.img, it will be taken care by fs_config. 231 i = self.ITEMS.get("system/recovery-from-boot.p", None) 232 if i: 233 i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0o644, None, None 234 i = self.ITEMS.get("system/etc/install-recovery.sh", None) 235 if i: 236 i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0o544, None, None 237 238 239class Item(object): 240 """Items represent the metadata (user, group, mode) of files and 241 directories in the system image.""" 242 def __init__(self, itemset, name, is_dir=False): 243 self.itemset = itemset 244 self.name = name 245 self.uid = None 246 self.gid = None 247 self.mode = None 248 self.selabel = None 249 self.capabilities = None 250 self.is_dir = is_dir 251 self.descendants = None 252 self.best_subtree = None 253 254 if name: 255 self.parent = itemset.Get(os.path.dirname(name), is_dir=True) 256 self.parent.children.append(self) 257 else: 258 self.parent = None 259 if self.is_dir: 260 self.children = [] 261 262 def Dump(self, indent=0): 263 if self.uid is not None: 264 print "%s%s %d %d %o" % ( 265 " " * indent, self.name, self.uid, self.gid, self.mode) 266 else: 267 print "%s%s %s %s %s" % ( 268 " " * indent, self.name, self.uid, self.gid, self.mode) 269 if self.is_dir: 270 print "%s%s" % (" "*indent, self.descendants) 271 print "%s%s" % (" "*indent, self.best_subtree) 272 for i in self.children: 273 i.Dump(indent=indent+1) 274 275 def CountChildMetadata(self): 276 """Count up the (uid, gid, mode, selabel, capabilities) tuples for 277 all children and determine the best strategy for using set_perm_recursive 278 and set_perm to correctly chown/chmod all the files to their desired 279 values. Recursively calls itself for all descendants. 280 281 Returns a dict of {(uid, gid, dmode, fmode, selabel, capabilities): count} 282 counting up all descendants of this node. (dmode or fmode may be None.) 283 Also sets the best_subtree of each directory Item to the (uid, gid, dmode, 284 fmode, selabel, capabilities) tuple that will match the most descendants of 285 that Item. 286 """ 287 288 assert self.is_dir 289 key = (self.uid, self.gid, self.mode, None, self.selabel, 290 self.capabilities) 291 self.descendants = {key: 1} 292 d = self.descendants 293 for i in self.children: 294 if i.is_dir: 295 for k, v in i.CountChildMetadata().iteritems(): 296 d[k] = d.get(k, 0) + v 297 else: 298 k = (i.uid, i.gid, None, i.mode, i.selabel, i.capabilities) 299 d[k] = d.get(k, 0) + 1 300 301 # Find the (uid, gid, dmode, fmode, selabel, capabilities) 302 # tuple that matches the most descendants. 303 304 # First, find the (uid, gid) pair that matches the most 305 # descendants. 306 ug = {} 307 for (uid, gid, _, _, _, _), count in d.iteritems(): 308 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 309 ug = MostPopularKey(ug, (0, 0)) 310 311 # Now find the dmode, fmode, selabel, and capabilities that match 312 # the most descendants with that (uid, gid), and choose those. 313 best_dmode = (0, 0o755) 314 best_fmode = (0, 0o644) 315 best_selabel = (0, None) 316 best_capabilities = (0, None) 317 for k, count in d.iteritems(): 318 if k[:2] != ug: 319 continue 320 if k[2] is not None and count >= best_dmode[0]: 321 best_dmode = (count, k[2]) 322 if k[3] is not None and count >= best_fmode[0]: 323 best_fmode = (count, k[3]) 324 if k[4] is not None and count >= best_selabel[0]: 325 best_selabel = (count, k[4]) 326 if k[5] is not None and count >= best_capabilities[0]: 327 best_capabilities = (count, k[5]) 328 self.best_subtree = ug + ( 329 best_dmode[1], best_fmode[1], best_selabel[1], best_capabilities[1]) 330 331 return d 332 333 def SetPermissions(self, script): 334 """Append set_perm/set_perm_recursive commands to 'script' to 335 set all permissions, users, and groups for the tree of files 336 rooted at 'self'.""" 337 338 self.CountChildMetadata() 339 340 def recurse(item, current): 341 # current is the (uid, gid, dmode, fmode, selabel, capabilities) tuple 342 # that the current item (and all its children) have already been set to. 343 # We only need to issue set_perm/set_perm_recursive commands if we're 344 # supposed to be something different. 345 if item.is_dir: 346 if current != item.best_subtree: 347 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 348 current = item.best_subtree 349 350 if item.uid != current[0] or item.gid != current[1] or \ 351 item.mode != current[2] or item.selabel != current[4] or \ 352 item.capabilities != current[5]: 353 script.SetPermissions("/"+item.name, item.uid, item.gid, 354 item.mode, item.selabel, item.capabilities) 355 356 for i in item.children: 357 recurse(i, current) 358 else: 359 if item.uid != current[0] or item.gid != current[1] or \ 360 item.mode != current[3] or item.selabel != current[4] or \ 361 item.capabilities != current[5]: 362 script.SetPermissions("/"+item.name, item.uid, item.gid, 363 item.mode, item.selabel, item.capabilities) 364 365 recurse(self, (-1, -1, -1, -1, None, None)) 366 367 368def CopyPartitionFiles(itemset, input_zip, output_zip=None, substitute=None): 369 """Copies files for the partition in the input zip to the output 370 zip. Populates the Item class with their metadata, and returns a 371 list of symlinks. output_zip may be None, in which case the copy is 372 skipped (but the other side effects still happen). substitute is an 373 optional dict of {output filename: contents} to be output instead of 374 certain input files. 375 """ 376 377 symlinks = [] 378 379 partition = itemset.partition 380 381 for info in input_zip.infolist(): 382 prefix = partition.upper() + "/" 383 if info.filename.startswith(prefix): 384 basefilename = info.filename[len(prefix):] 385 if IsSymlink(info): 386 symlinks.append((input_zip.read(info.filename), 387 "/" + partition + "/" + basefilename)) 388 else: 389 import copy 390 info2 = copy.copy(info) 391 fn = info2.filename = partition + "/" + basefilename 392 if substitute and fn in substitute and substitute[fn] is None: 393 continue 394 if output_zip is not None: 395 if substitute and fn in substitute: 396 data = substitute[fn] 397 else: 398 data = input_zip.read(info.filename) 399 common.ZipWriteStr(output_zip, info2, data) 400 if fn.endswith("/"): 401 itemset.Get(fn[:-1], is_dir=True) 402 else: 403 itemset.Get(fn) 404 405 symlinks.sort() 406 return symlinks 407 408 409def SignOutput(temp_zip_name, output_zip_name): 410 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 411 pw = key_passwords[OPTIONS.package_key] 412 413 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 414 whole_file=True) 415 416 417def AppendAssertions(script, info_dict, oem_dict=None): 418 oem_props = info_dict.get("oem_fingerprint_properties") 419 if oem_props is None or len(oem_props) == 0: 420 device = GetBuildProp("ro.product.device", info_dict) 421 script.AssertDevice(device) 422 else: 423 if oem_dict is None: 424 raise common.ExternalError( 425 "No OEM file provided to answer expected assertions") 426 for prop in oem_props.split(): 427 if oem_dict.get(prop) is None: 428 raise common.ExternalError( 429 "The OEM file is missing the property %s" % prop) 430 script.AssertOemProperty(prop, oem_dict.get(prop)) 431 432 433def HasRecoveryPatch(target_files_zip): 434 namelist = [name for name in target_files_zip.namelist()] 435 return ("SYSTEM/recovery-from-boot.p" in namelist or 436 "SYSTEM/etc/recovery.img" in namelist) 437 438def HasVendorPartition(target_files_zip): 439 try: 440 target_files_zip.getinfo("VENDOR/") 441 return True 442 except KeyError: 443 return False 444 445def GetOemProperty(name, oem_props, oem_dict, info_dict): 446 if oem_props is not None and name in oem_props: 447 return oem_dict[name] 448 return GetBuildProp(name, info_dict) 449 450 451def CalculateFingerprint(oem_props, oem_dict, info_dict): 452 if oem_props is None: 453 return GetBuildProp("ro.build.fingerprint", info_dict) 454 return "%s/%s/%s:%s" % ( 455 GetOemProperty("ro.product.brand", oem_props, oem_dict, info_dict), 456 GetOemProperty("ro.product.name", oem_props, oem_dict, info_dict), 457 GetOemProperty("ro.product.device", oem_props, oem_dict, info_dict), 458 GetBuildProp("ro.build.thumbprint", info_dict)) 459 460 461def GetImage(which, tmpdir, info_dict): 462 # Return an image object (suitable for passing to BlockImageDiff) 463 # for the 'which' partition (most be "system" or "vendor"). If a 464 # prebuilt image and file map are found in tmpdir they are used, 465 # otherwise they are reconstructed from the individual files. 466 467 assert which in ("system", "vendor") 468 469 path = os.path.join(tmpdir, "IMAGES", which + ".img") 470 mappath = os.path.join(tmpdir, "IMAGES", which + ".map") 471 if os.path.exists(path) and os.path.exists(mappath): 472 print "using %s.img from target-files" % (which,) 473 # This is a 'new' target-files, which already has the image in it. 474 475 else: 476 print "building %s.img from target-files" % (which,) 477 478 # This is an 'old' target-files, which does not contain images 479 # already built. Build them. 480 481 mappath = tempfile.mkstemp()[1] 482 OPTIONS.tempfiles.append(mappath) 483 484 import add_img_to_target_files 485 if which == "system": 486 path = add_img_to_target_files.BuildSystem( 487 tmpdir, info_dict, block_list=mappath) 488 elif which == "vendor": 489 path = add_img_to_target_files.BuildVendor( 490 tmpdir, info_dict, block_list=mappath) 491 492 # Bug: http://b/20939131 493 # In ext4 filesystems, block 0 might be changed even being mounted 494 # R/O. We add it to clobbered_blocks so that it will be written to the 495 # target unconditionally. Note that they are still part of care_map. 496 clobbered_blocks = "0" 497 498 return sparse_img.SparseImage(path, mappath, clobbered_blocks) 499 500 501def WriteFullOTAPackage(input_zip, output_zip): 502 # TODO: how to determine this? We don't know what version it will 503 # be installed on top of. For now, we expect the API just won't 504 # change very often. Similarly for fstab, it might have changed 505 # in the target build. 506 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 507 508 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 509 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 510 oem_dict = None 511 if oem_props is not None and len(oem_props) > 0: 512 if OPTIONS.oem_source is None: 513 raise common.ExternalError("OEM source required for this build") 514 script.Mount("/oem", recovery_mount_options) 515 oem_dict = common.LoadDictionaryFromLines( 516 open(OPTIONS.oem_source).readlines()) 517 518 metadata = { 519 "post-build": CalculateFingerprint(oem_props, oem_dict, 520 OPTIONS.info_dict), 521 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 522 OPTIONS.info_dict), 523 "post-timestamp": GetBuildProp("ro.build.date.utc", OPTIONS.info_dict), 524 } 525 526 device_specific = common.DeviceSpecificParams( 527 input_zip=input_zip, 528 input_version=OPTIONS.info_dict["recovery_api_version"], 529 output_zip=output_zip, 530 script=script, 531 input_tmp=OPTIONS.input_tmp, 532 metadata=metadata, 533 info_dict=OPTIONS.info_dict) 534 535 has_recovery_patch = HasRecoveryPatch(input_zip) 536 block_based = OPTIONS.block_based and has_recovery_patch 537 538 if not OPTIONS.omit_prereq: 539 ts = GetBuildProp("ro.build.date.utc", OPTIONS.info_dict) 540 ts_text = GetBuildProp("ro.build.date", OPTIONS.info_dict) 541 script.AssertOlderBuild(ts, ts_text) 542 543 AppendAssertions(script, OPTIONS.info_dict, oem_dict) 544 device_specific.FullOTA_Assertions() 545 546 # Two-step package strategy (in chronological order, which is *not* 547 # the order in which the generated script has things): 548 # 549 # if stage is not "2/3" or "3/3": 550 # write recovery image to boot partition 551 # set stage to "2/3" 552 # reboot to boot partition and restart recovery 553 # else if stage is "2/3": 554 # write recovery image to recovery partition 555 # set stage to "3/3" 556 # reboot to recovery partition and restart recovery 557 # else: 558 # (stage must be "3/3") 559 # set stage to "" 560 # do normal full package installation: 561 # wipe and install system, boot image, etc. 562 # set up system to update recovery partition on first boot 563 # complete script normally 564 # (allow recovery to mark itself finished and reboot) 565 566 recovery_img = common.GetBootableImage("recovery.img", "recovery.img", 567 OPTIONS.input_tmp, "RECOVERY") 568 if OPTIONS.two_step: 569 if not OPTIONS.info_dict.get("multistage_support", None): 570 assert False, "two-step packages not supported by this build" 571 fs = OPTIONS.info_dict["fstab"]["/misc"] 572 assert fs.fs_type.upper() == "EMMC", \ 573 "two-step packages only supported on devices with EMMC /misc partitions" 574 bcb_dev = {"bcb_dev": fs.device} 575 common.ZipWriteStr(output_zip, "recovery.img", recovery_img.data) 576 script.AppendExtra(""" 577if get_stage("%(bcb_dev)s") == "2/3" then 578""" % bcb_dev) 579 script.WriteRawImage("/recovery", "recovery.img") 580 script.AppendExtra(""" 581set_stage("%(bcb_dev)s", "3/3"); 582reboot_now("%(bcb_dev)s", "recovery"); 583else if get_stage("%(bcb_dev)s") == "3/3" then 584""" % bcb_dev) 585 586 # Dump fingerprints 587 script.Print("Target: %s" % CalculateFingerprint( 588 oem_props, oem_dict, OPTIONS.info_dict)) 589 590 device_specific.FullOTA_InstallBegin() 591 592 system_progress = 0.75 593 594 if OPTIONS.wipe_user_data: 595 system_progress -= 0.1 596 if HasVendorPartition(input_zip): 597 system_progress -= 0.1 598 599 # Place a copy of file_contexts.bin into the OTA package which will be used 600 # by the recovery program. 601 if "selinux_fc" in OPTIONS.info_dict: 602 WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip) 603 604 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 605 606 system_items = ItemSet("system", "META/filesystem_config.txt") 607 script.ShowProgress(system_progress, 0) 608 609 if block_based: 610 # Full OTA is done as an "incremental" against an empty source 611 # image. This has the effect of writing new data from the package 612 # to the entire partition, but lets us reuse the updater code that 613 # writes incrementals to do it. 614 system_tgt = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict) 615 system_tgt.ResetFileMap() 616 system_diff = common.BlockDifference("system", system_tgt, src=None) 617 system_diff.WriteScript(script, output_zip) 618 else: 619 script.FormatPartition("/system") 620 script.Mount("/system", recovery_mount_options) 621 if not has_recovery_patch: 622 script.UnpackPackageDir("recovery", "/system") 623 script.UnpackPackageDir("system", "/system") 624 625 symlinks = CopyPartitionFiles(system_items, input_zip, output_zip) 626 script.MakeSymlinks(symlinks) 627 628 boot_img = common.GetBootableImage( 629 "boot.img", "boot.img", OPTIONS.input_tmp, "BOOT") 630 631 if not block_based: 632 def output_sink(fn, data): 633 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 634 system_items.Get("system/" + fn) 635 636 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, 637 recovery_img, boot_img) 638 639 system_items.GetMetadata(input_zip) 640 system_items.Get("system").SetPermissions(script) 641 642 if HasVendorPartition(input_zip): 643 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 644 script.ShowProgress(0.1, 0) 645 646 if block_based: 647 vendor_tgt = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict) 648 vendor_tgt.ResetFileMap() 649 vendor_diff = common.BlockDifference("vendor", vendor_tgt) 650 vendor_diff.WriteScript(script, output_zip) 651 else: 652 script.FormatPartition("/vendor") 653 script.Mount("/vendor", recovery_mount_options) 654 script.UnpackPackageDir("vendor", "/vendor") 655 656 symlinks = CopyPartitionFiles(vendor_items, input_zip, output_zip) 657 script.MakeSymlinks(symlinks) 658 659 vendor_items.GetMetadata(input_zip) 660 vendor_items.Get("vendor").SetPermissions(script) 661 662 common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict) 663 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 664 665 script.ShowProgress(0.05, 5) 666 script.WriteRawImage("/boot", "boot.img") 667 668 script.ShowProgress(0.2, 10) 669 device_specific.FullOTA_InstallEnd() 670 671 if OPTIONS.extra_script is not None: 672 script.AppendExtra(OPTIONS.extra_script) 673 674 script.UnmountAll() 675 676 if OPTIONS.wipe_user_data: 677 script.ShowProgress(0.1, 10) 678 script.FormatPartition("/data") 679 680 if OPTIONS.two_step: 681 script.AppendExtra(""" 682set_stage("%(bcb_dev)s", ""); 683""" % bcb_dev) 684 script.AppendExtra("else\n") 685 script.WriteRawImage("/boot", "recovery.img") 686 script.AppendExtra(""" 687set_stage("%(bcb_dev)s", "2/3"); 688reboot_now("%(bcb_dev)s", ""); 689endif; 690endif; 691""" % bcb_dev) 692 script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary) 693 WriteMetadata(metadata, output_zip) 694 695 696def WritePolicyConfig(file_name, output_zip): 697 common.ZipWrite(output_zip, file_name, os.path.basename(file_name)) 698 699 700def WriteMetadata(metadata, output_zip): 701 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata", 702 "".join(["%s=%s\n" % kv 703 for kv in sorted(metadata.iteritems())])) 704 705 706def LoadPartitionFiles(z, partition): 707 """Load all the files from the given partition in a given target-files 708 ZipFile, and return a dict of {filename: File object}.""" 709 out = {} 710 prefix = partition.upper() + "/" 711 for info in z.infolist(): 712 if info.filename.startswith(prefix) and not IsSymlink(info): 713 basefilename = info.filename[len(prefix):] 714 fn = partition + "/" + basefilename 715 data = z.read(info.filename) 716 out[fn] = common.File(fn, data) 717 return out 718 719 720def GetBuildProp(prop, info_dict): 721 """Return the fingerprint of the build of a given target-files info_dict.""" 722 try: 723 return info_dict.get("build.prop", {})[prop] 724 except KeyError: 725 raise common.ExternalError("couldn't find %s in build.prop" % (prop,)) 726 727 728def AddToKnownPaths(filename, known_paths): 729 if filename[-1] == "/": 730 return 731 dirs = filename.split("/")[:-1] 732 while len(dirs) > 0: 733 path = "/".join(dirs) 734 if path in known_paths: 735 break 736 known_paths.add(path) 737 dirs.pop() 738 739 740def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 741 # TODO(tbao): We should factor out the common parts between 742 # WriteBlockIncrementalOTAPackage() and WriteIncrementalOTAPackage(). 743 source_version = OPTIONS.source_info_dict["recovery_api_version"] 744 target_version = OPTIONS.target_info_dict["recovery_api_version"] 745 746 if source_version == 0: 747 print ("WARNING: generating edify script for a source that " 748 "can't install it.") 749 script = edify_generator.EdifyGenerator( 750 source_version, OPTIONS.target_info_dict, 751 fstab=OPTIONS.source_info_dict["fstab"]) 752 753 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 754 recovery_mount_options = OPTIONS.source_info_dict.get( 755 "recovery_mount_options") 756 oem_dict = None 757 if oem_props is not None and len(oem_props) > 0: 758 if OPTIONS.oem_source is None: 759 raise common.ExternalError("OEM source required for this build") 760 script.Mount("/oem", recovery_mount_options) 761 oem_dict = common.LoadDictionaryFromLines( 762 open(OPTIONS.oem_source).readlines()) 763 764 metadata = { 765 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 766 OPTIONS.source_info_dict), 767 "post-timestamp": GetBuildProp("ro.build.date.utc", 768 OPTIONS.target_info_dict), 769 } 770 771 device_specific = common.DeviceSpecificParams( 772 source_zip=source_zip, 773 source_version=source_version, 774 target_zip=target_zip, 775 target_version=target_version, 776 output_zip=output_zip, 777 script=script, 778 metadata=metadata, 779 info_dict=OPTIONS.source_info_dict) 780 781 source_fp = CalculateFingerprint(oem_props, oem_dict, 782 OPTIONS.source_info_dict) 783 target_fp = CalculateFingerprint(oem_props, oem_dict, 784 OPTIONS.target_info_dict) 785 metadata["pre-build"] = source_fp 786 metadata["post-build"] = target_fp 787 788 source_boot = common.GetBootableImage( 789 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 790 OPTIONS.source_info_dict) 791 target_boot = common.GetBootableImage( 792 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 793 updating_boot = (not OPTIONS.two_step and 794 (source_boot.data != target_boot.data)) 795 796 target_recovery = common.GetBootableImage( 797 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 798 799 system_src = GetImage("system", OPTIONS.source_tmp, OPTIONS.source_info_dict) 800 system_tgt = GetImage("system", OPTIONS.target_tmp, OPTIONS.target_info_dict) 801 802 blockimgdiff_version = 1 803 if OPTIONS.info_dict: 804 blockimgdiff_version = max( 805 int(i) for i in 806 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(",")) 807 808 system_diff = common.BlockDifference("system", system_tgt, system_src, 809 version=blockimgdiff_version) 810 811 if HasVendorPartition(target_zip): 812 if not HasVendorPartition(source_zip): 813 raise RuntimeError("can't generate incremental that adds /vendor") 814 vendor_src = GetImage("vendor", OPTIONS.source_tmp, 815 OPTIONS.source_info_dict) 816 vendor_tgt = GetImage("vendor", OPTIONS.target_tmp, 817 OPTIONS.target_info_dict) 818 vendor_diff = common.BlockDifference("vendor", vendor_tgt, vendor_src, 819 version=blockimgdiff_version) 820 else: 821 vendor_diff = None 822 823 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 824 device_specific.IncrementalOTA_Assertions() 825 826 # Two-step incremental package strategy (in chronological order, 827 # which is *not* the order in which the generated script has 828 # things): 829 # 830 # if stage is not "2/3" or "3/3": 831 # do verification on current system 832 # write recovery image to boot partition 833 # set stage to "2/3" 834 # reboot to boot partition and restart recovery 835 # else if stage is "2/3": 836 # write recovery image to recovery partition 837 # set stage to "3/3" 838 # reboot to recovery partition and restart recovery 839 # else: 840 # (stage must be "3/3") 841 # perform update: 842 # patch system files, etc. 843 # force full install of new boot image 844 # set up system to update recovery partition on first boot 845 # complete script normally 846 # (allow recovery to mark itself finished and reboot) 847 848 if OPTIONS.two_step: 849 if not OPTIONS.source_info_dict.get("multistage_support", None): 850 assert False, "two-step packages not supported by this build" 851 fs = OPTIONS.source_info_dict["fstab"]["/misc"] 852 assert fs.fs_type.upper() == "EMMC", \ 853 "two-step packages only supported on devices with EMMC /misc partitions" 854 bcb_dev = {"bcb_dev": fs.device} 855 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 856 script.AppendExtra(""" 857if get_stage("%(bcb_dev)s") == "2/3" then 858""" % bcb_dev) 859 script.AppendExtra("sleep(20);\n") 860 script.WriteRawImage("/recovery", "recovery.img") 861 script.AppendExtra(""" 862set_stage("%(bcb_dev)s", "3/3"); 863reboot_now("%(bcb_dev)s", "recovery"); 864else if get_stage("%(bcb_dev)s") != "3/3" then 865""" % bcb_dev) 866 867 # Dump fingerprints 868 script.Print("Source: %s" % CalculateFingerprint( 869 oem_props, oem_dict, OPTIONS.source_info_dict)) 870 script.Print("Target: %s" % CalculateFingerprint( 871 oem_props, oem_dict, OPTIONS.target_info_dict)) 872 873 script.Print("Verifying current system...") 874 875 device_specific.IncrementalOTA_VerifyBegin() 876 877 if oem_props is None: 878 # When blockimgdiff version is less than 3 (non-resumable block-based OTA), 879 # patching on a device that's already on the target build will damage the 880 # system. Because operations like move don't check the block state, they 881 # always apply the changes unconditionally. 882 if blockimgdiff_version <= 2: 883 script.AssertSomeFingerprint(source_fp) 884 else: 885 script.AssertSomeFingerprint(source_fp, target_fp) 886 else: 887 if blockimgdiff_version <= 2: 888 script.AssertSomeThumbprint( 889 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 890 else: 891 script.AssertSomeThumbprint( 892 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 893 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 894 895 if updating_boot: 896 boot_type, boot_device = common.GetTypeAndDevice( 897 "/boot", OPTIONS.source_info_dict) 898 d = common.Difference(target_boot, source_boot) 899 _, _, d = d.ComputePatch() 900 if d is None: 901 include_full_boot = True 902 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 903 else: 904 include_full_boot = False 905 906 print "boot target: %d source: %d diff: %d" % ( 907 target_boot.size, source_boot.size, len(d)) 908 909 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 910 911 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 912 (boot_type, boot_device, 913 source_boot.size, source_boot.sha1, 914 target_boot.size, target_boot.sha1)) 915 916 device_specific.IncrementalOTA_VerifyEnd() 917 918 if OPTIONS.two_step: 919 script.WriteRawImage("/boot", "recovery.img") 920 script.AppendExtra(""" 921set_stage("%(bcb_dev)s", "2/3"); 922reboot_now("%(bcb_dev)s", ""); 923else 924""" % bcb_dev) 925 926 # Verify the existing partitions. 927 system_diff.WriteVerifyScript(script) 928 if vendor_diff: 929 vendor_diff.WriteVerifyScript(script) 930 931 script.Comment("---- start making changes here ----") 932 933 device_specific.IncrementalOTA_InstallBegin() 934 935 system_diff.WriteScript(script, output_zip, 936 progress=0.8 if vendor_diff else 0.9) 937 938 if vendor_diff: 939 vendor_diff.WriteScript(script, output_zip, progress=0.1) 940 941 if OPTIONS.two_step: 942 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 943 script.WriteRawImage("/boot", "boot.img") 944 print "writing full boot image (forced by two-step mode)" 945 946 if not OPTIONS.two_step: 947 if updating_boot: 948 if include_full_boot: 949 print "boot image changed; including full." 950 script.Print("Installing boot image...") 951 script.WriteRawImage("/boot", "boot.img") 952 else: 953 # Produce the boot image by applying a patch to the current 954 # contents of the boot partition, and write it back to the 955 # partition. 956 print "boot image changed; including patch." 957 script.Print("Patching boot image...") 958 script.ShowProgress(0.1, 10) 959 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 960 % (boot_type, boot_device, 961 source_boot.size, source_boot.sha1, 962 target_boot.size, target_boot.sha1), 963 "-", 964 target_boot.size, target_boot.sha1, 965 source_boot.sha1, "patch/boot.img.p") 966 else: 967 print "boot image unchanged; skipping." 968 969 # Do device-specific installation (eg, write radio image). 970 device_specific.IncrementalOTA_InstallEnd() 971 972 if OPTIONS.extra_script is not None: 973 script.AppendExtra(OPTIONS.extra_script) 974 975 if OPTIONS.wipe_user_data: 976 script.Print("Erasing user data...") 977 script.FormatPartition("/data") 978 979 if OPTIONS.two_step: 980 script.AppendExtra(""" 981set_stage("%(bcb_dev)s", ""); 982endif; 983endif; 984""" % bcb_dev) 985 986 script.SetProgress(1) 987 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 988 WriteMetadata(metadata, output_zip) 989 990 991def WriteVerifyPackage(input_zip, output_zip): 992 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 993 994 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 995 recovery_mount_options = OPTIONS.info_dict.get( 996 "recovery_mount_options") 997 oem_dict = None 998 if oem_props is not None and len(oem_props) > 0: 999 if OPTIONS.oem_source is None: 1000 raise common.ExternalError("OEM source required for this build") 1001 script.Mount("/oem", recovery_mount_options) 1002 oem_dict = common.LoadDictionaryFromLines( 1003 open(OPTIONS.oem_source).readlines()) 1004 1005 target_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.info_dict) 1006 metadata = { 1007 "post-build": target_fp, 1008 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 1009 OPTIONS.info_dict), 1010 "post-timestamp": GetBuildProp("ro.build.date.utc", OPTIONS.info_dict), 1011 } 1012 1013 device_specific = common.DeviceSpecificParams( 1014 input_zip=input_zip, 1015 input_version=OPTIONS.info_dict["recovery_api_version"], 1016 output_zip=output_zip, 1017 script=script, 1018 input_tmp=OPTIONS.input_tmp, 1019 metadata=metadata, 1020 info_dict=OPTIONS.info_dict) 1021 1022 AppendAssertions(script, OPTIONS.info_dict, oem_dict) 1023 1024 script.Print("Verifying device images against %s..." % target_fp) 1025 script.AppendExtra("") 1026 1027 script.Print("Verifying boot...") 1028 boot_img = common.GetBootableImage( 1029 "boot.img", "boot.img", OPTIONS.input_tmp, "BOOT") 1030 boot_type, boot_device = common.GetTypeAndDevice( 1031 "/boot", OPTIONS.info_dict) 1032 script.Verify("%s:%s:%d:%s" % ( 1033 boot_type, boot_device, boot_img.size, boot_img.sha1)) 1034 script.AppendExtra("") 1035 1036 script.Print("Verifying recovery...") 1037 recovery_img = common.GetBootableImage( 1038 "recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY") 1039 recovery_type, recovery_device = common.GetTypeAndDevice( 1040 "/recovery", OPTIONS.info_dict) 1041 script.Verify("%s:%s:%d:%s" % ( 1042 recovery_type, recovery_device, recovery_img.size, recovery_img.sha1)) 1043 script.AppendExtra("") 1044 1045 system_tgt = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict) 1046 system_tgt.ResetFileMap() 1047 system_diff = common.BlockDifference("system", system_tgt, src=None) 1048 system_diff.WriteStrictVerifyScript(script) 1049 1050 if HasVendorPartition(input_zip): 1051 vendor_tgt = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict) 1052 vendor_tgt.ResetFileMap() 1053 vendor_diff = common.BlockDifference("vendor", vendor_tgt, src=None) 1054 vendor_diff.WriteStrictVerifyScript(script) 1055 1056 # Device specific partitions, such as radio, bootloader and etc. 1057 device_specific.VerifyOTA_Assertions() 1058 1059 script.SetProgress(1.0) 1060 script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary) 1061 WriteMetadata(metadata, output_zip) 1062 1063 1064class FileDifference(object): 1065 def __init__(self, partition, source_zip, target_zip, output_zip): 1066 self.deferred_patch_list = None 1067 print "Loading target..." 1068 self.target_data = target_data = LoadPartitionFiles(target_zip, partition) 1069 print "Loading source..." 1070 self.source_data = source_data = LoadPartitionFiles(source_zip, partition) 1071 1072 self.verbatim_targets = verbatim_targets = [] 1073 self.patch_list = patch_list = [] 1074 diffs = [] 1075 self.renames = renames = {} 1076 known_paths = set() 1077 largest_source_size = 0 1078 1079 matching_file_cache = {} 1080 for fn, sf in source_data.items(): 1081 assert fn == sf.name 1082 matching_file_cache["path:" + fn] = sf 1083 if fn in target_data.keys(): 1084 AddToKnownPaths(fn, known_paths) 1085 # Only allow eligibility for filename/sha matching 1086 # if there isn't a perfect path match. 1087 if target_data.get(sf.name) is None: 1088 matching_file_cache["file:" + fn.split("/")[-1]] = sf 1089 matching_file_cache["sha:" + sf.sha1] = sf 1090 1091 for fn in sorted(target_data.keys()): 1092 tf = target_data[fn] 1093 assert fn == tf.name 1094 sf = ClosestFileMatch(tf, matching_file_cache, renames) 1095 if sf is not None and sf.name != tf.name: 1096 print "File has moved from " + sf.name + " to " + tf.name 1097 renames[sf.name] = tf 1098 1099 if sf is None or fn in OPTIONS.require_verbatim: 1100 # This file should be included verbatim 1101 if fn in OPTIONS.prohibit_verbatim: 1102 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 1103 print "send", fn, "verbatim" 1104 tf.AddToZip(output_zip) 1105 verbatim_targets.append((fn, tf.size, tf.sha1)) 1106 if fn in target_data.keys(): 1107 AddToKnownPaths(fn, known_paths) 1108 elif tf.sha1 != sf.sha1: 1109 # File is different; consider sending as a patch 1110 diffs.append(common.Difference(tf, sf)) 1111 else: 1112 # Target file data identical to source (may still be renamed) 1113 pass 1114 1115 common.ComputeDifferences(diffs) 1116 1117 for diff in diffs: 1118 tf, sf, d = diff.GetPatch() 1119 path = "/".join(tf.name.split("/")[:-1]) 1120 if d is None or len(d) > tf.size * OPTIONS.patch_threshold or \ 1121 path not in known_paths: 1122 # patch is almost as big as the file; don't bother patching 1123 # or a patch + rename cannot take place due to the target 1124 # directory not existing 1125 tf.AddToZip(output_zip) 1126 verbatim_targets.append((tf.name, tf.size, tf.sha1)) 1127 if sf.name in renames: 1128 del renames[sf.name] 1129 AddToKnownPaths(tf.name, known_paths) 1130 else: 1131 common.ZipWriteStr(output_zip, "patch/" + sf.name + ".p", d) 1132 patch_list.append((tf, sf, tf.size, common.sha1(d).hexdigest())) 1133 largest_source_size = max(largest_source_size, sf.size) 1134 1135 self.largest_source_size = largest_source_size 1136 1137 def EmitVerification(self, script): 1138 so_far = 0 1139 for tf, sf, _, _ in self.patch_list: 1140 if tf.name != sf.name: 1141 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1142 script.PatchCheck("/"+sf.name, tf.sha1, sf.sha1) 1143 so_far += sf.size 1144 return so_far 1145 1146 def EmitExplicitTargetVerification(self, script): 1147 for fn, _, sha1 in self.verbatim_targets: 1148 if fn[-1] != "/": 1149 script.FileCheck("/"+fn, sha1) 1150 for tf, _, _, _ in self.patch_list: 1151 script.FileCheck(tf.name, tf.sha1) 1152 1153 def RemoveUnneededFiles(self, script, extras=()): 1154 file_list = ["/" + i[0] for i in self.verbatim_targets] 1155 file_list += ["/" + i for i in self.source_data 1156 if i not in self.target_data and i not in self.renames] 1157 file_list += list(extras) 1158 # Sort the list in descending order, which removes all the files first 1159 # before attempting to remove the folder. (Bug: 22960996) 1160 script.DeleteFiles(sorted(file_list, reverse=True)) 1161 1162 def TotalPatchSize(self): 1163 return sum(i[1].size for i in self.patch_list) 1164 1165 def EmitPatches(self, script, total_patch_size, so_far): 1166 self.deferred_patch_list = deferred_patch_list = [] 1167 for item in self.patch_list: 1168 tf, sf, _, _ = item 1169 if tf.name == "system/build.prop": 1170 deferred_patch_list.append(item) 1171 continue 1172 if sf.name != tf.name: 1173 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1174 script.ApplyPatch("/" + sf.name, "-", tf.size, tf.sha1, sf.sha1, 1175 "patch/" + sf.name + ".p") 1176 so_far += tf.size 1177 script.SetProgress(so_far / total_patch_size) 1178 return so_far 1179 1180 def EmitDeferredPatches(self, script): 1181 for item in self.deferred_patch_list: 1182 tf, sf, _, _ = item 1183 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, 1184 "patch/" + sf.name + ".p") 1185 script.SetPermissions("/system/build.prop", 0, 0, 0o644, None, None) 1186 1187 def EmitRenames(self, script): 1188 if len(self.renames) > 0: 1189 script.Print("Renaming files...") 1190 for src, tgt in self.renames.iteritems(): 1191 print "Renaming " + src + " to " + tgt.name 1192 script.RenameFile(src, tgt.name) 1193 1194 1195def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): 1196 target_has_recovery_patch = HasRecoveryPatch(target_zip) 1197 source_has_recovery_patch = HasRecoveryPatch(source_zip) 1198 1199 if (OPTIONS.block_based and 1200 target_has_recovery_patch and 1201 source_has_recovery_patch): 1202 return WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip) 1203 1204 source_version = OPTIONS.source_info_dict["recovery_api_version"] 1205 target_version = OPTIONS.target_info_dict["recovery_api_version"] 1206 1207 if source_version == 0: 1208 print ("WARNING: generating edify script for a source that " 1209 "can't install it.") 1210 script = edify_generator.EdifyGenerator( 1211 source_version, OPTIONS.target_info_dict, 1212 fstab=OPTIONS.source_info_dict["fstab"]) 1213 1214 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 1215 recovery_mount_options = OPTIONS.source_info_dict.get( 1216 "recovery_mount_options") 1217 oem_dict = None 1218 if oem_props is not None and len(oem_props) > 0: 1219 if OPTIONS.oem_source is None: 1220 raise common.ExternalError("OEM source required for this build") 1221 script.Mount("/oem", recovery_mount_options) 1222 oem_dict = common.LoadDictionaryFromLines( 1223 open(OPTIONS.oem_source).readlines()) 1224 1225 metadata = { 1226 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 1227 OPTIONS.source_info_dict), 1228 "post-timestamp": GetBuildProp("ro.build.date.utc", 1229 OPTIONS.target_info_dict), 1230 } 1231 1232 device_specific = common.DeviceSpecificParams( 1233 source_zip=source_zip, 1234 source_version=source_version, 1235 target_zip=target_zip, 1236 target_version=target_version, 1237 output_zip=output_zip, 1238 script=script, 1239 metadata=metadata, 1240 info_dict=OPTIONS.source_info_dict) 1241 1242 system_diff = FileDifference("system", source_zip, target_zip, output_zip) 1243 script.Mount("/system", recovery_mount_options) 1244 if HasVendorPartition(target_zip): 1245 vendor_diff = FileDifference("vendor", source_zip, target_zip, output_zip) 1246 script.Mount("/vendor", recovery_mount_options) 1247 else: 1248 vendor_diff = None 1249 1250 target_fp = CalculateFingerprint(oem_props, oem_dict, 1251 OPTIONS.target_info_dict) 1252 source_fp = CalculateFingerprint(oem_props, oem_dict, 1253 OPTIONS.source_info_dict) 1254 1255 if oem_props is None: 1256 script.AssertSomeFingerprint(source_fp, target_fp) 1257 else: 1258 script.AssertSomeThumbprint( 1259 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 1260 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 1261 1262 metadata["pre-build"] = source_fp 1263 metadata["post-build"] = target_fp 1264 1265 source_boot = common.GetBootableImage( 1266 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 1267 OPTIONS.source_info_dict) 1268 target_boot = common.GetBootableImage( 1269 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 1270 updating_boot = (not OPTIONS.two_step and 1271 (source_boot.data != target_boot.data)) 1272 1273 source_recovery = common.GetBootableImage( 1274 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 1275 OPTIONS.source_info_dict) 1276 target_recovery = common.GetBootableImage( 1277 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 1278 updating_recovery = (source_recovery.data != target_recovery.data) 1279 1280 # Here's how we divide up the progress bar: 1281 # 0.1 for verifying the start state (PatchCheck calls) 1282 # 0.8 for applying patches (ApplyPatch calls) 1283 # 0.1 for unpacking verbatim files, symlinking, and doing the 1284 # device-specific commands. 1285 1286 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 1287 device_specific.IncrementalOTA_Assertions() 1288 1289 # Two-step incremental package strategy (in chronological order, 1290 # which is *not* the order in which the generated script has 1291 # things): 1292 # 1293 # if stage is not "2/3" or "3/3": 1294 # do verification on current system 1295 # write recovery image to boot partition 1296 # set stage to "2/3" 1297 # reboot to boot partition and restart recovery 1298 # else if stage is "2/3": 1299 # write recovery image to recovery partition 1300 # set stage to "3/3" 1301 # reboot to recovery partition and restart recovery 1302 # else: 1303 # (stage must be "3/3") 1304 # perform update: 1305 # patch system files, etc. 1306 # force full install of new boot image 1307 # set up system to update recovery partition on first boot 1308 # complete script normally 1309 # (allow recovery to mark itself finished and reboot) 1310 1311 if OPTIONS.two_step: 1312 if not OPTIONS.source_info_dict.get("multistage_support", None): 1313 assert False, "two-step packages not supported by this build" 1314 fs = OPTIONS.source_info_dict["fstab"]["/misc"] 1315 assert fs.fs_type.upper() == "EMMC", \ 1316 "two-step packages only supported on devices with EMMC /misc partitions" 1317 bcb_dev = {"bcb_dev": fs.device} 1318 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 1319 script.AppendExtra(""" 1320if get_stage("%(bcb_dev)s") == "2/3" then 1321""" % bcb_dev) 1322 script.AppendExtra("sleep(20);\n") 1323 script.WriteRawImage("/recovery", "recovery.img") 1324 script.AppendExtra(""" 1325set_stage("%(bcb_dev)s", "3/3"); 1326reboot_now("%(bcb_dev)s", "recovery"); 1327else if get_stage("%(bcb_dev)s") != "3/3" then 1328""" % bcb_dev) 1329 1330 # Dump fingerprints 1331 script.Print("Source: %s" % (source_fp,)) 1332 script.Print("Target: %s" % (target_fp,)) 1333 1334 script.Print("Verifying current system...") 1335 1336 device_specific.IncrementalOTA_VerifyBegin() 1337 1338 script.ShowProgress(0.1, 0) 1339 so_far = system_diff.EmitVerification(script) 1340 if vendor_diff: 1341 so_far += vendor_diff.EmitVerification(script) 1342 1343 if updating_boot: 1344 d = common.Difference(target_boot, source_boot) 1345 _, _, d = d.ComputePatch() 1346 print "boot target: %d source: %d diff: %d" % ( 1347 target_boot.size, source_boot.size, len(d)) 1348 1349 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 1350 1351 boot_type, boot_device = common.GetTypeAndDevice( 1352 "/boot", OPTIONS.source_info_dict) 1353 1354 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 1355 (boot_type, boot_device, 1356 source_boot.size, source_boot.sha1, 1357 target_boot.size, target_boot.sha1)) 1358 so_far += source_boot.size 1359 1360 size = [] 1361 if system_diff.patch_list: 1362 size.append(system_diff.largest_source_size) 1363 if vendor_diff: 1364 if vendor_diff.patch_list: 1365 size.append(vendor_diff.largest_source_size) 1366 if size or updating_recovery or updating_boot: 1367 script.CacheFreeSpaceCheck(max(size)) 1368 1369 device_specific.IncrementalOTA_VerifyEnd() 1370 1371 if OPTIONS.two_step: 1372 script.WriteRawImage("/boot", "recovery.img") 1373 script.AppendExtra(""" 1374set_stage("%(bcb_dev)s", "2/3"); 1375reboot_now("%(bcb_dev)s", ""); 1376else 1377""" % bcb_dev) 1378 1379 script.Comment("---- start making changes here ----") 1380 1381 device_specific.IncrementalOTA_InstallBegin() 1382 1383 if OPTIONS.two_step: 1384 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 1385 script.WriteRawImage("/boot", "boot.img") 1386 print "writing full boot image (forced by two-step mode)" 1387 1388 script.Print("Removing unneeded files...") 1389 system_diff.RemoveUnneededFiles(script, ("/system/recovery.img",)) 1390 if vendor_diff: 1391 vendor_diff.RemoveUnneededFiles(script) 1392 1393 script.ShowProgress(0.8, 0) 1394 total_patch_size = 1.0 + system_diff.TotalPatchSize() 1395 if vendor_diff: 1396 total_patch_size += vendor_diff.TotalPatchSize() 1397 if updating_boot: 1398 total_patch_size += target_boot.size 1399 1400 script.Print("Patching system files...") 1401 so_far = system_diff.EmitPatches(script, total_patch_size, 0) 1402 if vendor_diff: 1403 script.Print("Patching vendor files...") 1404 so_far = vendor_diff.EmitPatches(script, total_patch_size, so_far) 1405 1406 if not OPTIONS.two_step: 1407 if updating_boot: 1408 # Produce the boot image by applying a patch to the current 1409 # contents of the boot partition, and write it back to the 1410 # partition. 1411 script.Print("Patching boot image...") 1412 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 1413 % (boot_type, boot_device, 1414 source_boot.size, source_boot.sha1, 1415 target_boot.size, target_boot.sha1), 1416 "-", 1417 target_boot.size, target_boot.sha1, 1418 source_boot.sha1, "patch/boot.img.p") 1419 so_far += target_boot.size 1420 script.SetProgress(so_far / total_patch_size) 1421 print "boot image changed; including." 1422 else: 1423 print "boot image unchanged; skipping." 1424 1425 system_items = ItemSet("system", "META/filesystem_config.txt") 1426 if vendor_diff: 1427 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 1428 1429 if updating_recovery: 1430 # Recovery is generated as a patch using both the boot image 1431 # (which contains the same linux kernel as recovery) and the file 1432 # /system/etc/recovery-resource.dat (which contains all the images 1433 # used in the recovery UI) as sources. This lets us minimize the 1434 # size of the patch, which must be included in every OTA package. 1435 # 1436 # For older builds where recovery-resource.dat is not present, we 1437 # use only the boot image as the source. 1438 1439 if not target_has_recovery_patch: 1440 def output_sink(fn, data): 1441 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 1442 system_items.Get("system/" + fn) 1443 1444 common.MakeRecoveryPatch(OPTIONS.target_tmp, output_sink, 1445 target_recovery, target_boot) 1446 script.DeleteFiles(["/system/recovery-from-boot.p", 1447 "/system/etc/recovery.img", 1448 "/system/etc/install-recovery.sh"]) 1449 print "recovery image changed; including as patch from boot." 1450 else: 1451 print "recovery image unchanged; skipping." 1452 1453 script.ShowProgress(0.1, 10) 1454 1455 target_symlinks = CopyPartitionFiles(system_items, target_zip, None) 1456 if vendor_diff: 1457 target_symlinks.extend(CopyPartitionFiles(vendor_items, target_zip, None)) 1458 1459 temp_script = script.MakeTemporary() 1460 system_items.GetMetadata(target_zip) 1461 system_items.Get("system").SetPermissions(temp_script) 1462 if vendor_diff: 1463 vendor_items.GetMetadata(target_zip) 1464 vendor_items.Get("vendor").SetPermissions(temp_script) 1465 1466 # Note that this call will mess up the trees of Items, so make sure 1467 # we're done with them. 1468 source_symlinks = CopyPartitionFiles(system_items, source_zip, None) 1469 if vendor_diff: 1470 source_symlinks.extend(CopyPartitionFiles(vendor_items, source_zip, None)) 1471 1472 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 1473 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 1474 1475 # Delete all the symlinks in source that aren't in target. This 1476 # needs to happen before verbatim files are unpacked, in case a 1477 # symlink in the source is replaced by a real file in the target. 1478 1479 # If a symlink in the source will be replaced by a regular file, we cannot 1480 # delete the symlink/file in case the package gets applied again. For such 1481 # a symlink, we prepend a sha1_check() to detect if it has been updated. 1482 # (Bug: 23646151) 1483 replaced_symlinks = dict() 1484 if system_diff: 1485 for i in system_diff.verbatim_targets: 1486 replaced_symlinks["/%s" % (i[0],)] = i[2] 1487 if vendor_diff: 1488 for i in vendor_diff.verbatim_targets: 1489 replaced_symlinks["/%s" % (i[0],)] = i[2] 1490 1491 if system_diff: 1492 for tf in system_diff.renames.values(): 1493 replaced_symlinks["/%s" % (tf.name,)] = tf.sha1 1494 if vendor_diff: 1495 for tf in vendor_diff.renames.values(): 1496 replaced_symlinks["/%s" % (tf.name,)] = tf.sha1 1497 1498 always_delete = [] 1499 may_delete = [] 1500 for dest, link in source_symlinks: 1501 if link not in target_symlinks_d: 1502 if link in replaced_symlinks: 1503 may_delete.append((link, replaced_symlinks[link])) 1504 else: 1505 always_delete.append(link) 1506 script.DeleteFiles(always_delete) 1507 script.DeleteFilesIfNotMatching(may_delete) 1508 1509 if system_diff.verbatim_targets: 1510 script.Print("Unpacking new system files...") 1511 script.UnpackPackageDir("system", "/system") 1512 if vendor_diff and vendor_diff.verbatim_targets: 1513 script.Print("Unpacking new vendor files...") 1514 script.UnpackPackageDir("vendor", "/vendor") 1515 1516 if updating_recovery and not target_has_recovery_patch: 1517 script.Print("Unpacking new recovery...") 1518 script.UnpackPackageDir("recovery", "/system") 1519 1520 system_diff.EmitRenames(script) 1521 if vendor_diff: 1522 vendor_diff.EmitRenames(script) 1523 1524 script.Print("Symlinks and permissions...") 1525 1526 # Create all the symlinks that don't already exist, or point to 1527 # somewhere different than what we want. Delete each symlink before 1528 # creating it, since the 'symlink' command won't overwrite. 1529 to_create = [] 1530 for dest, link in target_symlinks: 1531 if link in source_symlinks_d: 1532 if dest != source_symlinks_d[link]: 1533 to_create.append((dest, link)) 1534 else: 1535 to_create.append((dest, link)) 1536 script.DeleteFiles([i[1] for i in to_create]) 1537 script.MakeSymlinks(to_create) 1538 1539 # Now that the symlinks are created, we can set all the 1540 # permissions. 1541 script.AppendScript(temp_script) 1542 1543 # Do device-specific installation (eg, write radio image). 1544 device_specific.IncrementalOTA_InstallEnd() 1545 1546 if OPTIONS.extra_script is not None: 1547 script.AppendExtra(OPTIONS.extra_script) 1548 1549 # Patch the build.prop file last, so if something fails but the 1550 # device can still come up, it appears to be the old build and will 1551 # get set the OTA package again to retry. 1552 script.Print("Patching remaining system files...") 1553 system_diff.EmitDeferredPatches(script) 1554 1555 if OPTIONS.wipe_user_data: 1556 script.Print("Erasing user data...") 1557 script.FormatPartition("/data") 1558 1559 if OPTIONS.two_step: 1560 script.AppendExtra(""" 1561set_stage("%(bcb_dev)s", ""); 1562endif; 1563endif; 1564""" % bcb_dev) 1565 1566 if OPTIONS.verify and system_diff: 1567 script.Print("Remounting and verifying system partition files...") 1568 script.Unmount("/system") 1569 script.Mount("/system") 1570 system_diff.EmitExplicitTargetVerification(script) 1571 1572 if OPTIONS.verify and vendor_diff: 1573 script.Print("Remounting and verifying vendor partition files...") 1574 script.Unmount("/vendor") 1575 script.Mount("/vendor") 1576 vendor_diff.EmitExplicitTargetVerification(script) 1577 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 1578 1579 WriteMetadata(metadata, output_zip) 1580 1581 1582def main(argv): 1583 1584 def option_handler(o, a): 1585 if o == "--board_config": 1586 pass # deprecated 1587 elif o in ("-k", "--package_key"): 1588 OPTIONS.package_key = a 1589 elif o in ("-i", "--incremental_from"): 1590 OPTIONS.incremental_source = a 1591 elif o == "--full_radio": 1592 OPTIONS.full_radio = True 1593 elif o == "--full_bootloader": 1594 OPTIONS.full_bootloader = True 1595 elif o in ("-w", "--wipe_user_data"): 1596 OPTIONS.wipe_user_data = True 1597 elif o in ("-n", "--no_prereq"): 1598 OPTIONS.omit_prereq = True 1599 elif o in ("-o", "--oem_settings"): 1600 OPTIONS.oem_source = a 1601 elif o in ("-e", "--extra_script"): 1602 OPTIONS.extra_script = a 1603 elif o in ("-a", "--aslr_mode"): 1604 if a in ("on", "On", "true", "True", "yes", "Yes"): 1605 OPTIONS.aslr_mode = True 1606 else: 1607 OPTIONS.aslr_mode = False 1608 elif o in ("-t", "--worker_threads"): 1609 if a.isdigit(): 1610 OPTIONS.worker_threads = int(a) 1611 else: 1612 raise ValueError("Cannot parse value %r for option %r - only " 1613 "integers are allowed." % (a, o)) 1614 elif o in ("-2", "--two_step"): 1615 OPTIONS.two_step = True 1616 elif o == "--no_signing": 1617 OPTIONS.no_signing = True 1618 elif o == "--verify": 1619 OPTIONS.verify = True 1620 elif o == "--block": 1621 OPTIONS.block_based = True 1622 elif o in ("-b", "--binary"): 1623 OPTIONS.updater_binary = a 1624 elif o in ("--no_fallback_to_full",): 1625 OPTIONS.fallback_to_full = False 1626 elif o == "--stash_threshold": 1627 try: 1628 OPTIONS.stash_threshold = float(a) 1629 except ValueError: 1630 raise ValueError("Cannot parse value %r for option %r - expecting " 1631 "a float" % (a, o)) 1632 elif o == "--gen_verify": 1633 OPTIONS.gen_verify = True 1634 else: 1635 return False 1636 return True 1637 1638 args = common.ParseOptions(argv, __doc__, 1639 extra_opts="b:k:i:d:wne:t:a:2o:", 1640 extra_long_opts=[ 1641 "board_config=", 1642 "package_key=", 1643 "incremental_from=", 1644 "full_radio", 1645 "full_bootloader", 1646 "wipe_user_data", 1647 "no_prereq", 1648 "extra_script=", 1649 "worker_threads=", 1650 "aslr_mode=", 1651 "two_step", 1652 "no_signing", 1653 "block", 1654 "binary=", 1655 "oem_settings=", 1656 "verify", 1657 "no_fallback_to_full", 1658 "stash_threshold=", 1659 "gen_verify" 1660 ], extra_option_handler=option_handler) 1661 1662 if len(args) != 2: 1663 common.Usage(__doc__) 1664 sys.exit(1) 1665 1666 if OPTIONS.extra_script is not None: 1667 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 1668 1669 print "unzipping target target-files..." 1670 OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0]) 1671 1672 OPTIONS.target_tmp = OPTIONS.input_tmp 1673 OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.target_tmp) 1674 1675 if OPTIONS.verbose: 1676 print "--- target info ---" 1677 common.DumpInfoDict(OPTIONS.info_dict) 1678 1679 # If the caller explicitly specified the device-specific extensions 1680 # path via -s/--device_specific, use that. Otherwise, use 1681 # META/releasetools.py if it is present in the target target_files. 1682 # Otherwise, take the path of the file from 'tool_extensions' in the 1683 # info dict and look for that in the local filesystem, relative to 1684 # the current directory. 1685 1686 if OPTIONS.device_specific is None: 1687 from_input = os.path.join(OPTIONS.input_tmp, "META", "releasetools.py") 1688 if os.path.exists(from_input): 1689 print "(using device-specific extensions from target_files)" 1690 OPTIONS.device_specific = from_input 1691 else: 1692 OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None) 1693 1694 if OPTIONS.device_specific is not None: 1695 OPTIONS.device_specific = os.path.abspath(OPTIONS.device_specific) 1696 1697 if OPTIONS.info_dict.get("no_recovery") == "true": 1698 raise common.ExternalError( 1699 "--- target build has specified no recovery ---") 1700 1701 # Use the default key to sign the package if not specified with package_key. 1702 if not OPTIONS.no_signing: 1703 if OPTIONS.package_key is None: 1704 OPTIONS.package_key = OPTIONS.info_dict.get( 1705 "default_system_dev_certificate", 1706 "build/target/product/security/testkey") 1707 1708 # Set up the output zip. Create a temporary zip file if signing is needed. 1709 if OPTIONS.no_signing: 1710 if os.path.exists(args[1]): 1711 os.unlink(args[1]) 1712 output_zip = zipfile.ZipFile(args[1], "w", 1713 compression=zipfile.ZIP_DEFLATED) 1714 else: 1715 temp_zip_file = tempfile.NamedTemporaryFile() 1716 output_zip = zipfile.ZipFile(temp_zip_file, "w", 1717 compression=zipfile.ZIP_DEFLATED) 1718 1719 cache_size = OPTIONS.info_dict.get("cache_size", None) 1720 if cache_size is None: 1721 print "--- can't determine the cache partition size ---" 1722 OPTIONS.cache_size = cache_size 1723 1724 # Generate a verify package. 1725 if OPTIONS.gen_verify: 1726 WriteVerifyPackage(input_zip, output_zip) 1727 1728 # Generate a full OTA. 1729 elif OPTIONS.incremental_source is None: 1730 WriteFullOTAPackage(input_zip, output_zip) 1731 1732 # Generate an incremental OTA. It will fall back to generate a full OTA on 1733 # failure unless no_fallback_to_full is specified. 1734 else: 1735 print "unzipping source target-files..." 1736 OPTIONS.source_tmp, source_zip = common.UnzipTemp( 1737 OPTIONS.incremental_source) 1738 OPTIONS.target_info_dict = OPTIONS.info_dict 1739 OPTIONS.source_info_dict = common.LoadInfoDict(source_zip, 1740 OPTIONS.source_tmp) 1741 if OPTIONS.verbose: 1742 print "--- source info ---" 1743 common.DumpInfoDict(OPTIONS.source_info_dict) 1744 try: 1745 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) 1746 except ValueError: 1747 if not OPTIONS.fallback_to_full: 1748 raise 1749 print "--- failed to build incremental; falling back to full ---" 1750 OPTIONS.incremental_source = None 1751 WriteFullOTAPackage(input_zip, output_zip) 1752 1753 common.ZipClose(output_zip) 1754 1755 # Sign the generated zip package unless no_signing is specified. 1756 if not OPTIONS.no_signing: 1757 SignOutput(temp_zip_file.name, args[1]) 1758 temp_zip_file.close() 1759 1760 print "done." 1761 1762 1763if __name__ == '__main__': 1764 try: 1765 common.CloseInheritedPipes() 1766 main(sys.argv[1:]) 1767 except common.ExternalError as e: 1768 print 1769 print " ERROR: %s" % (e,) 1770 print 1771 sys.exit(1) 1772 finally: 1773 common.Cleanup() 1774