ota_from_target_files.py revision 116977c7dba5ff79d1119a30966f918d72efedeb
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 --board_config <file> 25 Deprecated. 26 27 -k (--package_key) <key> Key to use to sign the package (default is 28 the value of default_system_dev_certificate from the input 29 target-files's META/misc_info.txt, or 30 "build/target/product/security/testkey" if that value is not 31 specified). 32 33 For incremental OTAs, the default value is based on the source 34 target-file, not the target build. 35 36 -i (--incremental_from) <file> 37 Generate an incremental OTA using the given target-files zip as 38 the starting build. 39 40 --full_radio 41 When generating an incremental OTA, always include a full copy of 42 radio image. This option is only meaningful when -i is specified, 43 because a full radio is always included in a full OTA if applicable. 44 45 --full_bootloader 46 When generating an incremental OTA, always include a full copy of 47 bootloader image. This option is only meaningful when -i is specified, 48 because a full bootloader is always included in a full OTA if applicable. 49 50 -v (--verify) 51 Remount and verify the checksums of the files written to the 52 system and vendor (if used) partitions. Incremental builds only. 53 54 -o (--oem_settings) <file> 55 Use the file to specify the expected OEM-specific properties 56 on the OEM partition of the intended device. 57 58 -w (--wipe_user_data) 59 Generate an OTA package that will wipe the user data partition 60 when installed. 61 62 -n (--no_prereq) 63 Omit the timestamp prereq check normally included at the top of 64 the build scripts (used for developer OTA packages which 65 legitimately need to go back and forth). 66 67 -e (--extra_script) <file> 68 Insert the contents of file at the end of the update script. 69 70 -a (--aslr_mode) <on|off> 71 Specify whether to turn on ASLR for the package (on by default). 72 73 -2 (--two_step) 74 Generate a 'two-step' OTA package, where recovery is updated 75 first, so that any changes made to the system partition are done 76 using the new recovery (new kernel, etc.). 77 78 --block 79 Generate a block-based OTA if possible. Will fall back to a 80 file-based OTA if the target_files is older and doesn't support 81 block-based OTAs. 82 83 -b (--binary) <file> 84 Use the given binary as the update-binary in the output package, 85 instead of the binary in the build's target_files. Use for 86 development only. 87 88 -t (--worker_threads) <int> 89 Specifies the number of worker-threads that will be used when 90 generating patches for incremental updates (defaults to 3). 91 92 --stash_threshold <float> 93 Specifies the threshold that will be used to compute the maximum 94 allowed stash size (defaults to 0.8). 95""" 96 97import sys 98 99if sys.hexversion < 0x02070000: 100 print >> sys.stderr, "Python 2.7 or newer is required." 101 sys.exit(1) 102 103import multiprocessing 104import os 105import tempfile 106import zipfile 107 108import common 109import edify_generator 110import sparse_img 111 112OPTIONS = common.OPTIONS 113OPTIONS.package_key = None 114OPTIONS.incremental_source = None 115OPTIONS.verify = False 116OPTIONS.require_verbatim = set() 117OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 118OPTIONS.patch_threshold = 0.95 119OPTIONS.wipe_user_data = False 120OPTIONS.omit_prereq = False 121OPTIONS.extra_script = None 122OPTIONS.aslr_mode = True 123OPTIONS.worker_threads = multiprocessing.cpu_count() // 2 124if OPTIONS.worker_threads == 0: 125 OPTIONS.worker_threads = 1 126OPTIONS.two_step = False 127OPTIONS.no_signing = False 128OPTIONS.block_based = False 129OPTIONS.updater_binary = None 130OPTIONS.oem_source = None 131OPTIONS.fallback_to_full = True 132OPTIONS.full_radio = False 133OPTIONS.full_bootloader = False 134# Stash size cannot exceed cache_size * threshold. 135OPTIONS.cache_size = None 136OPTIONS.stash_threshold = 0.8 137 138def MostPopularKey(d, default): 139 """Given a dict, return the key corresponding to the largest 140 value. Returns 'default' if the dict is empty.""" 141 x = [(v, k) for (k, v) in d.iteritems()] 142 if not x: 143 return default 144 x.sort() 145 return x[-1][1] 146 147 148def IsSymlink(info): 149 """Return true if the zipfile.ZipInfo object passed in represents a 150 symlink.""" 151 return (info.external_attr >> 16) == 0o120777 152 153def IsRegular(info): 154 """Return true if the zipfile.ZipInfo object passed in represents a 155 symlink.""" 156 return (info.external_attr >> 28) == 0o10 157 158def ClosestFileMatch(src, tgtfiles, existing): 159 """Returns the closest file match between a source file and list 160 of potential matches. The exact filename match is preferred, 161 then the sha1 is searched for, and finally a file with the same 162 basename is evaluated. Rename support in the updater-binary is 163 required for the latter checks to be used.""" 164 165 result = tgtfiles.get("path:" + src.name) 166 if result is not None: 167 return result 168 169 if not OPTIONS.target_info_dict.get("update_rename_support", False): 170 return None 171 172 if src.size < 1000: 173 return None 174 175 result = tgtfiles.get("sha1:" + src.sha1) 176 if result is not None and existing.get(result.name) is None: 177 return result 178 result = tgtfiles.get("file:" + src.name.split("/")[-1]) 179 if result is not None and existing.get(result.name) is None: 180 return result 181 return None 182 183class ItemSet(object): 184 def __init__(self, partition, fs_config): 185 self.partition = partition 186 self.fs_config = fs_config 187 self.ITEMS = {} 188 189 def Get(self, name, is_dir=False): 190 if name not in self.ITEMS: 191 self.ITEMS[name] = Item(self, name, is_dir=is_dir) 192 return self.ITEMS[name] 193 194 def GetMetadata(self, input_zip): 195 # The target_files contains a record of what the uid, 196 # gid, and mode are supposed to be. 197 output = input_zip.read(self.fs_config) 198 199 for line in output.split("\n"): 200 if not line: 201 continue 202 columns = line.split() 203 name, uid, gid, mode = columns[:4] 204 selabel = None 205 capabilities = None 206 207 # After the first 4 columns, there are a series of key=value 208 # pairs. Extract out the fields we care about. 209 for element in columns[4:]: 210 key, value = element.split("=") 211 if key == "selabel": 212 selabel = value 213 if key == "capabilities": 214 capabilities = value 215 216 i = self.ITEMS.get(name, None) 217 if i is not None: 218 i.uid = int(uid) 219 i.gid = int(gid) 220 i.mode = int(mode, 8) 221 i.selabel = selabel 222 i.capabilities = capabilities 223 if i.is_dir: 224 i.children.sort(key=lambda i: i.name) 225 226 # set metadata for the files generated by this script. 227 i = self.ITEMS.get("system/recovery-from-boot.p", None) 228 if i: 229 i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0o644, None, None 230 i = self.ITEMS.get("system/etc/install-recovery.sh", None) 231 if i: 232 i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0o544, None, None 233 234 235class Item(object): 236 """Items represent the metadata (user, group, mode) of files and 237 directories in the system image.""" 238 def __init__(self, itemset, name, is_dir=False): 239 self.itemset = itemset 240 self.name = name 241 self.uid = None 242 self.gid = None 243 self.mode = None 244 self.selabel = None 245 self.capabilities = None 246 self.is_dir = is_dir 247 self.descendants = None 248 self.best_subtree = None 249 250 if name: 251 self.parent = itemset.Get(os.path.dirname(name), is_dir=True) 252 self.parent.children.append(self) 253 else: 254 self.parent = None 255 if self.is_dir: 256 self.children = [] 257 258 def Dump(self, indent=0): 259 if self.uid is not None: 260 print "%s%s %d %d %o" % ( 261 " " * indent, self.name, self.uid, self.gid, self.mode) 262 else: 263 print "%s%s %s %s %s" % ( 264 " " * indent, self.name, self.uid, self.gid, self.mode) 265 if self.is_dir: 266 print "%s%s" % (" "*indent, self.descendants) 267 print "%s%s" % (" "*indent, self.best_subtree) 268 for i in self.children: 269 i.Dump(indent=indent+1) 270 271 def CountChildMetadata(self): 272 """Count up the (uid, gid, mode, selabel, capabilities) tuples for 273 all children and determine the best strategy for using set_perm_recursive 274 and set_perm to correctly chown/chmod all the files to their desired 275 values. Recursively calls itself for all descendants. 276 277 Returns a dict of {(uid, gid, dmode, fmode, selabel, capabilities): count} 278 counting up all descendants of this node. (dmode or fmode may be None.) 279 Also sets the best_subtree of each directory Item to the (uid, gid, dmode, 280 fmode, selabel, capabilities) tuple that will match the most descendants of 281 that Item. 282 """ 283 284 assert self.is_dir 285 key = (self.uid, self.gid, self.mode, None, self.selabel, 286 self.capabilities) 287 self.descendants = {key: 1} 288 d = self.descendants 289 for i in self.children: 290 if i.is_dir: 291 for k, v in i.CountChildMetadata().iteritems(): 292 d[k] = d.get(k, 0) + v 293 else: 294 k = (i.uid, i.gid, None, i.mode, i.selabel, i.capabilities) 295 d[k] = d.get(k, 0) + 1 296 297 # Find the (uid, gid, dmode, fmode, selabel, capabilities) 298 # tuple that matches the most descendants. 299 300 # First, find the (uid, gid) pair that matches the most 301 # descendants. 302 ug = {} 303 for (uid, gid, _, _, _, _), count in d.iteritems(): 304 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 305 ug = MostPopularKey(ug, (0, 0)) 306 307 # Now find the dmode, fmode, selabel, and capabilities that match 308 # the most descendants with that (uid, gid), and choose those. 309 best_dmode = (0, 0o755) 310 best_fmode = (0, 0o644) 311 best_selabel = (0, None) 312 best_capabilities = (0, None) 313 for k, count in d.iteritems(): 314 if k[:2] != ug: 315 continue 316 if k[2] is not None and count >= best_dmode[0]: 317 best_dmode = (count, k[2]) 318 if k[3] is not None and count >= best_fmode[0]: 319 best_fmode = (count, k[3]) 320 if k[4] is not None and count >= best_selabel[0]: 321 best_selabel = (count, k[4]) 322 if k[5] is not None and count >= best_capabilities[0]: 323 best_capabilities = (count, k[5]) 324 self.best_subtree = ug + ( 325 best_dmode[1], best_fmode[1], best_selabel[1], best_capabilities[1]) 326 327 return d 328 329 def SetPermissions(self, script): 330 """Append set_perm/set_perm_recursive commands to 'script' to 331 set all permissions, users, and groups for the tree of files 332 rooted at 'self'.""" 333 334 self.CountChildMetadata() 335 336 def recurse(item, current): 337 # current is the (uid, gid, dmode, fmode, selabel, capabilities) tuple 338 # that the current item (and all its children) have already been set to. 339 # We only need to issue set_perm/set_perm_recursive commands if we're 340 # supposed to be something different. 341 if item.is_dir: 342 if current != item.best_subtree: 343 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 344 current = item.best_subtree 345 346 if item.uid != current[0] or item.gid != current[1] or \ 347 item.mode != current[2] or item.selabel != current[4] or \ 348 item.capabilities != current[5]: 349 script.SetPermissions("/"+item.name, item.uid, item.gid, 350 item.mode, item.selabel, item.capabilities) 351 352 for i in item.children: 353 recurse(i, current) 354 else: 355 if item.uid != current[0] or item.gid != current[1] or \ 356 item.mode != current[3] or item.selabel != current[4] or \ 357 item.capabilities != current[5]: 358 script.SetPermissions("/"+item.name, item.uid, item.gid, 359 item.mode, item.selabel, item.capabilities) 360 361 recurse(self, (-1, -1, -1, -1, None, None)) 362 363 364def CopyPartitionFiles(itemset, input_zip, output_zip=None, substitute=None): 365 """Copies files for the partition in the input zip to the output 366 zip. Populates the Item class with their metadata, and returns a 367 list of symlinks. output_zip may be None, in which case the copy is 368 skipped (but the other side effects still happen). substitute is an 369 optional dict of {output filename: contents} to be output instead of 370 certain input files. 371 """ 372 373 symlinks = [] 374 375 partition = itemset.partition 376 377 for info in input_zip.infolist(): 378 prefix = partition.upper() + "/" 379 if info.filename.startswith(prefix): 380 basefilename = info.filename[len(prefix):] 381 if IsSymlink(info): 382 symlinks.append((input_zip.read(info.filename), 383 "/" + partition + "/" + basefilename)) 384 else: 385 import copy 386 info2 = copy.copy(info) 387 fn = info2.filename = partition + "/" + basefilename 388 if substitute and fn in substitute and substitute[fn] is None: 389 continue 390 if output_zip is not None: 391 if substitute and fn in substitute: 392 data = substitute[fn] 393 else: 394 data = input_zip.read(info.filename) 395 common.ZipWriteStr(output_zip, info2, data) 396 if fn.endswith("/"): 397 itemset.Get(fn[:-1], is_dir=True) 398 else: 399 itemset.Get(fn) 400 401 symlinks.sort() 402 return symlinks 403 404 405def SignOutput(temp_zip_name, output_zip_name): 406 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 407 pw = key_passwords[OPTIONS.package_key] 408 409 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 410 whole_file=True) 411 412 413def AppendAssertions(script, info_dict, oem_dict=None): 414 oem_props = info_dict.get("oem_fingerprint_properties") 415 if oem_props is None or len(oem_props) == 0: 416 device = GetBuildProp("ro.product.device", info_dict) 417 script.AssertDevice(device) 418 else: 419 if oem_dict is None: 420 raise common.ExternalError( 421 "No OEM file provided to answer expected assertions") 422 for prop in oem_props.split(): 423 if oem_dict.get(prop) is None: 424 raise common.ExternalError( 425 "The OEM file is missing the property %s" % prop) 426 script.AssertOemProperty(prop, oem_dict.get(prop)) 427 428 429def HasRecoveryPatch(target_files_zip): 430 try: 431 target_files_zip.getinfo("SYSTEM/recovery-from-boot.p") 432 return True 433 except KeyError: 434 return False 435 436def HasVendorPartition(target_files_zip): 437 try: 438 target_files_zip.getinfo("VENDOR/") 439 return True 440 except KeyError: 441 return False 442 443def GetOemProperty(name, oem_props, oem_dict, info_dict): 444 if oem_props is not None and name in oem_props: 445 return oem_dict[name] 446 return GetBuildProp(name, info_dict) 447 448 449def CalculateFingerprint(oem_props, oem_dict, info_dict): 450 if oem_props is None: 451 return GetBuildProp("ro.build.fingerprint", info_dict) 452 return "%s/%s/%s:%s" % ( 453 GetOemProperty("ro.product.brand", oem_props, oem_dict, info_dict), 454 GetOemProperty("ro.product.name", oem_props, oem_dict, info_dict), 455 GetOemProperty("ro.product.device", oem_props, oem_dict, info_dict), 456 GetBuildProp("ro.build.thumbprint", info_dict)) 457 458 459def GetImage(which, tmpdir, info_dict): 460 # Return an image object (suitable for passing to BlockImageDiff) 461 # for the 'which' partition (most be "system" or "vendor"). If a 462 # prebuilt image and file map are found in tmpdir they are used, 463 # otherwise they are reconstructed from the individual files. 464 465 assert which in ("system", "vendor") 466 467 path = os.path.join(tmpdir, "IMAGES", which + ".img") 468 mappath = os.path.join(tmpdir, "IMAGES", which + ".map") 469 if os.path.exists(path) and os.path.exists(mappath): 470 print "using %s.img from target-files" % (which,) 471 # This is a 'new' target-files, which already has the image in it. 472 473 else: 474 print "building %s.img from target-files" % (which,) 475 476 # This is an 'old' target-files, which does not contain images 477 # already built. Build them. 478 479 mappath = tempfile.mkstemp()[1] 480 OPTIONS.tempfiles.append(mappath) 481 482 import add_img_to_target_files 483 if which == "system": 484 path = add_img_to_target_files.BuildSystem( 485 tmpdir, info_dict, block_list=mappath) 486 elif which == "vendor": 487 path = add_img_to_target_files.BuildVendor( 488 tmpdir, info_dict, block_list=mappath) 489 490 # Bug: http://b/20939131 491 # In ext4 filesystems, block 0 might be changed even being mounted 492 # R/O. We add it to clobbered_blocks so that it will be written to the 493 # target unconditionally. Note that they are still part of care_map. 494 clobbered_blocks = "0" 495 496 return sparse_img.SparseImage(path, mappath, clobbered_blocks) 497 498 499def WriteFullOTAPackage(input_zip, output_zip): 500 # TODO: how to determine this? We don't know what version it will 501 # be installed on top of. For now, we expect the API just won't 502 # change very often. Similarly for fstab, it might have changed 503 # in the target build. 504 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 505 506 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 507 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 508 oem_dict = None 509 if oem_props is not None and len(oem_props) > 0: 510 if OPTIONS.oem_source is None: 511 raise common.ExternalError("OEM source required for this build") 512 script.Mount("/oem", recovery_mount_options) 513 oem_dict = common.LoadDictionaryFromLines( 514 open(OPTIONS.oem_source).readlines()) 515 516 metadata = { 517 "post-build": CalculateFingerprint(oem_props, oem_dict, 518 OPTIONS.info_dict), 519 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 520 OPTIONS.info_dict), 521 "post-timestamp": GetBuildProp("ro.build.date.utc", OPTIONS.info_dict), 522 } 523 524 device_specific = common.DeviceSpecificParams( 525 input_zip=input_zip, 526 input_version=OPTIONS.info_dict["recovery_api_version"], 527 output_zip=output_zip, 528 script=script, 529 input_tmp=OPTIONS.input_tmp, 530 metadata=metadata, 531 info_dict=OPTIONS.info_dict) 532 533 has_recovery_patch = HasRecoveryPatch(input_zip) 534 block_based = OPTIONS.block_based and has_recovery_patch 535 536 if not OPTIONS.omit_prereq: 537 ts = GetBuildProp("ro.build.date.utc", OPTIONS.info_dict) 538 ts_text = GetBuildProp("ro.build.date", OPTIONS.info_dict) 539 script.AssertOlderBuild(ts, ts_text) 540 541 AppendAssertions(script, OPTIONS.info_dict, oem_dict) 542 device_specific.FullOTA_Assertions() 543 544 # Two-step package strategy (in chronological order, which is *not* 545 # the order in which the generated script has things): 546 # 547 # if stage is not "2/3" or "3/3": 548 # write recovery image to boot partition 549 # set stage to "2/3" 550 # reboot to boot partition and restart recovery 551 # else if stage is "2/3": 552 # write recovery image to recovery partition 553 # set stage to "3/3" 554 # reboot to recovery partition and restart recovery 555 # else: 556 # (stage must be "3/3") 557 # set stage to "" 558 # do normal full package installation: 559 # wipe and install system, boot image, etc. 560 # set up system to update recovery partition on first boot 561 # complete script normally 562 # (allow recovery to mark itself finished and reboot) 563 564 recovery_img = common.GetBootableImage("recovery.img", "recovery.img", 565 OPTIONS.input_tmp, "RECOVERY") 566 if OPTIONS.two_step: 567 if not OPTIONS.info_dict.get("multistage_support", None): 568 assert False, "two-step packages not supported by this build" 569 fs = OPTIONS.info_dict["fstab"]["/misc"] 570 assert fs.fs_type.upper() == "EMMC", \ 571 "two-step packages only supported on devices with EMMC /misc partitions" 572 bcb_dev = {"bcb_dev": fs.device} 573 common.ZipWriteStr(output_zip, "recovery.img", recovery_img.data) 574 script.AppendExtra(""" 575if get_stage("%(bcb_dev)s") == "2/3" then 576""" % bcb_dev) 577 script.WriteRawImage("/recovery", "recovery.img") 578 script.AppendExtra(""" 579set_stage("%(bcb_dev)s", "3/3"); 580reboot_now("%(bcb_dev)s", "recovery"); 581else if get_stage("%(bcb_dev)s") == "3/3" then 582""" % bcb_dev) 583 584 # Dump fingerprints 585 script.Print("Target: %s" % CalculateFingerprint( 586 oem_props, oem_dict, OPTIONS.info_dict)) 587 588 device_specific.FullOTA_InstallBegin() 589 590 system_progress = 0.75 591 592 if OPTIONS.wipe_user_data: 593 system_progress -= 0.1 594 if HasVendorPartition(input_zip): 595 system_progress -= 0.1 596 597 if "selinux_fc" in OPTIONS.info_dict: 598 WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip) 599 600 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 601 602 system_items = ItemSet("system", "META/filesystem_config.txt") 603 script.ShowProgress(system_progress, 0) 604 605 if block_based: 606 # Full OTA is done as an "incremental" against an empty source 607 # image. This has the effect of writing new data from the package 608 # to the entire partition, but lets us reuse the updater code that 609 # writes incrementals to do it. 610 system_tgt = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict) 611 system_tgt.ResetFileMap() 612 system_diff = common.BlockDifference("system", system_tgt, src=None) 613 system_diff.WriteScript(script, output_zip) 614 else: 615 script.FormatPartition("/system") 616 script.Mount("/system", recovery_mount_options) 617 if not has_recovery_patch: 618 script.UnpackPackageDir("recovery", "/system") 619 script.UnpackPackageDir("system", "/system") 620 621 symlinks = CopyPartitionFiles(system_items, input_zip, output_zip) 622 script.MakeSymlinks(symlinks) 623 624 boot_img = common.GetBootableImage("boot.img", "boot.img", 625 OPTIONS.input_tmp, "BOOT") 626 627 if not block_based: 628 def output_sink(fn, data): 629 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 630 system_items.Get("system/" + fn) 631 632 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, 633 recovery_img, boot_img) 634 635 system_items.GetMetadata(input_zip) 636 system_items.Get("system").SetPermissions(script) 637 638 if HasVendorPartition(input_zip): 639 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 640 script.ShowProgress(0.1, 0) 641 642 if block_based: 643 vendor_tgt = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict) 644 vendor_tgt.ResetFileMap() 645 vendor_diff = common.BlockDifference("vendor", vendor_tgt) 646 vendor_diff.WriteScript(script, output_zip) 647 else: 648 script.FormatPartition("/vendor") 649 script.Mount("/vendor", recovery_mount_options) 650 script.UnpackPackageDir("vendor", "/vendor") 651 652 symlinks = CopyPartitionFiles(vendor_items, input_zip, output_zip) 653 script.MakeSymlinks(symlinks) 654 655 vendor_items.GetMetadata(input_zip) 656 vendor_items.Get("vendor").SetPermissions(script) 657 658 common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict) 659 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 660 661 script.ShowProgress(0.05, 5) 662 script.WriteRawImage("/boot", "boot.img") 663 664 script.ShowProgress(0.2, 10) 665 device_specific.FullOTA_InstallEnd() 666 667 if OPTIONS.extra_script is not None: 668 script.AppendExtra(OPTIONS.extra_script) 669 670 script.UnmountAll() 671 672 if OPTIONS.wipe_user_data: 673 script.ShowProgress(0.1, 10) 674 script.FormatPartition("/data") 675 676 if OPTIONS.two_step: 677 script.AppendExtra(""" 678set_stage("%(bcb_dev)s", ""); 679""" % bcb_dev) 680 script.AppendExtra("else\n") 681 script.WriteRawImage("/boot", "recovery.img") 682 script.AppendExtra(""" 683set_stage("%(bcb_dev)s", "2/3"); 684reboot_now("%(bcb_dev)s", ""); 685endif; 686endif; 687""" % bcb_dev) 688 script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary) 689 WriteMetadata(metadata, output_zip) 690 691 692def WritePolicyConfig(file_name, output_zip): 693 common.ZipWrite(output_zip, file_name, os.path.basename(file_name)) 694 695 696def WriteMetadata(metadata, output_zip): 697 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata", 698 "".join(["%s=%s\n" % kv 699 for kv in sorted(metadata.iteritems())])) 700 701 702def LoadPartitionFiles(z, partition): 703 """Load all the files from the given partition in a given target-files 704 ZipFile, and return a dict of {filename: File object}.""" 705 out = {} 706 prefix = partition.upper() + "/" 707 for info in z.infolist(): 708 if info.filename.startswith(prefix) and not IsSymlink(info): 709 basefilename = info.filename[len(prefix):] 710 fn = partition + "/" + basefilename 711 data = z.read(info.filename) 712 out[fn] = common.File(fn, data) 713 return out 714 715 716def GetBuildProp(prop, info_dict): 717 """Return the fingerprint of the build of a given target-files info_dict.""" 718 try: 719 return info_dict.get("build.prop", {})[prop] 720 except KeyError: 721 raise common.ExternalError("couldn't find %s in build.prop" % (prop,)) 722 723 724def AddToKnownPaths(filename, known_paths): 725 if filename[-1] == "/": 726 return 727 dirs = filename.split("/")[:-1] 728 while len(dirs) > 0: 729 path = "/".join(dirs) 730 if path in known_paths: 731 break 732 known_paths.add(path) 733 dirs.pop() 734 735 736def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 737 source_version = OPTIONS.source_info_dict["recovery_api_version"] 738 target_version = OPTIONS.target_info_dict["recovery_api_version"] 739 740 if source_version == 0: 741 print ("WARNING: generating edify script for a source that " 742 "can't install it.") 743 script = edify_generator.EdifyGenerator( 744 source_version, OPTIONS.target_info_dict, 745 fstab=OPTIONS.source_info_dict["fstab"]) 746 747 metadata = { 748 "pre-device": GetBuildProp("ro.product.device", 749 OPTIONS.source_info_dict), 750 "post-timestamp": GetBuildProp("ro.build.date.utc", 751 OPTIONS.target_info_dict), 752 } 753 754 device_specific = common.DeviceSpecificParams( 755 source_zip=source_zip, 756 source_version=source_version, 757 target_zip=target_zip, 758 target_version=target_version, 759 output_zip=output_zip, 760 script=script, 761 metadata=metadata, 762 info_dict=OPTIONS.source_info_dict) 763 764 # TODO: Currently this works differently from WriteIncrementalOTAPackage(). 765 # This function doesn't consider thumbprints when writing 766 # metadata["pre/post-build"]. One possible reason is that the current 767 # devices with thumbprints are all using file-based OTAs. Long term we 768 # should factor out the common parts into a shared one to avoid further 769 # divergence. 770 source_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.source_info_dict) 771 target_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.target_info_dict) 772 metadata["pre-build"] = source_fp 773 metadata["post-build"] = target_fp 774 775 source_boot = common.GetBootableImage( 776 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 777 OPTIONS.source_info_dict) 778 target_boot = common.GetBootableImage( 779 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 780 updating_boot = (not OPTIONS.two_step and 781 (source_boot.data != target_boot.data)) 782 783 target_recovery = common.GetBootableImage( 784 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 785 786 system_src = GetImage("system", OPTIONS.source_tmp, OPTIONS.source_info_dict) 787 system_tgt = GetImage("system", OPTIONS.target_tmp, OPTIONS.target_info_dict) 788 789 blockimgdiff_version = 1 790 if OPTIONS.info_dict: 791 blockimgdiff_version = max( 792 int(i) for i in 793 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(",")) 794 795 system_diff = common.BlockDifference("system", system_tgt, system_src, 796 version=blockimgdiff_version) 797 798 if HasVendorPartition(target_zip): 799 if not HasVendorPartition(source_zip): 800 raise RuntimeError("can't generate incremental that adds /vendor") 801 vendor_src = GetImage("vendor", OPTIONS.source_tmp, 802 OPTIONS.source_info_dict) 803 vendor_tgt = GetImage("vendor", OPTIONS.target_tmp, 804 OPTIONS.target_info_dict) 805 vendor_diff = common.BlockDifference("vendor", vendor_tgt, vendor_src, 806 version=blockimgdiff_version) 807 else: 808 vendor_diff = None 809 810 oem_props = OPTIONS.target_info_dict.get("oem_fingerprint_properties") 811 recovery_mount_options = OPTIONS.source_info_dict.get( 812 "recovery_mount_options") 813 oem_dict = None 814 if oem_props is not None and len(oem_props) > 0: 815 if OPTIONS.oem_source is None: 816 raise common.ExternalError("OEM source required for this build") 817 script.Mount("/oem", recovery_mount_options) 818 oem_dict = common.LoadDictionaryFromLines( 819 open(OPTIONS.oem_source).readlines()) 820 821 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 822 device_specific.IncrementalOTA_Assertions() 823 824 # Two-step incremental package strategy (in chronological order, 825 # which is *not* the order in which the generated script has 826 # things): 827 # 828 # if stage is not "2/3" or "3/3": 829 # do verification on current system 830 # write recovery image to boot partition 831 # set stage to "2/3" 832 # reboot to boot partition and restart recovery 833 # else if stage is "2/3": 834 # write recovery image to recovery partition 835 # set stage to "3/3" 836 # reboot to recovery partition and restart recovery 837 # else: 838 # (stage must be "3/3") 839 # perform update: 840 # patch system files, etc. 841 # force full install of new boot image 842 # set up system to update recovery partition on first boot 843 # complete script normally 844 # (allow recovery to mark itself finished and reboot) 845 846 if OPTIONS.two_step: 847 if not OPTIONS.source_info_dict.get("multistage_support", None): 848 assert False, "two-step packages not supported by this build" 849 fs = OPTIONS.source_info_dict["fstab"]["/misc"] 850 assert fs.fs_type.upper() == "EMMC", \ 851 "two-step packages only supported on devices with EMMC /misc partitions" 852 bcb_dev = {"bcb_dev": fs.device} 853 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 854 script.AppendExtra(""" 855if get_stage("%(bcb_dev)s") == "2/3" then 856""" % bcb_dev) 857 script.AppendExtra("sleep(20);\n") 858 script.WriteRawImage("/recovery", "recovery.img") 859 script.AppendExtra(""" 860set_stage("%(bcb_dev)s", "3/3"); 861reboot_now("%(bcb_dev)s", "recovery"); 862else if get_stage("%(bcb_dev)s") != "3/3" then 863""" % bcb_dev) 864 865 # Dump fingerprints 866 script.Print("Source: %s" % CalculateFingerprint( 867 oem_props, oem_dict, OPTIONS.source_info_dict)) 868 script.Print("Target: %s" % CalculateFingerprint( 869 oem_props, oem_dict, OPTIONS.target_info_dict)) 870 871 script.Print("Verifying current system...") 872 873 device_specific.IncrementalOTA_VerifyBegin() 874 875 if oem_props is None: 876 # When blockimgdiff version is less than 3 (non-resumable block-based OTA), 877 # patching on a device that's already on the target build will damage the 878 # system. Because operations like move don't check the block state, they 879 # always apply the changes unconditionally. 880 if blockimgdiff_version <= 2: 881 script.AssertSomeFingerprint(source_fp) 882 else: 883 script.AssertSomeFingerprint(source_fp, target_fp) 884 else: 885 if blockimgdiff_version <= 2: 886 script.AssertSomeThumbprint( 887 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 888 else: 889 script.AssertSomeThumbprint( 890 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 891 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 892 893 if updating_boot: 894 boot_type, boot_device = common.GetTypeAndDevice( 895 "/boot", OPTIONS.source_info_dict) 896 d = common.Difference(target_boot, source_boot) 897 _, _, d = d.ComputePatch() 898 if d is None: 899 include_full_boot = True 900 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 901 else: 902 include_full_boot = False 903 904 print "boot target: %d source: %d diff: %d" % ( 905 target_boot.size, source_boot.size, len(d)) 906 907 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 908 909 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 910 (boot_type, boot_device, 911 source_boot.size, source_boot.sha1, 912 target_boot.size, target_boot.sha1)) 913 914 device_specific.IncrementalOTA_VerifyEnd() 915 916 if OPTIONS.two_step: 917 script.WriteRawImage("/boot", "recovery.img") 918 script.AppendExtra(""" 919set_stage("%(bcb_dev)s", "2/3"); 920reboot_now("%(bcb_dev)s", ""); 921else 922""" % bcb_dev) 923 924 # Verify the existing partitions. 925 system_diff.WriteVerifyScript(script) 926 if vendor_diff: 927 vendor_diff.WriteVerifyScript(script) 928 929 script.Comment("---- start making changes here ----") 930 931 device_specific.IncrementalOTA_InstallBegin() 932 933 system_diff.WriteScript(script, output_zip, 934 progress=0.8 if vendor_diff else 0.9) 935 if vendor_diff: 936 vendor_diff.WriteScript(script, output_zip, progress=0.1) 937 938 if OPTIONS.two_step: 939 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 940 script.WriteRawImage("/boot", "boot.img") 941 print "writing full boot image (forced by two-step mode)" 942 943 if not OPTIONS.two_step: 944 if updating_boot: 945 if include_full_boot: 946 print "boot image changed; including full." 947 script.Print("Installing boot image...") 948 script.WriteRawImage("/boot", "boot.img") 949 else: 950 # Produce the boot image by applying a patch to the current 951 # contents of the boot partition, and write it back to the 952 # partition. 953 print "boot image changed; including patch." 954 script.Print("Patching boot image...") 955 script.ShowProgress(0.1, 10) 956 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 957 % (boot_type, boot_device, 958 source_boot.size, source_boot.sha1, 959 target_boot.size, target_boot.sha1), 960 "-", 961 target_boot.size, target_boot.sha1, 962 source_boot.sha1, "patch/boot.img.p") 963 else: 964 print "boot image unchanged; skipping." 965 966 # Do device-specific installation (eg, write radio image). 967 device_specific.IncrementalOTA_InstallEnd() 968 969 if OPTIONS.extra_script is not None: 970 script.AppendExtra(OPTIONS.extra_script) 971 972 if OPTIONS.wipe_user_data: 973 script.Print("Erasing user data...") 974 script.FormatPartition("/data") 975 976 if OPTIONS.two_step: 977 script.AppendExtra(""" 978set_stage("%(bcb_dev)s", ""); 979endif; 980endif; 981""" % bcb_dev) 982 983 script.SetProgress(1) 984 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 985 WriteMetadata(metadata, output_zip) 986 987 988class FileDifference(object): 989 def __init__(self, partition, source_zip, target_zip, output_zip): 990 self.deferred_patch_list = None 991 print "Loading target..." 992 self.target_data = target_data = LoadPartitionFiles(target_zip, partition) 993 print "Loading source..." 994 self.source_data = source_data = LoadPartitionFiles(source_zip, partition) 995 996 self.verbatim_targets = verbatim_targets = [] 997 self.patch_list = patch_list = [] 998 diffs = [] 999 self.renames = renames = {} 1000 known_paths = set() 1001 largest_source_size = 0 1002 1003 matching_file_cache = {} 1004 for fn, sf in source_data.items(): 1005 assert fn == sf.name 1006 matching_file_cache["path:" + fn] = sf 1007 if fn in target_data.keys(): 1008 AddToKnownPaths(fn, known_paths) 1009 # Only allow eligibility for filename/sha matching 1010 # if there isn't a perfect path match. 1011 if target_data.get(sf.name) is None: 1012 matching_file_cache["file:" + fn.split("/")[-1]] = sf 1013 matching_file_cache["sha:" + sf.sha1] = sf 1014 1015 for fn in sorted(target_data.keys()): 1016 tf = target_data[fn] 1017 assert fn == tf.name 1018 sf = ClosestFileMatch(tf, matching_file_cache, renames) 1019 if sf is not None and sf.name != tf.name: 1020 print "File has moved from " + sf.name + " to " + tf.name 1021 renames[sf.name] = tf 1022 1023 if sf is None or fn in OPTIONS.require_verbatim: 1024 # This file should be included verbatim 1025 if fn in OPTIONS.prohibit_verbatim: 1026 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 1027 print "send", fn, "verbatim" 1028 tf.AddToZip(output_zip) 1029 verbatim_targets.append((fn, tf.size, tf.sha1)) 1030 if fn in target_data.keys(): 1031 AddToKnownPaths(fn, known_paths) 1032 elif tf.sha1 != sf.sha1: 1033 # File is different; consider sending as a patch 1034 diffs.append(common.Difference(tf, sf)) 1035 else: 1036 # Target file data identical to source (may still be renamed) 1037 pass 1038 1039 common.ComputeDifferences(diffs) 1040 1041 for diff in diffs: 1042 tf, sf, d = diff.GetPatch() 1043 path = "/".join(tf.name.split("/")[:-1]) 1044 if d is None or len(d) > tf.size * OPTIONS.patch_threshold or \ 1045 path not in known_paths: 1046 # patch is almost as big as the file; don't bother patching 1047 # or a patch + rename cannot take place due to the target 1048 # directory not existing 1049 tf.AddToZip(output_zip) 1050 verbatim_targets.append((tf.name, tf.size, tf.sha1)) 1051 if sf.name in renames: 1052 del renames[sf.name] 1053 AddToKnownPaths(tf.name, known_paths) 1054 else: 1055 common.ZipWriteStr(output_zip, "patch/" + sf.name + ".p", d) 1056 patch_list.append((tf, sf, tf.size, common.sha1(d).hexdigest())) 1057 largest_source_size = max(largest_source_size, sf.size) 1058 1059 self.largest_source_size = largest_source_size 1060 1061 def EmitVerification(self, script): 1062 so_far = 0 1063 for tf, sf, _, _ in self.patch_list: 1064 if tf.name != sf.name: 1065 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1066 script.PatchCheck("/"+sf.name, tf.sha1, sf.sha1) 1067 so_far += sf.size 1068 return so_far 1069 1070 def EmitExplicitTargetVerification(self, script): 1071 for fn, _, sha1 in self.verbatim_targets: 1072 if fn[-1] != "/": 1073 script.FileCheck("/"+fn, sha1) 1074 for tf, _, _, _ in self.patch_list: 1075 script.FileCheck(tf.name, tf.sha1) 1076 1077 def RemoveUnneededFiles(self, script, extras=()): 1078 script.DeleteFiles( 1079 ["/" + i[0] for i in self.verbatim_targets] + 1080 ["/" + i for i in sorted(self.source_data) 1081 if i not in self.target_data and i not in self.renames] + 1082 list(extras)) 1083 1084 def TotalPatchSize(self): 1085 return sum(i[1].size for i in self.patch_list) 1086 1087 def EmitPatches(self, script, total_patch_size, so_far): 1088 self.deferred_patch_list = deferred_patch_list = [] 1089 for item in self.patch_list: 1090 tf, sf, _, _ = item 1091 if tf.name == "system/build.prop": 1092 deferred_patch_list.append(item) 1093 continue 1094 if sf.name != tf.name: 1095 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1096 script.ApplyPatch("/" + sf.name, "-", tf.size, tf.sha1, sf.sha1, 1097 "patch/" + sf.name + ".p") 1098 so_far += tf.size 1099 script.SetProgress(so_far / total_patch_size) 1100 return so_far 1101 1102 def EmitDeferredPatches(self, script): 1103 for item in self.deferred_patch_list: 1104 tf, sf, _, _ = item 1105 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, 1106 "patch/" + sf.name + ".p") 1107 script.SetPermissions("/system/build.prop", 0, 0, 0o644, None, None) 1108 1109 def EmitRenames(self, script): 1110 if len(self.renames) > 0: 1111 script.Print("Renaming files...") 1112 for src, tgt in self.renames.iteritems(): 1113 print "Renaming " + src + " to " + tgt.name 1114 script.RenameFile(src, tgt.name) 1115 1116 1117def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): 1118 target_has_recovery_patch = HasRecoveryPatch(target_zip) 1119 source_has_recovery_patch = HasRecoveryPatch(source_zip) 1120 1121 if (OPTIONS.block_based and 1122 target_has_recovery_patch and 1123 source_has_recovery_patch): 1124 return WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip) 1125 1126 source_version = OPTIONS.source_info_dict["recovery_api_version"] 1127 target_version = OPTIONS.target_info_dict["recovery_api_version"] 1128 1129 if source_version == 0: 1130 print ("WARNING: generating edify script for a source that " 1131 "can't install it.") 1132 script = edify_generator.EdifyGenerator( 1133 source_version, OPTIONS.target_info_dict, 1134 fstab=OPTIONS.source_info_dict["fstab"]) 1135 1136 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 1137 recovery_mount_options = OPTIONS.source_info_dict.get( 1138 "recovery_mount_options") 1139 oem_dict = None 1140 if oem_props is not None and len(oem_props) > 0: 1141 if OPTIONS.oem_source is None: 1142 raise common.ExternalError("OEM source required for this build") 1143 script.Mount("/oem", recovery_mount_options) 1144 oem_dict = common.LoadDictionaryFromLines( 1145 open(OPTIONS.oem_source).readlines()) 1146 1147 metadata = { 1148 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 1149 OPTIONS.source_info_dict), 1150 "post-timestamp": GetBuildProp("ro.build.date.utc", 1151 OPTIONS.target_info_dict), 1152 } 1153 1154 device_specific = common.DeviceSpecificParams( 1155 source_zip=source_zip, 1156 source_version=source_version, 1157 target_zip=target_zip, 1158 target_version=target_version, 1159 output_zip=output_zip, 1160 script=script, 1161 metadata=metadata, 1162 info_dict=OPTIONS.source_info_dict) 1163 1164 system_diff = FileDifference("system", source_zip, target_zip, output_zip) 1165 script.Mount("/system", recovery_mount_options) 1166 if HasVendorPartition(target_zip): 1167 vendor_diff = FileDifference("vendor", source_zip, target_zip, output_zip) 1168 script.Mount("/vendor", recovery_mount_options) 1169 else: 1170 vendor_diff = None 1171 1172 target_fp = CalculateFingerprint(oem_props, oem_dict, 1173 OPTIONS.target_info_dict) 1174 source_fp = CalculateFingerprint(oem_props, oem_dict, 1175 OPTIONS.source_info_dict) 1176 1177 if oem_props is None: 1178 script.AssertSomeFingerprint(source_fp, target_fp) 1179 else: 1180 script.AssertSomeThumbprint( 1181 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 1182 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 1183 1184 metadata["pre-build"] = source_fp 1185 metadata["post-build"] = target_fp 1186 1187 source_boot = common.GetBootableImage( 1188 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 1189 OPTIONS.source_info_dict) 1190 target_boot = common.GetBootableImage( 1191 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 1192 updating_boot = (not OPTIONS.two_step and 1193 (source_boot.data != target_boot.data)) 1194 1195 source_recovery = common.GetBootableImage( 1196 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 1197 OPTIONS.source_info_dict) 1198 target_recovery = common.GetBootableImage( 1199 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 1200 updating_recovery = (source_recovery.data != target_recovery.data) 1201 1202 # Here's how we divide up the progress bar: 1203 # 0.1 for verifying the start state (PatchCheck calls) 1204 # 0.8 for applying patches (ApplyPatch calls) 1205 # 0.1 for unpacking verbatim files, symlinking, and doing the 1206 # device-specific commands. 1207 1208 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 1209 device_specific.IncrementalOTA_Assertions() 1210 1211 # Two-step incremental package strategy (in chronological order, 1212 # which is *not* the order in which the generated script has 1213 # things): 1214 # 1215 # if stage is not "2/3" or "3/3": 1216 # do verification on current system 1217 # write recovery image to boot partition 1218 # set stage to "2/3" 1219 # reboot to boot partition and restart recovery 1220 # else if stage is "2/3": 1221 # write recovery image to recovery partition 1222 # set stage to "3/3" 1223 # reboot to recovery partition and restart recovery 1224 # else: 1225 # (stage must be "3/3") 1226 # perform update: 1227 # patch system files, etc. 1228 # force full install of new boot image 1229 # set up system to update recovery partition on first boot 1230 # complete script normally 1231 # (allow recovery to mark itself finished and reboot) 1232 1233 if OPTIONS.two_step: 1234 if not OPTIONS.source_info_dict.get("multistage_support", None): 1235 assert False, "two-step packages not supported by this build" 1236 fs = OPTIONS.source_info_dict["fstab"]["/misc"] 1237 assert fs.fs_type.upper() == "EMMC", \ 1238 "two-step packages only supported on devices with EMMC /misc partitions" 1239 bcb_dev = {"bcb_dev": fs.device} 1240 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 1241 script.AppendExtra(""" 1242if get_stage("%(bcb_dev)s") == "2/3" then 1243""" % bcb_dev) 1244 script.AppendExtra("sleep(20);\n") 1245 script.WriteRawImage("/recovery", "recovery.img") 1246 script.AppendExtra(""" 1247set_stage("%(bcb_dev)s", "3/3"); 1248reboot_now("%(bcb_dev)s", "recovery"); 1249else if get_stage("%(bcb_dev)s") != "3/3" then 1250""" % bcb_dev) 1251 1252 # Dump fingerprints 1253 script.Print("Source: %s" % (source_fp,)) 1254 script.Print("Target: %s" % (target_fp,)) 1255 1256 script.Print("Verifying current system...") 1257 1258 device_specific.IncrementalOTA_VerifyBegin() 1259 1260 script.ShowProgress(0.1, 0) 1261 so_far = system_diff.EmitVerification(script) 1262 if vendor_diff: 1263 so_far += vendor_diff.EmitVerification(script) 1264 1265 if updating_boot: 1266 d = common.Difference(target_boot, source_boot) 1267 _, _, d = d.ComputePatch() 1268 print "boot target: %d source: %d diff: %d" % ( 1269 target_boot.size, source_boot.size, len(d)) 1270 1271 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 1272 1273 boot_type, boot_device = common.GetTypeAndDevice( 1274 "/boot", OPTIONS.source_info_dict) 1275 1276 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 1277 (boot_type, boot_device, 1278 source_boot.size, source_boot.sha1, 1279 target_boot.size, target_boot.sha1)) 1280 so_far += source_boot.size 1281 1282 size = [] 1283 if system_diff.patch_list: 1284 size.append(system_diff.largest_source_size) 1285 if vendor_diff: 1286 if vendor_diff.patch_list: 1287 size.append(vendor_diff.largest_source_size) 1288 if size or updating_recovery or updating_boot: 1289 script.CacheFreeSpaceCheck(max(size)) 1290 1291 device_specific.IncrementalOTA_VerifyEnd() 1292 1293 if OPTIONS.two_step: 1294 script.WriteRawImage("/boot", "recovery.img") 1295 script.AppendExtra(""" 1296set_stage("%(bcb_dev)s", "2/3"); 1297reboot_now("%(bcb_dev)s", ""); 1298else 1299""" % bcb_dev) 1300 1301 script.Comment("---- start making changes here ----") 1302 1303 device_specific.IncrementalOTA_InstallBegin() 1304 1305 if OPTIONS.two_step: 1306 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 1307 script.WriteRawImage("/boot", "boot.img") 1308 print "writing full boot image (forced by two-step mode)" 1309 1310 script.Print("Removing unneeded files...") 1311 system_diff.RemoveUnneededFiles(script, ("/system/recovery.img",)) 1312 if vendor_diff: 1313 vendor_diff.RemoveUnneededFiles(script) 1314 1315 script.ShowProgress(0.8, 0) 1316 total_patch_size = 1.0 + system_diff.TotalPatchSize() 1317 if vendor_diff: 1318 total_patch_size += vendor_diff.TotalPatchSize() 1319 if updating_boot: 1320 total_patch_size += target_boot.size 1321 1322 script.Print("Patching system files...") 1323 so_far = system_diff.EmitPatches(script, total_patch_size, 0) 1324 if vendor_diff: 1325 script.Print("Patching vendor files...") 1326 so_far = vendor_diff.EmitPatches(script, total_patch_size, so_far) 1327 1328 if not OPTIONS.two_step: 1329 if updating_boot: 1330 # Produce the boot image by applying a patch to the current 1331 # contents of the boot partition, and write it back to the 1332 # partition. 1333 script.Print("Patching boot image...") 1334 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 1335 % (boot_type, boot_device, 1336 source_boot.size, source_boot.sha1, 1337 target_boot.size, target_boot.sha1), 1338 "-", 1339 target_boot.size, target_boot.sha1, 1340 source_boot.sha1, "patch/boot.img.p") 1341 so_far += target_boot.size 1342 script.SetProgress(so_far / total_patch_size) 1343 print "boot image changed; including." 1344 else: 1345 print "boot image unchanged; skipping." 1346 1347 system_items = ItemSet("system", "META/filesystem_config.txt") 1348 if vendor_diff: 1349 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 1350 1351 if updating_recovery: 1352 # Recovery is generated as a patch using both the boot image 1353 # (which contains the same linux kernel as recovery) and the file 1354 # /system/etc/recovery-resource.dat (which contains all the images 1355 # used in the recovery UI) as sources. This lets us minimize the 1356 # size of the patch, which must be included in every OTA package. 1357 # 1358 # For older builds where recovery-resource.dat is not present, we 1359 # use only the boot image as the source. 1360 1361 if not target_has_recovery_patch: 1362 def output_sink(fn, data): 1363 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 1364 system_items.Get("system/" + fn) 1365 1366 common.MakeRecoveryPatch(OPTIONS.target_tmp, output_sink, 1367 target_recovery, target_boot) 1368 script.DeleteFiles(["/system/recovery-from-boot.p", 1369 "/system/etc/install-recovery.sh"]) 1370 print "recovery image changed; including as patch from boot." 1371 else: 1372 print "recovery image unchanged; skipping." 1373 1374 script.ShowProgress(0.1, 10) 1375 1376 target_symlinks = CopyPartitionFiles(system_items, target_zip, None) 1377 if vendor_diff: 1378 target_symlinks.extend(CopyPartitionFiles(vendor_items, target_zip, None)) 1379 1380 temp_script = script.MakeTemporary() 1381 system_items.GetMetadata(target_zip) 1382 system_items.Get("system").SetPermissions(temp_script) 1383 if vendor_diff: 1384 vendor_items.GetMetadata(target_zip) 1385 vendor_items.Get("vendor").SetPermissions(temp_script) 1386 1387 # Note that this call will mess up the trees of Items, so make sure 1388 # we're done with them. 1389 source_symlinks = CopyPartitionFiles(system_items, source_zip, None) 1390 if vendor_diff: 1391 source_symlinks.extend(CopyPartitionFiles(vendor_items, source_zip, None)) 1392 1393 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 1394 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 1395 1396 # Delete all the symlinks in source that aren't in target. This 1397 # needs to happen before verbatim files are unpacked, in case a 1398 # symlink in the source is replaced by a real file in the target. 1399 1400 # If a symlink in the source will be replaced by a regular file, we cannot 1401 # delete the symlink/file in case the package gets applied again. For such 1402 # a symlink, we prepend a sha1_check() to detect if it has been updated. 1403 # (Bug: 23646151) 1404 replaced_symlinks = dict() 1405 if system_diff: 1406 for i in system_diff.verbatim_targets: 1407 replaced_symlinks["/%s" % (i[0],)] = i[2] 1408 if vendor_diff: 1409 for i in vendor_diff.verbatim_targets: 1410 replaced_symlinks["/%s" % (i[0],)] = i[2] 1411 1412 if system_diff: 1413 for tf in system_diff.renames.values(): 1414 replaced_symlinks["/%s" % (tf.name,)] = tf.sha1 1415 if vendor_diff: 1416 for tf in vendor_diff.renames.values(): 1417 replaced_symlinks["/%s" % (tf.name,)] = tf.sha1 1418 1419 always_delete = [] 1420 may_delete = [] 1421 for dest, link in source_symlinks: 1422 if link not in target_symlinks_d: 1423 if link in replaced_symlinks: 1424 may_delete.append((link, replaced_symlinks[link])) 1425 else: 1426 always_delete.append(link) 1427 script.DeleteFiles(always_delete) 1428 script.DeleteFilesIfNotMatching(may_delete) 1429 1430 if system_diff.verbatim_targets: 1431 script.Print("Unpacking new system files...") 1432 script.UnpackPackageDir("system", "/system") 1433 if vendor_diff and vendor_diff.verbatim_targets: 1434 script.Print("Unpacking new vendor files...") 1435 script.UnpackPackageDir("vendor", "/vendor") 1436 1437 if updating_recovery and not target_has_recovery_patch: 1438 script.Print("Unpacking new recovery...") 1439 script.UnpackPackageDir("recovery", "/system") 1440 1441 system_diff.EmitRenames(script) 1442 if vendor_diff: 1443 vendor_diff.EmitRenames(script) 1444 1445 script.Print("Symlinks and permissions...") 1446 1447 # Create all the symlinks that don't already exist, or point to 1448 # somewhere different than what we want. Delete each symlink before 1449 # creating it, since the 'symlink' command won't overwrite. 1450 to_create = [] 1451 for dest, link in target_symlinks: 1452 if link in source_symlinks_d: 1453 if dest != source_symlinks_d[link]: 1454 to_create.append((dest, link)) 1455 else: 1456 to_create.append((dest, link)) 1457 script.DeleteFiles([i[1] for i in to_create]) 1458 script.MakeSymlinks(to_create) 1459 1460 # Now that the symlinks are created, we can set all the 1461 # permissions. 1462 script.AppendScript(temp_script) 1463 1464 # Do device-specific installation (eg, write radio image). 1465 device_specific.IncrementalOTA_InstallEnd() 1466 1467 if OPTIONS.extra_script is not None: 1468 script.AppendExtra(OPTIONS.extra_script) 1469 1470 # Patch the build.prop file last, so if something fails but the 1471 # device can still come up, it appears to be the old build and will 1472 # get set the OTA package again to retry. 1473 script.Print("Patching remaining system files...") 1474 system_diff.EmitDeferredPatches(script) 1475 1476 if OPTIONS.wipe_user_data: 1477 script.Print("Erasing user data...") 1478 script.FormatPartition("/data") 1479 1480 if OPTIONS.two_step: 1481 script.AppendExtra(""" 1482set_stage("%(bcb_dev)s", ""); 1483endif; 1484endif; 1485""" % bcb_dev) 1486 1487 if OPTIONS.verify and system_diff: 1488 script.Print("Remounting and verifying system partition files...") 1489 script.Unmount("/system") 1490 script.Mount("/system") 1491 system_diff.EmitExplicitTargetVerification(script) 1492 1493 if OPTIONS.verify and vendor_diff: 1494 script.Print("Remounting and verifying vendor partition files...") 1495 script.Unmount("/vendor") 1496 script.Mount("/vendor") 1497 vendor_diff.EmitExplicitTargetVerification(script) 1498 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 1499 1500 WriteMetadata(metadata, output_zip) 1501 1502 1503def main(argv): 1504 1505 def option_handler(o, a): 1506 if o == "--board_config": 1507 pass # deprecated 1508 elif o in ("-k", "--package_key"): 1509 OPTIONS.package_key = a 1510 elif o in ("-i", "--incremental_from"): 1511 OPTIONS.incremental_source = a 1512 elif o == "--full_radio": 1513 OPTIONS.full_radio = True 1514 elif o == "--full_bootloader": 1515 OPTIONS.full_bootloader = True 1516 elif o in ("-w", "--wipe_user_data"): 1517 OPTIONS.wipe_user_data = True 1518 elif o in ("-n", "--no_prereq"): 1519 OPTIONS.omit_prereq = True 1520 elif o in ("-o", "--oem_settings"): 1521 OPTIONS.oem_source = a 1522 elif o in ("-e", "--extra_script"): 1523 OPTIONS.extra_script = a 1524 elif o in ("-a", "--aslr_mode"): 1525 if a in ("on", "On", "true", "True", "yes", "Yes"): 1526 OPTIONS.aslr_mode = True 1527 else: 1528 OPTIONS.aslr_mode = False 1529 elif o in ("-t", "--worker_threads"): 1530 if a.isdigit(): 1531 OPTIONS.worker_threads = int(a) 1532 else: 1533 raise ValueError("Cannot parse value %r for option %r - only " 1534 "integers are allowed." % (a, o)) 1535 elif o in ("-2", "--two_step"): 1536 OPTIONS.two_step = True 1537 elif o == "--no_signing": 1538 OPTIONS.no_signing = True 1539 elif o == "--verify": 1540 OPTIONS.verify = True 1541 elif o == "--block": 1542 OPTIONS.block_based = True 1543 elif o in ("-b", "--binary"): 1544 OPTIONS.updater_binary = a 1545 elif o in ("--no_fallback_to_full",): 1546 OPTIONS.fallback_to_full = False 1547 elif o == "--stash_threshold": 1548 try: 1549 OPTIONS.stash_threshold = float(a) 1550 except ValueError: 1551 raise ValueError("Cannot parse value %r for option %r - expecting " 1552 "a float" % (a, o)) 1553 else: 1554 return False 1555 return True 1556 1557 args = common.ParseOptions(argv, __doc__, 1558 extra_opts="b:k:i:d:wne:t:a:2o:", 1559 extra_long_opts=[ 1560 "board_config=", 1561 "package_key=", 1562 "incremental_from=", 1563 "full_radio", 1564 "full_bootloader", 1565 "wipe_user_data", 1566 "no_prereq", 1567 "extra_script=", 1568 "worker_threads=", 1569 "aslr_mode=", 1570 "two_step", 1571 "no_signing", 1572 "block", 1573 "binary=", 1574 "oem_settings=", 1575 "verify", 1576 "no_fallback_to_full", 1577 "stash_threshold=", 1578 ], extra_option_handler=option_handler) 1579 1580 if len(args) != 2: 1581 common.Usage(__doc__) 1582 sys.exit(1) 1583 1584 if OPTIONS.extra_script is not None: 1585 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 1586 1587 print "unzipping target target-files..." 1588 OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0]) 1589 1590 OPTIONS.target_tmp = OPTIONS.input_tmp 1591 OPTIONS.info_dict = common.LoadInfoDict(input_zip) 1592 1593 # If this image was originally labelled with SELinux contexts, make sure we 1594 # also apply the labels in our new image. During building, the "file_contexts" 1595 # is in the out/ directory tree, but for repacking from target-files.zip it's 1596 # in the root directory of the ramdisk. 1597 if "selinux_fc" in OPTIONS.info_dict: 1598 OPTIONS.info_dict["selinux_fc"] = os.path.join( 1599 OPTIONS.input_tmp, "BOOT", "RAMDISK", "file_contexts") 1600 1601 if OPTIONS.verbose: 1602 print "--- target info ---" 1603 common.DumpInfoDict(OPTIONS.info_dict) 1604 1605 # If the caller explicitly specified the device-specific extensions 1606 # path via -s/--device_specific, use that. Otherwise, use 1607 # META/releasetools.py if it is present in the target target_files. 1608 # Otherwise, take the path of the file from 'tool_extensions' in the 1609 # info dict and look for that in the local filesystem, relative to 1610 # the current directory. 1611 1612 if OPTIONS.device_specific is None: 1613 from_input = os.path.join(OPTIONS.input_tmp, "META", "releasetools.py") 1614 if os.path.exists(from_input): 1615 print "(using device-specific extensions from target_files)" 1616 OPTIONS.device_specific = from_input 1617 else: 1618 OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None) 1619 1620 if OPTIONS.device_specific is not None: 1621 OPTIONS.device_specific = os.path.abspath(OPTIONS.device_specific) 1622 1623 while True: 1624 1625 if OPTIONS.no_signing: 1626 if os.path.exists(args[1]): 1627 os.unlink(args[1]) 1628 output_zip = zipfile.ZipFile(args[1], "w", 1629 compression=zipfile.ZIP_DEFLATED) 1630 else: 1631 temp_zip_file = tempfile.NamedTemporaryFile() 1632 output_zip = zipfile.ZipFile(temp_zip_file, "w", 1633 compression=zipfile.ZIP_DEFLATED) 1634 1635 cache_size = OPTIONS.info_dict.get("cache_size", None) 1636 if cache_size is None: 1637 print "--- can't determine the cache partition size ---" 1638 OPTIONS.cache_size = cache_size 1639 1640 if OPTIONS.incremental_source is None: 1641 WriteFullOTAPackage(input_zip, output_zip) 1642 if OPTIONS.package_key is None: 1643 OPTIONS.package_key = OPTIONS.info_dict.get( 1644 "default_system_dev_certificate", 1645 "build/target/product/security/testkey") 1646 common.ZipClose(output_zip) 1647 break 1648 1649 else: 1650 print "unzipping source target-files..." 1651 OPTIONS.source_tmp, source_zip = common.UnzipTemp( 1652 OPTIONS.incremental_source) 1653 OPTIONS.target_info_dict = OPTIONS.info_dict 1654 OPTIONS.source_info_dict = common.LoadInfoDict(source_zip) 1655 if "selinux_fc" in OPTIONS.source_info_dict: 1656 OPTIONS.source_info_dict["selinux_fc"] = os.path.join( 1657 OPTIONS.source_tmp, "BOOT", "RAMDISK", "file_contexts") 1658 if OPTIONS.package_key is None: 1659 OPTIONS.package_key = OPTIONS.source_info_dict.get( 1660 "default_system_dev_certificate", 1661 "build/target/product/security/testkey") 1662 if OPTIONS.verbose: 1663 print "--- source info ---" 1664 common.DumpInfoDict(OPTIONS.source_info_dict) 1665 try: 1666 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) 1667 common.ZipClose(output_zip) 1668 break 1669 except ValueError: 1670 if not OPTIONS.fallback_to_full: 1671 raise 1672 print "--- failed to build incremental; falling back to full ---" 1673 OPTIONS.incremental_source = None 1674 common.ZipClose(output_zip) 1675 1676 if not OPTIONS.no_signing: 1677 SignOutput(temp_zip_file.name, args[1]) 1678 temp_zip_file.close() 1679 1680 print "done." 1681 1682 1683if __name__ == '__main__': 1684 try: 1685 common.CloseInheritedPipes() 1686 main(sys.argv[1:]) 1687 except common.ExternalError as e: 1688 print 1689 print " ERROR: %s" % (e,) 1690 print 1691 sys.exit(1) 1692 finally: 1693 common.Cleanup() 1694