ota_from_target_files.py revision 8b72aefb5a8ed4da28c6f83854e8babf53b9cb53
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 --board_config <file> 25 Deprecated. 26 27 -k (--package_key) <key> Key to use to sign the package (default is 28 the value of default_system_dev_certificate from the input 29 target-files's META/misc_info.txt, or 30 "build/target/product/security/testkey" if that value is not 31 specified). 32 33 For incremental OTAs, the default value is based on the source 34 target-file, not the target build. 35 36 -i (--incremental_from) <file> 37 Generate an incremental OTA using the given target-files zip as 38 the starting build. 39 40 -v (--verify) 41 Remount and verify the checksums of the files written to the 42 system and vendor (if used) partitions. Incremental builds only. 43 44 -o (--oem_settings) <file> 45 Use the file to specify the expected OEM-specific properties 46 on the OEM partition of the intended device. 47 48 -w (--wipe_user_data) 49 Generate an OTA package that will wipe the user data partition 50 when installed. 51 52 -n (--no_prereq) 53 Omit the timestamp prereq check normally included at the top of 54 the build scripts (used for developer OTA packages which 55 legitimately need to go back and forth). 56 57 -e (--extra_script) <file> 58 Insert the contents of file at the end of the update script. 59 60 -a (--aslr_mode) <on|off> 61 Specify whether to turn on ASLR for the package (on by default). 62 63 -2 (--two_step) 64 Generate a 'two-step' OTA package, where recovery is updated 65 first, so that any changes made to the system partition are done 66 using the new recovery (new kernel, etc.). 67 68 --block 69 Generate a block-based OTA if possible. Will fall back to a 70 file-based OTA if the target_files is older and doesn't support 71 block-based OTAs. 72 73 -b (--binary) <file> 74 Use the given binary as the update-binary in the output package, 75 instead of the binary in the build's target_files. Use for 76 development only. 77 78 -t (--worker_threads) <int> 79 Specifies the number of worker-threads that will be used when 80 generating patches for incremental updates (defaults to 3). 81 82""" 83 84import sys 85 86if sys.hexversion < 0x02070000: 87 print >> sys.stderr, "Python 2.7 or newer is required." 88 sys.exit(1) 89 90import copy 91import multiprocessing 92import os 93import tempfile 94import zipfile 95 96import common 97import edify_generator 98import sparse_img 99 100OPTIONS = common.OPTIONS 101OPTIONS.package_key = None 102OPTIONS.incremental_source = None 103OPTIONS.verify = False 104OPTIONS.require_verbatim = set() 105OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 106OPTIONS.patch_threshold = 0.95 107OPTIONS.wipe_user_data = False 108OPTIONS.omit_prereq = False 109OPTIONS.extra_script = None 110OPTIONS.aslr_mode = True 111OPTIONS.worker_threads = multiprocessing.cpu_count() // 2 112if OPTIONS.worker_threads == 0: 113 OPTIONS.worker_threads = 1 114OPTIONS.two_step = False 115OPTIONS.no_signing = False 116OPTIONS.block_based = False 117OPTIONS.updater_binary = None 118OPTIONS.oem_source = None 119OPTIONS.fallback_to_full = True 120 121def MostPopularKey(d, default): 122 """Given a dict, return the key corresponding to the largest 123 value. Returns 'default' if the dict is empty.""" 124 x = [(v, k) for (k, v) in d.iteritems()] 125 if not x: 126 return default 127 x.sort() 128 return x[-1][1] 129 130 131def IsSymlink(info): 132 """Return true if the zipfile.ZipInfo object passed in represents a 133 symlink.""" 134 return (info.external_attr >> 16) == 0o120777 135 136def IsRegular(info): 137 """Return true if the zipfile.ZipInfo object passed in represents a 138 symlink.""" 139 return (info.external_attr >> 28) == 0o10 140 141def ClosestFileMatch(src, tgtfiles, existing): 142 """Returns the closest file match between a source file and list 143 of potential matches. The exact filename match is preferred, 144 then the sha1 is searched for, and finally a file with the same 145 basename is evaluated. Rename support in the updater-binary is 146 required for the latter checks to be used.""" 147 148 result = tgtfiles.get("path:" + src.name) 149 if result is not None: 150 return result 151 152 if not OPTIONS.target_info_dict.get("update_rename_support", False): 153 return None 154 155 if src.size < 1000: 156 return None 157 158 result = tgtfiles.get("sha1:" + src.sha1) 159 if result is not None and existing.get(result.name) is None: 160 return result 161 result = tgtfiles.get("file:" + src.name.split("/")[-1]) 162 if result is not None and existing.get(result.name) is None: 163 return result 164 return None 165 166class ItemSet(object): 167 def __init__(self, partition, fs_config): 168 self.partition = partition 169 self.fs_config = fs_config 170 self.ITEMS = {} 171 172 def Get(self, name, is_dir=False): 173 if name not in self.ITEMS: 174 self.ITEMS[name] = Item(self, name, is_dir=is_dir) 175 return self.ITEMS[name] 176 177 def GetMetadata(self, input_zip): 178 # The target_files contains a record of what the uid, 179 # gid, and mode are supposed to be. 180 output = input_zip.read(self.fs_config) 181 182 for line in output.split("\n"): 183 if not line: 184 continue 185 columns = line.split() 186 name, uid, gid, mode = columns[:4] 187 selabel = None 188 capabilities = None 189 190 # After the first 4 columns, there are a series of key=value 191 # pairs. Extract out the fields we care about. 192 for element in columns[4:]: 193 key, value = element.split("=") 194 if key == "selabel": 195 selabel = value 196 if key == "capabilities": 197 capabilities = value 198 199 i = self.ITEMS.get(name, None) 200 if i is not None: 201 i.uid = int(uid) 202 i.gid = int(gid) 203 i.mode = int(mode, 8) 204 i.selabel = selabel 205 i.capabilities = capabilities 206 if i.is_dir: 207 i.children.sort(key=lambda i: i.name) 208 209 # set metadata for the files generated by this script. 210 i = self.ITEMS.get("system/recovery-from-boot.p", None) 211 if i: 212 i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0o644, None, None 213 i = self.ITEMS.get("system/etc/install-recovery.sh", None) 214 if i: 215 i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0o544, None, None 216 217 218class Item(object): 219 """Items represent the metadata (user, group, mode) of files and 220 directories in the system image.""" 221 def __init__(self, itemset, name, is_dir=False): 222 self.itemset = itemset 223 self.name = name 224 self.uid = None 225 self.gid = None 226 self.mode = None 227 self.selabel = None 228 self.capabilities = None 229 self.is_dir = is_dir 230 self.descendants = None 231 self.best_subtree = None 232 233 if name: 234 self.parent = itemset.Get(os.path.dirname(name), is_dir=True) 235 self.parent.children.append(self) 236 else: 237 self.parent = None 238 if self.is_dir: 239 self.children = [] 240 241 def Dump(self, indent=0): 242 if self.uid is not None: 243 print "%s%s %d %d %o" % ( 244 " " * indent, self.name, self.uid, self.gid, self.mode) 245 else: 246 print "%s%s %s %s %s" % ( 247 " " * indent, self.name, self.uid, self.gid, self.mode) 248 if self.is_dir: 249 print "%s%s" % (" "*indent, self.descendants) 250 print "%s%s" % (" "*indent, self.best_subtree) 251 for i in self.children: 252 i.Dump(indent=indent+1) 253 254 def CountChildMetadata(self): 255 """Count up the (uid, gid, mode, selabel, capabilities) tuples for 256 all children and determine the best strategy for using set_perm_recursive 257 and set_perm to correctly chown/chmod all the files to their desired 258 values. Recursively calls itself for all descendants. 259 260 Returns a dict of {(uid, gid, dmode, fmode, selabel, capabilities): count} 261 counting up all descendants of this node. (dmode or fmode may be None.) 262 Also sets the best_subtree of each directory Item to the (uid, gid, dmode, 263 fmode, selabel, capabilities) tuple that will match the most descendants of 264 that Item. 265 """ 266 267 assert self.is_dir 268 key = (self.uid, self.gid, self.mode, None, self.selabel, 269 self.capabilities) 270 self.descendants = {key: 1} 271 d = self.descendants 272 for i in self.children: 273 if i.is_dir: 274 for k, v in i.CountChildMetadata().iteritems(): 275 d[k] = d.get(k, 0) + v 276 else: 277 k = (i.uid, i.gid, None, i.mode, i.selabel, i.capabilities) 278 d[k] = d.get(k, 0) + 1 279 280 # Find the (uid, gid, dmode, fmode, selabel, capabilities) 281 # tuple that matches the most descendants. 282 283 # First, find the (uid, gid) pair that matches the most 284 # descendants. 285 ug = {} 286 for (uid, gid, _, _, _, _), count in d.iteritems(): 287 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 288 ug = MostPopularKey(ug, (0, 0)) 289 290 # Now find the dmode, fmode, selabel, and capabilities that match 291 # the most descendants with that (uid, gid), and choose those. 292 best_dmode = (0, 0o755) 293 best_fmode = (0, 0o644) 294 best_selabel = (0, None) 295 best_capabilities = (0, None) 296 for k, count in d.iteritems(): 297 if k[:2] != ug: 298 continue 299 if k[2] is not None and count >= best_dmode[0]: 300 best_dmode = (count, k[2]) 301 if k[3] is not None and count >= best_fmode[0]: 302 best_fmode = (count, k[3]) 303 if k[4] is not None and count >= best_selabel[0]: 304 best_selabel = (count, k[4]) 305 if k[5] is not None and count >= best_capabilities[0]: 306 best_capabilities = (count, k[5]) 307 self.best_subtree = ug + ( 308 best_dmode[1], best_fmode[1], best_selabel[1], best_capabilities[1]) 309 310 return d 311 312 def SetPermissions(self, script): 313 """Append set_perm/set_perm_recursive commands to 'script' to 314 set all permissions, users, and groups for the tree of files 315 rooted at 'self'.""" 316 317 self.CountChildMetadata() 318 319 def recurse(item, current): 320 # current is the (uid, gid, dmode, fmode, selabel, capabilities) tuple 321 # that the current item (and all its children) have already been set to. 322 # We only need to issue set_perm/set_perm_recursive commands if we're 323 # supposed to be something different. 324 if item.is_dir: 325 if current != item.best_subtree: 326 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 327 current = item.best_subtree 328 329 if item.uid != current[0] or item.gid != current[1] or \ 330 item.mode != current[2] or item.selabel != current[4] or \ 331 item.capabilities != current[5]: 332 script.SetPermissions("/"+item.name, item.uid, item.gid, 333 item.mode, item.selabel, item.capabilities) 334 335 for i in item.children: 336 recurse(i, current) 337 else: 338 if item.uid != current[0] or item.gid != current[1] or \ 339 item.mode != current[3] or item.selabel != current[4] or \ 340 item.capabilities != current[5]: 341 script.SetPermissions("/"+item.name, item.uid, item.gid, 342 item.mode, item.selabel, item.capabilities) 343 344 recurse(self, (-1, -1, -1, -1, None, None)) 345 346 347def CopyPartitionFiles(itemset, input_zip, output_zip=None, substitute=None): 348 """Copies files for the partition in the input zip to the output 349 zip. Populates the Item class with their metadata, and returns a 350 list of symlinks. output_zip may be None, in which case the copy is 351 skipped (but the other side effects still happen). substitute is an 352 optional dict of {output filename: contents} to be output instead of 353 certain input files. 354 """ 355 356 symlinks = [] 357 358 partition = itemset.partition 359 360 for info in input_zip.infolist(): 361 prefix = partition.upper() + "/" 362 if info.filename.startswith(prefix): 363 basefilename = info.filename[len(prefix):] 364 if IsSymlink(info): 365 symlinks.append((input_zip.read(info.filename), 366 "/" + partition + "/" + basefilename)) 367 else: 368 info2 = copy.copy(info) 369 fn = info2.filename = partition + "/" + basefilename 370 if substitute and fn in substitute and substitute[fn] is None: 371 continue 372 if output_zip is not None: 373 if substitute and fn in substitute: 374 data = substitute[fn] 375 else: 376 data = input_zip.read(info.filename) 377 output_zip.writestr(info2, data) 378 if fn.endswith("/"): 379 itemset.Get(fn[:-1], is_dir=True) 380 else: 381 itemset.Get(fn) 382 383 symlinks.sort() 384 return symlinks 385 386 387def SignOutput(temp_zip_name, output_zip_name): 388 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 389 pw = key_passwords[OPTIONS.package_key] 390 391 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 392 whole_file=True) 393 394 395def AppendAssertions(script, info_dict, oem_dict=None): 396 oem_props = info_dict.get("oem_fingerprint_properties") 397 if oem_props is None or len(oem_props) == 0: 398 device = GetBuildProp("ro.product.device", info_dict) 399 script.AssertDevice(device) 400 else: 401 if oem_dict is None: 402 raise common.ExternalError( 403 "No OEM file provided to answer expected assertions") 404 for prop in oem_props.split(): 405 if oem_dict.get(prop) is None: 406 raise common.ExternalError( 407 "The OEM file is missing the property %s" % prop) 408 script.AssertOemProperty(prop, oem_dict.get(prop)) 409 410 411def HasRecoveryPatch(target_files_zip): 412 try: 413 target_files_zip.getinfo("SYSTEM/recovery-from-boot.p") 414 return True 415 except KeyError: 416 return False 417 418def HasVendorPartition(target_files_zip): 419 try: 420 target_files_zip.getinfo("VENDOR/") 421 return True 422 except KeyError: 423 return False 424 425def GetOemProperty(name, oem_props, oem_dict, info_dict): 426 if oem_props is not None and name in oem_props: 427 return oem_dict[name] 428 return GetBuildProp(name, info_dict) 429 430 431def CalculateFingerprint(oem_props, oem_dict, info_dict): 432 if oem_props is None: 433 return GetBuildProp("ro.build.fingerprint", info_dict) 434 return "%s/%s/%s:%s" % ( 435 GetOemProperty("ro.product.brand", oem_props, oem_dict, info_dict), 436 GetOemProperty("ro.product.name", oem_props, oem_dict, info_dict), 437 GetOemProperty("ro.product.device", oem_props, oem_dict, info_dict), 438 GetBuildProp("ro.build.thumbprint", info_dict)) 439 440 441def GetImage(which, tmpdir, info_dict): 442 # Return an image object (suitable for passing to BlockImageDiff) 443 # for the 'which' partition (most be "system" or "vendor"). If a 444 # prebuilt image and file map are found in tmpdir they are used, 445 # otherwise they are reconstructed from the individual files. 446 447 assert which in ("system", "vendor") 448 449 path = os.path.join(tmpdir, "IMAGES", which + ".img") 450 mappath = os.path.join(tmpdir, "IMAGES", which + ".map") 451 if os.path.exists(path) and os.path.exists(mappath): 452 print "using %s.img from target-files" % (which,) 453 # This is a 'new' target-files, which already has the image in it. 454 455 else: 456 print "building %s.img from target-files" % (which,) 457 458 # This is an 'old' target-files, which does not contain images 459 # already built. Build them. 460 461 mappath = tempfile.mkstemp()[1] 462 OPTIONS.tempfiles.append(mappath) 463 464 import add_img_to_target_files 465 if which == "system": 466 path = add_img_to_target_files.BuildSystem( 467 tmpdir, info_dict, block_list=mappath) 468 elif which == "vendor": 469 path = add_img_to_target_files.BuildVendor( 470 tmpdir, info_dict, block_list=mappath) 471 472 return sparse_img.SparseImage(path, mappath) 473 474 475def WriteFullOTAPackage(input_zip, output_zip): 476 # TODO: how to determine this? We don't know what version it will 477 # be installed on top of. For now, we expect the API just won't 478 # change very often. 479 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 480 481 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 482 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 483 oem_dict = None 484 if oem_props is not None and len(oem_props) > 0: 485 if OPTIONS.oem_source is None: 486 raise common.ExternalError("OEM source required for this build") 487 script.Mount("/oem", recovery_mount_options) 488 oem_dict = common.LoadDictionaryFromLines( 489 open(OPTIONS.oem_source).readlines()) 490 491 metadata = { 492 "post-build": CalculateFingerprint(oem_props, oem_dict, 493 OPTIONS.info_dict), 494 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 495 OPTIONS.info_dict), 496 "post-timestamp": GetBuildProp("ro.build.date.utc", OPTIONS.info_dict), 497 } 498 499 device_specific = common.DeviceSpecificParams( 500 input_zip=input_zip, 501 input_version=OPTIONS.info_dict["recovery_api_version"], 502 output_zip=output_zip, 503 script=script, 504 input_tmp=OPTIONS.input_tmp, 505 metadata=metadata, 506 info_dict=OPTIONS.info_dict) 507 508 has_recovery_patch = HasRecoveryPatch(input_zip) 509 block_based = OPTIONS.block_based and has_recovery_patch 510 511 if not OPTIONS.omit_prereq: 512 ts = GetBuildProp("ro.build.date.utc", OPTIONS.info_dict) 513 ts_text = GetBuildProp("ro.build.date", OPTIONS.info_dict) 514 script.AssertOlderBuild(ts, ts_text) 515 516 AppendAssertions(script, OPTIONS.info_dict, oem_dict) 517 device_specific.FullOTA_Assertions() 518 519 # Two-step package strategy (in chronological order, which is *not* 520 # the order in which the generated script has things): 521 # 522 # if stage is not "2/3" or "3/3": 523 # write recovery image to boot partition 524 # set stage to "2/3" 525 # reboot to boot partition and restart recovery 526 # else if stage is "2/3": 527 # write recovery image to recovery partition 528 # set stage to "3/3" 529 # reboot to recovery partition and restart recovery 530 # else: 531 # (stage must be "3/3") 532 # set stage to "" 533 # do normal full package installation: 534 # wipe and install system, boot image, etc. 535 # set up system to update recovery partition on first boot 536 # complete script normally 537 # (allow recovery to mark itself finished and reboot) 538 539 recovery_img = common.GetBootableImage("recovery.img", "recovery.img", 540 OPTIONS.input_tmp, "RECOVERY") 541 if OPTIONS.two_step: 542 if not OPTIONS.info_dict.get("multistage_support", None): 543 assert False, "two-step packages not supported by this build" 544 fs = OPTIONS.info_dict["fstab"]["/misc"] 545 assert fs.fs_type.upper() == "EMMC", \ 546 "two-step packages only supported on devices with EMMC /misc partitions" 547 bcb_dev = {"bcb_dev": fs.device} 548 common.ZipWriteStr(output_zip, "recovery.img", recovery_img.data) 549 script.AppendExtra(""" 550if get_stage("%(bcb_dev)s") == "2/3" then 551""" % bcb_dev) 552 script.WriteRawImage("/recovery", "recovery.img") 553 script.AppendExtra(""" 554set_stage("%(bcb_dev)s", "3/3"); 555reboot_now("%(bcb_dev)s", "recovery"); 556else if get_stage("%(bcb_dev)s") == "3/3" then 557""" % bcb_dev) 558 559 device_specific.FullOTA_InstallBegin() 560 561 system_progress = 0.75 562 563 if OPTIONS.wipe_user_data: 564 system_progress -= 0.1 565 if HasVendorPartition(input_zip): 566 system_progress -= 0.1 567 568 if "selinux_fc" in OPTIONS.info_dict: 569 WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip) 570 571 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 572 573 system_items = ItemSet("system", "META/filesystem_config.txt") 574 script.ShowProgress(system_progress, 0) 575 576 if block_based: 577 # Full OTA is done as an "incremental" against an empty source 578 # image. This has the effect of writing new data from the package 579 # to the entire partition, but lets us reuse the updater code that 580 # writes incrementals to do it. 581 system_tgt = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict) 582 system_tgt.ResetFileMap() 583 system_diff = common.BlockDifference("system", system_tgt, src=None) 584 system_diff.WriteScript(script, output_zip) 585 else: 586 script.FormatPartition("/system") 587 script.Mount("/system", recovery_mount_options) 588 if not has_recovery_patch: 589 script.UnpackPackageDir("recovery", "/system") 590 script.UnpackPackageDir("system", "/system") 591 592 symlinks = CopyPartitionFiles(system_items, input_zip, output_zip) 593 script.MakeSymlinks(symlinks) 594 595 boot_img = common.GetBootableImage("boot.img", "boot.img", 596 OPTIONS.input_tmp, "BOOT") 597 598 if not block_based: 599 def output_sink(fn, data): 600 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 601 system_items.Get("system/" + fn) 602 603 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, 604 recovery_img, boot_img) 605 606 system_items.GetMetadata(input_zip) 607 system_items.Get("system").SetPermissions(script) 608 609 if HasVendorPartition(input_zip): 610 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 611 script.ShowProgress(0.1, 0) 612 613 if block_based: 614 vendor_tgt = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict) 615 vendor_tgt.ResetFileMap() 616 vendor_diff = common.BlockDifference("vendor", vendor_tgt) 617 vendor_diff.WriteScript(script, output_zip) 618 else: 619 script.FormatPartition("/vendor") 620 script.Mount("/vendor", recovery_mount_options) 621 script.UnpackPackageDir("vendor", "/vendor") 622 623 symlinks = CopyPartitionFiles(vendor_items, input_zip, output_zip) 624 script.MakeSymlinks(symlinks) 625 626 vendor_items.GetMetadata(input_zip) 627 vendor_items.Get("vendor").SetPermissions(script) 628 629 common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict) 630 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 631 632 script.ShowProgress(0.05, 5) 633 script.WriteRawImage("/boot", "boot.img") 634 635 script.ShowProgress(0.2, 10) 636 device_specific.FullOTA_InstallEnd() 637 638 if OPTIONS.extra_script is not None: 639 script.AppendExtra(OPTIONS.extra_script) 640 641 script.UnmountAll() 642 643 if OPTIONS.wipe_user_data: 644 script.ShowProgress(0.1, 10) 645 script.FormatPartition("/data") 646 647 if OPTIONS.two_step: 648 script.AppendExtra(""" 649set_stage("%(bcb_dev)s", ""); 650""" % bcb_dev) 651 script.AppendExtra("else\n") 652 script.WriteRawImage("/boot", "recovery.img") 653 script.AppendExtra(""" 654set_stage("%(bcb_dev)s", "2/3"); 655reboot_now("%(bcb_dev)s", ""); 656endif; 657endif; 658""" % bcb_dev) 659 script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary) 660 WriteMetadata(metadata, output_zip) 661 662 663def WritePolicyConfig(file_name, output_zip): 664 common.ZipWrite(output_zip, file_name, os.path.basename(file_name)) 665 666 667def WriteMetadata(metadata, output_zip): 668 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata", 669 "".join(["%s=%s\n" % kv 670 for kv in sorted(metadata.iteritems())])) 671 672 673def LoadPartitionFiles(z, partition): 674 """Load all the files from the given partition in a given target-files 675 ZipFile, and return a dict of {filename: File object}.""" 676 out = {} 677 prefix = partition.upper() + "/" 678 for info in z.infolist(): 679 if info.filename.startswith(prefix) and not IsSymlink(info): 680 basefilename = info.filename[len(prefix):] 681 fn = partition + "/" + basefilename 682 data = z.read(info.filename) 683 out[fn] = common.File(fn, data) 684 return out 685 686 687def GetBuildProp(prop, info_dict): 688 """Return the fingerprint of the build of a given target-files info_dict.""" 689 try: 690 return info_dict.get("build.prop", {})[prop] 691 except KeyError: 692 raise common.ExternalError("couldn't find %s in build.prop" % (prop,)) 693 694 695def AddToKnownPaths(filename, known_paths): 696 if filename[-1] == "/": 697 return 698 dirs = filename.split("/")[:-1] 699 while len(dirs) > 0: 700 path = "/".join(dirs) 701 if path in known_paths: 702 break 703 known_paths.add(path) 704 dirs.pop() 705 706 707def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 708 source_version = OPTIONS.source_info_dict["recovery_api_version"] 709 target_version = OPTIONS.target_info_dict["recovery_api_version"] 710 711 if source_version == 0: 712 print ("WARNING: generating edify script for a source that " 713 "can't install it.") 714 script = edify_generator.EdifyGenerator(source_version, 715 OPTIONS.target_info_dict) 716 717 metadata = { 718 "pre-device": GetBuildProp("ro.product.device", 719 OPTIONS.source_info_dict), 720 "post-timestamp": GetBuildProp("ro.build.date.utc", 721 OPTIONS.target_info_dict), 722 } 723 724 device_specific = common.DeviceSpecificParams( 725 source_zip=source_zip, 726 source_version=source_version, 727 target_zip=target_zip, 728 target_version=target_version, 729 output_zip=output_zip, 730 script=script, 731 metadata=metadata, 732 info_dict=OPTIONS.info_dict) 733 734 source_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.source_info_dict) 735 target_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.target_info_dict) 736 metadata["pre-build"] = source_fp 737 metadata["post-build"] = target_fp 738 739 source_boot = common.GetBootableImage( 740 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 741 OPTIONS.source_info_dict) 742 target_boot = common.GetBootableImage( 743 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 744 updating_boot = (not OPTIONS.two_step and 745 (source_boot.data != target_boot.data)) 746 747 target_recovery = common.GetBootableImage( 748 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 749 750 system_src = GetImage("system", OPTIONS.source_tmp, OPTIONS.source_info_dict) 751 system_tgt = GetImage("system", OPTIONS.target_tmp, OPTIONS.target_info_dict) 752 753 blockimgdiff_version = 1 754 if OPTIONS.info_dict: 755 blockimgdiff_version = max( 756 int(i) for i in 757 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(",")) 758 759 system_diff = common.BlockDifference("system", system_tgt, system_src, 760 check_first_block=True, 761 version=blockimgdiff_version) 762 763 if HasVendorPartition(target_zip): 764 if not HasVendorPartition(source_zip): 765 raise RuntimeError("can't generate incremental that adds /vendor") 766 vendor_src = GetImage("vendor", OPTIONS.source_tmp, 767 OPTIONS.source_info_dict) 768 vendor_tgt = GetImage("vendor", OPTIONS.target_tmp, 769 OPTIONS.target_info_dict) 770 vendor_diff = common.BlockDifference("vendor", vendor_tgt, vendor_src, 771 check_first_block=True, 772 version=blockimgdiff_version) 773 else: 774 vendor_diff = None 775 776 oem_props = OPTIONS.target_info_dict.get("oem_fingerprint_properties") 777 recovery_mount_options = OPTIONS.target_info_dict.get( 778 "recovery_mount_options") 779 oem_dict = None 780 if oem_props is not None and len(oem_props) > 0: 781 if OPTIONS.oem_source is None: 782 raise common.ExternalError("OEM source required for this build") 783 script.Mount("/oem", recovery_mount_options) 784 oem_dict = common.LoadDictionaryFromLines( 785 open(OPTIONS.oem_source).readlines()) 786 787 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 788 device_specific.IncrementalOTA_Assertions() 789 790 # Two-step incremental package strategy (in chronological order, 791 # which is *not* the order in which the generated script has 792 # things): 793 # 794 # if stage is not "2/3" or "3/3": 795 # do verification on current system 796 # write recovery image to boot partition 797 # set stage to "2/3" 798 # reboot to boot partition and restart recovery 799 # else if stage is "2/3": 800 # write recovery image to recovery partition 801 # set stage to "3/3" 802 # reboot to recovery partition and restart recovery 803 # else: 804 # (stage must be "3/3") 805 # perform update: 806 # patch system files, etc. 807 # force full install of new boot image 808 # set up system to update recovery partition on first boot 809 # complete script normally 810 # (allow recovery to mark itself finished and reboot) 811 812 if OPTIONS.two_step: 813 if not OPTIONS.info_dict.get("multistage_support", None): 814 assert False, "two-step packages not supported by this build" 815 fs = OPTIONS.info_dict["fstab"]["/misc"] 816 assert fs.fs_type.upper() == "EMMC", \ 817 "two-step packages only supported on devices with EMMC /misc partitions" 818 bcb_dev = {"bcb_dev": fs.device} 819 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 820 script.AppendExtra(""" 821if get_stage("%(bcb_dev)s") == "2/3" then 822""" % bcb_dev) 823 script.AppendExtra("sleep(20);\n") 824 script.WriteRawImage("/recovery", "recovery.img") 825 script.AppendExtra(""" 826set_stage("%(bcb_dev)s", "3/3"); 827reboot_now("%(bcb_dev)s", "recovery"); 828else if get_stage("%(bcb_dev)s") != "3/3" then 829""" % bcb_dev) 830 831 script.Print("Verifying current system...") 832 833 device_specific.IncrementalOTA_VerifyBegin() 834 835 if oem_props is None: 836 # When blockimgdiff version is less than 3 (non-resumable block-based OTA), 837 # patching on a device that's already on the target build will damage the 838 # system. Because operations like move don't check the block state, they 839 # always apply the changes unconditionally. 840 if blockimgdiff_version <= 2: 841 script.AssertSomeFingerprint(source_fp) 842 else: 843 script.AssertSomeFingerprint(source_fp, target_fp) 844 else: 845 if blockimgdiff_version <= 2: 846 script.AssertSomeThumbprint( 847 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 848 else: 849 script.AssertSomeThumbprint( 850 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 851 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 852 853 if updating_boot: 854 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 855 d = common.Difference(target_boot, source_boot) 856 _, _, d = d.ComputePatch() 857 if d is None: 858 include_full_boot = True 859 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 860 else: 861 include_full_boot = False 862 863 print "boot target: %d source: %d diff: %d" % ( 864 target_boot.size, source_boot.size, len(d)) 865 866 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 867 868 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 869 (boot_type, boot_device, 870 source_boot.size, source_boot.sha1, 871 target_boot.size, target_boot.sha1)) 872 873 device_specific.IncrementalOTA_VerifyEnd() 874 875 if OPTIONS.two_step: 876 script.WriteRawImage("/boot", "recovery.img") 877 script.AppendExtra(""" 878set_stage("%(bcb_dev)s", "2/3"); 879reboot_now("%(bcb_dev)s", ""); 880else 881""" % bcb_dev) 882 883 # Verify the existing partitions. 884 system_diff.WriteVerifyScript(script) 885 if vendor_diff: 886 vendor_diff.WriteVerifyScript(script) 887 888 script.Comment("---- start making changes here ----") 889 890 device_specific.IncrementalOTA_InstallBegin() 891 892 system_diff.WriteScript(script, output_zip, 893 progress=0.8 if vendor_diff else 0.9) 894 if vendor_diff: 895 vendor_diff.WriteScript(script, output_zip, progress=0.1) 896 897 if OPTIONS.two_step: 898 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 899 script.WriteRawImage("/boot", "boot.img") 900 print "writing full boot image (forced by two-step mode)" 901 902 if not OPTIONS.two_step: 903 if updating_boot: 904 if include_full_boot: 905 print "boot image changed; including full." 906 script.Print("Installing boot image...") 907 script.WriteRawImage("/boot", "boot.img") 908 else: 909 # Produce the boot image by applying a patch to the current 910 # contents of the boot partition, and write it back to the 911 # partition. 912 print "boot image changed; including patch." 913 script.Print("Patching boot image...") 914 script.ShowProgress(0.1, 10) 915 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 916 % (boot_type, boot_device, 917 source_boot.size, source_boot.sha1, 918 target_boot.size, target_boot.sha1), 919 "-", 920 target_boot.size, target_boot.sha1, 921 source_boot.sha1, "patch/boot.img.p") 922 else: 923 print "boot image unchanged; skipping." 924 925 # Do device-specific installation (eg, write radio image). 926 device_specific.IncrementalOTA_InstallEnd() 927 928 if OPTIONS.extra_script is not None: 929 script.AppendExtra(OPTIONS.extra_script) 930 931 if OPTIONS.wipe_user_data: 932 script.Print("Erasing user data...") 933 script.FormatPartition("/data") 934 935 if OPTIONS.two_step: 936 script.AppendExtra(""" 937set_stage("%(bcb_dev)s", ""); 938endif; 939endif; 940""" % bcb_dev) 941 942 script.SetProgress(1) 943 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 944 WriteMetadata(metadata, output_zip) 945 946 947class FileDifference(object): 948 def __init__(self, partition, source_zip, target_zip, output_zip): 949 self.deferred_patch_list = None 950 print "Loading target..." 951 self.target_data = target_data = LoadPartitionFiles(target_zip, partition) 952 print "Loading source..." 953 self.source_data = source_data = LoadPartitionFiles(source_zip, partition) 954 955 self.verbatim_targets = verbatim_targets = [] 956 self.patch_list = patch_list = [] 957 diffs = [] 958 self.renames = renames = {} 959 known_paths = set() 960 largest_source_size = 0 961 962 matching_file_cache = {} 963 for fn, sf in source_data.items(): 964 assert fn == sf.name 965 matching_file_cache["path:" + fn] = sf 966 if fn in target_data.keys(): 967 AddToKnownPaths(fn, known_paths) 968 # Only allow eligibility for filename/sha matching 969 # if there isn't a perfect path match. 970 if target_data.get(sf.name) is None: 971 matching_file_cache["file:" + fn.split("/")[-1]] = sf 972 matching_file_cache["sha:" + sf.sha1] = sf 973 974 for fn in sorted(target_data.keys()): 975 tf = target_data[fn] 976 assert fn == tf.name 977 sf = ClosestFileMatch(tf, matching_file_cache, renames) 978 if sf is not None and sf.name != tf.name: 979 print "File has moved from " + sf.name + " to " + tf.name 980 renames[sf.name] = tf 981 982 if sf is None or fn in OPTIONS.require_verbatim: 983 # This file should be included verbatim 984 if fn in OPTIONS.prohibit_verbatim: 985 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 986 print "send", fn, "verbatim" 987 tf.AddToZip(output_zip) 988 verbatim_targets.append((fn, tf.size, tf.sha1)) 989 if fn in target_data.keys(): 990 AddToKnownPaths(fn, known_paths) 991 elif tf.sha1 != sf.sha1: 992 # File is different; consider sending as a patch 993 diffs.append(common.Difference(tf, sf)) 994 else: 995 # Target file data identical to source (may still be renamed) 996 pass 997 998 common.ComputeDifferences(diffs) 999 1000 for diff in diffs: 1001 tf, sf, d = diff.GetPatch() 1002 path = "/".join(tf.name.split("/")[:-1]) 1003 if d is None or len(d) > tf.size * OPTIONS.patch_threshold or \ 1004 path not in known_paths: 1005 # patch is almost as big as the file; don't bother patching 1006 # or a patch + rename cannot take place due to the target 1007 # directory not existing 1008 tf.AddToZip(output_zip) 1009 verbatim_targets.append((tf.name, tf.size, tf.sha1)) 1010 if sf.name in renames: 1011 del renames[sf.name] 1012 AddToKnownPaths(tf.name, known_paths) 1013 else: 1014 common.ZipWriteStr(output_zip, "patch/" + sf.name + ".p", d) 1015 patch_list.append((tf, sf, tf.size, common.sha1(d).hexdigest())) 1016 largest_source_size = max(largest_source_size, sf.size) 1017 1018 self.largest_source_size = largest_source_size 1019 1020 def EmitVerification(self, script): 1021 so_far = 0 1022 for tf, sf, _, _ in self.patch_list: 1023 if tf.name != sf.name: 1024 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1025 script.PatchCheck("/"+sf.name, tf.sha1, sf.sha1) 1026 so_far += sf.size 1027 return so_far 1028 1029 def EmitExplicitTargetVerification(self, script): 1030 for fn, _, sha1 in self.verbatim_targets: 1031 if fn[-1] != "/": 1032 script.FileCheck("/"+fn, sha1) 1033 for tf, _, _, _ in self.patch_list: 1034 script.FileCheck(tf.name, tf.sha1) 1035 1036 def RemoveUnneededFiles(self, script, extras=()): 1037 script.DeleteFiles( 1038 ["/" + i[0] for i in self.verbatim_targets] + 1039 ["/" + i for i in sorted(self.source_data) 1040 if i not in self.target_data and i not in self.renames] + 1041 list(extras)) 1042 1043 def TotalPatchSize(self): 1044 return sum(i[1].size for i in self.patch_list) 1045 1046 def EmitPatches(self, script, total_patch_size, so_far): 1047 self.deferred_patch_list = deferred_patch_list = [] 1048 for item in self.patch_list: 1049 tf, sf, _, _ = item 1050 if tf.name == "system/build.prop": 1051 deferred_patch_list.append(item) 1052 continue 1053 if sf.name != tf.name: 1054 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1055 script.ApplyPatch("/" + sf.name, "-", tf.size, tf.sha1, sf.sha1, 1056 "patch/" + sf.name + ".p") 1057 so_far += tf.size 1058 script.SetProgress(so_far / total_patch_size) 1059 return so_far 1060 1061 def EmitDeferredPatches(self, script): 1062 for item in self.deferred_patch_list: 1063 tf, sf, _, _ = item 1064 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, 1065 "patch/" + sf.name + ".p") 1066 script.SetPermissions("/system/build.prop", 0, 0, 0o644, None, None) 1067 1068 def EmitRenames(self, script): 1069 if len(self.renames) > 0: 1070 script.Print("Renaming files...") 1071 for src, tgt in self.renames.iteritems(): 1072 print "Renaming " + src + " to " + tgt.name 1073 script.RenameFile(src, tgt.name) 1074 1075 1076def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): 1077 target_has_recovery_patch = HasRecoveryPatch(target_zip) 1078 source_has_recovery_patch = HasRecoveryPatch(source_zip) 1079 1080 if (OPTIONS.block_based and 1081 target_has_recovery_patch and 1082 source_has_recovery_patch): 1083 return WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip) 1084 1085 source_version = OPTIONS.source_info_dict["recovery_api_version"] 1086 target_version = OPTIONS.target_info_dict["recovery_api_version"] 1087 1088 if source_version == 0: 1089 print ("WARNING: generating edify script for a source that " 1090 "can't install it.") 1091 script = edify_generator.EdifyGenerator(source_version, 1092 OPTIONS.target_info_dict) 1093 1094 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 1095 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 1096 oem_dict = None 1097 if oem_props is not None and len(oem_props) > 0: 1098 if OPTIONS.oem_source is None: 1099 raise common.ExternalError("OEM source required for this build") 1100 script.Mount("/oem", recovery_mount_options) 1101 oem_dict = common.LoadDictionaryFromLines( 1102 open(OPTIONS.oem_source).readlines()) 1103 1104 metadata = { 1105 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 1106 OPTIONS.source_info_dict), 1107 "post-timestamp": GetBuildProp("ro.build.date.utc", 1108 OPTIONS.target_info_dict), 1109 } 1110 1111 device_specific = common.DeviceSpecificParams( 1112 source_zip=source_zip, 1113 source_version=source_version, 1114 target_zip=target_zip, 1115 target_version=target_version, 1116 output_zip=output_zip, 1117 script=script, 1118 metadata=metadata, 1119 info_dict=OPTIONS.info_dict) 1120 1121 system_diff = FileDifference("system", source_zip, target_zip, output_zip) 1122 script.Mount("/system", recovery_mount_options) 1123 if HasVendorPartition(target_zip): 1124 vendor_diff = FileDifference("vendor", source_zip, target_zip, output_zip) 1125 script.Mount("/vendor", recovery_mount_options) 1126 else: 1127 vendor_diff = None 1128 1129 target_fp = CalculateFingerprint(oem_props, oem_dict, 1130 OPTIONS.target_info_dict) 1131 source_fp = CalculateFingerprint(oem_props, oem_dict, 1132 OPTIONS.source_info_dict) 1133 1134 if oem_props is None: 1135 script.AssertSomeFingerprint(source_fp, target_fp) 1136 else: 1137 script.AssertSomeThumbprint( 1138 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 1139 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 1140 1141 metadata["pre-build"] = source_fp 1142 metadata["post-build"] = target_fp 1143 1144 source_boot = common.GetBootableImage( 1145 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 1146 OPTIONS.source_info_dict) 1147 target_boot = common.GetBootableImage( 1148 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 1149 updating_boot = (not OPTIONS.two_step and 1150 (source_boot.data != target_boot.data)) 1151 1152 source_recovery = common.GetBootableImage( 1153 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 1154 OPTIONS.source_info_dict) 1155 target_recovery = common.GetBootableImage( 1156 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 1157 updating_recovery = (source_recovery.data != target_recovery.data) 1158 1159 # Here's how we divide up the progress bar: 1160 # 0.1 for verifying the start state (PatchCheck calls) 1161 # 0.8 for applying patches (ApplyPatch calls) 1162 # 0.1 for unpacking verbatim files, symlinking, and doing the 1163 # device-specific commands. 1164 1165 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 1166 device_specific.IncrementalOTA_Assertions() 1167 1168 # Two-step incremental package strategy (in chronological order, 1169 # which is *not* the order in which the generated script has 1170 # things): 1171 # 1172 # if stage is not "2/3" or "3/3": 1173 # do verification on current system 1174 # write recovery image to boot partition 1175 # set stage to "2/3" 1176 # reboot to boot partition and restart recovery 1177 # else if stage is "2/3": 1178 # write recovery image to recovery partition 1179 # set stage to "3/3" 1180 # reboot to recovery partition and restart recovery 1181 # else: 1182 # (stage must be "3/3") 1183 # perform update: 1184 # patch system files, etc. 1185 # force full install of new boot image 1186 # set up system to update recovery partition on first boot 1187 # complete script normally 1188 # (allow recovery to mark itself finished and reboot) 1189 1190 if OPTIONS.two_step: 1191 if not OPTIONS.info_dict.get("multistage_support", None): 1192 assert False, "two-step packages not supported by this build" 1193 fs = OPTIONS.info_dict["fstab"]["/misc"] 1194 assert fs.fs_type.upper() == "EMMC", \ 1195 "two-step packages only supported on devices with EMMC /misc partitions" 1196 bcb_dev = {"bcb_dev": fs.device} 1197 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 1198 script.AppendExtra(""" 1199if get_stage("%(bcb_dev)s") == "2/3" then 1200""" % bcb_dev) 1201 script.AppendExtra("sleep(20);\n") 1202 script.WriteRawImage("/recovery", "recovery.img") 1203 script.AppendExtra(""" 1204set_stage("%(bcb_dev)s", "3/3"); 1205reboot_now("%(bcb_dev)s", "recovery"); 1206else if get_stage("%(bcb_dev)s") != "3/3" then 1207""" % bcb_dev) 1208 1209 script.Print("Verifying current system...") 1210 1211 device_specific.IncrementalOTA_VerifyBegin() 1212 1213 script.ShowProgress(0.1, 0) 1214 so_far = system_diff.EmitVerification(script) 1215 if vendor_diff: 1216 so_far += vendor_diff.EmitVerification(script) 1217 1218 if updating_boot: 1219 d = common.Difference(target_boot, source_boot) 1220 _, _, d = d.ComputePatch() 1221 print "boot target: %d source: %d diff: %d" % ( 1222 target_boot.size, source_boot.size, len(d)) 1223 1224 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 1225 1226 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 1227 1228 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 1229 (boot_type, boot_device, 1230 source_boot.size, source_boot.sha1, 1231 target_boot.size, target_boot.sha1)) 1232 so_far += source_boot.size 1233 1234 size = [] 1235 if system_diff.patch_list: 1236 size.append(system_diff.largest_source_size) 1237 if vendor_diff: 1238 if vendor_diff.patch_list: 1239 size.append(vendor_diff.largest_source_size) 1240 if size or updating_recovery or updating_boot: 1241 script.CacheFreeSpaceCheck(max(size)) 1242 1243 device_specific.IncrementalOTA_VerifyEnd() 1244 1245 if OPTIONS.two_step: 1246 script.WriteRawImage("/boot", "recovery.img") 1247 script.AppendExtra(""" 1248set_stage("%(bcb_dev)s", "2/3"); 1249reboot_now("%(bcb_dev)s", ""); 1250else 1251""" % bcb_dev) 1252 1253 script.Comment("---- start making changes here ----") 1254 1255 device_specific.IncrementalOTA_InstallBegin() 1256 1257 if OPTIONS.two_step: 1258 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 1259 script.WriteRawImage("/boot", "boot.img") 1260 print "writing full boot image (forced by two-step mode)" 1261 1262 script.Print("Removing unneeded files...") 1263 system_diff.RemoveUnneededFiles(script, ("/system/recovery.img",)) 1264 if vendor_diff: 1265 vendor_diff.RemoveUnneededFiles(script) 1266 1267 script.ShowProgress(0.8, 0) 1268 total_patch_size = 1.0 + system_diff.TotalPatchSize() 1269 if vendor_diff: 1270 total_patch_size += vendor_diff.TotalPatchSize() 1271 if updating_boot: 1272 total_patch_size += target_boot.size 1273 1274 script.Print("Patching system files...") 1275 so_far = system_diff.EmitPatches(script, total_patch_size, 0) 1276 if vendor_diff: 1277 script.Print("Patching vendor files...") 1278 so_far = vendor_diff.EmitPatches(script, total_patch_size, so_far) 1279 1280 if not OPTIONS.two_step: 1281 if updating_boot: 1282 # Produce the boot image by applying a patch to the current 1283 # contents of the boot partition, and write it back to the 1284 # partition. 1285 script.Print("Patching boot image...") 1286 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 1287 % (boot_type, boot_device, 1288 source_boot.size, source_boot.sha1, 1289 target_boot.size, target_boot.sha1), 1290 "-", 1291 target_boot.size, target_boot.sha1, 1292 source_boot.sha1, "patch/boot.img.p") 1293 so_far += target_boot.size 1294 script.SetProgress(so_far / total_patch_size) 1295 print "boot image changed; including." 1296 else: 1297 print "boot image unchanged; skipping." 1298 1299 system_items = ItemSet("system", "META/filesystem_config.txt") 1300 if vendor_diff: 1301 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 1302 1303 if updating_recovery: 1304 # Recovery is generated as a patch using both the boot image 1305 # (which contains the same linux kernel as recovery) and the file 1306 # /system/etc/recovery-resource.dat (which contains all the images 1307 # used in the recovery UI) as sources. This lets us minimize the 1308 # size of the patch, which must be included in every OTA package. 1309 # 1310 # For older builds where recovery-resource.dat is not present, we 1311 # use only the boot image as the source. 1312 1313 if not target_has_recovery_patch: 1314 def output_sink(fn, data): 1315 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 1316 system_items.Get("system/" + fn) 1317 1318 common.MakeRecoveryPatch(OPTIONS.target_tmp, output_sink, 1319 target_recovery, target_boot) 1320 script.DeleteFiles(["/system/recovery-from-boot.p", 1321 "/system/etc/install-recovery.sh"]) 1322 print "recovery image changed; including as patch from boot." 1323 else: 1324 print "recovery image unchanged; skipping." 1325 1326 script.ShowProgress(0.1, 10) 1327 1328 target_symlinks = CopyPartitionFiles(system_items, target_zip, None) 1329 if vendor_diff: 1330 target_symlinks.extend(CopyPartitionFiles(vendor_items, target_zip, None)) 1331 1332 temp_script = script.MakeTemporary() 1333 system_items.GetMetadata(target_zip) 1334 system_items.Get("system").SetPermissions(temp_script) 1335 if vendor_diff: 1336 vendor_items.GetMetadata(target_zip) 1337 vendor_items.Get("vendor").SetPermissions(temp_script) 1338 1339 # Note that this call will mess up the trees of Items, so make sure 1340 # we're done with them. 1341 source_symlinks = CopyPartitionFiles(system_items, source_zip, None) 1342 if vendor_diff: 1343 source_symlinks.extend(CopyPartitionFiles(vendor_items, source_zip, None)) 1344 1345 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 1346 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 1347 1348 # Delete all the symlinks in source that aren't in target. This 1349 # needs to happen before verbatim files are unpacked, in case a 1350 # symlink in the source is replaced by a real file in the target. 1351 to_delete = [] 1352 for dest, link in source_symlinks: 1353 if link not in target_symlinks_d: 1354 to_delete.append(link) 1355 script.DeleteFiles(to_delete) 1356 1357 if system_diff.verbatim_targets: 1358 script.Print("Unpacking new system files...") 1359 script.UnpackPackageDir("system", "/system") 1360 if vendor_diff and vendor_diff.verbatim_targets: 1361 script.Print("Unpacking new vendor files...") 1362 script.UnpackPackageDir("vendor", "/vendor") 1363 1364 if updating_recovery and not target_has_recovery_patch: 1365 script.Print("Unpacking new recovery...") 1366 script.UnpackPackageDir("recovery", "/system") 1367 1368 system_diff.EmitRenames(script) 1369 if vendor_diff: 1370 vendor_diff.EmitRenames(script) 1371 1372 script.Print("Symlinks and permissions...") 1373 1374 # Create all the symlinks that don't already exist, or point to 1375 # somewhere different than what we want. Delete each symlink before 1376 # creating it, since the 'symlink' command won't overwrite. 1377 to_create = [] 1378 for dest, link in target_symlinks: 1379 if link in source_symlinks_d: 1380 if dest != source_symlinks_d[link]: 1381 to_create.append((dest, link)) 1382 else: 1383 to_create.append((dest, link)) 1384 script.DeleteFiles([i[1] for i in to_create]) 1385 script.MakeSymlinks(to_create) 1386 1387 # Now that the symlinks are created, we can set all the 1388 # permissions. 1389 script.AppendScript(temp_script) 1390 1391 # Do device-specific installation (eg, write radio image). 1392 device_specific.IncrementalOTA_InstallEnd() 1393 1394 if OPTIONS.extra_script is not None: 1395 script.AppendExtra(OPTIONS.extra_script) 1396 1397 # Patch the build.prop file last, so if something fails but the 1398 # device can still come up, it appears to be the old build and will 1399 # get set the OTA package again to retry. 1400 script.Print("Patching remaining system files...") 1401 system_diff.EmitDeferredPatches(script) 1402 1403 if OPTIONS.wipe_user_data: 1404 script.Print("Erasing user data...") 1405 script.FormatPartition("/data") 1406 1407 if OPTIONS.two_step: 1408 script.AppendExtra(""" 1409set_stage("%(bcb_dev)s", ""); 1410endif; 1411endif; 1412""" % bcb_dev) 1413 1414 if OPTIONS.verify and system_diff: 1415 script.Print("Remounting and verifying system partition files...") 1416 script.Unmount("/system") 1417 script.Mount("/system") 1418 system_diff.EmitExplicitTargetVerification(script) 1419 1420 if OPTIONS.verify and vendor_diff: 1421 script.Print("Remounting and verifying vendor partition files...") 1422 script.Unmount("/vendor") 1423 script.Mount("/vendor") 1424 vendor_diff.EmitExplicitTargetVerification(script) 1425 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 1426 1427 WriteMetadata(metadata, output_zip) 1428 1429 1430def main(argv): 1431 1432 def option_handler(o, a): 1433 if o == "--board_config": 1434 pass # deprecated 1435 elif o in ("-k", "--package_key"): 1436 OPTIONS.package_key = a 1437 elif o in ("-i", "--incremental_from"): 1438 OPTIONS.incremental_source = a 1439 elif o in ("-w", "--wipe_user_data"): 1440 OPTIONS.wipe_user_data = True 1441 elif o in ("-n", "--no_prereq"): 1442 OPTIONS.omit_prereq = True 1443 elif o in ("-o", "--oem_settings"): 1444 OPTIONS.oem_source = a 1445 elif o in ("-e", "--extra_script"): 1446 OPTIONS.extra_script = a 1447 elif o in ("-a", "--aslr_mode"): 1448 if a in ("on", "On", "true", "True", "yes", "Yes"): 1449 OPTIONS.aslr_mode = True 1450 else: 1451 OPTIONS.aslr_mode = False 1452 elif o in ("-t", "--worker_threads"): 1453 if a.isdigit(): 1454 OPTIONS.worker_threads = int(a) 1455 else: 1456 raise ValueError("Cannot parse value %r for option %r - only " 1457 "integers are allowed." % (a, o)) 1458 elif o in ("-2", "--two_step"): 1459 OPTIONS.two_step = True 1460 elif o == "--no_signing": 1461 OPTIONS.no_signing = True 1462 elif o == "--verify": 1463 OPTIONS.verify = True 1464 elif o == "--block": 1465 OPTIONS.block_based = True 1466 elif o in ("-b", "--binary"): 1467 OPTIONS.updater_binary = a 1468 elif o in ("--no_fallback_to_full",): 1469 OPTIONS.fallback_to_full = False 1470 else: 1471 return False 1472 return True 1473 1474 args = common.ParseOptions(argv, __doc__, 1475 extra_opts="b:k:i:d:wne:t:a:2o:", 1476 extra_long_opts=[ 1477 "board_config=", 1478 "package_key=", 1479 "incremental_from=", 1480 "wipe_user_data", 1481 "no_prereq", 1482 "extra_script=", 1483 "worker_threads=", 1484 "aslr_mode=", 1485 "two_step", 1486 "no_signing", 1487 "block", 1488 "binary=", 1489 "oem_settings=", 1490 "verify", 1491 "no_fallback_to_full", 1492 ], extra_option_handler=option_handler) 1493 1494 if len(args) != 2: 1495 common.Usage(__doc__) 1496 sys.exit(1) 1497 1498 if OPTIONS.extra_script is not None: 1499 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 1500 1501 print "unzipping target target-files..." 1502 OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0]) 1503 1504 OPTIONS.target_tmp = OPTIONS.input_tmp 1505 OPTIONS.info_dict = common.LoadInfoDict(input_zip) 1506 1507 # If this image was originally labelled with SELinux contexts, make sure we 1508 # also apply the labels in our new image. During building, the "file_contexts" 1509 # is in the out/ directory tree, but for repacking from target-files.zip it's 1510 # in the root directory of the ramdisk. 1511 if "selinux_fc" in OPTIONS.info_dict: 1512 OPTIONS.info_dict["selinux_fc"] = os.path.join( 1513 OPTIONS.input_tmp, "BOOT", "RAMDISK", "file_contexts") 1514 1515 if OPTIONS.verbose: 1516 print "--- target info ---" 1517 common.DumpInfoDict(OPTIONS.info_dict) 1518 1519 # If the caller explicitly specified the device-specific extensions 1520 # path via -s/--device_specific, use that. Otherwise, use 1521 # META/releasetools.py if it is present in the target target_files. 1522 # Otherwise, take the path of the file from 'tool_extensions' in the 1523 # info dict and look for that in the local filesystem, relative to 1524 # the current directory. 1525 1526 if OPTIONS.device_specific is None: 1527 from_input = os.path.join(OPTIONS.input_tmp, "META", "releasetools.py") 1528 if os.path.exists(from_input): 1529 print "(using device-specific extensions from target_files)" 1530 OPTIONS.device_specific = from_input 1531 else: 1532 OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None) 1533 1534 if OPTIONS.device_specific is not None: 1535 OPTIONS.device_specific = os.path.abspath(OPTIONS.device_specific) 1536 1537 while True: 1538 1539 if OPTIONS.no_signing: 1540 if os.path.exists(args[1]): 1541 os.unlink(args[1]) 1542 output_zip = zipfile.ZipFile(args[1], "w", 1543 compression=zipfile.ZIP_DEFLATED) 1544 else: 1545 temp_zip_file = tempfile.NamedTemporaryFile() 1546 output_zip = zipfile.ZipFile(temp_zip_file, "w", 1547 compression=zipfile.ZIP_DEFLATED) 1548 1549 if OPTIONS.incremental_source is None: 1550 WriteFullOTAPackage(input_zip, output_zip) 1551 if OPTIONS.package_key is None: 1552 OPTIONS.package_key = OPTIONS.info_dict.get( 1553 "default_system_dev_certificate", 1554 "build/target/product/security/testkey") 1555 break 1556 1557 else: 1558 print "unzipping source target-files..." 1559 OPTIONS.source_tmp, source_zip = common.UnzipTemp( 1560 OPTIONS.incremental_source) 1561 OPTIONS.target_info_dict = OPTIONS.info_dict 1562 OPTIONS.source_info_dict = common.LoadInfoDict(source_zip) 1563 if "selinux_fc" in OPTIONS.source_info_dict: 1564 OPTIONS.source_info_dict["selinux_fc"] = os.path.join( 1565 OPTIONS.source_tmp, "BOOT", "RAMDISK", "file_contexts") 1566 if OPTIONS.package_key is None: 1567 OPTIONS.package_key = OPTIONS.source_info_dict.get( 1568 "default_system_dev_certificate", 1569 "build/target/product/security/testkey") 1570 if OPTIONS.verbose: 1571 print "--- source info ---" 1572 common.DumpInfoDict(OPTIONS.source_info_dict) 1573 try: 1574 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) 1575 break 1576 except ValueError: 1577 if not OPTIONS.fallback_to_full: 1578 raise 1579 print "--- failed to build incremental; falling back to full ---" 1580 OPTIONS.incremental_source = None 1581 output_zip.close() 1582 1583 output_zip.close() 1584 1585 if not OPTIONS.no_signing: 1586 SignOutput(temp_zip_file.name, args[1]) 1587 temp_zip_file.close() 1588 1589 print "done." 1590 1591 1592if __name__ == '__main__': 1593 try: 1594 common.CloseInheritedPipes() 1595 main(sys.argv[1:]) 1596 except common.ExternalError as e: 1597 print 1598 print " ERROR: %s" % (e,) 1599 print 1600 sys.exit(1) 1601 finally: 1602 common.Cleanup() 1603