ota_from_target_files.py revision dd2a5892e500e9c07f933e3f511b18e67df076c4
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 --board_config <file> 25 Deprecated. 26 27 -k (--package_key) <key> Key to use to sign the package (default is 28 the value of default_system_dev_certificate from the input 29 target-files's META/misc_info.txt, or 30 "build/target/product/security/testkey" if that value is not 31 specified). 32 33 For incremental OTAs, the default value is based on the source 34 target-file, not the target build. 35 36 -i (--incremental_from) <file> 37 Generate an incremental OTA using the given target-files zip as 38 the starting build. 39 40 -v (--verify) 41 Remount and verify the checksums of the files written to the 42 system and vendor (if used) partitions. Incremental builds only. 43 44 -o (--oem_settings) <file> 45 Use the file to specify the expected OEM-specific properties 46 on the OEM partition of the intended device. 47 48 -w (--wipe_user_data) 49 Generate an OTA package that will wipe the user data partition 50 when installed. 51 52 -n (--no_prereq) 53 Omit the timestamp prereq check normally included at the top of 54 the build scripts (used for developer OTA packages which 55 legitimately need to go back and forth). 56 57 -e (--extra_script) <file> 58 Insert the contents of file at the end of the update script. 59 60 -a (--aslr_mode) <on|off> 61 Specify whether to turn on ASLR for the package (on by default). 62 63 -2 (--two_step) 64 Generate a 'two-step' OTA package, where recovery is updated 65 first, so that any changes made to the system partition are done 66 using the new recovery (new kernel, etc.). 67 68 --block 69 Generate a block-based OTA if possible. Will fall back to a 70 file-based OTA if the target_files is older and doesn't support 71 block-based OTAs. 72 73 -b (--binary) <file> 74 Use the given binary as the update-binary in the output package, 75 instead of the binary in the build's target_files. Use for 76 development only. 77 78 -t (--worker_threads) <int> 79 Specifies the number of worker-threads that will be used when 80 generating patches for incremental updates (defaults to 3). 81 82""" 83 84import sys 85 86if sys.hexversion < 0x02070000: 87 print >> sys.stderr, "Python 2.7 or newer is required." 88 sys.exit(1) 89 90import copy 91import errno 92import multiprocessing 93import os 94import re 95import subprocess 96import tempfile 97import time 98import zipfile 99 100from hashlib import sha1 as sha1 101 102import common 103import edify_generator 104import build_image 105import blockimgdiff 106import sparse_img 107 108OPTIONS = common.OPTIONS 109OPTIONS.package_key = None 110OPTIONS.incremental_source = None 111OPTIONS.verify = False 112OPTIONS.require_verbatim = set() 113OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 114OPTIONS.patch_threshold = 0.95 115OPTIONS.wipe_user_data = False 116OPTIONS.omit_prereq = False 117OPTIONS.extra_script = None 118OPTIONS.aslr_mode = True 119OPTIONS.worker_threads = multiprocessing.cpu_count() // 2 120if OPTIONS.worker_threads == 0: 121 OPTIONS.worker_threads = 1 122OPTIONS.two_step = False 123OPTIONS.no_signing = False 124OPTIONS.block_based = False 125OPTIONS.updater_binary = None 126OPTIONS.oem_source = None 127OPTIONS.fallback_to_full = True 128 129def MostPopularKey(d, default): 130 """Given a dict, return the key corresponding to the largest 131 value. Returns 'default' if the dict is empty.""" 132 x = [(v, k) for (k, v) in d.iteritems()] 133 if not x: return default 134 x.sort() 135 return x[-1][1] 136 137 138def IsSymlink(info): 139 """Return true if the zipfile.ZipInfo object passed in represents a 140 symlink.""" 141 return (info.external_attr >> 16) == 0120777 142 143def IsRegular(info): 144 """Return true if the zipfile.ZipInfo object passed in represents a 145 symlink.""" 146 return (info.external_attr >> 28) == 010 147 148def ClosestFileMatch(src, tgtfiles, existing): 149 """Returns the closest file match between a source file and list 150 of potential matches. The exact filename match is preferred, 151 then the sha1 is searched for, and finally a file with the same 152 basename is evaluated. Rename support in the updater-binary is 153 required for the latter checks to be used.""" 154 155 result = tgtfiles.get("path:" + src.name) 156 if result is not None: 157 return result 158 159 if not OPTIONS.target_info_dict.get("update_rename_support", False): 160 return None 161 162 if src.size < 1000: 163 return None 164 165 result = tgtfiles.get("sha1:" + src.sha1) 166 if result is not None and existing.get(result.name) is None: 167 return result 168 result = tgtfiles.get("file:" + src.name.split("/")[-1]) 169 if result is not None and existing.get(result.name) is None: 170 return result 171 return None 172 173class ItemSet: 174 def __init__(self, partition, fs_config): 175 self.partition = partition 176 self.fs_config = fs_config 177 self.ITEMS = {} 178 179 def Get(self, name, dir=False): 180 if name not in self.ITEMS: 181 self.ITEMS[name] = Item(self, name, dir=dir) 182 return self.ITEMS[name] 183 184 def GetMetadata(self, input_zip): 185 # The target_files contains a record of what the uid, 186 # gid, and mode are supposed to be. 187 output = input_zip.read(self.fs_config) 188 189 for line in output.split("\n"): 190 if not line: continue 191 columns = line.split() 192 name, uid, gid, mode = columns[:4] 193 selabel = None 194 capabilities = None 195 196 # After the first 4 columns, there are a series of key=value 197 # pairs. Extract out the fields we care about. 198 for element in columns[4:]: 199 key, value = element.split("=") 200 if key == "selabel": 201 selabel = value 202 if key == "capabilities": 203 capabilities = value 204 205 i = self.ITEMS.get(name, None) 206 if i is not None: 207 i.uid = int(uid) 208 i.gid = int(gid) 209 i.mode = int(mode, 8) 210 i.selabel = selabel 211 i.capabilities = capabilities 212 if i.dir: 213 i.children.sort(key=lambda i: i.name) 214 215 # set metadata for the files generated by this script. 216 i = self.ITEMS.get("system/recovery-from-boot.p", None) 217 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0644, None, None 218 i = self.ITEMS.get("system/etc/install-recovery.sh", None) 219 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0544, None, None 220 221 222class Item: 223 """Items represent the metadata (user, group, mode) of files and 224 directories in the system image.""" 225 def __init__(self, itemset, name, dir=False): 226 self.itemset = itemset 227 self.name = name 228 self.uid = None 229 self.gid = None 230 self.mode = None 231 self.selabel = None 232 self.capabilities = None 233 self.dir = dir 234 235 if name: 236 self.parent = itemset.Get(os.path.dirname(name), dir=True) 237 self.parent.children.append(self) 238 else: 239 self.parent = None 240 if dir: 241 self.children = [] 242 243 def Dump(self, indent=0): 244 if self.uid is not None: 245 print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode) 246 else: 247 print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode) 248 if self.dir: 249 print "%s%s" % (" "*indent, self.descendants) 250 print "%s%s" % (" "*indent, self.best_subtree) 251 for i in self.children: 252 i.Dump(indent=indent+1) 253 254 def CountChildMetadata(self): 255 """Count up the (uid, gid, mode, selabel, capabilities) tuples for 256 all children and determine the best strategy for using set_perm_recursive and 257 set_perm to correctly chown/chmod all the files to their desired 258 values. Recursively calls itself for all descendants. 259 260 Returns a dict of {(uid, gid, dmode, fmode, selabel, capabilities): count} counting up 261 all descendants of this node. (dmode or fmode may be None.) Also 262 sets the best_subtree of each directory Item to the (uid, gid, 263 dmode, fmode, selabel, capabilities) tuple that will match the most 264 descendants of that Item. 265 """ 266 267 assert self.dir 268 d = self.descendants = {(self.uid, self.gid, self.mode, None, self.selabel, self.capabilities): 1} 269 for i in self.children: 270 if i.dir: 271 for k, v in i.CountChildMetadata().iteritems(): 272 d[k] = d.get(k, 0) + v 273 else: 274 k = (i.uid, i.gid, None, i.mode, i.selabel, i.capabilities) 275 d[k] = d.get(k, 0) + 1 276 277 # Find the (uid, gid, dmode, fmode, selabel, capabilities) 278 # tuple that matches the most descendants. 279 280 # First, find the (uid, gid) pair that matches the most 281 # descendants. 282 ug = {} 283 for (uid, gid, _, _, _, _), count in d.iteritems(): 284 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 285 ug = MostPopularKey(ug, (0, 0)) 286 287 # Now find the dmode, fmode, selabel, and capabilities that match 288 # the most descendants with that (uid, gid), and choose those. 289 best_dmode = (0, 0755) 290 best_fmode = (0, 0644) 291 best_selabel = (0, None) 292 best_capabilities = (0, None) 293 for k, count in d.iteritems(): 294 if k[:2] != ug: continue 295 if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2]) 296 if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3]) 297 if k[4] is not None and count >= best_selabel[0]: best_selabel = (count, k[4]) 298 if k[5] is not None and count >= best_capabilities[0]: best_capabilities = (count, k[5]) 299 self.best_subtree = ug + (best_dmode[1], best_fmode[1], best_selabel[1], best_capabilities[1]) 300 301 return d 302 303 def SetPermissions(self, script): 304 """Append set_perm/set_perm_recursive commands to 'script' to 305 set all permissions, users, and groups for the tree of files 306 rooted at 'self'.""" 307 308 self.CountChildMetadata() 309 310 def recurse(item, current): 311 # current is the (uid, gid, dmode, fmode, selabel, capabilities) tuple that the current 312 # item (and all its children) have already been set to. We only 313 # need to issue set_perm/set_perm_recursive commands if we're 314 # supposed to be something different. 315 if item.dir: 316 if current != item.best_subtree: 317 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 318 current = item.best_subtree 319 320 if item.uid != current[0] or item.gid != current[1] or \ 321 item.mode != current[2] or item.selabel != current[4] or \ 322 item.capabilities != current[5]: 323 script.SetPermissions("/"+item.name, item.uid, item.gid, 324 item.mode, item.selabel, item.capabilities) 325 326 for i in item.children: 327 recurse(i, current) 328 else: 329 if item.uid != current[0] or item.gid != current[1] or \ 330 item.mode != current[3] or item.selabel != current[4] or \ 331 item.capabilities != current[5]: 332 script.SetPermissions("/"+item.name, item.uid, item.gid, 333 item.mode, item.selabel, item.capabilities) 334 335 recurse(self, (-1, -1, -1, -1, None, None)) 336 337 338def CopyPartitionFiles(itemset, input_zip, output_zip=None, substitute=None): 339 """Copies files for the partition in the input zip to the output 340 zip. Populates the Item class with their metadata, and returns a 341 list of symlinks. output_zip may be None, in which case the copy is 342 skipped (but the other side effects still happen). substitute is an 343 optional dict of {output filename: contents} to be output instead of 344 certain input files. 345 """ 346 347 symlinks = [] 348 349 partition = itemset.partition 350 351 for info in input_zip.infolist(): 352 if info.filename.startswith(partition.upper() + "/"): 353 basefilename = info.filename[7:] 354 if IsSymlink(info): 355 symlinks.append((input_zip.read(info.filename), 356 "/" + partition + "/" + basefilename)) 357 else: 358 info2 = copy.copy(info) 359 fn = info2.filename = partition + "/" + basefilename 360 if substitute and fn in substitute and substitute[fn] is None: 361 continue 362 if output_zip is not None: 363 if substitute and fn in substitute: 364 data = substitute[fn] 365 else: 366 data = input_zip.read(info.filename) 367 output_zip.writestr(info2, data) 368 if fn.endswith("/"): 369 itemset.Get(fn[:-1], dir=True) 370 else: 371 itemset.Get(fn, dir=False) 372 373 symlinks.sort() 374 return symlinks 375 376 377def SignOutput(temp_zip_name, output_zip_name): 378 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 379 pw = key_passwords[OPTIONS.package_key] 380 381 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 382 whole_file=True) 383 384 385def AppendAssertions(script, info_dict, oem_dict = None): 386 oem_props = info_dict.get("oem_fingerprint_properties") 387 if oem_props is None or len(oem_props) == 0: 388 device = GetBuildProp("ro.product.device", info_dict) 389 script.AssertDevice(device) 390 else: 391 if oem_dict is None: 392 raise common.ExternalError("No OEM file provided to answer expected assertions") 393 for prop in oem_props.split(): 394 if oem_dict.get(prop) is None: 395 raise common.ExternalError("The OEM file is missing the property %s" % prop) 396 script.AssertOemProperty(prop, oem_dict.get(prop)) 397 398 399def HasRecoveryPatch(target_files_zip): 400 try: 401 target_files_zip.getinfo("SYSTEM/recovery-from-boot.p") 402 return True 403 except KeyError: 404 return False 405 406def HasVendorPartition(target_files_zip): 407 try: 408 target_files_zip.getinfo("VENDOR/") 409 return True 410 except KeyError: 411 return False 412 413def GetOemProperty(name, oem_props, oem_dict, info_dict): 414 if oem_props is not None and name in oem_props: 415 return oem_dict[name] 416 return GetBuildProp(name, info_dict) 417 418 419def CalculateFingerprint(oem_props, oem_dict, info_dict): 420 if oem_props is None: 421 return GetBuildProp("ro.build.fingerprint", info_dict) 422 return "%s/%s/%s:%s" % ( 423 GetOemProperty("ro.product.brand", oem_props, oem_dict, info_dict), 424 GetOemProperty("ro.product.name", oem_props, oem_dict, info_dict), 425 GetOemProperty("ro.product.device", oem_props, oem_dict, info_dict), 426 GetBuildProp("ro.build.thumbprint", info_dict)) 427 428 429def GetImage(which, tmpdir, info_dict): 430 # Return an image object (suitable for passing to BlockImageDiff) 431 # for the 'which' partition (most be "system" or "vendor"). If a 432 # prebuilt image and file map are found in tmpdir they are used, 433 # otherwise they are reconstructed from the individual files. 434 435 assert which in ("system", "vendor") 436 437 path = os.path.join(tmpdir, "IMAGES", which + ".img") 438 mappath = os.path.join(tmpdir, "IMAGES", which + ".map") 439 if os.path.exists(path) and os.path.exists(mappath): 440 print "using %s.img from target-files" % (which,) 441 # This is a 'new' target-files, which already has the image in it. 442 443 else: 444 print "building %s.img from target-files" % (which,) 445 446 # This is an 'old' target-files, which does not contain images 447 # already built. Build them. 448 449 mappath = tempfile.mkstemp()[1] 450 OPTIONS.tempfiles.append(mappath) 451 452 import add_img_to_target_files 453 if which == "system": 454 path = add_img_to_target_files.BuildSystem( 455 tmpdir, info_dict, block_list=mappath) 456 elif which == "vendor": 457 path = add_img_to_target_files.BuildVendor( 458 tmpdir, info_dict, block_list=mappath) 459 460 return sparse_img.SparseImage(path, mappath) 461 462 463def WriteFullOTAPackage(input_zip, output_zip): 464 # TODO: how to determine this? We don't know what version it will 465 # be installed on top of. For now, we expect the API just won't 466 # change very often. 467 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 468 469 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 470 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 471 oem_dict = None 472 if oem_props is not None and len(oem_props) > 0: 473 if OPTIONS.oem_source is None: 474 raise common.ExternalError("OEM source required for this build") 475 script.Mount("/oem", recovery_mount_options) 476 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 477 478 metadata = {"post-build": CalculateFingerprint( 479 oem_props, oem_dict, OPTIONS.info_dict), 480 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 481 OPTIONS.info_dict), 482 "post-timestamp": GetBuildProp("ro.build.date.utc", 483 OPTIONS.info_dict), 484 } 485 486 device_specific = common.DeviceSpecificParams( 487 input_zip=input_zip, 488 input_version=OPTIONS.info_dict["recovery_api_version"], 489 output_zip=output_zip, 490 script=script, 491 input_tmp=OPTIONS.input_tmp, 492 metadata=metadata, 493 info_dict=OPTIONS.info_dict) 494 495 has_recovery_patch = HasRecoveryPatch(input_zip) 496 block_based = OPTIONS.block_based and has_recovery_patch 497 498 if not OPTIONS.omit_prereq: 499 ts = GetBuildProp("ro.build.date.utc", OPTIONS.info_dict) 500 ts_text = GetBuildProp("ro.build.date", OPTIONS.info_dict) 501 script.AssertOlderBuild(ts, ts_text) 502 503 AppendAssertions(script, OPTIONS.info_dict, oem_dict) 504 device_specific.FullOTA_Assertions() 505 506 # Two-step package strategy (in chronological order, which is *not* 507 # the order in which the generated script has things): 508 # 509 # if stage is not "2/3" or "3/3": 510 # write recovery image to boot partition 511 # set stage to "2/3" 512 # reboot to boot partition and restart recovery 513 # else if stage is "2/3": 514 # write recovery image to recovery partition 515 # set stage to "3/3" 516 # reboot to recovery partition and restart recovery 517 # else: 518 # (stage must be "3/3") 519 # set stage to "" 520 # do normal full package installation: 521 # wipe and install system, boot image, etc. 522 # set up system to update recovery partition on first boot 523 # complete script normally (allow recovery to mark itself finished and reboot) 524 525 recovery_img = common.GetBootableImage("recovery.img", "recovery.img", 526 OPTIONS.input_tmp, "RECOVERY") 527 if OPTIONS.two_step: 528 if not OPTIONS.info_dict.get("multistage_support", None): 529 assert False, "two-step packages not supported by this build" 530 fs = OPTIONS.info_dict["fstab"]["/misc"] 531 assert fs.fs_type.upper() == "EMMC", \ 532 "two-step packages only supported on devices with EMMC /misc partitions" 533 bcb_dev = {"bcb_dev": fs.device} 534 common.ZipWriteStr(output_zip, "recovery.img", recovery_img.data) 535 script.AppendExtra(""" 536if get_stage("%(bcb_dev)s") == "2/3" then 537""" % bcb_dev) 538 script.WriteRawImage("/recovery", "recovery.img") 539 script.AppendExtra(""" 540set_stage("%(bcb_dev)s", "3/3"); 541reboot_now("%(bcb_dev)s", "recovery"); 542else if get_stage("%(bcb_dev)s") == "3/3" then 543""" % bcb_dev) 544 545 device_specific.FullOTA_InstallBegin() 546 547 system_progress = 0.75 548 549 if OPTIONS.wipe_user_data: 550 system_progress -= 0.1 551 if HasVendorPartition(input_zip): 552 system_progress -= 0.1 553 554 if "selinux_fc" in OPTIONS.info_dict: 555 WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip) 556 557 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 558 559 system_items = ItemSet("system", "META/filesystem_config.txt") 560 script.ShowProgress(system_progress, 0) 561 562 if block_based: 563 # Full OTA is done as an "incremental" against an empty source 564 # image. This has the effect of writing new data from the package 565 # to the entire partition, but lets us reuse the updater code that 566 # writes incrementals to do it. 567 system_tgt = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict) 568 system_tgt.ResetFileMap() 569 system_diff = common.BlockDifference("system", system_tgt, src=None) 570 system_diff.WriteScript(script, output_zip) 571 else: 572 script.FormatPartition("/system") 573 script.Mount("/system", recovery_mount_options) 574 if not has_recovery_patch: 575 script.UnpackPackageDir("recovery", "/system") 576 script.UnpackPackageDir("system", "/system") 577 578 symlinks = CopyPartitionFiles(system_items, input_zip, output_zip) 579 script.MakeSymlinks(symlinks) 580 581 boot_img = common.GetBootableImage("boot.img", "boot.img", 582 OPTIONS.input_tmp, "BOOT") 583 584 if not block_based: 585 def output_sink(fn, data): 586 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 587 system_items.Get("system/" + fn, dir=False) 588 589 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, 590 recovery_img, boot_img) 591 592 system_items.GetMetadata(input_zip) 593 system_items.Get("system").SetPermissions(script) 594 595 if HasVendorPartition(input_zip): 596 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 597 script.ShowProgress(0.1, 0) 598 599 if block_based: 600 vendor_tgt = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict) 601 vendor_tgt.ResetFileMap() 602 vendor_diff = common.BlockDifference("vendor", vendor_tgt) 603 vendor_diff.WriteScript(script, output_zip) 604 else: 605 script.FormatPartition("/vendor") 606 script.Mount("/vendor", recovery_mount_options) 607 script.UnpackPackageDir("vendor", "/vendor") 608 609 symlinks = CopyPartitionFiles(vendor_items, input_zip, output_zip) 610 script.MakeSymlinks(symlinks) 611 612 vendor_items.GetMetadata(input_zip) 613 vendor_items.Get("vendor").SetPermissions(script) 614 615 common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict) 616 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 617 618 script.ShowProgress(0.05, 5) 619 script.WriteRawImage("/boot", "boot.img") 620 621 script.ShowProgress(0.2, 10) 622 device_specific.FullOTA_InstallEnd() 623 624 if OPTIONS.extra_script is not None: 625 script.AppendExtra(OPTIONS.extra_script) 626 627 script.UnmountAll() 628 629 if OPTIONS.wipe_user_data: 630 script.ShowProgress(0.1, 10) 631 script.FormatPartition("/data") 632 633 if OPTIONS.two_step: 634 script.AppendExtra(""" 635set_stage("%(bcb_dev)s", ""); 636""" % bcb_dev) 637 script.AppendExtra("else\n") 638 script.WriteRawImage("/boot", "recovery.img") 639 script.AppendExtra(""" 640set_stage("%(bcb_dev)s", "2/3"); 641reboot_now("%(bcb_dev)s", ""); 642endif; 643endif; 644""" % bcb_dev) 645 script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary) 646 WriteMetadata(metadata, output_zip) 647 648 649def WritePolicyConfig(file_context, output_zip): 650 f = open(file_context, 'r'); 651 basename = os.path.basename(file_context) 652 common.ZipWriteStr(output_zip, basename, f.read()) 653 654 655def WriteMetadata(metadata, output_zip): 656 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata", 657 "".join(["%s=%s\n" % kv 658 for kv in sorted(metadata.iteritems())])) 659 660 661def LoadPartitionFiles(z, partition): 662 """Load all the files from the given partition in a given target-files 663 ZipFile, and return a dict of {filename: File object}.""" 664 out = {} 665 prefix = partition.upper() + "/" 666 for info in z.infolist(): 667 if info.filename.startswith(prefix) and not IsSymlink(info): 668 basefilename = info.filename[7:] 669 fn = partition + "/" + basefilename 670 data = z.read(info.filename) 671 out[fn] = common.File(fn, data) 672 return out 673 674 675def GetBuildProp(prop, info_dict): 676 """Return the fingerprint of the build of a given target-files info_dict.""" 677 try: 678 return info_dict.get("build.prop", {})[prop] 679 except KeyError: 680 raise common.ExternalError("couldn't find %s in build.prop" % (prop,)) 681 682 683def AddToKnownPaths(filename, known_paths): 684 if filename[-1] == "/": 685 return 686 dirs = filename.split("/")[:-1] 687 while len(dirs) > 0: 688 path = "/".join(dirs) 689 if path in known_paths: 690 break; 691 known_paths.add(path) 692 dirs.pop() 693 694 695def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 696 source_version = OPTIONS.source_info_dict["recovery_api_version"] 697 target_version = OPTIONS.target_info_dict["recovery_api_version"] 698 699 if source_version == 0: 700 print ("WARNING: generating edify script for a source that " 701 "can't install it.") 702 script = edify_generator.EdifyGenerator(source_version, 703 OPTIONS.target_info_dict) 704 705 metadata = {"pre-device": GetBuildProp("ro.product.device", 706 OPTIONS.source_info_dict), 707 "post-timestamp": GetBuildProp("ro.build.date.utc", 708 OPTIONS.target_info_dict), 709 } 710 711 device_specific = common.DeviceSpecificParams( 712 source_zip=source_zip, 713 source_version=source_version, 714 target_zip=target_zip, 715 target_version=target_version, 716 output_zip=output_zip, 717 script=script, 718 metadata=metadata, 719 info_dict=OPTIONS.info_dict) 720 721 source_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.source_info_dict) 722 target_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.target_info_dict) 723 metadata["pre-build"] = source_fp 724 metadata["post-build"] = target_fp 725 726 source_boot = common.GetBootableImage( 727 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 728 OPTIONS.source_info_dict) 729 target_boot = common.GetBootableImage( 730 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 731 updating_boot = (not OPTIONS.two_step and 732 (source_boot.data != target_boot.data)) 733 734 source_recovery = common.GetBootableImage( 735 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 736 OPTIONS.source_info_dict) 737 target_recovery = common.GetBootableImage( 738 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 739 updating_recovery = (source_recovery.data != target_recovery.data) 740 741 system_src = GetImage("system", OPTIONS.source_tmp, OPTIONS.source_info_dict) 742 system_tgt = GetImage("system", OPTIONS.target_tmp, OPTIONS.target_info_dict) 743 744 blockimgdiff_version = 1 745 if OPTIONS.info_dict: 746 blockimgdiff_version = max( 747 int(i) for i in 748 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(",")) 749 750 system_diff = common.BlockDifference("system", system_tgt, system_src, 751 check_first_block=True, 752 version=blockimgdiff_version) 753 754 if HasVendorPartition(target_zip): 755 if not HasVendorPartition(source_zip): 756 raise RuntimeError("can't generate incremental that adds /vendor") 757 vendor_src = GetImage("vendor", OPTIONS.source_tmp, OPTIONS.source_info_dict) 758 vendor_tgt = GetImage("vendor", OPTIONS.target_tmp, OPTIONS.target_info_dict) 759 vendor_diff = common.BlockDifference("vendor", vendor_tgt, vendor_src, 760 check_first_block=True, 761 version=blockimgdiff_version) 762 else: 763 vendor_diff = None 764 765 oem_props = OPTIONS.target_info_dict.get("oem_fingerprint_properties") 766 recovery_mount_options = OPTIONS.target_info_dict.get("recovery_mount_options") 767 oem_dict = None 768 if oem_props is not None and len(oem_props) > 0: 769 if OPTIONS.oem_source is None: 770 raise common.ExternalError("OEM source required for this build") 771 script.Mount("/oem", recovery_mount_options) 772 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 773 774 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 775 device_specific.IncrementalOTA_Assertions() 776 777 # Two-step incremental package strategy (in chronological order, 778 # which is *not* the order in which the generated script has 779 # things): 780 # 781 # if stage is not "2/3" or "3/3": 782 # do verification on current system 783 # write recovery image to boot partition 784 # set stage to "2/3" 785 # reboot to boot partition and restart recovery 786 # else if stage is "2/3": 787 # write recovery image to recovery partition 788 # set stage to "3/3" 789 # reboot to recovery partition and restart recovery 790 # else: 791 # (stage must be "3/3") 792 # perform update: 793 # patch system files, etc. 794 # force full install of new boot image 795 # set up system to update recovery partition on first boot 796 # complete script normally (allow recovery to mark itself finished and reboot) 797 798 if OPTIONS.two_step: 799 if not OPTIONS.info_dict.get("multistage_support", None): 800 assert False, "two-step packages not supported by this build" 801 fs = OPTIONS.info_dict["fstab"]["/misc"] 802 assert fs.fs_type.upper() == "EMMC", \ 803 "two-step packages only supported on devices with EMMC /misc partitions" 804 bcb_dev = {"bcb_dev": fs.device} 805 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 806 script.AppendExtra(""" 807if get_stage("%(bcb_dev)s") == "2/3" then 808""" % bcb_dev) 809 script.AppendExtra("sleep(20);\n"); 810 script.WriteRawImage("/recovery", "recovery.img") 811 script.AppendExtra(""" 812set_stage("%(bcb_dev)s", "3/3"); 813reboot_now("%(bcb_dev)s", "recovery"); 814else if get_stage("%(bcb_dev)s") != "3/3" then 815""" % bcb_dev) 816 817 script.Print("Verifying current system...") 818 819 device_specific.IncrementalOTA_VerifyBegin() 820 821 if oem_props is None: 822 # When blockimgdiff version is less than 3 (non-resumable block-based OTA), 823 # patching on a device that's already on the target build will damage the 824 # system. Because operations like move don't check the block state, they 825 # always apply the changes unconditionally. 826 if blockimgdiff_version <= 2: 827 script.AssertSomeFingerprint(source_fp) 828 else: 829 script.AssertSomeFingerprint(source_fp, target_fp) 830 else: 831 if blockimgdiff_version <= 2: 832 script.AssertSomeThumbprint( 833 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 834 else: 835 script.AssertSomeThumbprint( 836 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 837 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 838 839 if updating_boot: 840 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 841 d = common.Difference(target_boot, source_boot) 842 _, _, d = d.ComputePatch() 843 if d is None: 844 include_full_boot = True 845 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 846 else: 847 include_full_boot = False 848 849 print "boot target: %d source: %d diff: %d" % ( 850 target_boot.size, source_boot.size, len(d)) 851 852 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 853 854 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 855 (boot_type, boot_device, 856 source_boot.size, source_boot.sha1, 857 target_boot.size, target_boot.sha1)) 858 859 device_specific.IncrementalOTA_VerifyEnd() 860 861 if OPTIONS.two_step: 862 script.WriteRawImage("/boot", "recovery.img") 863 script.AppendExtra(""" 864set_stage("%(bcb_dev)s", "2/3"); 865reboot_now("%(bcb_dev)s", ""); 866else 867""" % bcb_dev) 868 869 # Verify the existing partitions. 870 system_diff.WriteVerifyScript(script) 871 if vendor_diff: 872 vendor_diff.WriteVerifyScript(script) 873 874 script.Comment("---- start making changes here ----") 875 876 device_specific.IncrementalOTA_InstallBegin() 877 878 system_diff.WriteScript(script, output_zip, 879 progress=0.8 if vendor_diff else 0.9) 880 if vendor_diff: 881 vendor_diff.WriteScript(script, output_zip, progress=0.1) 882 883 if OPTIONS.two_step: 884 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 885 script.WriteRawImage("/boot", "boot.img") 886 print "writing full boot image (forced by two-step mode)" 887 888 if not OPTIONS.two_step: 889 if updating_boot: 890 if include_full_boot: 891 print "boot image changed; including full." 892 script.Print("Installing boot image...") 893 script.WriteRawImage("/boot", "boot.img") 894 else: 895 # Produce the boot image by applying a patch to the current 896 # contents of the boot partition, and write it back to the 897 # partition. 898 print "boot image changed; including patch." 899 script.Print("Patching boot image...") 900 script.ShowProgress(0.1, 10) 901 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 902 % (boot_type, boot_device, 903 source_boot.size, source_boot.sha1, 904 target_boot.size, target_boot.sha1), 905 "-", 906 target_boot.size, target_boot.sha1, 907 source_boot.sha1, "patch/boot.img.p") 908 else: 909 print "boot image unchanged; skipping." 910 911 # Do device-specific installation (eg, write radio image). 912 device_specific.IncrementalOTA_InstallEnd() 913 914 if OPTIONS.extra_script is not None: 915 script.AppendExtra(OPTIONS.extra_script) 916 917 if OPTIONS.wipe_user_data: 918 script.Print("Erasing user data...") 919 script.FormatPartition("/data") 920 921 if OPTIONS.two_step: 922 script.AppendExtra(""" 923set_stage("%(bcb_dev)s", ""); 924endif; 925endif; 926""" % bcb_dev) 927 928 script.SetProgress(1) 929 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 930 WriteMetadata(metadata, output_zip) 931 932 933class FileDifference: 934 def __init__(self, partition, source_zip, target_zip, output_zip): 935 print "Loading target..." 936 self.target_data = target_data = LoadPartitionFiles(target_zip, partition) 937 print "Loading source..." 938 self.source_data = source_data = LoadPartitionFiles(source_zip, partition) 939 940 self.verbatim_targets = verbatim_targets = [] 941 self.patch_list = patch_list = [] 942 diffs = [] 943 self.renames = renames = {} 944 known_paths = set() 945 largest_source_size = 0 946 947 matching_file_cache = {} 948 for fn, sf in source_data.items(): 949 assert fn == sf.name 950 matching_file_cache["path:" + fn] = sf 951 if fn in target_data.keys(): 952 AddToKnownPaths(fn, known_paths) 953 # Only allow eligibility for filename/sha matching 954 # if there isn't a perfect path match. 955 if target_data.get(sf.name) is None: 956 matching_file_cache["file:" + fn.split("/")[-1]] = sf 957 matching_file_cache["sha:" + sf.sha1] = sf 958 959 for fn in sorted(target_data.keys()): 960 tf = target_data[fn] 961 assert fn == tf.name 962 sf = ClosestFileMatch(tf, matching_file_cache, renames) 963 if sf is not None and sf.name != tf.name: 964 print "File has moved from " + sf.name + " to " + tf.name 965 renames[sf.name] = tf 966 967 if sf is None or fn in OPTIONS.require_verbatim: 968 # This file should be included verbatim 969 if fn in OPTIONS.prohibit_verbatim: 970 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 971 print "send", fn, "verbatim" 972 tf.AddToZip(output_zip) 973 verbatim_targets.append((fn, tf.size, tf.sha1)) 974 if fn in target_data.keys(): 975 AddToKnownPaths(fn, known_paths) 976 elif tf.sha1 != sf.sha1: 977 # File is different; consider sending as a patch 978 diffs.append(common.Difference(tf, sf)) 979 else: 980 # Target file data identical to source (may still be renamed) 981 pass 982 983 common.ComputeDifferences(diffs) 984 985 for diff in diffs: 986 tf, sf, d = diff.GetPatch() 987 path = "/".join(tf.name.split("/")[:-1]) 988 if d is None or len(d) > tf.size * OPTIONS.patch_threshold or \ 989 path not in known_paths: 990 # patch is almost as big as the file; don't bother patching 991 # or a patch + rename cannot take place due to the target 992 # directory not existing 993 tf.AddToZip(output_zip) 994 verbatim_targets.append((tf.name, tf.size, tf.sha1)) 995 if sf.name in renames: 996 del renames[sf.name] 997 AddToKnownPaths(tf.name, known_paths) 998 else: 999 common.ZipWriteStr(output_zip, "patch/" + sf.name + ".p", d) 1000 patch_list.append((tf, sf, tf.size, common.sha1(d).hexdigest())) 1001 largest_source_size = max(largest_source_size, sf.size) 1002 1003 self.largest_source_size = largest_source_size 1004 1005 def EmitVerification(self, script): 1006 so_far = 0 1007 for tf, sf, size, patch_sha in self.patch_list: 1008 if tf.name != sf.name: 1009 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1010 script.PatchCheck("/"+sf.name, tf.sha1, sf.sha1) 1011 so_far += sf.size 1012 return so_far 1013 1014 def EmitExplicitTargetVerification(self, script): 1015 for fn, size, sha1 in self.verbatim_targets: 1016 if (fn[-1] != "/"): 1017 script.FileCheck("/"+fn, sha1) 1018 for tf, _, _, _ in self.patch_list: 1019 script.FileCheck(tf.name, tf.sha1) 1020 1021 def RemoveUnneededFiles(self, script, extras=()): 1022 script.DeleteFiles(["/"+i[0] for i in self.verbatim_targets] + 1023 ["/"+i for i in sorted(self.source_data) 1024 if i not in self.target_data and 1025 i not in self.renames] + 1026 list(extras)) 1027 1028 def TotalPatchSize(self): 1029 return sum(i[1].size for i in self.patch_list) 1030 1031 def EmitPatches(self, script, total_patch_size, so_far): 1032 self.deferred_patch_list = deferred_patch_list = [] 1033 for item in self.patch_list: 1034 tf, sf, size, _ = item 1035 if tf.name == "system/build.prop": 1036 deferred_patch_list.append(item) 1037 continue 1038 if (sf.name != tf.name): 1039 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1040 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p") 1041 so_far += tf.size 1042 script.SetProgress(so_far / total_patch_size) 1043 return so_far 1044 1045 def EmitDeferredPatches(self, script): 1046 for item in self.deferred_patch_list: 1047 tf, sf, size, _ = item 1048 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p") 1049 script.SetPermissions("/system/build.prop", 0, 0, 0644, None, None) 1050 1051 def EmitRenames(self, script): 1052 if len(self.renames) > 0: 1053 script.Print("Renaming files...") 1054 for src, tgt in self.renames.iteritems(): 1055 print "Renaming " + src + " to " + tgt.name 1056 script.RenameFile(src, tgt.name) 1057 1058 1059 1060 1061def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): 1062 target_has_recovery_patch = HasRecoveryPatch(target_zip) 1063 source_has_recovery_patch = HasRecoveryPatch(source_zip) 1064 1065 if (OPTIONS.block_based and 1066 target_has_recovery_patch and 1067 source_has_recovery_patch): 1068 return WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip) 1069 1070 source_version = OPTIONS.source_info_dict["recovery_api_version"] 1071 target_version = OPTIONS.target_info_dict["recovery_api_version"] 1072 1073 if source_version == 0: 1074 print ("WARNING: generating edify script for a source that " 1075 "can't install it.") 1076 script = edify_generator.EdifyGenerator(source_version, 1077 OPTIONS.target_info_dict) 1078 1079 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 1080 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 1081 oem_dict = None 1082 if oem_props is not None and len(oem_props) > 0: 1083 if OPTIONS.oem_source is None: 1084 raise common.ExternalError("OEM source required for this build") 1085 script.Mount("/oem", recovery_mount_options) 1086 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 1087 1088 metadata = {"pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 1089 OPTIONS.source_info_dict), 1090 "post-timestamp": GetBuildProp("ro.build.date.utc", 1091 OPTIONS.target_info_dict), 1092 } 1093 1094 device_specific = common.DeviceSpecificParams( 1095 source_zip=source_zip, 1096 source_version=source_version, 1097 target_zip=target_zip, 1098 target_version=target_version, 1099 output_zip=output_zip, 1100 script=script, 1101 metadata=metadata, 1102 info_dict=OPTIONS.info_dict) 1103 1104 system_diff = FileDifference("system", source_zip, target_zip, output_zip) 1105 script.Mount("/system", recovery_mount_options) 1106 if HasVendorPartition(target_zip): 1107 vendor_diff = FileDifference("vendor", source_zip, target_zip, output_zip) 1108 script.Mount("/vendor", recovery_mount_options) 1109 else: 1110 vendor_diff = None 1111 1112 target_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.target_info_dict) 1113 source_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.source_info_dict) 1114 1115 if oem_props is None: 1116 script.AssertSomeFingerprint(source_fp, target_fp) 1117 else: 1118 script.AssertSomeThumbprint( 1119 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 1120 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 1121 1122 metadata["pre-build"] = source_fp 1123 metadata["post-build"] = target_fp 1124 1125 source_boot = common.GetBootableImage( 1126 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 1127 OPTIONS.source_info_dict) 1128 target_boot = common.GetBootableImage( 1129 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 1130 updating_boot = (not OPTIONS.two_step and 1131 (source_boot.data != target_boot.data)) 1132 1133 source_recovery = common.GetBootableImage( 1134 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 1135 OPTIONS.source_info_dict) 1136 target_recovery = common.GetBootableImage( 1137 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 1138 updating_recovery = (source_recovery.data != target_recovery.data) 1139 1140 # Here's how we divide up the progress bar: 1141 # 0.1 for verifying the start state (PatchCheck calls) 1142 # 0.8 for applying patches (ApplyPatch calls) 1143 # 0.1 for unpacking verbatim files, symlinking, and doing the 1144 # device-specific commands. 1145 1146 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 1147 device_specific.IncrementalOTA_Assertions() 1148 1149 # Two-step incremental package strategy (in chronological order, 1150 # which is *not* the order in which the generated script has 1151 # things): 1152 # 1153 # if stage is not "2/3" or "3/3": 1154 # do verification on current system 1155 # write recovery image to boot partition 1156 # set stage to "2/3" 1157 # reboot to boot partition and restart recovery 1158 # else if stage is "2/3": 1159 # write recovery image to recovery partition 1160 # set stage to "3/3" 1161 # reboot to recovery partition and restart recovery 1162 # else: 1163 # (stage must be "3/3") 1164 # perform update: 1165 # patch system files, etc. 1166 # force full install of new boot image 1167 # set up system to update recovery partition on first boot 1168 # complete script normally (allow recovery to mark itself finished and reboot) 1169 1170 if OPTIONS.two_step: 1171 if not OPTIONS.info_dict.get("multistage_support", None): 1172 assert False, "two-step packages not supported by this build" 1173 fs = OPTIONS.info_dict["fstab"]["/misc"] 1174 assert fs.fs_type.upper() == "EMMC", \ 1175 "two-step packages only supported on devices with EMMC /misc partitions" 1176 bcb_dev = {"bcb_dev": fs.device} 1177 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 1178 script.AppendExtra(""" 1179if get_stage("%(bcb_dev)s") == "2/3" then 1180""" % bcb_dev) 1181 script.AppendExtra("sleep(20);\n"); 1182 script.WriteRawImage("/recovery", "recovery.img") 1183 script.AppendExtra(""" 1184set_stage("%(bcb_dev)s", "3/3"); 1185reboot_now("%(bcb_dev)s", "recovery"); 1186else if get_stage("%(bcb_dev)s") != "3/3" then 1187""" % bcb_dev) 1188 1189 script.Print("Verifying current system...") 1190 1191 device_specific.IncrementalOTA_VerifyBegin() 1192 1193 script.ShowProgress(0.1, 0) 1194 so_far = system_diff.EmitVerification(script) 1195 if vendor_diff: 1196 so_far += vendor_diff.EmitVerification(script) 1197 1198 if updating_boot: 1199 d = common.Difference(target_boot, source_boot) 1200 _, _, d = d.ComputePatch() 1201 print "boot target: %d source: %d diff: %d" % ( 1202 target_boot.size, source_boot.size, len(d)) 1203 1204 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 1205 1206 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 1207 1208 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 1209 (boot_type, boot_device, 1210 source_boot.size, source_boot.sha1, 1211 target_boot.size, target_boot.sha1)) 1212 so_far += source_boot.size 1213 1214 size = [] 1215 if system_diff.patch_list: size.append(system_diff.largest_source_size) 1216 if vendor_diff: 1217 if vendor_diff.patch_list: size.append(vendor_diff.largest_source_size) 1218 if size or updating_recovery or updating_boot: 1219 script.CacheFreeSpaceCheck(max(size)) 1220 1221 device_specific.IncrementalOTA_VerifyEnd() 1222 1223 if OPTIONS.two_step: 1224 script.WriteRawImage("/boot", "recovery.img") 1225 script.AppendExtra(""" 1226set_stage("%(bcb_dev)s", "2/3"); 1227reboot_now("%(bcb_dev)s", ""); 1228else 1229""" % bcb_dev) 1230 1231 script.Comment("---- start making changes here ----") 1232 1233 device_specific.IncrementalOTA_InstallBegin() 1234 1235 if OPTIONS.two_step: 1236 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 1237 script.WriteRawImage("/boot", "boot.img") 1238 print "writing full boot image (forced by two-step mode)" 1239 1240 script.Print("Removing unneeded files...") 1241 system_diff.RemoveUnneededFiles(script, ("/system/recovery.img",)) 1242 if vendor_diff: 1243 vendor_diff.RemoveUnneededFiles(script) 1244 1245 script.ShowProgress(0.8, 0) 1246 total_patch_size = 1.0 + system_diff.TotalPatchSize() 1247 if vendor_diff: 1248 total_patch_size += vendor_diff.TotalPatchSize() 1249 if updating_boot: 1250 total_patch_size += target_boot.size 1251 1252 script.Print("Patching system files...") 1253 so_far = system_diff.EmitPatches(script, total_patch_size, 0) 1254 if vendor_diff: 1255 script.Print("Patching vendor files...") 1256 so_far = vendor_diff.EmitPatches(script, total_patch_size, so_far) 1257 1258 if not OPTIONS.two_step: 1259 if updating_boot: 1260 # Produce the boot image by applying a patch to the current 1261 # contents of the boot partition, and write it back to the 1262 # partition. 1263 script.Print("Patching boot image...") 1264 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 1265 % (boot_type, boot_device, 1266 source_boot.size, source_boot.sha1, 1267 target_boot.size, target_boot.sha1), 1268 "-", 1269 target_boot.size, target_boot.sha1, 1270 source_boot.sha1, "patch/boot.img.p") 1271 so_far += target_boot.size 1272 script.SetProgress(so_far / total_patch_size) 1273 print "boot image changed; including." 1274 else: 1275 print "boot image unchanged; skipping." 1276 1277 system_items = ItemSet("system", "META/filesystem_config.txt") 1278 if vendor_diff: 1279 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 1280 1281 if updating_recovery: 1282 # Recovery is generated as a patch using both the boot image 1283 # (which contains the same linux kernel as recovery) and the file 1284 # /system/etc/recovery-resource.dat (which contains all the images 1285 # used in the recovery UI) as sources. This lets us minimize the 1286 # size of the patch, which must be included in every OTA package. 1287 # 1288 # For older builds where recovery-resource.dat is not present, we 1289 # use only the boot image as the source. 1290 1291 if not target_has_recovery_patch: 1292 def output_sink(fn, data): 1293 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 1294 system_items.Get("system/" + fn, dir=False) 1295 1296 common.MakeRecoveryPatch(OPTIONS.target_tmp, output_sink, 1297 target_recovery, target_boot) 1298 script.DeleteFiles(["/system/recovery-from-boot.p", 1299 "/system/etc/install-recovery.sh"]) 1300 print "recovery image changed; including as patch from boot." 1301 else: 1302 print "recovery image unchanged; skipping." 1303 1304 script.ShowProgress(0.1, 10) 1305 1306 target_symlinks = CopyPartitionFiles(system_items, target_zip, None) 1307 if vendor_diff: 1308 target_symlinks.extend(CopyPartitionFiles(vendor_items, target_zip, None)) 1309 1310 temp_script = script.MakeTemporary() 1311 system_items.GetMetadata(target_zip) 1312 system_items.Get("system").SetPermissions(temp_script) 1313 if vendor_diff: 1314 vendor_items.GetMetadata(target_zip) 1315 vendor_items.Get("vendor").SetPermissions(temp_script) 1316 1317 # Note that this call will mess up the trees of Items, so make sure 1318 # we're done with them. 1319 source_symlinks = CopyPartitionFiles(system_items, source_zip, None) 1320 if vendor_diff: 1321 source_symlinks.extend(CopyPartitionFiles(vendor_items, source_zip, None)) 1322 1323 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 1324 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 1325 1326 # Delete all the symlinks in source that aren't in target. This 1327 # needs to happen before verbatim files are unpacked, in case a 1328 # symlink in the source is replaced by a real file in the target. 1329 to_delete = [] 1330 for dest, link in source_symlinks: 1331 if link not in target_symlinks_d: 1332 to_delete.append(link) 1333 script.DeleteFiles(to_delete) 1334 1335 if system_diff.verbatim_targets: 1336 script.Print("Unpacking new system files...") 1337 script.UnpackPackageDir("system", "/system") 1338 if vendor_diff and vendor_diff.verbatim_targets: 1339 script.Print("Unpacking new vendor files...") 1340 script.UnpackPackageDir("vendor", "/vendor") 1341 1342 if updating_recovery and not target_has_recovery_patch: 1343 script.Print("Unpacking new recovery...") 1344 script.UnpackPackageDir("recovery", "/system") 1345 1346 system_diff.EmitRenames(script) 1347 if vendor_diff: 1348 vendor_diff.EmitRenames(script) 1349 1350 script.Print("Symlinks and permissions...") 1351 1352 # Create all the symlinks that don't already exist, or point to 1353 # somewhere different than what we want. Delete each symlink before 1354 # creating it, since the 'symlink' command won't overwrite. 1355 to_create = [] 1356 for dest, link in target_symlinks: 1357 if link in source_symlinks_d: 1358 if dest != source_symlinks_d[link]: 1359 to_create.append((dest, link)) 1360 else: 1361 to_create.append((dest, link)) 1362 script.DeleteFiles([i[1] for i in to_create]) 1363 script.MakeSymlinks(to_create) 1364 1365 # Now that the symlinks are created, we can set all the 1366 # permissions. 1367 script.AppendScript(temp_script) 1368 1369 # Do device-specific installation (eg, write radio image). 1370 device_specific.IncrementalOTA_InstallEnd() 1371 1372 if OPTIONS.extra_script is not None: 1373 script.AppendExtra(OPTIONS.extra_script) 1374 1375 # Patch the build.prop file last, so if something fails but the 1376 # device can still come up, it appears to be the old build and will 1377 # get set the OTA package again to retry. 1378 script.Print("Patching remaining system files...") 1379 system_diff.EmitDeferredPatches(script) 1380 1381 if OPTIONS.wipe_user_data: 1382 script.Print("Erasing user data...") 1383 script.FormatPartition("/data") 1384 1385 if OPTIONS.two_step: 1386 script.AppendExtra(""" 1387set_stage("%(bcb_dev)s", ""); 1388endif; 1389endif; 1390""" % bcb_dev) 1391 1392 if OPTIONS.verify and system_diff: 1393 script.Print("Remounting and verifying system partition files...") 1394 script.Unmount("/system") 1395 script.Mount("/system") 1396 system_diff.EmitExplicitTargetVerification(script) 1397 1398 if OPTIONS.verify and vendor_diff: 1399 script.Print("Remounting and verifying vendor partition files...") 1400 script.Unmount("/vendor") 1401 script.Mount("/vendor") 1402 vendor_diff.EmitExplicitTargetVerification(script) 1403 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 1404 1405 WriteMetadata(metadata, output_zip) 1406 1407 1408def main(argv): 1409 1410 def option_handler(o, a): 1411 if o == "--board_config": 1412 pass # deprecated 1413 elif o in ("-k", "--package_key"): 1414 OPTIONS.package_key = a 1415 elif o in ("-i", "--incremental_from"): 1416 OPTIONS.incremental_source = a 1417 elif o in ("-w", "--wipe_user_data"): 1418 OPTIONS.wipe_user_data = True 1419 elif o in ("-n", "--no_prereq"): 1420 OPTIONS.omit_prereq = True 1421 elif o in ("-o", "--oem_settings"): 1422 OPTIONS.oem_source = a 1423 elif o in ("-e", "--extra_script"): 1424 OPTIONS.extra_script = a 1425 elif o in ("-a", "--aslr_mode"): 1426 if a in ("on", "On", "true", "True", "yes", "Yes"): 1427 OPTIONS.aslr_mode = True 1428 else: 1429 OPTIONS.aslr_mode = False 1430 elif o in ("-t", "--worker_threads"): 1431 if a.isdigit(): 1432 OPTIONS.worker_threads = int(a) 1433 else: 1434 raise ValueError("Cannot parse value %r for option %r - only " 1435 "integers are allowed." % (a, o)) 1436 elif o in ("-2", "--two_step"): 1437 OPTIONS.two_step = True 1438 elif o == "--no_signing": 1439 OPTIONS.no_signing = True 1440 elif o in ("--verify"): 1441 OPTIONS.verify = True 1442 elif o == "--block": 1443 OPTIONS.block_based = True 1444 elif o in ("-b", "--binary"): 1445 OPTIONS.updater_binary = a 1446 elif o in ("--no_fallback_to_full",): 1447 OPTIONS.fallback_to_full = False 1448 else: 1449 return False 1450 return True 1451 1452 args = common.ParseOptions(argv, __doc__, 1453 extra_opts="b:k:i:d:wne:t:a:2o:", 1454 extra_long_opts=["board_config=", 1455 "package_key=", 1456 "incremental_from=", 1457 "wipe_user_data", 1458 "no_prereq", 1459 "extra_script=", 1460 "worker_threads=", 1461 "aslr_mode=", 1462 "two_step", 1463 "no_signing", 1464 "block", 1465 "binary=", 1466 "oem_settings=", 1467 "verify", 1468 "no_fallback_to_full", 1469 ], 1470 extra_option_handler=option_handler) 1471 1472 if len(args) != 2: 1473 common.Usage(__doc__) 1474 sys.exit(1) 1475 1476 if OPTIONS.extra_script is not None: 1477 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 1478 1479 print "unzipping target target-files..." 1480 OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0]) 1481 1482 OPTIONS.target_tmp = OPTIONS.input_tmp 1483 OPTIONS.info_dict = common.LoadInfoDict(input_zip) 1484 1485 # If this image was originally labelled with SELinux contexts, make sure we 1486 # also apply the labels in our new image. During building, the "file_contexts" 1487 # is in the out/ directory tree, but for repacking from target-files.zip it's 1488 # in the root directory of the ramdisk. 1489 if "selinux_fc" in OPTIONS.info_dict: 1490 OPTIONS.info_dict["selinux_fc"] = os.path.join(OPTIONS.input_tmp, "BOOT", "RAMDISK", 1491 "file_contexts") 1492 1493 if OPTIONS.verbose: 1494 print "--- target info ---" 1495 common.DumpInfoDict(OPTIONS.info_dict) 1496 1497 # If the caller explicitly specified the device-specific extensions 1498 # path via -s/--device_specific, use that. Otherwise, use 1499 # META/releasetools.py if it is present in the target target_files. 1500 # Otherwise, take the path of the file from 'tool_extensions' in the 1501 # info dict and look for that in the local filesystem, relative to 1502 # the current directory. 1503 1504 if OPTIONS.device_specific is None: 1505 from_input = os.path.join(OPTIONS.input_tmp, "META", "releasetools.py") 1506 if os.path.exists(from_input): 1507 print "(using device-specific extensions from target_files)" 1508 OPTIONS.device_specific = from_input 1509 else: 1510 OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None) 1511 1512 if OPTIONS.device_specific is not None: 1513 OPTIONS.device_specific = os.path.abspath(OPTIONS.device_specific) 1514 1515 while True: 1516 1517 if OPTIONS.no_signing: 1518 if os.path.exists(args[1]): os.unlink(args[1]) 1519 output_zip = zipfile.ZipFile(args[1], "w", compression=zipfile.ZIP_DEFLATED) 1520 else: 1521 temp_zip_file = tempfile.NamedTemporaryFile() 1522 output_zip = zipfile.ZipFile(temp_zip_file, "w", 1523 compression=zipfile.ZIP_DEFLATED) 1524 1525 if OPTIONS.incremental_source is None: 1526 WriteFullOTAPackage(input_zip, output_zip) 1527 if OPTIONS.package_key is None: 1528 OPTIONS.package_key = OPTIONS.info_dict.get( 1529 "default_system_dev_certificate", 1530 "build/target/product/security/testkey") 1531 break 1532 1533 else: 1534 print "unzipping source target-files..." 1535 OPTIONS.source_tmp, source_zip = common.UnzipTemp(OPTIONS.incremental_source) 1536 OPTIONS.target_info_dict = OPTIONS.info_dict 1537 OPTIONS.source_info_dict = common.LoadInfoDict(source_zip) 1538 if "selinux_fc" in OPTIONS.source_info_dict: 1539 OPTIONS.source_info_dict["selinux_fc"] = os.path.join(OPTIONS.source_tmp, "BOOT", "RAMDISK", 1540 "file_contexts") 1541 if OPTIONS.package_key is None: 1542 OPTIONS.package_key = OPTIONS.source_info_dict.get( 1543 "default_system_dev_certificate", 1544 "build/target/product/security/testkey") 1545 if OPTIONS.verbose: 1546 print "--- source info ---" 1547 common.DumpInfoDict(OPTIONS.source_info_dict) 1548 try: 1549 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) 1550 break 1551 except ValueError: 1552 if not OPTIONS.fallback_to_full: raise 1553 print "--- failed to build incremental; falling back to full ---" 1554 OPTIONS.incremental_source = None 1555 output_zip.close() 1556 1557 output_zip.close() 1558 1559 if not OPTIONS.no_signing: 1560 SignOutput(temp_zip_file.name, args[1]) 1561 temp_zip_file.close() 1562 1563 print "done." 1564 1565 1566if __name__ == '__main__': 1567 try: 1568 common.CloseInheritedPipes() 1569 main(sys.argv[1:]) 1570 except common.ExternalError, e: 1571 print 1572 print " ERROR: %s" % (e,) 1573 print 1574 sys.exit(1) 1575 finally: 1576 common.Cleanup() 1577