ota_from_target_files.py revision 8e0178d41b9eeb6754eda07292d78762e3169140
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 --board_config <file> 25 Deprecated. 26 27 -k (--package_key) <key> Key to use to sign the package (default is 28 the value of default_system_dev_certificate from the input 29 target-files's META/misc_info.txt, or 30 "build/target/product/security/testkey" if that value is not 31 specified). 32 33 For incremental OTAs, the default value is based on the source 34 target-file, not the target build. 35 36 -i (--incremental_from) <file> 37 Generate an incremental OTA using the given target-files zip as 38 the starting build. 39 40 -v (--verify) 41 Remount and verify the checksums of the files written to the 42 system and vendor (if used) partitions. Incremental builds only. 43 44 -o (--oem_settings) <file> 45 Use the file to specify the expected OEM-specific properties 46 on the OEM partition of the intended device. 47 48 -w (--wipe_user_data) 49 Generate an OTA package that will wipe the user data partition 50 when installed. 51 52 -n (--no_prereq) 53 Omit the timestamp prereq check normally included at the top of 54 the build scripts (used for developer OTA packages which 55 legitimately need to go back and forth). 56 57 -e (--extra_script) <file> 58 Insert the contents of file at the end of the update script. 59 60 -a (--aslr_mode) <on|off> 61 Specify whether to turn on ASLR for the package (on by default). 62 63 -2 (--two_step) 64 Generate a 'two-step' OTA package, where recovery is updated 65 first, so that any changes made to the system partition are done 66 using the new recovery (new kernel, etc.). 67 68 --block 69 Generate a block-based OTA if possible. Will fall back to a 70 file-based OTA if the target_files is older and doesn't support 71 block-based OTAs. 72 73 -b (--binary) <file> 74 Use the given binary as the update-binary in the output package, 75 instead of the binary in the build's target_files. Use for 76 development only. 77 78 -t (--worker_threads) <int> 79 Specifies the number of worker-threads that will be used when 80 generating patches for incremental updates (defaults to 3). 81 82""" 83 84import sys 85 86if sys.hexversion < 0x02070000: 87 print >> sys.stderr, "Python 2.7 or newer is required." 88 sys.exit(1) 89 90import copy 91import errno 92import multiprocessing 93import os 94import re 95import subprocess 96import tempfile 97import time 98import zipfile 99 100from hashlib import sha1 as sha1 101 102import common 103import edify_generator 104import build_image 105import blockimgdiff 106import sparse_img 107 108OPTIONS = common.OPTIONS 109OPTIONS.package_key = None 110OPTIONS.incremental_source = None 111OPTIONS.verify = False 112OPTIONS.require_verbatim = set() 113OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 114OPTIONS.patch_threshold = 0.95 115OPTIONS.wipe_user_data = False 116OPTIONS.omit_prereq = False 117OPTIONS.extra_script = None 118OPTIONS.aslr_mode = True 119OPTIONS.worker_threads = multiprocessing.cpu_count() // 2 120if OPTIONS.worker_threads == 0: 121 OPTIONS.worker_threads = 1 122OPTIONS.two_step = False 123OPTIONS.no_signing = False 124OPTIONS.block_based = False 125OPTIONS.updater_binary = None 126OPTIONS.oem_source = None 127OPTIONS.fallback_to_full = True 128 129def MostPopularKey(d, default): 130 """Given a dict, return the key corresponding to the largest 131 value. Returns 'default' if the dict is empty.""" 132 x = [(v, k) for (k, v) in d.iteritems()] 133 if not x: return default 134 x.sort() 135 return x[-1][1] 136 137 138def IsSymlink(info): 139 """Return true if the zipfile.ZipInfo object passed in represents a 140 symlink.""" 141 return (info.external_attr >> 16) == 0120777 142 143def IsRegular(info): 144 """Return true if the zipfile.ZipInfo object passed in represents a 145 symlink.""" 146 return (info.external_attr >> 28) == 010 147 148def ClosestFileMatch(src, tgtfiles, existing): 149 """Returns the closest file match between a source file and list 150 of potential matches. The exact filename match is preferred, 151 then the sha1 is searched for, and finally a file with the same 152 basename is evaluated. Rename support in the updater-binary is 153 required for the latter checks to be used.""" 154 155 result = tgtfiles.get("path:" + src.name) 156 if result is not None: 157 return result 158 159 if not OPTIONS.target_info_dict.get("update_rename_support", False): 160 return None 161 162 if src.size < 1000: 163 return None 164 165 result = tgtfiles.get("sha1:" + src.sha1) 166 if result is not None and existing.get(result.name) is None: 167 return result 168 result = tgtfiles.get("file:" + src.name.split("/")[-1]) 169 if result is not None and existing.get(result.name) is None: 170 return result 171 return None 172 173class ItemSet: 174 def __init__(self, partition, fs_config): 175 self.partition = partition 176 self.fs_config = fs_config 177 self.ITEMS = {} 178 179 def Get(self, name, dir=False): 180 if name not in self.ITEMS: 181 self.ITEMS[name] = Item(self, name, dir=dir) 182 return self.ITEMS[name] 183 184 def GetMetadata(self, input_zip): 185 # The target_files contains a record of what the uid, 186 # gid, and mode are supposed to be. 187 output = input_zip.read(self.fs_config) 188 189 for line in output.split("\n"): 190 if not line: continue 191 columns = line.split() 192 name, uid, gid, mode = columns[:4] 193 selabel = None 194 capabilities = None 195 196 # After the first 4 columns, there are a series of key=value 197 # pairs. Extract out the fields we care about. 198 for element in columns[4:]: 199 key, value = element.split("=") 200 if key == "selabel": 201 selabel = value 202 if key == "capabilities": 203 capabilities = value 204 205 i = self.ITEMS.get(name, None) 206 if i is not None: 207 i.uid = int(uid) 208 i.gid = int(gid) 209 i.mode = int(mode, 8) 210 i.selabel = selabel 211 i.capabilities = capabilities 212 if i.dir: 213 i.children.sort(key=lambda i: i.name) 214 215 # set metadata for the files generated by this script. 216 i = self.ITEMS.get("system/recovery-from-boot.p", None) 217 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0644, None, None 218 i = self.ITEMS.get("system/etc/install-recovery.sh", None) 219 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0544, None, None 220 221 222class Item: 223 """Items represent the metadata (user, group, mode) of files and 224 directories in the system image.""" 225 def __init__(self, itemset, name, dir=False): 226 self.itemset = itemset 227 self.name = name 228 self.uid = None 229 self.gid = None 230 self.mode = None 231 self.selabel = None 232 self.capabilities = None 233 self.dir = dir 234 235 if name: 236 self.parent = itemset.Get(os.path.dirname(name), dir=True) 237 self.parent.children.append(self) 238 else: 239 self.parent = None 240 if dir: 241 self.children = [] 242 243 def Dump(self, indent=0): 244 if self.uid is not None: 245 print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode) 246 else: 247 print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode) 248 if self.dir: 249 print "%s%s" % (" "*indent, self.descendants) 250 print "%s%s" % (" "*indent, self.best_subtree) 251 for i in self.children: 252 i.Dump(indent=indent+1) 253 254 def CountChildMetadata(self): 255 """Count up the (uid, gid, mode, selabel, capabilities) tuples for 256 all children and determine the best strategy for using set_perm_recursive and 257 set_perm to correctly chown/chmod all the files to their desired 258 values. Recursively calls itself for all descendants. 259 260 Returns a dict of {(uid, gid, dmode, fmode, selabel, capabilities): count} counting up 261 all descendants of this node. (dmode or fmode may be None.) Also 262 sets the best_subtree of each directory Item to the (uid, gid, 263 dmode, fmode, selabel, capabilities) tuple that will match the most 264 descendants of that Item. 265 """ 266 267 assert self.dir 268 d = self.descendants = {(self.uid, self.gid, self.mode, None, self.selabel, self.capabilities): 1} 269 for i in self.children: 270 if i.dir: 271 for k, v in i.CountChildMetadata().iteritems(): 272 d[k] = d.get(k, 0) + v 273 else: 274 k = (i.uid, i.gid, None, i.mode, i.selabel, i.capabilities) 275 d[k] = d.get(k, 0) + 1 276 277 # Find the (uid, gid, dmode, fmode, selabel, capabilities) 278 # tuple that matches the most descendants. 279 280 # First, find the (uid, gid) pair that matches the most 281 # descendants. 282 ug = {} 283 for (uid, gid, _, _, _, _), count in d.iteritems(): 284 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 285 ug = MostPopularKey(ug, (0, 0)) 286 287 # Now find the dmode, fmode, selabel, and capabilities that match 288 # the most descendants with that (uid, gid), and choose those. 289 best_dmode = (0, 0755) 290 best_fmode = (0, 0644) 291 best_selabel = (0, None) 292 best_capabilities = (0, None) 293 for k, count in d.iteritems(): 294 if k[:2] != ug: continue 295 if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2]) 296 if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3]) 297 if k[4] is not None and count >= best_selabel[0]: best_selabel = (count, k[4]) 298 if k[5] is not None and count >= best_capabilities[0]: best_capabilities = (count, k[5]) 299 self.best_subtree = ug + (best_dmode[1], best_fmode[1], best_selabel[1], best_capabilities[1]) 300 301 return d 302 303 def SetPermissions(self, script): 304 """Append set_perm/set_perm_recursive commands to 'script' to 305 set all permissions, users, and groups for the tree of files 306 rooted at 'self'.""" 307 308 self.CountChildMetadata() 309 310 def recurse(item, current): 311 # current is the (uid, gid, dmode, fmode, selabel, capabilities) tuple that the current 312 # item (and all its children) have already been set to. We only 313 # need to issue set_perm/set_perm_recursive commands if we're 314 # supposed to be something different. 315 if item.dir: 316 if current != item.best_subtree: 317 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 318 current = item.best_subtree 319 320 if item.uid != current[0] or item.gid != current[1] or \ 321 item.mode != current[2] or item.selabel != current[4] or \ 322 item.capabilities != current[5]: 323 script.SetPermissions("/"+item.name, item.uid, item.gid, 324 item.mode, item.selabel, item.capabilities) 325 326 for i in item.children: 327 recurse(i, current) 328 else: 329 if item.uid != current[0] or item.gid != current[1] or \ 330 item.mode != current[3] or item.selabel != current[4] or \ 331 item.capabilities != current[5]: 332 script.SetPermissions("/"+item.name, item.uid, item.gid, 333 item.mode, item.selabel, item.capabilities) 334 335 recurse(self, (-1, -1, -1, -1, None, None)) 336 337 338def CopyPartitionFiles(itemset, input_zip, output_zip=None, substitute=None): 339 """Copies files for the partition in the input zip to the output 340 zip. Populates the Item class with their metadata, and returns a 341 list of symlinks. output_zip may be None, in which case the copy is 342 skipped (but the other side effects still happen). substitute is an 343 optional dict of {output filename: contents} to be output instead of 344 certain input files. 345 """ 346 347 symlinks = [] 348 349 partition = itemset.partition 350 351 for info in input_zip.infolist(): 352 if info.filename.startswith(partition.upper() + "/"): 353 basefilename = info.filename[7:] 354 if IsSymlink(info): 355 symlinks.append((input_zip.read(info.filename), 356 "/" + partition + "/" + basefilename)) 357 else: 358 info2 = copy.copy(info) 359 fn = info2.filename = partition + "/" + basefilename 360 if substitute and fn in substitute and substitute[fn] is None: 361 continue 362 if output_zip is not None: 363 if substitute and fn in substitute: 364 data = substitute[fn] 365 else: 366 data = input_zip.read(info.filename) 367 output_zip.writestr(info2, data) 368 if fn.endswith("/"): 369 itemset.Get(fn[:-1], dir=True) 370 else: 371 itemset.Get(fn, dir=False) 372 373 symlinks.sort() 374 return symlinks 375 376 377def SignOutput(temp_zip_name, output_zip_name): 378 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 379 pw = key_passwords[OPTIONS.package_key] 380 381 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 382 whole_file=True) 383 384 385def AppendAssertions(script, info_dict, oem_dict = None): 386 oem_props = info_dict.get("oem_fingerprint_properties") 387 if oem_props is None or len(oem_props) == 0: 388 device = GetBuildProp("ro.product.device", info_dict) 389 script.AssertDevice(device) 390 else: 391 if oem_dict is None: 392 raise common.ExternalError("No OEM file provided to answer expected assertions") 393 for prop in oem_props.split(): 394 if oem_dict.get(prop) is None: 395 raise common.ExternalError("The OEM file is missing the property %s" % prop) 396 script.AssertOemProperty(prop, oem_dict.get(prop)) 397 398 399def HasRecoveryPatch(target_files_zip): 400 try: 401 target_files_zip.getinfo("SYSTEM/recovery-from-boot.p") 402 return True 403 except KeyError: 404 return False 405 406def HasVendorPartition(target_files_zip): 407 try: 408 target_files_zip.getinfo("VENDOR/") 409 return True 410 except KeyError: 411 return False 412 413def GetOemProperty(name, oem_props, oem_dict, info_dict): 414 if oem_props is not None and name in oem_props: 415 return oem_dict[name] 416 return GetBuildProp(name, info_dict) 417 418 419def CalculateFingerprint(oem_props, oem_dict, info_dict): 420 if oem_props is None: 421 return GetBuildProp("ro.build.fingerprint", info_dict) 422 return "%s/%s/%s:%s" % ( 423 GetOemProperty("ro.product.brand", oem_props, oem_dict, info_dict), 424 GetOemProperty("ro.product.name", oem_props, oem_dict, info_dict), 425 GetOemProperty("ro.product.device", oem_props, oem_dict, info_dict), 426 GetBuildProp("ro.build.thumbprint", info_dict)) 427 428 429def GetImage(which, tmpdir, info_dict): 430 # Return an image object (suitable for passing to BlockImageDiff) 431 # for the 'which' partition (most be "system" or "vendor"). If a 432 # prebuilt image and file map are found in tmpdir they are used, 433 # otherwise they are reconstructed from the individual files. 434 435 assert which in ("system", "vendor") 436 437 path = os.path.join(tmpdir, "IMAGES", which + ".img") 438 mappath = os.path.join(tmpdir, "IMAGES", which + ".map") 439 if os.path.exists(path) and os.path.exists(mappath): 440 print "using %s.img from target-files" % (which,) 441 # This is a 'new' target-files, which already has the image in it. 442 443 else: 444 print "building %s.img from target-files" % (which,) 445 446 # This is an 'old' target-files, which does not contain images 447 # already built. Build them. 448 449 mappath = tempfile.mkstemp()[1] 450 OPTIONS.tempfiles.append(mappath) 451 452 import add_img_to_target_files 453 if which == "system": 454 path = add_img_to_target_files.BuildSystem( 455 tmpdir, info_dict, block_list=mappath) 456 elif which == "vendor": 457 path = add_img_to_target_files.BuildVendor( 458 tmpdir, info_dict, block_list=mappath) 459 460 return sparse_img.SparseImage(path, mappath) 461 462 463def WriteFullOTAPackage(input_zip, output_zip): 464 # TODO: how to determine this? We don't know what version it will 465 # be installed on top of. For now, we expect the API just won't 466 # change very often. 467 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 468 469 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 470 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 471 oem_dict = None 472 if oem_props is not None and len(oem_props) > 0: 473 if OPTIONS.oem_source is None: 474 raise common.ExternalError("OEM source required for this build") 475 script.Mount("/oem", recovery_mount_options) 476 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 477 478 metadata = {"post-build": CalculateFingerprint( 479 oem_props, oem_dict, OPTIONS.info_dict), 480 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 481 OPTIONS.info_dict), 482 "post-timestamp": GetBuildProp("ro.build.date.utc", 483 OPTIONS.info_dict), 484 } 485 486 device_specific = common.DeviceSpecificParams( 487 input_zip=input_zip, 488 input_version=OPTIONS.info_dict["recovery_api_version"], 489 output_zip=output_zip, 490 script=script, 491 input_tmp=OPTIONS.input_tmp, 492 metadata=metadata, 493 info_dict=OPTIONS.info_dict) 494 495 has_recovery_patch = HasRecoveryPatch(input_zip) 496 block_based = OPTIONS.block_based and has_recovery_patch 497 498 if not OPTIONS.omit_prereq: 499 ts = GetBuildProp("ro.build.date.utc", OPTIONS.info_dict) 500 ts_text = GetBuildProp("ro.build.date", OPTIONS.info_dict) 501 script.AssertOlderBuild(ts, ts_text) 502 503 AppendAssertions(script, OPTIONS.info_dict, oem_dict) 504 device_specific.FullOTA_Assertions() 505 506 # Two-step package strategy (in chronological order, which is *not* 507 # the order in which the generated script has things): 508 # 509 # if stage is not "2/3" or "3/3": 510 # write recovery image to boot partition 511 # set stage to "2/3" 512 # reboot to boot partition and restart recovery 513 # else if stage is "2/3": 514 # write recovery image to recovery partition 515 # set stage to "3/3" 516 # reboot to recovery partition and restart recovery 517 # else: 518 # (stage must be "3/3") 519 # set stage to "" 520 # do normal full package installation: 521 # wipe and install system, boot image, etc. 522 # set up system to update recovery partition on first boot 523 # complete script normally (allow recovery to mark itself finished and reboot) 524 525 recovery_img = common.GetBootableImage("recovery.img", "recovery.img", 526 OPTIONS.input_tmp, "RECOVERY") 527 if OPTIONS.two_step: 528 if not OPTIONS.info_dict.get("multistage_support", None): 529 assert False, "two-step packages not supported by this build" 530 fs = OPTIONS.info_dict["fstab"]["/misc"] 531 assert fs.fs_type.upper() == "EMMC", \ 532 "two-step packages only supported on devices with EMMC /misc partitions" 533 bcb_dev = {"bcb_dev": fs.device} 534 common.ZipWriteStr(output_zip, "recovery.img", recovery_img.data) 535 script.AppendExtra(""" 536if get_stage("%(bcb_dev)s") == "2/3" then 537""" % bcb_dev) 538 script.WriteRawImage("/recovery", "recovery.img") 539 script.AppendExtra(""" 540set_stage("%(bcb_dev)s", "3/3"); 541reboot_now("%(bcb_dev)s", "recovery"); 542else if get_stage("%(bcb_dev)s") == "3/3" then 543""" % bcb_dev) 544 545 device_specific.FullOTA_InstallBegin() 546 547 system_progress = 0.75 548 549 if OPTIONS.wipe_user_data: 550 system_progress -= 0.1 551 if HasVendorPartition(input_zip): 552 system_progress -= 0.1 553 554 if "selinux_fc" in OPTIONS.info_dict: 555 WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip) 556 557 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 558 559 system_items = ItemSet("system", "META/filesystem_config.txt") 560 script.ShowProgress(system_progress, 0) 561 562 if block_based: 563 # Full OTA is done as an "incremental" against an empty source 564 # image. This has the effect of writing new data from the package 565 # to the entire partition, but lets us reuse the updater code that 566 # writes incrementals to do it. 567 system_tgt = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict) 568 system_tgt.ResetFileMap() 569 system_diff = common.BlockDifference("system", system_tgt, src=None) 570 system_diff.WriteScript(script, output_zip) 571 else: 572 script.FormatPartition("/system") 573 script.Mount("/system", recovery_mount_options) 574 if not has_recovery_patch: 575 script.UnpackPackageDir("recovery", "/system") 576 script.UnpackPackageDir("system", "/system") 577 578 symlinks = CopyPartitionFiles(system_items, input_zip, output_zip) 579 script.MakeSymlinks(symlinks) 580 581 boot_img = common.GetBootableImage("boot.img", "boot.img", 582 OPTIONS.input_tmp, "BOOT") 583 584 if not block_based: 585 def output_sink(fn, data): 586 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 587 system_items.Get("system/" + fn, dir=False) 588 589 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, 590 recovery_img, boot_img) 591 592 system_items.GetMetadata(input_zip) 593 system_items.Get("system").SetPermissions(script) 594 595 if HasVendorPartition(input_zip): 596 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 597 script.ShowProgress(0.1, 0) 598 599 if block_based: 600 vendor_tgt = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict) 601 vendor_tgt.ResetFileMap() 602 vendor_diff = common.BlockDifference("vendor", vendor_tgt) 603 vendor_diff.WriteScript(script, output_zip) 604 else: 605 script.FormatPartition("/vendor") 606 script.Mount("/vendor", recovery_mount_options) 607 script.UnpackPackageDir("vendor", "/vendor") 608 609 symlinks = CopyPartitionFiles(vendor_items, input_zip, output_zip) 610 script.MakeSymlinks(symlinks) 611 612 vendor_items.GetMetadata(input_zip) 613 vendor_items.Get("vendor").SetPermissions(script) 614 615 common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict) 616 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 617 618 script.ShowProgress(0.05, 5) 619 script.WriteRawImage("/boot", "boot.img") 620 621 script.ShowProgress(0.2, 10) 622 device_specific.FullOTA_InstallEnd() 623 624 if OPTIONS.extra_script is not None: 625 script.AppendExtra(OPTIONS.extra_script) 626 627 script.UnmountAll() 628 629 if OPTIONS.wipe_user_data: 630 script.ShowProgress(0.1, 10) 631 script.FormatPartition("/data") 632 633 if OPTIONS.two_step: 634 script.AppendExtra(""" 635set_stage("%(bcb_dev)s", ""); 636""" % bcb_dev) 637 script.AppendExtra("else\n") 638 script.WriteRawImage("/boot", "recovery.img") 639 script.AppendExtra(""" 640set_stage("%(bcb_dev)s", "2/3"); 641reboot_now("%(bcb_dev)s", ""); 642endif; 643endif; 644""" % bcb_dev) 645 script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary) 646 WriteMetadata(metadata, output_zip) 647 648 649def WritePolicyConfig(file_name, output_zip): 650 common.ZipWrite(output_zip, file_name, os.path.basename(file_name)) 651 652 653def WriteMetadata(metadata, output_zip): 654 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata", 655 "".join(["%s=%s\n" % kv 656 for kv in sorted(metadata.iteritems())])) 657 658 659def LoadPartitionFiles(z, partition): 660 """Load all the files from the given partition in a given target-files 661 ZipFile, and return a dict of {filename: File object}.""" 662 out = {} 663 prefix = partition.upper() + "/" 664 for info in z.infolist(): 665 if info.filename.startswith(prefix) and not IsSymlink(info): 666 basefilename = info.filename[7:] 667 fn = partition + "/" + basefilename 668 data = z.read(info.filename) 669 out[fn] = common.File(fn, data) 670 return out 671 672 673def GetBuildProp(prop, info_dict): 674 """Return the fingerprint of the build of a given target-files info_dict.""" 675 try: 676 return info_dict.get("build.prop", {})[prop] 677 except KeyError: 678 raise common.ExternalError("couldn't find %s in build.prop" % (prop,)) 679 680 681def AddToKnownPaths(filename, known_paths): 682 if filename[-1] == "/": 683 return 684 dirs = filename.split("/")[:-1] 685 while len(dirs) > 0: 686 path = "/".join(dirs) 687 if path in known_paths: 688 break; 689 known_paths.add(path) 690 dirs.pop() 691 692 693def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 694 source_version = OPTIONS.source_info_dict["recovery_api_version"] 695 target_version = OPTIONS.target_info_dict["recovery_api_version"] 696 697 if source_version == 0: 698 print ("WARNING: generating edify script for a source that " 699 "can't install it.") 700 script = edify_generator.EdifyGenerator(source_version, 701 OPTIONS.target_info_dict) 702 703 metadata = {"pre-device": GetBuildProp("ro.product.device", 704 OPTIONS.source_info_dict), 705 "post-timestamp": GetBuildProp("ro.build.date.utc", 706 OPTIONS.target_info_dict), 707 } 708 709 device_specific = common.DeviceSpecificParams( 710 source_zip=source_zip, 711 source_version=source_version, 712 target_zip=target_zip, 713 target_version=target_version, 714 output_zip=output_zip, 715 script=script, 716 metadata=metadata, 717 info_dict=OPTIONS.info_dict) 718 719 source_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.source_info_dict) 720 target_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.target_info_dict) 721 metadata["pre-build"] = source_fp 722 metadata["post-build"] = target_fp 723 724 source_boot = common.GetBootableImage( 725 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 726 OPTIONS.source_info_dict) 727 target_boot = common.GetBootableImage( 728 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 729 updating_boot = (not OPTIONS.two_step and 730 (source_boot.data != target_boot.data)) 731 732 source_recovery = common.GetBootableImage( 733 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 734 OPTIONS.source_info_dict) 735 target_recovery = common.GetBootableImage( 736 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 737 updating_recovery = (source_recovery.data != target_recovery.data) 738 739 system_src = GetImage("system", OPTIONS.source_tmp, OPTIONS.source_info_dict) 740 system_tgt = GetImage("system", OPTIONS.target_tmp, OPTIONS.target_info_dict) 741 742 blockimgdiff_version = 1 743 if OPTIONS.info_dict: 744 blockimgdiff_version = max( 745 int(i) for i in 746 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(",")) 747 748 system_diff = common.BlockDifference("system", system_tgt, system_src, 749 check_first_block=True, 750 version=blockimgdiff_version) 751 752 if HasVendorPartition(target_zip): 753 if not HasVendorPartition(source_zip): 754 raise RuntimeError("can't generate incremental that adds /vendor") 755 vendor_src = GetImage("vendor", OPTIONS.source_tmp, OPTIONS.source_info_dict) 756 vendor_tgt = GetImage("vendor", OPTIONS.target_tmp, OPTIONS.target_info_dict) 757 vendor_diff = common.BlockDifference("vendor", vendor_tgt, vendor_src, 758 check_first_block=True, 759 version=blockimgdiff_version) 760 else: 761 vendor_diff = None 762 763 oem_props = OPTIONS.target_info_dict.get("oem_fingerprint_properties") 764 recovery_mount_options = OPTIONS.target_info_dict.get("recovery_mount_options") 765 oem_dict = None 766 if oem_props is not None and len(oem_props) > 0: 767 if OPTIONS.oem_source is None: 768 raise common.ExternalError("OEM source required for this build") 769 script.Mount("/oem", recovery_mount_options) 770 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 771 772 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 773 device_specific.IncrementalOTA_Assertions() 774 775 # Two-step incremental package strategy (in chronological order, 776 # which is *not* the order in which the generated script has 777 # things): 778 # 779 # if stage is not "2/3" or "3/3": 780 # do verification on current system 781 # write recovery image to boot partition 782 # set stage to "2/3" 783 # reboot to boot partition and restart recovery 784 # else if stage is "2/3": 785 # write recovery image to recovery partition 786 # set stage to "3/3" 787 # reboot to recovery partition and restart recovery 788 # else: 789 # (stage must be "3/3") 790 # perform update: 791 # patch system files, etc. 792 # force full install of new boot image 793 # set up system to update recovery partition on first boot 794 # complete script normally (allow recovery to mark itself finished and reboot) 795 796 if OPTIONS.two_step: 797 if not OPTIONS.info_dict.get("multistage_support", None): 798 assert False, "two-step packages not supported by this build" 799 fs = OPTIONS.info_dict["fstab"]["/misc"] 800 assert fs.fs_type.upper() == "EMMC", \ 801 "two-step packages only supported on devices with EMMC /misc partitions" 802 bcb_dev = {"bcb_dev": fs.device} 803 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 804 script.AppendExtra(""" 805if get_stage("%(bcb_dev)s") == "2/3" then 806""" % bcb_dev) 807 script.AppendExtra("sleep(20);\n"); 808 script.WriteRawImage("/recovery", "recovery.img") 809 script.AppendExtra(""" 810set_stage("%(bcb_dev)s", "3/3"); 811reboot_now("%(bcb_dev)s", "recovery"); 812else if get_stage("%(bcb_dev)s") != "3/3" then 813""" % bcb_dev) 814 815 script.Print("Verifying current system...") 816 817 device_specific.IncrementalOTA_VerifyBegin() 818 819 if oem_props is None: 820 # When blockimgdiff version is less than 3 (non-resumable block-based OTA), 821 # patching on a device that's already on the target build will damage the 822 # system. Because operations like move don't check the block state, they 823 # always apply the changes unconditionally. 824 if blockimgdiff_version <= 2: 825 script.AssertSomeFingerprint(source_fp) 826 else: 827 script.AssertSomeFingerprint(source_fp, target_fp) 828 else: 829 if blockimgdiff_version <= 2: 830 script.AssertSomeThumbprint( 831 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 832 else: 833 script.AssertSomeThumbprint( 834 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 835 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 836 837 if updating_boot: 838 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 839 d = common.Difference(target_boot, source_boot) 840 _, _, d = d.ComputePatch() 841 if d is None: 842 include_full_boot = True 843 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 844 else: 845 include_full_boot = False 846 847 print "boot target: %d source: %d diff: %d" % ( 848 target_boot.size, source_boot.size, len(d)) 849 850 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 851 852 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 853 (boot_type, boot_device, 854 source_boot.size, source_boot.sha1, 855 target_boot.size, target_boot.sha1)) 856 857 device_specific.IncrementalOTA_VerifyEnd() 858 859 if OPTIONS.two_step: 860 script.WriteRawImage("/boot", "recovery.img") 861 script.AppendExtra(""" 862set_stage("%(bcb_dev)s", "2/3"); 863reboot_now("%(bcb_dev)s", ""); 864else 865""" % bcb_dev) 866 867 # Verify the existing partitions. 868 system_diff.WriteVerifyScript(script) 869 if vendor_diff: 870 vendor_diff.WriteVerifyScript(script) 871 872 script.Comment("---- start making changes here ----") 873 874 device_specific.IncrementalOTA_InstallBegin() 875 876 system_diff.WriteScript(script, output_zip, 877 progress=0.8 if vendor_diff else 0.9) 878 if vendor_diff: 879 vendor_diff.WriteScript(script, output_zip, progress=0.1) 880 881 if OPTIONS.two_step: 882 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 883 script.WriteRawImage("/boot", "boot.img") 884 print "writing full boot image (forced by two-step mode)" 885 886 if not OPTIONS.two_step: 887 if updating_boot: 888 if include_full_boot: 889 print "boot image changed; including full." 890 script.Print("Installing boot image...") 891 script.WriteRawImage("/boot", "boot.img") 892 else: 893 # Produce the boot image by applying a patch to the current 894 # contents of the boot partition, and write it back to the 895 # partition. 896 print "boot image changed; including patch." 897 script.Print("Patching boot image...") 898 script.ShowProgress(0.1, 10) 899 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 900 % (boot_type, boot_device, 901 source_boot.size, source_boot.sha1, 902 target_boot.size, target_boot.sha1), 903 "-", 904 target_boot.size, target_boot.sha1, 905 source_boot.sha1, "patch/boot.img.p") 906 else: 907 print "boot image unchanged; skipping." 908 909 # Do device-specific installation (eg, write radio image). 910 device_specific.IncrementalOTA_InstallEnd() 911 912 if OPTIONS.extra_script is not None: 913 script.AppendExtra(OPTIONS.extra_script) 914 915 if OPTIONS.wipe_user_data: 916 script.Print("Erasing user data...") 917 script.FormatPartition("/data") 918 919 if OPTIONS.two_step: 920 script.AppendExtra(""" 921set_stage("%(bcb_dev)s", ""); 922endif; 923endif; 924""" % bcb_dev) 925 926 script.SetProgress(1) 927 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 928 WriteMetadata(metadata, output_zip) 929 930 931class FileDifference: 932 def __init__(self, partition, source_zip, target_zip, output_zip): 933 print "Loading target..." 934 self.target_data = target_data = LoadPartitionFiles(target_zip, partition) 935 print "Loading source..." 936 self.source_data = source_data = LoadPartitionFiles(source_zip, partition) 937 938 self.verbatim_targets = verbatim_targets = [] 939 self.patch_list = patch_list = [] 940 diffs = [] 941 self.renames = renames = {} 942 known_paths = set() 943 largest_source_size = 0 944 945 matching_file_cache = {} 946 for fn, sf in source_data.items(): 947 assert fn == sf.name 948 matching_file_cache["path:" + fn] = sf 949 if fn in target_data.keys(): 950 AddToKnownPaths(fn, known_paths) 951 # Only allow eligibility for filename/sha matching 952 # if there isn't a perfect path match. 953 if target_data.get(sf.name) is None: 954 matching_file_cache["file:" + fn.split("/")[-1]] = sf 955 matching_file_cache["sha:" + sf.sha1] = sf 956 957 for fn in sorted(target_data.keys()): 958 tf = target_data[fn] 959 assert fn == tf.name 960 sf = ClosestFileMatch(tf, matching_file_cache, renames) 961 if sf is not None and sf.name != tf.name: 962 print "File has moved from " + sf.name + " to " + tf.name 963 renames[sf.name] = tf 964 965 if sf is None or fn in OPTIONS.require_verbatim: 966 # This file should be included verbatim 967 if fn in OPTIONS.prohibit_verbatim: 968 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 969 print "send", fn, "verbatim" 970 tf.AddToZip(output_zip) 971 verbatim_targets.append((fn, tf.size, tf.sha1)) 972 if fn in target_data.keys(): 973 AddToKnownPaths(fn, known_paths) 974 elif tf.sha1 != sf.sha1: 975 # File is different; consider sending as a patch 976 diffs.append(common.Difference(tf, sf)) 977 else: 978 # Target file data identical to source (may still be renamed) 979 pass 980 981 common.ComputeDifferences(diffs) 982 983 for diff in diffs: 984 tf, sf, d = diff.GetPatch() 985 path = "/".join(tf.name.split("/")[:-1]) 986 if d is None or len(d) > tf.size * OPTIONS.patch_threshold or \ 987 path not in known_paths: 988 # patch is almost as big as the file; don't bother patching 989 # or a patch + rename cannot take place due to the target 990 # directory not existing 991 tf.AddToZip(output_zip) 992 verbatim_targets.append((tf.name, tf.size, tf.sha1)) 993 if sf.name in renames: 994 del renames[sf.name] 995 AddToKnownPaths(tf.name, known_paths) 996 else: 997 common.ZipWriteStr(output_zip, "patch/" + sf.name + ".p", d) 998 patch_list.append((tf, sf, tf.size, common.sha1(d).hexdigest())) 999 largest_source_size = max(largest_source_size, sf.size) 1000 1001 self.largest_source_size = largest_source_size 1002 1003 def EmitVerification(self, script): 1004 so_far = 0 1005 for tf, sf, size, patch_sha in self.patch_list: 1006 if tf.name != sf.name: 1007 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1008 script.PatchCheck("/"+sf.name, tf.sha1, sf.sha1) 1009 so_far += sf.size 1010 return so_far 1011 1012 def EmitExplicitTargetVerification(self, script): 1013 for fn, size, sha1 in self.verbatim_targets: 1014 if (fn[-1] != "/"): 1015 script.FileCheck("/"+fn, sha1) 1016 for tf, _, _, _ in self.patch_list: 1017 script.FileCheck(tf.name, tf.sha1) 1018 1019 def RemoveUnneededFiles(self, script, extras=()): 1020 script.DeleteFiles(["/"+i[0] for i in self.verbatim_targets] + 1021 ["/"+i for i in sorted(self.source_data) 1022 if i not in self.target_data and 1023 i not in self.renames] + 1024 list(extras)) 1025 1026 def TotalPatchSize(self): 1027 return sum(i[1].size for i in self.patch_list) 1028 1029 def EmitPatches(self, script, total_patch_size, so_far): 1030 self.deferred_patch_list = deferred_patch_list = [] 1031 for item in self.patch_list: 1032 tf, sf, size, _ = item 1033 if tf.name == "system/build.prop": 1034 deferred_patch_list.append(item) 1035 continue 1036 if (sf.name != tf.name): 1037 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1038 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p") 1039 so_far += tf.size 1040 script.SetProgress(so_far / total_patch_size) 1041 return so_far 1042 1043 def EmitDeferredPatches(self, script): 1044 for item in self.deferred_patch_list: 1045 tf, sf, size, _ = item 1046 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p") 1047 script.SetPermissions("/system/build.prop", 0, 0, 0644, None, None) 1048 1049 def EmitRenames(self, script): 1050 if len(self.renames) > 0: 1051 script.Print("Renaming files...") 1052 for src, tgt in self.renames.iteritems(): 1053 print "Renaming " + src + " to " + tgt.name 1054 script.RenameFile(src, tgt.name) 1055 1056 1057 1058 1059def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): 1060 target_has_recovery_patch = HasRecoveryPatch(target_zip) 1061 source_has_recovery_patch = HasRecoveryPatch(source_zip) 1062 1063 if (OPTIONS.block_based and 1064 target_has_recovery_patch and 1065 source_has_recovery_patch): 1066 return WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip) 1067 1068 source_version = OPTIONS.source_info_dict["recovery_api_version"] 1069 target_version = OPTIONS.target_info_dict["recovery_api_version"] 1070 1071 if source_version == 0: 1072 print ("WARNING: generating edify script for a source that " 1073 "can't install it.") 1074 script = edify_generator.EdifyGenerator(source_version, 1075 OPTIONS.target_info_dict) 1076 1077 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 1078 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 1079 oem_dict = None 1080 if oem_props is not None and len(oem_props) > 0: 1081 if OPTIONS.oem_source is None: 1082 raise common.ExternalError("OEM source required for this build") 1083 script.Mount("/oem", recovery_mount_options) 1084 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 1085 1086 metadata = {"pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 1087 OPTIONS.source_info_dict), 1088 "post-timestamp": GetBuildProp("ro.build.date.utc", 1089 OPTIONS.target_info_dict), 1090 } 1091 1092 device_specific = common.DeviceSpecificParams( 1093 source_zip=source_zip, 1094 source_version=source_version, 1095 target_zip=target_zip, 1096 target_version=target_version, 1097 output_zip=output_zip, 1098 script=script, 1099 metadata=metadata, 1100 info_dict=OPTIONS.info_dict) 1101 1102 system_diff = FileDifference("system", source_zip, target_zip, output_zip) 1103 script.Mount("/system", recovery_mount_options) 1104 if HasVendorPartition(target_zip): 1105 vendor_diff = FileDifference("vendor", source_zip, target_zip, output_zip) 1106 script.Mount("/vendor", recovery_mount_options) 1107 else: 1108 vendor_diff = None 1109 1110 target_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.target_info_dict) 1111 source_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.source_info_dict) 1112 1113 if oem_props is None: 1114 script.AssertSomeFingerprint(source_fp, target_fp) 1115 else: 1116 script.AssertSomeThumbprint( 1117 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 1118 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 1119 1120 metadata["pre-build"] = source_fp 1121 metadata["post-build"] = target_fp 1122 1123 source_boot = common.GetBootableImage( 1124 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 1125 OPTIONS.source_info_dict) 1126 target_boot = common.GetBootableImage( 1127 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 1128 updating_boot = (not OPTIONS.two_step and 1129 (source_boot.data != target_boot.data)) 1130 1131 source_recovery = common.GetBootableImage( 1132 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 1133 OPTIONS.source_info_dict) 1134 target_recovery = common.GetBootableImage( 1135 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 1136 updating_recovery = (source_recovery.data != target_recovery.data) 1137 1138 # Here's how we divide up the progress bar: 1139 # 0.1 for verifying the start state (PatchCheck calls) 1140 # 0.8 for applying patches (ApplyPatch calls) 1141 # 0.1 for unpacking verbatim files, symlinking, and doing the 1142 # device-specific commands. 1143 1144 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 1145 device_specific.IncrementalOTA_Assertions() 1146 1147 # Two-step incremental package strategy (in chronological order, 1148 # which is *not* the order in which the generated script has 1149 # things): 1150 # 1151 # if stage is not "2/3" or "3/3": 1152 # do verification on current system 1153 # write recovery image to boot partition 1154 # set stage to "2/3" 1155 # reboot to boot partition and restart recovery 1156 # else if stage is "2/3": 1157 # write recovery image to recovery partition 1158 # set stage to "3/3" 1159 # reboot to recovery partition and restart recovery 1160 # else: 1161 # (stage must be "3/3") 1162 # perform update: 1163 # patch system files, etc. 1164 # force full install of new boot image 1165 # set up system to update recovery partition on first boot 1166 # complete script normally (allow recovery to mark itself finished and reboot) 1167 1168 if OPTIONS.two_step: 1169 if not OPTIONS.info_dict.get("multistage_support", None): 1170 assert False, "two-step packages not supported by this build" 1171 fs = OPTIONS.info_dict["fstab"]["/misc"] 1172 assert fs.fs_type.upper() == "EMMC", \ 1173 "two-step packages only supported on devices with EMMC /misc partitions" 1174 bcb_dev = {"bcb_dev": fs.device} 1175 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 1176 script.AppendExtra(""" 1177if get_stage("%(bcb_dev)s") == "2/3" then 1178""" % bcb_dev) 1179 script.AppendExtra("sleep(20);\n"); 1180 script.WriteRawImage("/recovery", "recovery.img") 1181 script.AppendExtra(""" 1182set_stage("%(bcb_dev)s", "3/3"); 1183reboot_now("%(bcb_dev)s", "recovery"); 1184else if get_stage("%(bcb_dev)s") != "3/3" then 1185""" % bcb_dev) 1186 1187 script.Print("Verifying current system...") 1188 1189 device_specific.IncrementalOTA_VerifyBegin() 1190 1191 script.ShowProgress(0.1, 0) 1192 so_far = system_diff.EmitVerification(script) 1193 if vendor_diff: 1194 so_far += vendor_diff.EmitVerification(script) 1195 1196 if updating_boot: 1197 d = common.Difference(target_boot, source_boot) 1198 _, _, d = d.ComputePatch() 1199 print "boot target: %d source: %d diff: %d" % ( 1200 target_boot.size, source_boot.size, len(d)) 1201 1202 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 1203 1204 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 1205 1206 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 1207 (boot_type, boot_device, 1208 source_boot.size, source_boot.sha1, 1209 target_boot.size, target_boot.sha1)) 1210 so_far += source_boot.size 1211 1212 size = [] 1213 if system_diff.patch_list: size.append(system_diff.largest_source_size) 1214 if vendor_diff: 1215 if vendor_diff.patch_list: size.append(vendor_diff.largest_source_size) 1216 if size or updating_recovery or updating_boot: 1217 script.CacheFreeSpaceCheck(max(size)) 1218 1219 device_specific.IncrementalOTA_VerifyEnd() 1220 1221 if OPTIONS.two_step: 1222 script.WriteRawImage("/boot", "recovery.img") 1223 script.AppendExtra(""" 1224set_stage("%(bcb_dev)s", "2/3"); 1225reboot_now("%(bcb_dev)s", ""); 1226else 1227""" % bcb_dev) 1228 1229 script.Comment("---- start making changes here ----") 1230 1231 device_specific.IncrementalOTA_InstallBegin() 1232 1233 if OPTIONS.two_step: 1234 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 1235 script.WriteRawImage("/boot", "boot.img") 1236 print "writing full boot image (forced by two-step mode)" 1237 1238 script.Print("Removing unneeded files...") 1239 system_diff.RemoveUnneededFiles(script, ("/system/recovery.img",)) 1240 if vendor_diff: 1241 vendor_diff.RemoveUnneededFiles(script) 1242 1243 script.ShowProgress(0.8, 0) 1244 total_patch_size = 1.0 + system_diff.TotalPatchSize() 1245 if vendor_diff: 1246 total_patch_size += vendor_diff.TotalPatchSize() 1247 if updating_boot: 1248 total_patch_size += target_boot.size 1249 1250 script.Print("Patching system files...") 1251 so_far = system_diff.EmitPatches(script, total_patch_size, 0) 1252 if vendor_diff: 1253 script.Print("Patching vendor files...") 1254 so_far = vendor_diff.EmitPatches(script, total_patch_size, so_far) 1255 1256 if not OPTIONS.two_step: 1257 if updating_boot: 1258 # Produce the boot image by applying a patch to the current 1259 # contents of the boot partition, and write it back to the 1260 # partition. 1261 script.Print("Patching boot image...") 1262 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 1263 % (boot_type, boot_device, 1264 source_boot.size, source_boot.sha1, 1265 target_boot.size, target_boot.sha1), 1266 "-", 1267 target_boot.size, target_boot.sha1, 1268 source_boot.sha1, "patch/boot.img.p") 1269 so_far += target_boot.size 1270 script.SetProgress(so_far / total_patch_size) 1271 print "boot image changed; including." 1272 else: 1273 print "boot image unchanged; skipping." 1274 1275 system_items = ItemSet("system", "META/filesystem_config.txt") 1276 if vendor_diff: 1277 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 1278 1279 if updating_recovery: 1280 # Recovery is generated as a patch using both the boot image 1281 # (which contains the same linux kernel as recovery) and the file 1282 # /system/etc/recovery-resource.dat (which contains all the images 1283 # used in the recovery UI) as sources. This lets us minimize the 1284 # size of the patch, which must be included in every OTA package. 1285 # 1286 # For older builds where recovery-resource.dat is not present, we 1287 # use only the boot image as the source. 1288 1289 if not target_has_recovery_patch: 1290 def output_sink(fn, data): 1291 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 1292 system_items.Get("system/" + fn, dir=False) 1293 1294 common.MakeRecoveryPatch(OPTIONS.target_tmp, output_sink, 1295 target_recovery, target_boot) 1296 script.DeleteFiles(["/system/recovery-from-boot.p", 1297 "/system/etc/install-recovery.sh"]) 1298 print "recovery image changed; including as patch from boot." 1299 else: 1300 print "recovery image unchanged; skipping." 1301 1302 script.ShowProgress(0.1, 10) 1303 1304 target_symlinks = CopyPartitionFiles(system_items, target_zip, None) 1305 if vendor_diff: 1306 target_symlinks.extend(CopyPartitionFiles(vendor_items, target_zip, None)) 1307 1308 temp_script = script.MakeTemporary() 1309 system_items.GetMetadata(target_zip) 1310 system_items.Get("system").SetPermissions(temp_script) 1311 if vendor_diff: 1312 vendor_items.GetMetadata(target_zip) 1313 vendor_items.Get("vendor").SetPermissions(temp_script) 1314 1315 # Note that this call will mess up the trees of Items, so make sure 1316 # we're done with them. 1317 source_symlinks = CopyPartitionFiles(system_items, source_zip, None) 1318 if vendor_diff: 1319 source_symlinks.extend(CopyPartitionFiles(vendor_items, source_zip, None)) 1320 1321 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 1322 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 1323 1324 # Delete all the symlinks in source that aren't in target. This 1325 # needs to happen before verbatim files are unpacked, in case a 1326 # symlink in the source is replaced by a real file in the target. 1327 to_delete = [] 1328 for dest, link in source_symlinks: 1329 if link not in target_symlinks_d: 1330 to_delete.append(link) 1331 script.DeleteFiles(to_delete) 1332 1333 if system_diff.verbatim_targets: 1334 script.Print("Unpacking new system files...") 1335 script.UnpackPackageDir("system", "/system") 1336 if vendor_diff and vendor_diff.verbatim_targets: 1337 script.Print("Unpacking new vendor files...") 1338 script.UnpackPackageDir("vendor", "/vendor") 1339 1340 if updating_recovery and not target_has_recovery_patch: 1341 script.Print("Unpacking new recovery...") 1342 script.UnpackPackageDir("recovery", "/system") 1343 1344 system_diff.EmitRenames(script) 1345 if vendor_diff: 1346 vendor_diff.EmitRenames(script) 1347 1348 script.Print("Symlinks and permissions...") 1349 1350 # Create all the symlinks that don't already exist, or point to 1351 # somewhere different than what we want. Delete each symlink before 1352 # creating it, since the 'symlink' command won't overwrite. 1353 to_create = [] 1354 for dest, link in target_symlinks: 1355 if link in source_symlinks_d: 1356 if dest != source_symlinks_d[link]: 1357 to_create.append((dest, link)) 1358 else: 1359 to_create.append((dest, link)) 1360 script.DeleteFiles([i[1] for i in to_create]) 1361 script.MakeSymlinks(to_create) 1362 1363 # Now that the symlinks are created, we can set all the 1364 # permissions. 1365 script.AppendScript(temp_script) 1366 1367 # Do device-specific installation (eg, write radio image). 1368 device_specific.IncrementalOTA_InstallEnd() 1369 1370 if OPTIONS.extra_script is not None: 1371 script.AppendExtra(OPTIONS.extra_script) 1372 1373 # Patch the build.prop file last, so if something fails but the 1374 # device can still come up, it appears to be the old build and will 1375 # get set the OTA package again to retry. 1376 script.Print("Patching remaining system files...") 1377 system_diff.EmitDeferredPatches(script) 1378 1379 if OPTIONS.wipe_user_data: 1380 script.Print("Erasing user data...") 1381 script.FormatPartition("/data") 1382 1383 if OPTIONS.two_step: 1384 script.AppendExtra(""" 1385set_stage("%(bcb_dev)s", ""); 1386endif; 1387endif; 1388""" % bcb_dev) 1389 1390 if OPTIONS.verify and system_diff: 1391 script.Print("Remounting and verifying system partition files...") 1392 script.Unmount("/system") 1393 script.Mount("/system") 1394 system_diff.EmitExplicitTargetVerification(script) 1395 1396 if OPTIONS.verify and vendor_diff: 1397 script.Print("Remounting and verifying vendor partition files...") 1398 script.Unmount("/vendor") 1399 script.Mount("/vendor") 1400 vendor_diff.EmitExplicitTargetVerification(script) 1401 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 1402 1403 WriteMetadata(metadata, output_zip) 1404 1405 1406def main(argv): 1407 1408 def option_handler(o, a): 1409 if o == "--board_config": 1410 pass # deprecated 1411 elif o in ("-k", "--package_key"): 1412 OPTIONS.package_key = a 1413 elif o in ("-i", "--incremental_from"): 1414 OPTIONS.incremental_source = a 1415 elif o in ("-w", "--wipe_user_data"): 1416 OPTIONS.wipe_user_data = True 1417 elif o in ("-n", "--no_prereq"): 1418 OPTIONS.omit_prereq = True 1419 elif o in ("-o", "--oem_settings"): 1420 OPTIONS.oem_source = a 1421 elif o in ("-e", "--extra_script"): 1422 OPTIONS.extra_script = a 1423 elif o in ("-a", "--aslr_mode"): 1424 if a in ("on", "On", "true", "True", "yes", "Yes"): 1425 OPTIONS.aslr_mode = True 1426 else: 1427 OPTIONS.aslr_mode = False 1428 elif o in ("-t", "--worker_threads"): 1429 if a.isdigit(): 1430 OPTIONS.worker_threads = int(a) 1431 else: 1432 raise ValueError("Cannot parse value %r for option %r - only " 1433 "integers are allowed." % (a, o)) 1434 elif o in ("-2", "--two_step"): 1435 OPTIONS.two_step = True 1436 elif o == "--no_signing": 1437 OPTIONS.no_signing = True 1438 elif o in ("--verify"): 1439 OPTIONS.verify = True 1440 elif o == "--block": 1441 OPTIONS.block_based = True 1442 elif o in ("-b", "--binary"): 1443 OPTIONS.updater_binary = a 1444 elif o in ("--no_fallback_to_full",): 1445 OPTIONS.fallback_to_full = False 1446 else: 1447 return False 1448 return True 1449 1450 args = common.ParseOptions(argv, __doc__, 1451 extra_opts="b:k:i:d:wne:t:a:2o:", 1452 extra_long_opts=["board_config=", 1453 "package_key=", 1454 "incremental_from=", 1455 "wipe_user_data", 1456 "no_prereq", 1457 "extra_script=", 1458 "worker_threads=", 1459 "aslr_mode=", 1460 "two_step", 1461 "no_signing", 1462 "block", 1463 "binary=", 1464 "oem_settings=", 1465 "verify", 1466 "no_fallback_to_full", 1467 ], 1468 extra_option_handler=option_handler) 1469 1470 if len(args) != 2: 1471 common.Usage(__doc__) 1472 sys.exit(1) 1473 1474 if OPTIONS.extra_script is not None: 1475 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 1476 1477 print "unzipping target target-files..." 1478 OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0]) 1479 1480 OPTIONS.target_tmp = OPTIONS.input_tmp 1481 OPTIONS.info_dict = common.LoadInfoDict(input_zip) 1482 1483 # If this image was originally labelled with SELinux contexts, make sure we 1484 # also apply the labels in our new image. During building, the "file_contexts" 1485 # is in the out/ directory tree, but for repacking from target-files.zip it's 1486 # in the root directory of the ramdisk. 1487 if "selinux_fc" in OPTIONS.info_dict: 1488 OPTIONS.info_dict["selinux_fc"] = os.path.join(OPTIONS.input_tmp, "BOOT", "RAMDISK", 1489 "file_contexts") 1490 1491 if OPTIONS.verbose: 1492 print "--- target info ---" 1493 common.DumpInfoDict(OPTIONS.info_dict) 1494 1495 # If the caller explicitly specified the device-specific extensions 1496 # path via -s/--device_specific, use that. Otherwise, use 1497 # META/releasetools.py if it is present in the target target_files. 1498 # Otherwise, take the path of the file from 'tool_extensions' in the 1499 # info dict and look for that in the local filesystem, relative to 1500 # the current directory. 1501 1502 if OPTIONS.device_specific is None: 1503 from_input = os.path.join(OPTIONS.input_tmp, "META", "releasetools.py") 1504 if os.path.exists(from_input): 1505 print "(using device-specific extensions from target_files)" 1506 OPTIONS.device_specific = from_input 1507 else: 1508 OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None) 1509 1510 if OPTIONS.device_specific is not None: 1511 OPTIONS.device_specific = os.path.abspath(OPTIONS.device_specific) 1512 1513 while True: 1514 1515 if OPTIONS.no_signing: 1516 if os.path.exists(args[1]): os.unlink(args[1]) 1517 output_zip = zipfile.ZipFile(args[1], "w", compression=zipfile.ZIP_DEFLATED) 1518 else: 1519 temp_zip_file = tempfile.NamedTemporaryFile() 1520 output_zip = zipfile.ZipFile(temp_zip_file, "w", 1521 compression=zipfile.ZIP_DEFLATED) 1522 1523 if OPTIONS.incremental_source is None: 1524 WriteFullOTAPackage(input_zip, output_zip) 1525 if OPTIONS.package_key is None: 1526 OPTIONS.package_key = OPTIONS.info_dict.get( 1527 "default_system_dev_certificate", 1528 "build/target/product/security/testkey") 1529 break 1530 1531 else: 1532 print "unzipping source target-files..." 1533 OPTIONS.source_tmp, source_zip = common.UnzipTemp(OPTIONS.incremental_source) 1534 OPTIONS.target_info_dict = OPTIONS.info_dict 1535 OPTIONS.source_info_dict = common.LoadInfoDict(source_zip) 1536 if "selinux_fc" in OPTIONS.source_info_dict: 1537 OPTIONS.source_info_dict["selinux_fc"] = os.path.join(OPTIONS.source_tmp, "BOOT", "RAMDISK", 1538 "file_contexts") 1539 if OPTIONS.package_key is None: 1540 OPTIONS.package_key = OPTIONS.source_info_dict.get( 1541 "default_system_dev_certificate", 1542 "build/target/product/security/testkey") 1543 if OPTIONS.verbose: 1544 print "--- source info ---" 1545 common.DumpInfoDict(OPTIONS.source_info_dict) 1546 try: 1547 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) 1548 break 1549 except ValueError: 1550 if not OPTIONS.fallback_to_full: raise 1551 print "--- failed to build incremental; falling back to full ---" 1552 OPTIONS.incremental_source = None 1553 output_zip.close() 1554 1555 output_zip.close() 1556 1557 if not OPTIONS.no_signing: 1558 SignOutput(temp_zip_file.name, args[1]) 1559 temp_zip_file.close() 1560 1561 print "done." 1562 1563 1564if __name__ == '__main__': 1565 try: 1566 common.CloseInheritedPipes() 1567 main(sys.argv[1:]) 1568 except common.ExternalError, e: 1569 print 1570 print " ERROR: %s" % (e,) 1571 print 1572 sys.exit(1) 1573 finally: 1574 common.Cleanup() 1575