ota_from_target_files.py revision 7cd99bad210455dc57702b7cbf674a588a5eb028
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 --board_config <file> 25 Deprecated. 26 27 -k (--package_key) <key> Key to use to sign the package (default is 28 the value of default_system_dev_certificate from the input 29 target-files's META/misc_info.txt, or 30 "build/target/product/security/testkey" if that value is not 31 specified). 32 33 For incremental OTAs, the default value is based on the source 34 target-file, not the target build. 35 36 -i (--incremental_from) <file> 37 Generate an incremental OTA using the given target-files zip as 38 the starting build. 39 40 -o (--oem_settings) <file> 41 Use the file to specify the expected OEM-specific properties 42 on the OEM partition of the intended device. 43 44 -w (--wipe_user_data) 45 Generate an OTA package that will wipe the user data partition 46 when installed. 47 48 -n (--no_prereq) 49 Omit the timestamp prereq check normally included at the top of 50 the build scripts (used for developer OTA packages which 51 legitimately need to go back and forth). 52 53 -e (--extra_script) <file> 54 Insert the contents of file at the end of the update script. 55 56 -a (--aslr_mode) <on|off> 57 Specify whether to turn on ASLR for the package (on by default). 58 59 -2 (--two_step) 60 Generate a 'two-step' OTA package, where recovery is updated 61 first, so that any changes made to the system partition are done 62 using the new recovery (new kernel, etc.). 63 64 --block 65 Generate a block-based OTA if possible. Will fall back to a 66 file-based OTA if the target_files is older and doesn't support 67 block-based OTAs. 68 69 -b (--binary) <file> 70 Use the given binary as the update-binary in the output package, 71 instead of the binary in the build's target_files. Use for 72 development only. 73 74 -t (--worker_threads) <int> 75 Specifies the number of worker-threads that will be used when 76 generating patches for incremental updates (defaults to 3). 77 78""" 79 80import sys 81 82if sys.hexversion < 0x02070000: 83 print >> sys.stderr, "Python 2.7 or newer is required." 84 sys.exit(1) 85 86import copy 87import errno 88import multiprocessing 89import os 90import re 91import subprocess 92import tempfile 93import time 94import zipfile 95 96from hashlib import sha1 as sha1 97 98import common 99import edify_generator 100import build_image 101import blockimgdiff 102import sparse_img 103 104OPTIONS = common.OPTIONS 105OPTIONS.package_key = None 106OPTIONS.incremental_source = None 107OPTIONS.require_verbatim = set() 108OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 109OPTIONS.patch_threshold = 0.95 110OPTIONS.wipe_user_data = False 111OPTIONS.omit_prereq = False 112OPTIONS.extra_script = None 113OPTIONS.aslr_mode = True 114OPTIONS.worker_threads = multiprocessing.cpu_count() // 2 115if OPTIONS.worker_threads == 0: 116 OPTIONS.worker_threads = 1 117OPTIONS.two_step = False 118OPTIONS.no_signing = False 119OPTIONS.block_based = False 120OPTIONS.updater_binary = None 121OPTIONS.oem_source = None 122OPTIONS.fallback_to_full = True 123 124def MostPopularKey(d, default): 125 """Given a dict, return the key corresponding to the largest 126 value. Returns 'default' if the dict is empty.""" 127 x = [(v, k) for (k, v) in d.iteritems()] 128 if not x: return default 129 x.sort() 130 return x[-1][1] 131 132 133def IsSymlink(info): 134 """Return true if the zipfile.ZipInfo object passed in represents a 135 symlink.""" 136 return (info.external_attr >> 16) == 0120777 137 138def IsRegular(info): 139 """Return true if the zipfile.ZipInfo object passed in represents a 140 symlink.""" 141 return (info.external_attr >> 28) == 010 142 143def ClosestFileMatch(src, tgtfiles, existing): 144 """Returns the closest file match between a source file and list 145 of potential matches. The exact filename match is preferred, 146 then the sha1 is searched for, and finally a file with the same 147 basename is evaluated. Rename support in the updater-binary is 148 required for the latter checks to be used.""" 149 150 result = tgtfiles.get("path:" + src.name) 151 if result is not None: 152 return result 153 154 if not OPTIONS.target_info_dict.get("update_rename_support", False): 155 return None 156 157 if src.size < 1000: 158 return None 159 160 result = tgtfiles.get("sha1:" + src.sha1) 161 if result is not None and existing.get(result.name) is None: 162 return result 163 result = tgtfiles.get("file:" + src.name.split("/")[-1]) 164 if result is not None and existing.get(result.name) is None: 165 return result 166 return None 167 168class ItemSet: 169 def __init__(self, partition, fs_config): 170 self.partition = partition 171 self.fs_config = fs_config 172 self.ITEMS = {} 173 174 def Get(self, name, dir=False): 175 if name not in self.ITEMS: 176 self.ITEMS[name] = Item(self, name, dir=dir) 177 return self.ITEMS[name] 178 179 def GetMetadata(self, input_zip): 180 # The target_files contains a record of what the uid, 181 # gid, and mode are supposed to be. 182 output = input_zip.read(self.fs_config) 183 184 for line in output.split("\n"): 185 if not line: continue 186 columns = line.split() 187 name, uid, gid, mode = columns[:4] 188 selabel = None 189 capabilities = None 190 191 # After the first 4 columns, there are a series of key=value 192 # pairs. Extract out the fields we care about. 193 for element in columns[4:]: 194 key, value = element.split("=") 195 if key == "selabel": 196 selabel = value 197 if key == "capabilities": 198 capabilities = value 199 200 i = self.ITEMS.get(name, None) 201 if i is not None: 202 i.uid = int(uid) 203 i.gid = int(gid) 204 i.mode = int(mode, 8) 205 i.selabel = selabel 206 i.capabilities = capabilities 207 if i.dir: 208 i.children.sort(key=lambda i: i.name) 209 210 # set metadata for the files generated by this script. 211 i = self.ITEMS.get("system/recovery-from-boot.p", None) 212 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0644, None, None 213 i = self.ITEMS.get("system/etc/install-recovery.sh", None) 214 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0544, None, None 215 216 217class Item: 218 """Items represent the metadata (user, group, mode) of files and 219 directories in the system image.""" 220 def __init__(self, itemset, name, dir=False): 221 self.itemset = itemset 222 self.name = name 223 self.uid = None 224 self.gid = None 225 self.mode = None 226 self.selabel = None 227 self.capabilities = None 228 self.dir = dir 229 230 if name: 231 self.parent = itemset.Get(os.path.dirname(name), dir=True) 232 self.parent.children.append(self) 233 else: 234 self.parent = None 235 if dir: 236 self.children = [] 237 238 def Dump(self, indent=0): 239 if self.uid is not None: 240 print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode) 241 else: 242 print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode) 243 if self.dir: 244 print "%s%s" % (" "*indent, self.descendants) 245 print "%s%s" % (" "*indent, self.best_subtree) 246 for i in self.children: 247 i.Dump(indent=indent+1) 248 249 def CountChildMetadata(self): 250 """Count up the (uid, gid, mode, selabel, capabilities) tuples for 251 all children and determine the best strategy for using set_perm_recursive and 252 set_perm to correctly chown/chmod all the files to their desired 253 values. Recursively calls itself for all descendants. 254 255 Returns a dict of {(uid, gid, dmode, fmode, selabel, capabilities): count} counting up 256 all descendants of this node. (dmode or fmode may be None.) Also 257 sets the best_subtree of each directory Item to the (uid, gid, 258 dmode, fmode, selabel, capabilities) tuple that will match the most 259 descendants of that Item. 260 """ 261 262 assert self.dir 263 d = self.descendants = {(self.uid, self.gid, self.mode, None, self.selabel, self.capabilities): 1} 264 for i in self.children: 265 if i.dir: 266 for k, v in i.CountChildMetadata().iteritems(): 267 d[k] = d.get(k, 0) + v 268 else: 269 k = (i.uid, i.gid, None, i.mode, i.selabel, i.capabilities) 270 d[k] = d.get(k, 0) + 1 271 272 # Find the (uid, gid, dmode, fmode, selabel, capabilities) 273 # tuple that matches the most descendants. 274 275 # First, find the (uid, gid) pair that matches the most 276 # descendants. 277 ug = {} 278 for (uid, gid, _, _, _, _), count in d.iteritems(): 279 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 280 ug = MostPopularKey(ug, (0, 0)) 281 282 # Now find the dmode, fmode, selabel, and capabilities that match 283 # the most descendants with that (uid, gid), and choose those. 284 best_dmode = (0, 0755) 285 best_fmode = (0, 0644) 286 best_selabel = (0, None) 287 best_capabilities = (0, None) 288 for k, count in d.iteritems(): 289 if k[:2] != ug: continue 290 if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2]) 291 if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3]) 292 if k[4] is not None and count >= best_selabel[0]: best_selabel = (count, k[4]) 293 if k[5] is not None and count >= best_capabilities[0]: best_capabilities = (count, k[5]) 294 self.best_subtree = ug + (best_dmode[1], best_fmode[1], best_selabel[1], best_capabilities[1]) 295 296 return d 297 298 def SetPermissions(self, script): 299 """Append set_perm/set_perm_recursive commands to 'script' to 300 set all permissions, users, and groups for the tree of files 301 rooted at 'self'.""" 302 303 self.CountChildMetadata() 304 305 def recurse(item, current): 306 # current is the (uid, gid, dmode, fmode, selabel, capabilities) tuple that the current 307 # item (and all its children) have already been set to. We only 308 # need to issue set_perm/set_perm_recursive commands if we're 309 # supposed to be something different. 310 if item.dir: 311 if current != item.best_subtree: 312 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 313 current = item.best_subtree 314 315 if item.uid != current[0] or item.gid != current[1] or \ 316 item.mode != current[2] or item.selabel != current[4] or \ 317 item.capabilities != current[5]: 318 script.SetPermissions("/"+item.name, item.uid, item.gid, 319 item.mode, item.selabel, item.capabilities) 320 321 for i in item.children: 322 recurse(i, current) 323 else: 324 if item.uid != current[0] or item.gid != current[1] or \ 325 item.mode != current[3] or item.selabel != current[4] or \ 326 item.capabilities != current[5]: 327 script.SetPermissions("/"+item.name, item.uid, item.gid, 328 item.mode, item.selabel, item.capabilities) 329 330 recurse(self, (-1, -1, -1, -1, None, None)) 331 332 333def CopyPartitionFiles(itemset, input_zip, output_zip=None, substitute=None): 334 """Copies files for the partition in the input zip to the output 335 zip. Populates the Item class with their metadata, and returns a 336 list of symlinks. output_zip may be None, in which case the copy is 337 skipped (but the other side effects still happen). substitute is an 338 optional dict of {output filename: contents} to be output instead of 339 certain input files. 340 """ 341 342 symlinks = [] 343 344 partition = itemset.partition 345 346 for info in input_zip.infolist(): 347 if info.filename.startswith(partition.upper() + "/"): 348 basefilename = info.filename[7:] 349 if IsSymlink(info): 350 symlinks.append((input_zip.read(info.filename), 351 "/" + partition + "/" + basefilename)) 352 else: 353 info2 = copy.copy(info) 354 fn = info2.filename = partition + "/" + basefilename 355 if substitute and fn in substitute and substitute[fn] is None: 356 continue 357 if output_zip is not None: 358 if substitute and fn in substitute: 359 data = substitute[fn] 360 else: 361 data = input_zip.read(info.filename) 362 output_zip.writestr(info2, data) 363 if fn.endswith("/"): 364 itemset.Get(fn[:-1], dir=True) 365 else: 366 itemset.Get(fn, dir=False) 367 368 symlinks.sort() 369 return symlinks 370 371 372def SignOutput(temp_zip_name, output_zip_name): 373 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 374 pw = key_passwords[OPTIONS.package_key] 375 376 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 377 whole_file=True) 378 379 380def AppendAssertions(script, info_dict, oem_dict = None): 381 oem_props = info_dict.get("oem_fingerprint_properties") 382 if oem_props is None or len(oem_props) == 0: 383 device = GetBuildProp("ro.product.device", info_dict) 384 script.AssertDevice(device) 385 else: 386 if oem_dict is None: 387 raise common.ExternalError("No OEM file provided to answer expected assertions") 388 for prop in oem_props.split(): 389 if oem_dict.get(prop) is None: 390 raise common.ExternalError("The OEM file is missing the property %s" % prop) 391 script.AssertOemProperty(prop, oem_dict.get(prop)) 392 393 394def HasRecoveryPatch(target_files_zip): 395 try: 396 target_files_zip.getinfo("SYSTEM/recovery-from-boot.p") 397 return True 398 except KeyError: 399 return False 400 401def HasVendorPartition(target_files_zip): 402 try: 403 target_files_zip.getinfo("VENDOR/") 404 return True 405 except KeyError: 406 return False 407 408def GetOemProperty(name, oem_props, oem_dict, info_dict): 409 if oem_props is not None and name in oem_props: 410 return oem_dict[name] 411 return GetBuildProp(name, info_dict) 412 413 414def CalculateFingerprint(oem_props, oem_dict, info_dict): 415 if oem_props is None: 416 return GetBuildProp("ro.build.fingerprint", info_dict) 417 return "%s/%s/%s:%s" % ( 418 GetOemProperty("ro.product.brand", oem_props, oem_dict, info_dict), 419 GetOemProperty("ro.product.name", oem_props, oem_dict, info_dict), 420 GetOemProperty("ro.product.device", oem_props, oem_dict, info_dict), 421 GetBuildProp("ro.build.thumbprint", info_dict)) 422 423 424def GetImage(which, tmpdir, info_dict): 425 # Return an image object (suitable for passing to BlockImageDiff) 426 # for the 'which' partition (most be "system" or "vendor"). If a 427 # prebuilt image and file map are found in tmpdir they are used, 428 # otherwise they are reconstructed from the individual files. 429 430 assert which in ("system", "vendor") 431 432 path = os.path.join(tmpdir, "IMAGES", which + ".img") 433 mappath = os.path.join(tmpdir, "IMAGES", which + ".map") 434 if os.path.exists(path) and os.path.exists(mappath): 435 print "using %s.img from target-files" % (which,) 436 # This is a 'new' target-files, which already has the image in it. 437 438 else: 439 print "building %s.img from target-files" % (which,) 440 441 # This is an 'old' target-files, which does not contain images 442 # already built. Build them. 443 444 mappath = tempfile.mkstemp()[1] 445 OPTIONS.tempfiles.append(mappath) 446 447 import add_img_to_target_files 448 if which == "system": 449 path = add_img_to_target_files.BuildSystem( 450 tmpdir, info_dict, block_list=mappath) 451 elif which == "vendor": 452 path = add_img_to_target_files.BuildVendor( 453 tmpdir, info_dict, block_list=mappath) 454 455 return sparse_img.SparseImage(path, mappath) 456 457 458def WriteFullOTAPackage(input_zip, output_zip): 459 # TODO: how to determine this? We don't know what version it will 460 # be installed on top of. For now, we expect the API just won't 461 # change very often. 462 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 463 464 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 465 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 466 oem_dict = None 467 if oem_props is not None and len(oem_props) > 0: 468 if OPTIONS.oem_source is None: 469 raise common.ExternalError("OEM source required for this build") 470 script.Mount("/oem", recovery_mount_options) 471 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 472 473 metadata = {"post-build": CalculateFingerprint( 474 oem_props, oem_dict, OPTIONS.info_dict), 475 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 476 OPTIONS.info_dict), 477 "post-timestamp": GetBuildProp("ro.build.date.utc", 478 OPTIONS.info_dict), 479 } 480 481 device_specific = common.DeviceSpecificParams( 482 input_zip=input_zip, 483 input_version=OPTIONS.info_dict["recovery_api_version"], 484 output_zip=output_zip, 485 script=script, 486 input_tmp=OPTIONS.input_tmp, 487 metadata=metadata, 488 info_dict=OPTIONS.info_dict) 489 490 has_recovery_patch = HasRecoveryPatch(input_zip) 491 block_based = OPTIONS.block_based and has_recovery_patch 492 493 if not OPTIONS.omit_prereq: 494 ts = GetBuildProp("ro.build.date.utc", OPTIONS.info_dict) 495 ts_text = GetBuildProp("ro.build.date", OPTIONS.info_dict) 496 script.AssertOlderBuild(ts, ts_text) 497 498 AppendAssertions(script, OPTIONS.info_dict, oem_dict) 499 device_specific.FullOTA_Assertions() 500 501 # Two-step package strategy (in chronological order, which is *not* 502 # the order in which the generated script has things): 503 # 504 # if stage is not "2/3" or "3/3": 505 # write recovery image to boot partition 506 # set stage to "2/3" 507 # reboot to boot partition and restart recovery 508 # else if stage is "2/3": 509 # write recovery image to recovery partition 510 # set stage to "3/3" 511 # reboot to recovery partition and restart recovery 512 # else: 513 # (stage must be "3/3") 514 # set stage to "" 515 # do normal full package installation: 516 # wipe and install system, boot image, etc. 517 # set up system to update recovery partition on first boot 518 # complete script normally (allow recovery to mark itself finished and reboot) 519 520 recovery_img = common.GetBootableImage("recovery.img", "recovery.img", 521 OPTIONS.input_tmp, "RECOVERY") 522 if OPTIONS.two_step: 523 if not OPTIONS.info_dict.get("multistage_support", None): 524 assert False, "two-step packages not supported by this build" 525 fs = OPTIONS.info_dict["fstab"]["/misc"] 526 assert fs.fs_type.upper() == "EMMC", \ 527 "two-step packages only supported on devices with EMMC /misc partitions" 528 bcb_dev = {"bcb_dev": fs.device} 529 common.ZipWriteStr(output_zip, "recovery.img", recovery_img.data) 530 script.AppendExtra(""" 531if get_stage("%(bcb_dev)s", "stage") == "2/3" then 532""" % bcb_dev) 533 script.WriteRawImage("/recovery", "recovery.img") 534 script.AppendExtra(""" 535set_stage("%(bcb_dev)s", "3/3"); 536reboot_now("%(bcb_dev)s", "recovery"); 537else if get_stage("%(bcb_dev)s", "stage") == "3/3" then 538""" % bcb_dev) 539 540 device_specific.FullOTA_InstallBegin() 541 542 system_progress = 0.75 543 544 if OPTIONS.wipe_user_data: 545 system_progress -= 0.1 546 if HasVendorPartition(input_zip): 547 system_progress -= 0.1 548 549 if "selinux_fc" in OPTIONS.info_dict: 550 WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip) 551 552 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 553 554 system_items = ItemSet("system", "META/filesystem_config.txt") 555 script.ShowProgress(system_progress, 0) 556 if block_based: 557 # Full OTA is done as an "incremental" against an empty source 558 # image. This has the effect of writing new data from the package 559 # to the entire partition, but lets us reuse the updater code that 560 # writes incrementals to do it. 561 system_tgt = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict) 562 system_tgt.ResetFileMap() 563 system_diff = common.BlockDifference("system", system_tgt, src=None) 564 system_diff.WriteScript(script, output_zip) 565 else: 566 script.FormatPartition("/system") 567 script.Mount("/system", recovery_mount_options) 568 if not has_recovery_patch: 569 script.UnpackPackageDir("recovery", "/system") 570 script.UnpackPackageDir("system", "/system") 571 572 symlinks = CopyPartitionFiles(system_items, input_zip, output_zip) 573 script.MakeSymlinks(symlinks) 574 575 boot_img = common.GetBootableImage("boot.img", "boot.img", 576 OPTIONS.input_tmp, "BOOT") 577 578 if not block_based: 579 def output_sink(fn, data): 580 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 581 system_items.Get("system/" + fn, dir=False) 582 583 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, 584 recovery_img, boot_img) 585 586 system_items.GetMetadata(input_zip) 587 system_items.Get("system").SetPermissions(script) 588 589 if HasVendorPartition(input_zip): 590 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 591 script.ShowProgress(0.1, 0) 592 593 if block_based: 594 vendor_tgt = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict) 595 vendor_tgt.ResetFileMap() 596 vendor_diff = common.BlockDifference("vendor", vendor_tgt) 597 vendor_diff.WriteScript(script, output_zip) 598 else: 599 script.FormatPartition("/vendor") 600 script.Mount("/vendor", recovery_mount_options) 601 script.UnpackPackageDir("vendor", "/vendor") 602 603 symlinks = CopyPartitionFiles(vendor_items, input_zip, output_zip) 604 script.MakeSymlinks(symlinks) 605 606 vendor_items.GetMetadata(input_zip) 607 vendor_items.Get("vendor").SetPermissions(script) 608 609 common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict) 610 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 611 612 script.ShowProgress(0.05, 5) 613 script.WriteRawImage("/boot", "boot.img") 614 615 script.ShowProgress(0.2, 10) 616 device_specific.FullOTA_InstallEnd() 617 618 if OPTIONS.extra_script is not None: 619 script.AppendExtra(OPTIONS.extra_script) 620 621 script.UnmountAll() 622 623 if OPTIONS.wipe_user_data: 624 script.ShowProgress(0.1, 10) 625 script.FormatPartition("/data") 626 627 if OPTIONS.two_step: 628 script.AppendExtra(""" 629set_stage("%(bcb_dev)s", ""); 630""" % bcb_dev) 631 script.AppendExtra("else\n") 632 script.WriteRawImage("/boot", "recovery.img") 633 script.AppendExtra(""" 634set_stage("%(bcb_dev)s", "2/3"); 635reboot_now("%(bcb_dev)s", ""); 636endif; 637endif; 638""" % bcb_dev) 639 script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary) 640 WriteMetadata(metadata, output_zip) 641 642 643def WritePolicyConfig(file_context, output_zip): 644 f = open(file_context, 'r'); 645 basename = os.path.basename(file_context) 646 common.ZipWriteStr(output_zip, basename, f.read()) 647 648 649def WriteMetadata(metadata, output_zip): 650 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata", 651 "".join(["%s=%s\n" % kv 652 for kv in sorted(metadata.iteritems())])) 653 654 655def LoadPartitionFiles(z, partition): 656 """Load all the files from the given partition in a given target-files 657 ZipFile, and return a dict of {filename: File object}.""" 658 out = {} 659 prefix = partition.upper() + "/" 660 for info in z.infolist(): 661 if info.filename.startswith(prefix) and not IsSymlink(info): 662 basefilename = info.filename[7:] 663 fn = partition + "/" + basefilename 664 data = z.read(info.filename) 665 out[fn] = common.File(fn, data) 666 return out 667 668 669def GetBuildProp(prop, info_dict): 670 """Return the fingerprint of the build of a given target-files info_dict.""" 671 try: 672 return info_dict.get("build.prop", {})[prop] 673 except KeyError: 674 raise common.ExternalError("couldn't find %s in build.prop" % (prop,)) 675 676 677def AddToKnownPaths(filename, known_paths): 678 if filename[-1] == "/": 679 return 680 dirs = filename.split("/")[:-1] 681 while len(dirs) > 0: 682 path = "/".join(dirs) 683 if path in known_paths: 684 break; 685 known_paths.add(path) 686 dirs.pop() 687 688 689def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 690 source_version = OPTIONS.source_info_dict["recovery_api_version"] 691 target_version = OPTIONS.target_info_dict["recovery_api_version"] 692 693 if source_version == 0: 694 print ("WARNING: generating edify script for a source that " 695 "can't install it.") 696 script = edify_generator.EdifyGenerator(source_version, 697 OPTIONS.target_info_dict) 698 699 metadata = {"pre-device": GetBuildProp("ro.product.device", 700 OPTIONS.source_info_dict), 701 "post-timestamp": GetBuildProp("ro.build.date.utc", 702 OPTIONS.target_info_dict), 703 } 704 705 device_specific = common.DeviceSpecificParams( 706 source_zip=source_zip, 707 source_version=source_version, 708 target_zip=target_zip, 709 target_version=target_version, 710 output_zip=output_zip, 711 script=script, 712 metadata=metadata, 713 info_dict=OPTIONS.info_dict) 714 715 source_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.source_info_dict) 716 target_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.target_info_dict) 717 metadata["pre-build"] = source_fp 718 metadata["post-build"] = target_fp 719 720 source_boot = common.GetBootableImage( 721 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 722 OPTIONS.source_info_dict) 723 target_boot = common.GetBootableImage( 724 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 725 updating_boot = (not OPTIONS.two_step and 726 (source_boot.data != target_boot.data)) 727 728 source_recovery = common.GetBootableImage( 729 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 730 OPTIONS.source_info_dict) 731 target_recovery = common.GetBootableImage( 732 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 733 updating_recovery = (source_recovery.data != target_recovery.data) 734 735 system_src = GetImage("system", OPTIONS.source_tmp, OPTIONS.source_info_dict) 736 system_tgt = GetImage("system", OPTIONS.target_tmp, OPTIONS.target_info_dict) 737 system_diff = common.BlockDifference("system", system_tgt, system_src, 738 check_first_block=True) 739 740 if HasVendorPartition(target_zip): 741 if not HasVendorPartition(source_zip): 742 raise RuntimeError("can't generate incremental that adds /vendor") 743 vendor_src = GetImage("vendor", OPTIONS.source_tmp, OPTIONS.source_info_dict) 744 vendor_tgt = GetImage("vendor", OPTIONS.target_tmp, OPTIONS.target_info_dict) 745 vendor_diff = common.BlockDifference("vendor", vendor_tgt, vendor_src, 746 check_first_block=True) 747 else: 748 vendor_diff = None 749 750 oem_props = OPTIONS.target_info_dict.get("oem_fingerprint_properties") 751 recovery_mount_options = OPTIONS.target_info_dict.get("recovery_mount_options") 752 oem_dict = None 753 if oem_props is not None and len(oem_props) > 0: 754 if OPTIONS.oem_source is None: 755 raise common.ExternalError("OEM source required for this build") 756 script.Mount("/oem", recovery_mount_options) 757 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 758 759 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 760 device_specific.IncrementalOTA_Assertions() 761 762 # Two-step incremental package strategy (in chronological order, 763 # which is *not* the order in which the generated script has 764 # things): 765 # 766 # if stage is not "2/3" or "3/3": 767 # do verification on current system 768 # write recovery image to boot partition 769 # set stage to "2/3" 770 # reboot to boot partition and restart recovery 771 # else if stage is "2/3": 772 # write recovery image to recovery partition 773 # set stage to "3/3" 774 # reboot to recovery partition and restart recovery 775 # else: 776 # (stage must be "3/3") 777 # perform update: 778 # patch system files, etc. 779 # force full install of new boot image 780 # set up system to update recovery partition on first boot 781 # complete script normally (allow recovery to mark itself finished and reboot) 782 783 if OPTIONS.two_step: 784 if not OPTIONS.info_dict.get("multistage_support", None): 785 assert False, "two-step packages not supported by this build" 786 fs = OPTIONS.info_dict["fstab"]["/misc"] 787 assert fs.fs_type.upper() == "EMMC", \ 788 "two-step packages only supported on devices with EMMC /misc partitions" 789 bcb_dev = {"bcb_dev": fs.device} 790 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 791 script.AppendExtra(""" 792if get_stage("%(bcb_dev)s", "stage") == "2/3" then 793""" % bcb_dev) 794 script.AppendExtra("sleep(20);\n"); 795 script.WriteRawImage("/recovery", "recovery.img") 796 script.AppendExtra(""" 797set_stage("%(bcb_dev)s", "3/3"); 798reboot_now("%(bcb_dev)s", "recovery"); 799else if get_stage("%(bcb_dev)s", "stage") != "3/3" then 800""" % bcb_dev) 801 802 script.Print("Verifying current system...") 803 804 device_specific.IncrementalOTA_VerifyBegin() 805 806 if oem_props is None: 807 script.AssertSomeFingerprint(source_fp, target_fp) 808 else: 809 script.AssertSomeThumbprint( 810 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 811 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 812 813 if updating_boot: 814 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 815 d = common.Difference(target_boot, source_boot) 816 _, _, d = d.ComputePatch() 817 if d is None: 818 include_full_boot = True 819 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 820 else: 821 include_full_boot = False 822 823 print "boot target: %d source: %d diff: %d" % ( 824 target_boot.size, source_boot.size, len(d)) 825 826 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 827 828 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 829 (boot_type, boot_device, 830 source_boot.size, source_boot.sha1, 831 target_boot.size, target_boot.sha1)) 832 833 device_specific.IncrementalOTA_VerifyEnd() 834 835 if OPTIONS.two_step: 836 script.WriteRawImage("/boot", "recovery.img") 837 script.AppendExtra(""" 838set_stage("%(bcb_dev)s", "2/3"); 839reboot_now("%(bcb_dev)s", ""); 840else 841""" % bcb_dev) 842 843 script.Comment("---- start making changes here ----") 844 845 device_specific.IncrementalOTA_InstallBegin() 846 847 system_diff.WriteScript(script, output_zip, 848 progress=0.8 if vendor_diff else 0.9) 849 if vendor_diff: 850 vendor_diff.WriteScript(script, output_zip, progress=0.1) 851 852 if OPTIONS.two_step: 853 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 854 script.WriteRawImage("/boot", "boot.img") 855 print "writing full boot image (forced by two-step mode)" 856 857 if not OPTIONS.two_step: 858 if updating_boot: 859 if include_full_boot: 860 print "boot image changed; including full." 861 script.Print("Installing boot image...") 862 script.WriteRawImage("/boot", "boot.img") 863 else: 864 # Produce the boot image by applying a patch to the current 865 # contents of the boot partition, and write it back to the 866 # partition. 867 print "boot image changed; including patch." 868 script.Print("Patching boot image...") 869 script.ShowProgress(0.1, 10) 870 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 871 % (boot_type, boot_device, 872 source_boot.size, source_boot.sha1, 873 target_boot.size, target_boot.sha1), 874 "-", 875 target_boot.size, target_boot.sha1, 876 source_boot.sha1, "patch/boot.img.p") 877 else: 878 print "boot image unchanged; skipping." 879 880 # Do device-specific installation (eg, write radio image). 881 device_specific.IncrementalOTA_InstallEnd() 882 883 if OPTIONS.extra_script is not None: 884 script.AppendExtra(OPTIONS.extra_script) 885 886 if OPTIONS.wipe_user_data: 887 script.Print("Erasing user data...") 888 script.FormatPartition("/data") 889 890 if OPTIONS.two_step: 891 script.AppendExtra(""" 892set_stage("%(bcb_dev)s", ""); 893endif; 894endif; 895""" % bcb_dev) 896 897 script.SetProgress(1) 898 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 899 WriteMetadata(metadata, output_zip) 900 901 902class FileDifference: 903 def __init__(self, partition, source_zip, target_zip, output_zip): 904 print "Loading target..." 905 self.target_data = target_data = LoadPartitionFiles(target_zip, partition) 906 print "Loading source..." 907 self.source_data = source_data = LoadPartitionFiles(source_zip, partition) 908 909 self.verbatim_targets = verbatim_targets = [] 910 self.patch_list = patch_list = [] 911 diffs = [] 912 self.renames = renames = {} 913 known_paths = set() 914 largest_source_size = 0 915 916 matching_file_cache = {} 917 for fn, sf in source_data.items(): 918 assert fn == sf.name 919 matching_file_cache["path:" + fn] = sf 920 if fn in target_data.keys(): 921 AddToKnownPaths(fn, known_paths) 922 # Only allow eligibility for filename/sha matching 923 # if there isn't a perfect path match. 924 if target_data.get(sf.name) is None: 925 matching_file_cache["file:" + fn.split("/")[-1]] = sf 926 matching_file_cache["sha:" + sf.sha1] = sf 927 928 for fn in sorted(target_data.keys()): 929 tf = target_data[fn] 930 assert fn == tf.name 931 sf = ClosestFileMatch(tf, matching_file_cache, renames) 932 if sf is not None and sf.name != tf.name: 933 print "File has moved from " + sf.name + " to " + tf.name 934 renames[sf.name] = tf 935 936 if sf is None or fn in OPTIONS.require_verbatim: 937 # This file should be included verbatim 938 if fn in OPTIONS.prohibit_verbatim: 939 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 940 print "send", fn, "verbatim" 941 tf.AddToZip(output_zip) 942 verbatim_targets.append((fn, tf.size)) 943 if fn in target_data.keys(): 944 AddToKnownPaths(fn, known_paths) 945 elif tf.sha1 != sf.sha1: 946 # File is different; consider sending as a patch 947 diffs.append(common.Difference(tf, sf)) 948 else: 949 # Target file data identical to source (may still be renamed) 950 pass 951 952 common.ComputeDifferences(diffs) 953 954 for diff in diffs: 955 tf, sf, d = diff.GetPatch() 956 path = "/".join(tf.name.split("/")[:-1]) 957 if d is None or len(d) > tf.size * OPTIONS.patch_threshold or \ 958 path not in known_paths: 959 # patch is almost as big as the file; don't bother patching 960 # or a patch + rename cannot take place due to the target 961 # directory not existing 962 tf.AddToZip(output_zip) 963 verbatim_targets.append((tf.name, tf.size)) 964 if sf.name in renames: 965 del renames[sf.name] 966 AddToKnownPaths(tf.name, known_paths) 967 else: 968 common.ZipWriteStr(output_zip, "patch/" + sf.name + ".p", d) 969 patch_list.append((tf, sf, tf.size, common.sha1(d).hexdigest())) 970 largest_source_size = max(largest_source_size, sf.size) 971 972 self.largest_source_size = largest_source_size 973 974 def EmitVerification(self, script): 975 so_far = 0 976 for tf, sf, size, patch_sha in self.patch_list: 977 if tf.name != sf.name: 978 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 979 script.PatchCheck("/"+sf.name, tf.sha1, sf.sha1) 980 so_far += sf.size 981 return so_far 982 983 def RemoveUnneededFiles(self, script, extras=()): 984 script.DeleteFiles(["/"+i[0] for i in self.verbatim_targets] + 985 ["/"+i for i in sorted(self.source_data) 986 if i not in self.target_data and 987 i not in self.renames] + 988 list(extras)) 989 990 def TotalPatchSize(self): 991 return sum(i[1].size for i in self.patch_list) 992 993 def EmitPatches(self, script, total_patch_size, so_far): 994 self.deferred_patch_list = deferred_patch_list = [] 995 for item in self.patch_list: 996 tf, sf, size, _ = item 997 if tf.name == "system/build.prop": 998 deferred_patch_list.append(item) 999 continue 1000 if (sf.name != tf.name): 1001 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1002 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p") 1003 so_far += tf.size 1004 script.SetProgress(so_far / total_patch_size) 1005 return so_far 1006 1007 def EmitDeferredPatches(self, script): 1008 for item in self.deferred_patch_list: 1009 tf, sf, size, _ = item 1010 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p") 1011 script.SetPermissions("/system/build.prop", 0, 0, 0644, None, None) 1012 1013 def EmitRenames(self, script): 1014 if len(self.renames) > 0: 1015 script.Print("Renaming files...") 1016 for src, tgt in self.renames.iteritems(): 1017 print "Renaming " + src + " to " + tgt.name 1018 script.RenameFile(src, tgt.name) 1019 1020 1021 1022 1023def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): 1024 target_has_recovery_patch = HasRecoveryPatch(target_zip) 1025 source_has_recovery_patch = HasRecoveryPatch(source_zip) 1026 1027 if (OPTIONS.block_based and 1028 target_has_recovery_patch and 1029 source_has_recovery_patch): 1030 return WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip) 1031 1032 source_version = OPTIONS.source_info_dict["recovery_api_version"] 1033 target_version = OPTIONS.target_info_dict["recovery_api_version"] 1034 1035 if source_version == 0: 1036 print ("WARNING: generating edify script for a source that " 1037 "can't install it.") 1038 script = edify_generator.EdifyGenerator(source_version, 1039 OPTIONS.target_info_dict) 1040 1041 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 1042 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 1043 oem_dict = None 1044 if oem_props is not None and len(oem_props) > 0: 1045 if OPTIONS.oem_source is None: 1046 raise common.ExternalError("OEM source required for this build") 1047 script.Mount("/oem", recovery_mount_options) 1048 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 1049 1050 metadata = {"pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 1051 OPTIONS.source_info_dict), 1052 "post-timestamp": GetBuildProp("ro.build.date.utc", 1053 OPTIONS.target_info_dict), 1054 } 1055 1056 device_specific = common.DeviceSpecificParams( 1057 source_zip=source_zip, 1058 source_version=source_version, 1059 target_zip=target_zip, 1060 target_version=target_version, 1061 output_zip=output_zip, 1062 script=script, 1063 metadata=metadata, 1064 info_dict=OPTIONS.info_dict) 1065 1066 system_diff = FileDifference("system", source_zip, target_zip, output_zip) 1067 script.Mount("/system", recovery_mount_options) 1068 if HasVendorPartition(target_zip): 1069 vendor_diff = FileDifference("vendor", source_zip, target_zip, output_zip) 1070 script.Mount("/vendor", recovery_mount_options) 1071 else: 1072 vendor_diff = None 1073 1074 target_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.target_info_dict) 1075 source_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.source_info_dict) 1076 1077 if oem_props is None: 1078 script.AssertSomeFingerprint(source_fp, target_fp) 1079 else: 1080 script.AssertSomeThumbprint( 1081 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 1082 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 1083 1084 metadata["pre-build"] = source_fp 1085 metadata["post-build"] = target_fp 1086 1087 source_boot = common.GetBootableImage( 1088 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 1089 OPTIONS.source_info_dict) 1090 target_boot = common.GetBootableImage( 1091 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 1092 updating_boot = (not OPTIONS.two_step and 1093 (source_boot.data != target_boot.data)) 1094 1095 source_recovery = common.GetBootableImage( 1096 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 1097 OPTIONS.source_info_dict) 1098 target_recovery = common.GetBootableImage( 1099 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 1100 updating_recovery = (source_recovery.data != target_recovery.data) 1101 1102 # Here's how we divide up the progress bar: 1103 # 0.1 for verifying the start state (PatchCheck calls) 1104 # 0.8 for applying patches (ApplyPatch calls) 1105 # 0.1 for unpacking verbatim files, symlinking, and doing the 1106 # device-specific commands. 1107 1108 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 1109 device_specific.IncrementalOTA_Assertions() 1110 1111 # Two-step incremental package strategy (in chronological order, 1112 # which is *not* the order in which the generated script has 1113 # things): 1114 # 1115 # if stage is not "2/3" or "3/3": 1116 # do verification on current system 1117 # write recovery image to boot partition 1118 # set stage to "2/3" 1119 # reboot to boot partition and restart recovery 1120 # else if stage is "2/3": 1121 # write recovery image to recovery partition 1122 # set stage to "3/3" 1123 # reboot to recovery partition and restart recovery 1124 # else: 1125 # (stage must be "3/3") 1126 # perform update: 1127 # patch system files, etc. 1128 # force full install of new boot image 1129 # set up system to update recovery partition on first boot 1130 # complete script normally (allow recovery to mark itself finished and reboot) 1131 1132 if OPTIONS.two_step: 1133 if not OPTIONS.info_dict.get("multistage_support", None): 1134 assert False, "two-step packages not supported by this build" 1135 fs = OPTIONS.info_dict["fstab"]["/misc"] 1136 assert fs.fs_type.upper() == "EMMC", \ 1137 "two-step packages only supported on devices with EMMC /misc partitions" 1138 bcb_dev = {"bcb_dev": fs.device} 1139 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 1140 script.AppendExtra(""" 1141if get_stage("%(bcb_dev)s", "stage") == "2/3" then 1142""" % bcb_dev) 1143 script.AppendExtra("sleep(20);\n"); 1144 script.WriteRawImage("/recovery", "recovery.img") 1145 script.AppendExtra(""" 1146set_stage("%(bcb_dev)s", "3/3"); 1147reboot_now("%(bcb_dev)s", "recovery"); 1148else if get_stage("%(bcb_dev)s", "stage") != "3/3" then 1149""" % bcb_dev) 1150 1151 script.Print("Verifying current system...") 1152 1153 device_specific.IncrementalOTA_VerifyBegin() 1154 1155 script.ShowProgress(0.1, 0) 1156 so_far = system_diff.EmitVerification(script) 1157 if vendor_diff: 1158 so_far += vendor_diff.EmitVerification(script) 1159 1160 if updating_boot: 1161 d = common.Difference(target_boot, source_boot) 1162 _, _, d = d.ComputePatch() 1163 print "boot target: %d source: %d diff: %d" % ( 1164 target_boot.size, source_boot.size, len(d)) 1165 1166 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 1167 1168 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 1169 1170 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 1171 (boot_type, boot_device, 1172 source_boot.size, source_boot.sha1, 1173 target_boot.size, target_boot.sha1)) 1174 so_far += source_boot.size 1175 1176 size = [] 1177 if system_diff.patch_list: size.append(system_diff.largest_source_size) 1178 if vendor_diff: 1179 if vendor_diff.patch_list: size.append(vendor_diff.largest_source_size) 1180 if size or updating_recovery or updating_boot: 1181 script.CacheFreeSpaceCheck(max(size)) 1182 1183 device_specific.IncrementalOTA_VerifyEnd() 1184 1185 if OPTIONS.two_step: 1186 script.WriteRawImage("/boot", "recovery.img") 1187 script.AppendExtra(""" 1188set_stage("%(bcb_dev)s", "2/3"); 1189reboot_now("%(bcb_dev)s", ""); 1190else 1191""" % bcb_dev) 1192 1193 script.Comment("---- start making changes here ----") 1194 1195 device_specific.IncrementalOTA_InstallBegin() 1196 1197 if OPTIONS.two_step: 1198 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 1199 script.WriteRawImage("/boot", "boot.img") 1200 print "writing full boot image (forced by two-step mode)" 1201 1202 script.Print("Removing unneeded files...") 1203 system_diff.RemoveUnneededFiles(script, ("/system/recovery.img",)) 1204 if vendor_diff: 1205 vendor_diff.RemoveUnneededFiles(script) 1206 1207 script.ShowProgress(0.8, 0) 1208 total_patch_size = 1.0 + system_diff.TotalPatchSize() 1209 if vendor_diff: 1210 total_patch_size += vendor_diff.TotalPatchSize() 1211 if updating_boot: 1212 total_patch_size += target_boot.size 1213 1214 script.Print("Patching system files...") 1215 so_far = system_diff.EmitPatches(script, total_patch_size, 0) 1216 if vendor_diff: 1217 script.Print("Patching vendor files...") 1218 so_far = vendor_diff.EmitPatches(script, total_patch_size, so_far) 1219 1220 if not OPTIONS.two_step: 1221 if updating_boot: 1222 # Produce the boot image by applying a patch to the current 1223 # contents of the boot partition, and write it back to the 1224 # partition. 1225 script.Print("Patching boot image...") 1226 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 1227 % (boot_type, boot_device, 1228 source_boot.size, source_boot.sha1, 1229 target_boot.size, target_boot.sha1), 1230 "-", 1231 target_boot.size, target_boot.sha1, 1232 source_boot.sha1, "patch/boot.img.p") 1233 so_far += target_boot.size 1234 script.SetProgress(so_far / total_patch_size) 1235 print "boot image changed; including." 1236 else: 1237 print "boot image unchanged; skipping." 1238 1239 system_items = ItemSet("system", "META/filesystem_config.txt") 1240 if vendor_diff: 1241 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 1242 1243 if updating_recovery: 1244 # Recovery is generated as a patch using both the boot image 1245 # (which contains the same linux kernel as recovery) and the file 1246 # /system/etc/recovery-resource.dat (which contains all the images 1247 # used in the recovery UI) as sources. This lets us minimize the 1248 # size of the patch, which must be included in every OTA package. 1249 # 1250 # For older builds where recovery-resource.dat is not present, we 1251 # use only the boot image as the source. 1252 1253 if not target_has_recovery_patch: 1254 def output_sink(fn, data): 1255 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 1256 system_items.Get("system/" + fn, dir=False) 1257 1258 common.MakeRecoveryPatch(OPTIONS.target_tmp, output_sink, 1259 target_recovery, target_boot) 1260 script.DeleteFiles(["/system/recovery-from-boot.p", 1261 "/system/etc/install-recovery.sh"]) 1262 print "recovery image changed; including as patch from boot." 1263 else: 1264 print "recovery image unchanged; skipping." 1265 1266 script.ShowProgress(0.1, 10) 1267 1268 target_symlinks = CopyPartitionFiles(system_items, target_zip, None) 1269 if vendor_diff: 1270 target_symlinks.extend(CopyPartitionFiles(vendor_items, target_zip, None)) 1271 1272 temp_script = script.MakeTemporary() 1273 system_items.GetMetadata(target_zip) 1274 system_items.Get("system").SetPermissions(temp_script) 1275 if vendor_diff: 1276 vendor_items.GetMetadata(target_zip) 1277 vendor_items.Get("vendor").SetPermissions(temp_script) 1278 1279 # Note that this call will mess up the trees of Items, so make sure 1280 # we're done with them. 1281 source_symlinks = CopyPartitionFiles(system_items, source_zip, None) 1282 if vendor_diff: 1283 source_symlinks.extend(CopyPartitionFiles(vendor_items, source_zip, None)) 1284 1285 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 1286 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 1287 1288 # Delete all the symlinks in source that aren't in target. This 1289 # needs to happen before verbatim files are unpacked, in case a 1290 # symlink in the source is replaced by a real file in the target. 1291 to_delete = [] 1292 for dest, link in source_symlinks: 1293 if link not in target_symlinks_d: 1294 to_delete.append(link) 1295 script.DeleteFiles(to_delete) 1296 1297 if system_diff.verbatim_targets: 1298 script.Print("Unpacking new system files...") 1299 script.UnpackPackageDir("system", "/system") 1300 if vendor_diff and vendor_diff.verbatim_targets: 1301 script.Print("Unpacking new vendor files...") 1302 script.UnpackPackageDir("vendor", "/vendor") 1303 1304 if updating_recovery and not target_has_recovery_patch: 1305 script.Print("Unpacking new recovery...") 1306 script.UnpackPackageDir("recovery", "/system") 1307 1308 system_diff.EmitRenames(script) 1309 if vendor_diff: 1310 vendor_diff.EmitRenames(script) 1311 1312 script.Print("Symlinks and permissions...") 1313 1314 # Create all the symlinks that don't already exist, or point to 1315 # somewhere different than what we want. Delete each symlink before 1316 # creating it, since the 'symlink' command won't overwrite. 1317 to_create = [] 1318 for dest, link in target_symlinks: 1319 if link in source_symlinks_d: 1320 if dest != source_symlinks_d[link]: 1321 to_create.append((dest, link)) 1322 else: 1323 to_create.append((dest, link)) 1324 script.DeleteFiles([i[1] for i in to_create]) 1325 script.MakeSymlinks(to_create) 1326 1327 # Now that the symlinks are created, we can set all the 1328 # permissions. 1329 script.AppendScript(temp_script) 1330 1331 # Do device-specific installation (eg, write radio image). 1332 device_specific.IncrementalOTA_InstallEnd() 1333 1334 if OPTIONS.extra_script is not None: 1335 script.AppendExtra(OPTIONS.extra_script) 1336 1337 # Patch the build.prop file last, so if something fails but the 1338 # device can still come up, it appears to be the old build and will 1339 # get set the OTA package again to retry. 1340 script.Print("Patching remaining system files...") 1341 system_diff.EmitDeferredPatches(script) 1342 1343 if OPTIONS.wipe_user_data: 1344 script.Print("Erasing user data...") 1345 script.FormatPartition("/data") 1346 1347 if OPTIONS.two_step: 1348 script.AppendExtra(""" 1349set_stage("%(bcb_dev)s", ""); 1350endif; 1351endif; 1352""" % bcb_dev) 1353 1354 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 1355 WriteMetadata(metadata, output_zip) 1356 1357 1358def main(argv): 1359 1360 def option_handler(o, a): 1361 if o == "--board_config": 1362 pass # deprecated 1363 elif o in ("-k", "--package_key"): 1364 OPTIONS.package_key = a 1365 elif o in ("-i", "--incremental_from"): 1366 OPTIONS.incremental_source = a 1367 elif o in ("-w", "--wipe_user_data"): 1368 OPTIONS.wipe_user_data = True 1369 elif o in ("-n", "--no_prereq"): 1370 OPTIONS.omit_prereq = True 1371 elif o in ("-o", "--oem_settings"): 1372 OPTIONS.oem_source = a 1373 elif o in ("-e", "--extra_script"): 1374 OPTIONS.extra_script = a 1375 elif o in ("-a", "--aslr_mode"): 1376 if a in ("on", "On", "true", "True", "yes", "Yes"): 1377 OPTIONS.aslr_mode = True 1378 else: 1379 OPTIONS.aslr_mode = False 1380 elif o in ("-t", "--worker_threads"): 1381 if a.isdigit(): 1382 OPTIONS.worker_threads = int(a) 1383 else: 1384 raise ValueError("Cannot parse value %r for option %r - only " 1385 "integers are allowed." % (a, o)) 1386 elif o in ("-2", "--two_step"): 1387 OPTIONS.two_step = True 1388 elif o == "--no_signing": 1389 OPTIONS.no_signing = True 1390 elif o == "--block": 1391 OPTIONS.block_based = True 1392 elif o in ("-b", "--binary"): 1393 OPTIONS.updater_binary = a 1394 elif o in ("--no_fallback_to_full",): 1395 OPTIONS.fallback_to_full = False 1396 else: 1397 return False 1398 return True 1399 1400 args = common.ParseOptions(argv, __doc__, 1401 extra_opts="b:k:i:d:wne:t:a:2o:", 1402 extra_long_opts=["board_config=", 1403 "package_key=", 1404 "incremental_from=", 1405 "wipe_user_data", 1406 "no_prereq", 1407 "extra_script=", 1408 "worker_threads=", 1409 "aslr_mode=", 1410 "two_step", 1411 "no_signing", 1412 "block", 1413 "binary=", 1414 "oem_settings=", 1415 "no_fallback_to_full", 1416 ], 1417 extra_option_handler=option_handler) 1418 1419 if len(args) != 2: 1420 common.Usage(__doc__) 1421 sys.exit(1) 1422 1423 if OPTIONS.extra_script is not None: 1424 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 1425 1426 print "unzipping target target-files..." 1427 OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0]) 1428 1429 OPTIONS.target_tmp = OPTIONS.input_tmp 1430 OPTIONS.info_dict = common.LoadInfoDict(input_zip) 1431 1432 # If this image was originally labelled with SELinux contexts, make sure we 1433 # also apply the labels in our new image. During building, the "file_contexts" 1434 # is in the out/ directory tree, but for repacking from target-files.zip it's 1435 # in the root directory of the ramdisk. 1436 if "selinux_fc" in OPTIONS.info_dict: 1437 OPTIONS.info_dict["selinux_fc"] = os.path.join(OPTIONS.input_tmp, "BOOT", "RAMDISK", 1438 "file_contexts") 1439 1440 if OPTIONS.verbose: 1441 print "--- target info ---" 1442 common.DumpInfoDict(OPTIONS.info_dict) 1443 1444 # If the caller explicitly specified the device-specific extensions 1445 # path via -s/--device_specific, use that. Otherwise, use 1446 # META/releasetools.py if it is present in the target target_files. 1447 # Otherwise, take the path of the file from 'tool_extensions' in the 1448 # info dict and look for that in the local filesystem, relative to 1449 # the current directory. 1450 1451 if OPTIONS.device_specific is None: 1452 from_input = os.path.join(OPTIONS.input_tmp, "META", "releasetools.py") 1453 if os.path.exists(from_input): 1454 print "(using device-specific extensions from target_files)" 1455 OPTIONS.device_specific = from_input 1456 else: 1457 OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None) 1458 1459 if OPTIONS.device_specific is not None: 1460 OPTIONS.device_specific = os.path.abspath(OPTIONS.device_specific) 1461 1462 while True: 1463 1464 if OPTIONS.no_signing: 1465 if os.path.exists(args[1]): os.unlink(args[1]) 1466 output_zip = zipfile.ZipFile(args[1], "w", compression=zipfile.ZIP_DEFLATED) 1467 else: 1468 temp_zip_file = tempfile.NamedTemporaryFile() 1469 output_zip = zipfile.ZipFile(temp_zip_file, "w", 1470 compression=zipfile.ZIP_DEFLATED) 1471 1472 if OPTIONS.incremental_source is None: 1473 WriteFullOTAPackage(input_zip, output_zip) 1474 if OPTIONS.package_key is None: 1475 OPTIONS.package_key = OPTIONS.info_dict.get( 1476 "default_system_dev_certificate", 1477 "build/target/product/security/testkey") 1478 break 1479 1480 else: 1481 print "unzipping source target-files..." 1482 OPTIONS.source_tmp, source_zip = common.UnzipTemp(OPTIONS.incremental_source) 1483 OPTIONS.target_info_dict = OPTIONS.info_dict 1484 OPTIONS.source_info_dict = common.LoadInfoDict(source_zip) 1485 if "selinux_fc" in OPTIONS.source_info_dict: 1486 OPTIONS.source_info_dict["selinux_fc"] = os.path.join(OPTIONS.source_tmp, "BOOT", "RAMDISK", 1487 "file_contexts") 1488 if OPTIONS.package_key is None: 1489 OPTIONS.package_key = OPTIONS.source_info_dict.get( 1490 "default_system_dev_certificate", 1491 "build/target/product/security/testkey") 1492 if OPTIONS.verbose: 1493 print "--- source info ---" 1494 common.DumpInfoDict(OPTIONS.source_info_dict) 1495 try: 1496 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) 1497 break 1498 except ValueError: 1499 if not OPTIONS.fallback_to_full: raise 1500 print "--- failed to build incremental; falling back to full ---" 1501 OPTIONS.incremental_source = None 1502 output_zip.close() 1503 1504 output_zip.close() 1505 1506 if not OPTIONS.no_signing: 1507 SignOutput(temp_zip_file.name, args[1]) 1508 temp_zip_file.close() 1509 1510 print "done." 1511 1512 1513if __name__ == '__main__': 1514 try: 1515 common.CloseInheritedPipes() 1516 main(sys.argv[1:]) 1517 except common.ExternalError, e: 1518 print 1519 print " ERROR: %s" % (e,) 1520 print 1521 sys.exit(1) 1522 finally: 1523 common.Cleanup() 1524