ota_from_target_files.py revision eaf885b1d0f5f841b769b269ce3ac4ed35e6b22d
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 --board_config <file> 25 Deprecated. 26 27 -k (--package_key) <key> Key to use to sign the package (default is 28 the value of default_system_dev_certificate from the input 29 target-files's META/misc_info.txt, or 30 "build/target/product/security/testkey" if that value is not 31 specified). 32 33 For incremental OTAs, the default value is based on the source 34 target-file, not the target build. 35 36 -i (--incremental_from) <file> 37 Generate an incremental OTA using the given target-files zip as 38 the starting build. 39 40 -v (--verify) 41 Remount and verify the checksums of the files written to the 42 system and vendor (if used) partitions. Incremental builds only. 43 44 -o (--oem_settings) <file> 45 Use the file to specify the expected OEM-specific properties 46 on the OEM partition of the intended device. 47 48 -w (--wipe_user_data) 49 Generate an OTA package that will wipe the user data partition 50 when installed. 51 52 -n (--no_prereq) 53 Omit the timestamp prereq check normally included at the top of 54 the build scripts (used for developer OTA packages which 55 legitimately need to go back and forth). 56 57 -e (--extra_script) <file> 58 Insert the contents of file at the end of the update script. 59 60 -a (--aslr_mode) <on|off> 61 Specify whether to turn on ASLR for the package (on by default). 62 63 -2 (--two_step) 64 Generate a 'two-step' OTA package, where recovery is updated 65 first, so that any changes made to the system partition are done 66 using the new recovery (new kernel, etc.). 67 68 --block 69 Generate a block-based OTA if possible. Will fall back to a 70 file-based OTA if the target_files is older and doesn't support 71 block-based OTAs. 72 73 -b (--binary) <file> 74 Use the given binary as the update-binary in the output package, 75 instead of the binary in the build's target_files. Use for 76 development only. 77 78 -t (--worker_threads) <int> 79 Specifies the number of worker-threads that will be used when 80 generating patches for incremental updates (defaults to 3). 81 82""" 83 84import sys 85 86if sys.hexversion < 0x02070000: 87 print >> sys.stderr, "Python 2.7 or newer is required." 88 sys.exit(1) 89 90import copy 91import errno 92import multiprocessing 93import os 94import re 95import subprocess 96import tempfile 97import time 98import zipfile 99 100from hashlib import sha1 as sha1 101 102import common 103import edify_generator 104import build_image 105import blockimgdiff 106import sparse_img 107 108OPTIONS = common.OPTIONS 109OPTIONS.package_key = None 110OPTIONS.incremental_source = None 111OPTIONS.verify = False 112OPTIONS.require_verbatim = set() 113OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 114OPTIONS.patch_threshold = 0.95 115OPTIONS.wipe_user_data = False 116OPTIONS.omit_prereq = False 117OPTIONS.extra_script = None 118OPTIONS.aslr_mode = True 119OPTIONS.worker_threads = multiprocessing.cpu_count() // 2 120if OPTIONS.worker_threads == 0: 121 OPTIONS.worker_threads = 1 122OPTIONS.two_step = False 123OPTIONS.no_signing = False 124OPTIONS.block_based = False 125OPTIONS.updater_binary = None 126OPTIONS.oem_source = None 127OPTIONS.fallback_to_full = True 128 129def MostPopularKey(d, default): 130 """Given a dict, return the key corresponding to the largest 131 value. Returns 'default' if the dict is empty.""" 132 x = [(v, k) for (k, v) in d.iteritems()] 133 if not x: return default 134 x.sort() 135 return x[-1][1] 136 137 138def IsSymlink(info): 139 """Return true if the zipfile.ZipInfo object passed in represents a 140 symlink.""" 141 return (info.external_attr >> 16) == 0120777 142 143def IsRegular(info): 144 """Return true if the zipfile.ZipInfo object passed in represents a 145 symlink.""" 146 return (info.external_attr >> 28) == 010 147 148def ClosestFileMatch(src, tgtfiles, existing): 149 """Returns the closest file match between a source file and list 150 of potential matches. The exact filename match is preferred, 151 then the sha1 is searched for, and finally a file with the same 152 basename is evaluated. Rename support in the updater-binary is 153 required for the latter checks to be used.""" 154 155 result = tgtfiles.get("path:" + src.name) 156 if result is not None: 157 return result 158 159 if not OPTIONS.target_info_dict.get("update_rename_support", False): 160 return None 161 162 if src.size < 1000: 163 return None 164 165 result = tgtfiles.get("sha1:" + src.sha1) 166 if result is not None and existing.get(result.name) is None: 167 return result 168 result = tgtfiles.get("file:" + src.name.split("/")[-1]) 169 if result is not None and existing.get(result.name) is None: 170 return result 171 return None 172 173class ItemSet: 174 def __init__(self, partition, fs_config): 175 self.partition = partition 176 self.fs_config = fs_config 177 self.ITEMS = {} 178 179 def Get(self, name, dir=False): 180 if name not in self.ITEMS: 181 self.ITEMS[name] = Item(self, name, dir=dir) 182 return self.ITEMS[name] 183 184 def GetMetadata(self, input_zip): 185 # The target_files contains a record of what the uid, 186 # gid, and mode are supposed to be. 187 output = input_zip.read(self.fs_config) 188 189 for line in output.split("\n"): 190 if not line: continue 191 columns = line.split() 192 name, uid, gid, mode = columns[:4] 193 selabel = None 194 capabilities = None 195 196 # After the first 4 columns, there are a series of key=value 197 # pairs. Extract out the fields we care about. 198 for element in columns[4:]: 199 key, value = element.split("=") 200 if key == "selabel": 201 selabel = value 202 if key == "capabilities": 203 capabilities = value 204 205 i = self.ITEMS.get(name, None) 206 if i is not None: 207 i.uid = int(uid) 208 i.gid = int(gid) 209 i.mode = int(mode, 8) 210 i.selabel = selabel 211 i.capabilities = capabilities 212 if i.dir: 213 i.children.sort(key=lambda i: i.name) 214 215 # set metadata for the files generated by this script. 216 i = self.ITEMS.get("system/recovery-from-boot.p", None) 217 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0644, None, None 218 i = self.ITEMS.get("system/etc/install-recovery.sh", None) 219 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0544, None, None 220 221 222class Item: 223 """Items represent the metadata (user, group, mode) of files and 224 directories in the system image.""" 225 def __init__(self, itemset, name, dir=False): 226 self.itemset = itemset 227 self.name = name 228 self.uid = None 229 self.gid = None 230 self.mode = None 231 self.selabel = None 232 self.capabilities = None 233 self.dir = dir 234 235 if name: 236 self.parent = itemset.Get(os.path.dirname(name), dir=True) 237 self.parent.children.append(self) 238 else: 239 self.parent = None 240 if dir: 241 self.children = [] 242 243 def Dump(self, indent=0): 244 if self.uid is not None: 245 print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode) 246 else: 247 print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode) 248 if self.dir: 249 print "%s%s" % (" "*indent, self.descendants) 250 print "%s%s" % (" "*indent, self.best_subtree) 251 for i in self.children: 252 i.Dump(indent=indent+1) 253 254 def CountChildMetadata(self): 255 """Count up the (uid, gid, mode, selabel, capabilities) tuples for 256 all children and determine the best strategy for using set_perm_recursive and 257 set_perm to correctly chown/chmod all the files to their desired 258 values. Recursively calls itself for all descendants. 259 260 Returns a dict of {(uid, gid, dmode, fmode, selabel, capabilities): count} counting up 261 all descendants of this node. (dmode or fmode may be None.) Also 262 sets the best_subtree of each directory Item to the (uid, gid, 263 dmode, fmode, selabel, capabilities) tuple that will match the most 264 descendants of that Item. 265 """ 266 267 assert self.dir 268 d = self.descendants = {(self.uid, self.gid, self.mode, None, self.selabel, self.capabilities): 1} 269 for i in self.children: 270 if i.dir: 271 for k, v in i.CountChildMetadata().iteritems(): 272 d[k] = d.get(k, 0) + v 273 else: 274 k = (i.uid, i.gid, None, i.mode, i.selabel, i.capabilities) 275 d[k] = d.get(k, 0) + 1 276 277 # Find the (uid, gid, dmode, fmode, selabel, capabilities) 278 # tuple that matches the most descendants. 279 280 # First, find the (uid, gid) pair that matches the most 281 # descendants. 282 ug = {} 283 for (uid, gid, _, _, _, _), count in d.iteritems(): 284 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 285 ug = MostPopularKey(ug, (0, 0)) 286 287 # Now find the dmode, fmode, selabel, and capabilities that match 288 # the most descendants with that (uid, gid), and choose those. 289 best_dmode = (0, 0755) 290 best_fmode = (0, 0644) 291 best_selabel = (0, None) 292 best_capabilities = (0, None) 293 for k, count in d.iteritems(): 294 if k[:2] != ug: continue 295 if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2]) 296 if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3]) 297 if k[4] is not None and count >= best_selabel[0]: best_selabel = (count, k[4]) 298 if k[5] is not None and count >= best_capabilities[0]: best_capabilities = (count, k[5]) 299 self.best_subtree = ug + (best_dmode[1], best_fmode[1], best_selabel[1], best_capabilities[1]) 300 301 return d 302 303 def SetPermissions(self, script): 304 """Append set_perm/set_perm_recursive commands to 'script' to 305 set all permissions, users, and groups for the tree of files 306 rooted at 'self'.""" 307 308 self.CountChildMetadata() 309 310 def recurse(item, current): 311 # current is the (uid, gid, dmode, fmode, selabel, capabilities) tuple that the current 312 # item (and all its children) have already been set to. We only 313 # need to issue set_perm/set_perm_recursive commands if we're 314 # supposed to be something different. 315 if item.dir: 316 if current != item.best_subtree: 317 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 318 current = item.best_subtree 319 320 if item.uid != current[0] or item.gid != current[1] or \ 321 item.mode != current[2] or item.selabel != current[4] or \ 322 item.capabilities != current[5]: 323 script.SetPermissions("/"+item.name, item.uid, item.gid, 324 item.mode, item.selabel, item.capabilities) 325 326 for i in item.children: 327 recurse(i, current) 328 else: 329 if item.uid != current[0] or item.gid != current[1] or \ 330 item.mode != current[3] or item.selabel != current[4] or \ 331 item.capabilities != current[5]: 332 script.SetPermissions("/"+item.name, item.uid, item.gid, 333 item.mode, item.selabel, item.capabilities) 334 335 recurse(self, (-1, -1, -1, -1, None, None)) 336 337 338def CopyPartitionFiles(itemset, input_zip, output_zip=None, substitute=None): 339 """Copies files for the partition in the input zip to the output 340 zip. Populates the Item class with their metadata, and returns a 341 list of symlinks. output_zip may be None, in which case the copy is 342 skipped (but the other side effects still happen). substitute is an 343 optional dict of {output filename: contents} to be output instead of 344 certain input files. 345 """ 346 347 symlinks = [] 348 349 partition = itemset.partition 350 351 for info in input_zip.infolist(): 352 prefix = partition.upper() + "/" 353 if info.filename.startswith(prefix): 354 basefilename = info.filename[len(prefix):] 355 if IsSymlink(info): 356 symlinks.append((input_zip.read(info.filename), 357 "/" + partition + "/" + basefilename)) 358 else: 359 info2 = copy.copy(info) 360 fn = info2.filename = partition + "/" + basefilename 361 if substitute and fn in substitute and substitute[fn] is None: 362 continue 363 if output_zip is not None: 364 if substitute and fn in substitute: 365 data = substitute[fn] 366 else: 367 data = input_zip.read(info.filename) 368 output_zip.writestr(info2, data) 369 if fn.endswith("/"): 370 itemset.Get(fn[:-1], dir=True) 371 else: 372 itemset.Get(fn, dir=False) 373 374 symlinks.sort() 375 return symlinks 376 377 378def SignOutput(temp_zip_name, output_zip_name): 379 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 380 pw = key_passwords[OPTIONS.package_key] 381 382 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 383 whole_file=True) 384 385 386def AppendAssertions(script, info_dict, oem_dict = None): 387 oem_props = info_dict.get("oem_fingerprint_properties") 388 if oem_props is None or len(oem_props) == 0: 389 device = GetBuildProp("ro.product.device", info_dict) 390 script.AssertDevice(device) 391 else: 392 if oem_dict is None: 393 raise common.ExternalError("No OEM file provided to answer expected assertions") 394 for prop in oem_props.split(): 395 if oem_dict.get(prop) is None: 396 raise common.ExternalError("The OEM file is missing the property %s" % prop) 397 script.AssertOemProperty(prop, oem_dict.get(prop)) 398 399 400def HasRecoveryPatch(target_files_zip): 401 try: 402 target_files_zip.getinfo("SYSTEM/recovery-from-boot.p") 403 return True 404 except KeyError: 405 return False 406 407def HasVendorPartition(target_files_zip): 408 try: 409 target_files_zip.getinfo("VENDOR/") 410 return True 411 except KeyError: 412 return False 413 414def GetOemProperty(name, oem_props, oem_dict, info_dict): 415 if oem_props is not None and name in oem_props: 416 return oem_dict[name] 417 return GetBuildProp(name, info_dict) 418 419 420def CalculateFingerprint(oem_props, oem_dict, info_dict): 421 if oem_props is None: 422 return GetBuildProp("ro.build.fingerprint", info_dict) 423 return "%s/%s/%s:%s" % ( 424 GetOemProperty("ro.product.brand", oem_props, oem_dict, info_dict), 425 GetOemProperty("ro.product.name", oem_props, oem_dict, info_dict), 426 GetOemProperty("ro.product.device", oem_props, oem_dict, info_dict), 427 GetBuildProp("ro.build.thumbprint", info_dict)) 428 429 430def GetImage(which, tmpdir, info_dict): 431 # Return an image object (suitable for passing to BlockImageDiff) 432 # for the 'which' partition (most be "system" or "vendor"). If a 433 # prebuilt image and file map are found in tmpdir they are used, 434 # otherwise they are reconstructed from the individual files. 435 436 assert which in ("system", "vendor") 437 438 path = os.path.join(tmpdir, "IMAGES", which + ".img") 439 mappath = os.path.join(tmpdir, "IMAGES", which + ".map") 440 if os.path.exists(path) and os.path.exists(mappath): 441 print "using %s.img from target-files" % (which,) 442 # This is a 'new' target-files, which already has the image in it. 443 444 else: 445 print "building %s.img from target-files" % (which,) 446 447 # This is an 'old' target-files, which does not contain images 448 # already built. Build them. 449 450 mappath = tempfile.mkstemp()[1] 451 OPTIONS.tempfiles.append(mappath) 452 453 import add_img_to_target_files 454 if which == "system": 455 path = add_img_to_target_files.BuildSystem( 456 tmpdir, info_dict, block_list=mappath) 457 elif which == "vendor": 458 path = add_img_to_target_files.BuildVendor( 459 tmpdir, info_dict, block_list=mappath) 460 461 return sparse_img.SparseImage(path, mappath) 462 463 464def WriteFullOTAPackage(input_zip, output_zip): 465 # TODO: how to determine this? We don't know what version it will 466 # be installed on top of. For now, we expect the API just won't 467 # change very often. 468 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 469 470 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 471 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 472 oem_dict = None 473 if oem_props is not None and len(oem_props) > 0: 474 if OPTIONS.oem_source is None: 475 raise common.ExternalError("OEM source required for this build") 476 script.Mount("/oem", recovery_mount_options) 477 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 478 479 metadata = {"post-build": CalculateFingerprint( 480 oem_props, oem_dict, OPTIONS.info_dict), 481 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 482 OPTIONS.info_dict), 483 "post-timestamp": GetBuildProp("ro.build.date.utc", 484 OPTIONS.info_dict), 485 } 486 487 device_specific = common.DeviceSpecificParams( 488 input_zip=input_zip, 489 input_version=OPTIONS.info_dict["recovery_api_version"], 490 output_zip=output_zip, 491 script=script, 492 input_tmp=OPTIONS.input_tmp, 493 metadata=metadata, 494 info_dict=OPTIONS.info_dict) 495 496 has_recovery_patch = HasRecoveryPatch(input_zip) 497 block_based = OPTIONS.block_based and has_recovery_patch 498 499 if not OPTIONS.omit_prereq: 500 ts = GetBuildProp("ro.build.date.utc", OPTIONS.info_dict) 501 ts_text = GetBuildProp("ro.build.date", OPTIONS.info_dict) 502 script.AssertOlderBuild(ts, ts_text) 503 504 AppendAssertions(script, OPTIONS.info_dict, oem_dict) 505 device_specific.FullOTA_Assertions() 506 507 # Two-step package strategy (in chronological order, which is *not* 508 # the order in which the generated script has things): 509 # 510 # if stage is not "2/3" or "3/3": 511 # write recovery image to boot partition 512 # set stage to "2/3" 513 # reboot to boot partition and restart recovery 514 # else if stage is "2/3": 515 # write recovery image to recovery partition 516 # set stage to "3/3" 517 # reboot to recovery partition and restart recovery 518 # else: 519 # (stage must be "3/3") 520 # set stage to "" 521 # do normal full package installation: 522 # wipe and install system, boot image, etc. 523 # set up system to update recovery partition on first boot 524 # complete script normally (allow recovery to mark itself finished and reboot) 525 526 recovery_img = common.GetBootableImage("recovery.img", "recovery.img", 527 OPTIONS.input_tmp, "RECOVERY") 528 if OPTIONS.two_step: 529 if not OPTIONS.info_dict.get("multistage_support", None): 530 assert False, "two-step packages not supported by this build" 531 fs = OPTIONS.info_dict["fstab"]["/misc"] 532 assert fs.fs_type.upper() == "EMMC", \ 533 "two-step packages only supported on devices with EMMC /misc partitions" 534 bcb_dev = {"bcb_dev": fs.device} 535 common.ZipWriteStr(output_zip, "recovery.img", recovery_img.data) 536 script.AppendExtra(""" 537if get_stage("%(bcb_dev)s") == "2/3" then 538""" % bcb_dev) 539 script.WriteRawImage("/recovery", "recovery.img") 540 script.AppendExtra(""" 541set_stage("%(bcb_dev)s", "3/3"); 542reboot_now("%(bcb_dev)s", "recovery"); 543else if get_stage("%(bcb_dev)s") == "3/3" then 544""" % bcb_dev) 545 546 device_specific.FullOTA_InstallBegin() 547 548 system_progress = 0.75 549 550 if OPTIONS.wipe_user_data: 551 system_progress -= 0.1 552 if HasVendorPartition(input_zip): 553 system_progress -= 0.1 554 555 if "selinux_fc" in OPTIONS.info_dict: 556 WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip) 557 558 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 559 560 system_items = ItemSet("system", "META/filesystem_config.txt") 561 script.ShowProgress(system_progress, 0) 562 563 if block_based: 564 # Full OTA is done as an "incremental" against an empty source 565 # image. This has the effect of writing new data from the package 566 # to the entire partition, but lets us reuse the updater code that 567 # writes incrementals to do it. 568 system_tgt = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict) 569 system_tgt.ResetFileMap() 570 system_diff = common.BlockDifference("system", system_tgt, src=None) 571 system_diff.WriteScript(script, output_zip) 572 else: 573 script.FormatPartition("/system") 574 script.Mount("/system", recovery_mount_options) 575 if not has_recovery_patch: 576 script.UnpackPackageDir("recovery", "/system") 577 script.UnpackPackageDir("system", "/system") 578 579 symlinks = CopyPartitionFiles(system_items, input_zip, output_zip) 580 script.MakeSymlinks(symlinks) 581 582 boot_img = common.GetBootableImage("boot.img", "boot.img", 583 OPTIONS.input_tmp, "BOOT") 584 585 if not block_based: 586 def output_sink(fn, data): 587 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 588 system_items.Get("system/" + fn, dir=False) 589 590 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, 591 recovery_img, boot_img) 592 593 system_items.GetMetadata(input_zip) 594 system_items.Get("system").SetPermissions(script) 595 596 if HasVendorPartition(input_zip): 597 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 598 script.ShowProgress(0.1, 0) 599 600 if block_based: 601 vendor_tgt = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict) 602 vendor_tgt.ResetFileMap() 603 vendor_diff = common.BlockDifference("vendor", vendor_tgt) 604 vendor_diff.WriteScript(script, output_zip) 605 else: 606 script.FormatPartition("/vendor") 607 script.Mount("/vendor", recovery_mount_options) 608 script.UnpackPackageDir("vendor", "/vendor") 609 610 symlinks = CopyPartitionFiles(vendor_items, input_zip, output_zip) 611 script.MakeSymlinks(symlinks) 612 613 vendor_items.GetMetadata(input_zip) 614 vendor_items.Get("vendor").SetPermissions(script) 615 616 common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict) 617 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 618 619 script.ShowProgress(0.05, 5) 620 script.WriteRawImage("/boot", "boot.img") 621 622 script.ShowProgress(0.2, 10) 623 device_specific.FullOTA_InstallEnd() 624 625 if OPTIONS.extra_script is not None: 626 script.AppendExtra(OPTIONS.extra_script) 627 628 script.UnmountAll() 629 630 if OPTIONS.wipe_user_data: 631 script.ShowProgress(0.1, 10) 632 script.FormatPartition("/data") 633 634 if OPTIONS.two_step: 635 script.AppendExtra(""" 636set_stage("%(bcb_dev)s", ""); 637""" % bcb_dev) 638 script.AppendExtra("else\n") 639 script.WriteRawImage("/boot", "recovery.img") 640 script.AppendExtra(""" 641set_stage("%(bcb_dev)s", "2/3"); 642reboot_now("%(bcb_dev)s", ""); 643endif; 644endif; 645""" % bcb_dev) 646 script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary) 647 WriteMetadata(metadata, output_zip) 648 649 650def WritePolicyConfig(file_name, output_zip): 651 common.ZipWrite(output_zip, file_name, os.path.basename(file_name)) 652 653 654def WriteMetadata(metadata, output_zip): 655 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata", 656 "".join(["%s=%s\n" % kv 657 for kv in sorted(metadata.iteritems())])) 658 659 660def LoadPartitionFiles(z, partition): 661 """Load all the files from the given partition in a given target-files 662 ZipFile, and return a dict of {filename: File object}.""" 663 out = {} 664 prefix = partition.upper() + "/" 665 for info in z.infolist(): 666 if info.filename.startswith(prefix) and not IsSymlink(info): 667 basefilename = info.filename[len(prefix):] 668 fn = partition + "/" + basefilename 669 data = z.read(info.filename) 670 out[fn] = common.File(fn, data) 671 return out 672 673 674def GetBuildProp(prop, info_dict): 675 """Return the fingerprint of the build of a given target-files info_dict.""" 676 try: 677 return info_dict.get("build.prop", {})[prop] 678 except KeyError: 679 raise common.ExternalError("couldn't find %s in build.prop" % (prop,)) 680 681 682def AddToKnownPaths(filename, known_paths): 683 if filename[-1] == "/": 684 return 685 dirs = filename.split("/")[:-1] 686 while len(dirs) > 0: 687 path = "/".join(dirs) 688 if path in known_paths: 689 break; 690 known_paths.add(path) 691 dirs.pop() 692 693 694def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 695 source_version = OPTIONS.source_info_dict["recovery_api_version"] 696 target_version = OPTIONS.target_info_dict["recovery_api_version"] 697 698 if source_version == 0: 699 print ("WARNING: generating edify script for a source that " 700 "can't install it.") 701 script = edify_generator.EdifyGenerator(source_version, 702 OPTIONS.target_info_dict) 703 704 metadata = {"pre-device": GetBuildProp("ro.product.device", 705 OPTIONS.source_info_dict), 706 "post-timestamp": GetBuildProp("ro.build.date.utc", 707 OPTIONS.target_info_dict), 708 } 709 710 device_specific = common.DeviceSpecificParams( 711 source_zip=source_zip, 712 source_version=source_version, 713 target_zip=target_zip, 714 target_version=target_version, 715 output_zip=output_zip, 716 script=script, 717 metadata=metadata, 718 info_dict=OPTIONS.info_dict) 719 720 source_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.source_info_dict) 721 target_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.target_info_dict) 722 metadata["pre-build"] = source_fp 723 metadata["post-build"] = target_fp 724 725 source_boot = common.GetBootableImage( 726 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 727 OPTIONS.source_info_dict) 728 target_boot = common.GetBootableImage( 729 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 730 updating_boot = (not OPTIONS.two_step and 731 (source_boot.data != target_boot.data)) 732 733 source_recovery = common.GetBootableImage( 734 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 735 OPTIONS.source_info_dict) 736 target_recovery = common.GetBootableImage( 737 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 738 updating_recovery = (source_recovery.data != target_recovery.data) 739 740 system_src = GetImage("system", OPTIONS.source_tmp, OPTIONS.source_info_dict) 741 system_tgt = GetImage("system", OPTIONS.target_tmp, OPTIONS.target_info_dict) 742 743 blockimgdiff_version = 1 744 if OPTIONS.info_dict: 745 blockimgdiff_version = max( 746 int(i) for i in 747 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(",")) 748 749 system_diff = common.BlockDifference("system", system_tgt, system_src, 750 check_first_block=True, 751 version=blockimgdiff_version) 752 753 if HasVendorPartition(target_zip): 754 if not HasVendorPartition(source_zip): 755 raise RuntimeError("can't generate incremental that adds /vendor") 756 vendor_src = GetImage("vendor", OPTIONS.source_tmp, OPTIONS.source_info_dict) 757 vendor_tgt = GetImage("vendor", OPTIONS.target_tmp, OPTIONS.target_info_dict) 758 vendor_diff = common.BlockDifference("vendor", vendor_tgt, vendor_src, 759 check_first_block=True, 760 version=blockimgdiff_version) 761 else: 762 vendor_diff = None 763 764 oem_props = OPTIONS.target_info_dict.get("oem_fingerprint_properties") 765 recovery_mount_options = OPTIONS.target_info_dict.get("recovery_mount_options") 766 oem_dict = None 767 if oem_props is not None and len(oem_props) > 0: 768 if OPTIONS.oem_source is None: 769 raise common.ExternalError("OEM source required for this build") 770 script.Mount("/oem", recovery_mount_options) 771 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 772 773 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 774 device_specific.IncrementalOTA_Assertions() 775 776 # Two-step incremental package strategy (in chronological order, 777 # which is *not* the order in which the generated script has 778 # things): 779 # 780 # if stage is not "2/3" or "3/3": 781 # do verification on current system 782 # write recovery image to boot partition 783 # set stage to "2/3" 784 # reboot to boot partition and restart recovery 785 # else if stage is "2/3": 786 # write recovery image to recovery partition 787 # set stage to "3/3" 788 # reboot to recovery partition and restart recovery 789 # else: 790 # (stage must be "3/3") 791 # perform update: 792 # patch system files, etc. 793 # force full install of new boot image 794 # set up system to update recovery partition on first boot 795 # complete script normally (allow recovery to mark itself finished and reboot) 796 797 if OPTIONS.two_step: 798 if not OPTIONS.info_dict.get("multistage_support", None): 799 assert False, "two-step packages not supported by this build" 800 fs = OPTIONS.info_dict["fstab"]["/misc"] 801 assert fs.fs_type.upper() == "EMMC", \ 802 "two-step packages only supported on devices with EMMC /misc partitions" 803 bcb_dev = {"bcb_dev": fs.device} 804 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 805 script.AppendExtra(""" 806if get_stage("%(bcb_dev)s") == "2/3" then 807""" % bcb_dev) 808 script.AppendExtra("sleep(20);\n"); 809 script.WriteRawImage("/recovery", "recovery.img") 810 script.AppendExtra(""" 811set_stage("%(bcb_dev)s", "3/3"); 812reboot_now("%(bcb_dev)s", "recovery"); 813else if get_stage("%(bcb_dev)s") != "3/3" then 814""" % bcb_dev) 815 816 script.Print("Verifying current system...") 817 818 device_specific.IncrementalOTA_VerifyBegin() 819 820 if oem_props is None: 821 # When blockimgdiff version is less than 3 (non-resumable block-based OTA), 822 # patching on a device that's already on the target build will damage the 823 # system. Because operations like move don't check the block state, they 824 # always apply the changes unconditionally. 825 if blockimgdiff_version <= 2: 826 script.AssertSomeFingerprint(source_fp) 827 else: 828 script.AssertSomeFingerprint(source_fp, target_fp) 829 else: 830 if blockimgdiff_version <= 2: 831 script.AssertSomeThumbprint( 832 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 833 else: 834 script.AssertSomeThumbprint( 835 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 836 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 837 838 if updating_boot: 839 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 840 d = common.Difference(target_boot, source_boot) 841 _, _, d = d.ComputePatch() 842 if d is None: 843 include_full_boot = True 844 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 845 else: 846 include_full_boot = False 847 848 print "boot target: %d source: %d diff: %d" % ( 849 target_boot.size, source_boot.size, len(d)) 850 851 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 852 853 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 854 (boot_type, boot_device, 855 source_boot.size, source_boot.sha1, 856 target_boot.size, target_boot.sha1)) 857 858 device_specific.IncrementalOTA_VerifyEnd() 859 860 if OPTIONS.two_step: 861 script.WriteRawImage("/boot", "recovery.img") 862 script.AppendExtra(""" 863set_stage("%(bcb_dev)s", "2/3"); 864reboot_now("%(bcb_dev)s", ""); 865else 866""" % bcb_dev) 867 868 # Verify the existing partitions. 869 system_diff.WriteVerifyScript(script) 870 if vendor_diff: 871 vendor_diff.WriteVerifyScript(script) 872 873 script.Comment("---- start making changes here ----") 874 875 device_specific.IncrementalOTA_InstallBegin() 876 877 system_diff.WriteScript(script, output_zip, 878 progress=0.8 if vendor_diff else 0.9) 879 if vendor_diff: 880 vendor_diff.WriteScript(script, output_zip, progress=0.1) 881 882 if OPTIONS.two_step: 883 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 884 script.WriteRawImage("/boot", "boot.img") 885 print "writing full boot image (forced by two-step mode)" 886 887 if not OPTIONS.two_step: 888 if updating_boot: 889 if include_full_boot: 890 print "boot image changed; including full." 891 script.Print("Installing boot image...") 892 script.WriteRawImage("/boot", "boot.img") 893 else: 894 # Produce the boot image by applying a patch to the current 895 # contents of the boot partition, and write it back to the 896 # partition. 897 print "boot image changed; including patch." 898 script.Print("Patching boot image...") 899 script.ShowProgress(0.1, 10) 900 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 901 % (boot_type, boot_device, 902 source_boot.size, source_boot.sha1, 903 target_boot.size, target_boot.sha1), 904 "-", 905 target_boot.size, target_boot.sha1, 906 source_boot.sha1, "patch/boot.img.p") 907 else: 908 print "boot image unchanged; skipping." 909 910 # Do device-specific installation (eg, write radio image). 911 device_specific.IncrementalOTA_InstallEnd() 912 913 if OPTIONS.extra_script is not None: 914 script.AppendExtra(OPTIONS.extra_script) 915 916 if OPTIONS.wipe_user_data: 917 script.Print("Erasing user data...") 918 script.FormatPartition("/data") 919 920 if OPTIONS.two_step: 921 script.AppendExtra(""" 922set_stage("%(bcb_dev)s", ""); 923endif; 924endif; 925""" % bcb_dev) 926 927 script.SetProgress(1) 928 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 929 WriteMetadata(metadata, output_zip) 930 931 932class FileDifference: 933 def __init__(self, partition, source_zip, target_zip, output_zip): 934 print "Loading target..." 935 self.target_data = target_data = LoadPartitionFiles(target_zip, partition) 936 print "Loading source..." 937 self.source_data = source_data = LoadPartitionFiles(source_zip, partition) 938 939 self.verbatim_targets = verbatim_targets = [] 940 self.patch_list = patch_list = [] 941 diffs = [] 942 self.renames = renames = {} 943 known_paths = set() 944 largest_source_size = 0 945 946 matching_file_cache = {} 947 for fn, sf in source_data.items(): 948 assert fn == sf.name 949 matching_file_cache["path:" + fn] = sf 950 if fn in target_data.keys(): 951 AddToKnownPaths(fn, known_paths) 952 # Only allow eligibility for filename/sha matching 953 # if there isn't a perfect path match. 954 if target_data.get(sf.name) is None: 955 matching_file_cache["file:" + fn.split("/")[-1]] = sf 956 matching_file_cache["sha:" + sf.sha1] = sf 957 958 for fn in sorted(target_data.keys()): 959 tf = target_data[fn] 960 assert fn == tf.name 961 sf = ClosestFileMatch(tf, matching_file_cache, renames) 962 if sf is not None and sf.name != tf.name: 963 print "File has moved from " + sf.name + " to " + tf.name 964 renames[sf.name] = tf 965 966 if sf is None or fn in OPTIONS.require_verbatim: 967 # This file should be included verbatim 968 if fn in OPTIONS.prohibit_verbatim: 969 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 970 print "send", fn, "verbatim" 971 tf.AddToZip(output_zip) 972 verbatim_targets.append((fn, tf.size, tf.sha1)) 973 if fn in target_data.keys(): 974 AddToKnownPaths(fn, known_paths) 975 elif tf.sha1 != sf.sha1: 976 # File is different; consider sending as a patch 977 diffs.append(common.Difference(tf, sf)) 978 else: 979 # Target file data identical to source (may still be renamed) 980 pass 981 982 common.ComputeDifferences(diffs) 983 984 for diff in diffs: 985 tf, sf, d = diff.GetPatch() 986 path = "/".join(tf.name.split("/")[:-1]) 987 if d is None or len(d) > tf.size * OPTIONS.patch_threshold or \ 988 path not in known_paths: 989 # patch is almost as big as the file; don't bother patching 990 # or a patch + rename cannot take place due to the target 991 # directory not existing 992 tf.AddToZip(output_zip) 993 verbatim_targets.append((tf.name, tf.size, tf.sha1)) 994 if sf.name in renames: 995 del renames[sf.name] 996 AddToKnownPaths(tf.name, known_paths) 997 else: 998 common.ZipWriteStr(output_zip, "patch/" + sf.name + ".p", d) 999 patch_list.append((tf, sf, tf.size, common.sha1(d).hexdigest())) 1000 largest_source_size = max(largest_source_size, sf.size) 1001 1002 self.largest_source_size = largest_source_size 1003 1004 def EmitVerification(self, script): 1005 so_far = 0 1006 for tf, sf, size, patch_sha in self.patch_list: 1007 if tf.name != sf.name: 1008 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1009 script.PatchCheck("/"+sf.name, tf.sha1, sf.sha1) 1010 so_far += sf.size 1011 return so_far 1012 1013 def EmitExplicitTargetVerification(self, script): 1014 for fn, size, sha1 in self.verbatim_targets: 1015 if (fn[-1] != "/"): 1016 script.FileCheck("/"+fn, sha1) 1017 for tf, _, _, _ in self.patch_list: 1018 script.FileCheck(tf.name, tf.sha1) 1019 1020 def RemoveUnneededFiles(self, script, extras=()): 1021 script.DeleteFiles(["/"+i[0] for i in self.verbatim_targets] + 1022 ["/"+i for i in sorted(self.source_data) 1023 if i not in self.target_data and 1024 i not in self.renames] + 1025 list(extras)) 1026 1027 def TotalPatchSize(self): 1028 return sum(i[1].size for i in self.patch_list) 1029 1030 def EmitPatches(self, script, total_patch_size, so_far): 1031 self.deferred_patch_list = deferred_patch_list = [] 1032 for item in self.patch_list: 1033 tf, sf, size, _ = item 1034 if tf.name == "system/build.prop": 1035 deferred_patch_list.append(item) 1036 continue 1037 if (sf.name != tf.name): 1038 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1039 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p") 1040 so_far += tf.size 1041 script.SetProgress(so_far / total_patch_size) 1042 return so_far 1043 1044 def EmitDeferredPatches(self, script): 1045 for item in self.deferred_patch_list: 1046 tf, sf, size, _ = item 1047 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p") 1048 script.SetPermissions("/system/build.prop", 0, 0, 0644, None, None) 1049 1050 def EmitRenames(self, script): 1051 if len(self.renames) > 0: 1052 script.Print("Renaming files...") 1053 for src, tgt in self.renames.iteritems(): 1054 print "Renaming " + src + " to " + tgt.name 1055 script.RenameFile(src, tgt.name) 1056 1057 1058 1059 1060def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): 1061 target_has_recovery_patch = HasRecoveryPatch(target_zip) 1062 source_has_recovery_patch = HasRecoveryPatch(source_zip) 1063 1064 if (OPTIONS.block_based and 1065 target_has_recovery_patch and 1066 source_has_recovery_patch): 1067 return WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip) 1068 1069 source_version = OPTIONS.source_info_dict["recovery_api_version"] 1070 target_version = OPTIONS.target_info_dict["recovery_api_version"] 1071 1072 if source_version == 0: 1073 print ("WARNING: generating edify script for a source that " 1074 "can't install it.") 1075 script = edify_generator.EdifyGenerator(source_version, 1076 OPTIONS.target_info_dict) 1077 1078 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 1079 recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options") 1080 oem_dict = None 1081 if oem_props is not None and len(oem_props) > 0: 1082 if OPTIONS.oem_source is None: 1083 raise common.ExternalError("OEM source required for this build") 1084 script.Mount("/oem", recovery_mount_options) 1085 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 1086 1087 metadata = {"pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 1088 OPTIONS.source_info_dict), 1089 "post-timestamp": GetBuildProp("ro.build.date.utc", 1090 OPTIONS.target_info_dict), 1091 } 1092 1093 device_specific = common.DeviceSpecificParams( 1094 source_zip=source_zip, 1095 source_version=source_version, 1096 target_zip=target_zip, 1097 target_version=target_version, 1098 output_zip=output_zip, 1099 script=script, 1100 metadata=metadata, 1101 info_dict=OPTIONS.info_dict) 1102 1103 system_diff = FileDifference("system", source_zip, target_zip, output_zip) 1104 script.Mount("/system", recovery_mount_options) 1105 if HasVendorPartition(target_zip): 1106 vendor_diff = FileDifference("vendor", source_zip, target_zip, output_zip) 1107 script.Mount("/vendor", recovery_mount_options) 1108 else: 1109 vendor_diff = None 1110 1111 target_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.target_info_dict) 1112 source_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.source_info_dict) 1113 1114 if oem_props is None: 1115 script.AssertSomeFingerprint(source_fp, target_fp) 1116 else: 1117 script.AssertSomeThumbprint( 1118 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 1119 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 1120 1121 metadata["pre-build"] = source_fp 1122 metadata["post-build"] = target_fp 1123 1124 source_boot = common.GetBootableImage( 1125 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 1126 OPTIONS.source_info_dict) 1127 target_boot = common.GetBootableImage( 1128 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 1129 updating_boot = (not OPTIONS.two_step and 1130 (source_boot.data != target_boot.data)) 1131 1132 source_recovery = common.GetBootableImage( 1133 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 1134 OPTIONS.source_info_dict) 1135 target_recovery = common.GetBootableImage( 1136 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 1137 updating_recovery = (source_recovery.data != target_recovery.data) 1138 1139 # Here's how we divide up the progress bar: 1140 # 0.1 for verifying the start state (PatchCheck calls) 1141 # 0.8 for applying patches (ApplyPatch calls) 1142 # 0.1 for unpacking verbatim files, symlinking, and doing the 1143 # device-specific commands. 1144 1145 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 1146 device_specific.IncrementalOTA_Assertions() 1147 1148 # Two-step incremental package strategy (in chronological order, 1149 # which is *not* the order in which the generated script has 1150 # things): 1151 # 1152 # if stage is not "2/3" or "3/3": 1153 # do verification on current system 1154 # write recovery image to boot partition 1155 # set stage to "2/3" 1156 # reboot to boot partition and restart recovery 1157 # else if stage is "2/3": 1158 # write recovery image to recovery partition 1159 # set stage to "3/3" 1160 # reboot to recovery partition and restart recovery 1161 # else: 1162 # (stage must be "3/3") 1163 # perform update: 1164 # patch system files, etc. 1165 # force full install of new boot image 1166 # set up system to update recovery partition on first boot 1167 # complete script normally (allow recovery to mark itself finished and reboot) 1168 1169 if OPTIONS.two_step: 1170 if not OPTIONS.info_dict.get("multistage_support", None): 1171 assert False, "two-step packages not supported by this build" 1172 fs = OPTIONS.info_dict["fstab"]["/misc"] 1173 assert fs.fs_type.upper() == "EMMC", \ 1174 "two-step packages only supported on devices with EMMC /misc partitions" 1175 bcb_dev = {"bcb_dev": fs.device} 1176 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 1177 script.AppendExtra(""" 1178if get_stage("%(bcb_dev)s") == "2/3" then 1179""" % bcb_dev) 1180 script.AppendExtra("sleep(20);\n"); 1181 script.WriteRawImage("/recovery", "recovery.img") 1182 script.AppendExtra(""" 1183set_stage("%(bcb_dev)s", "3/3"); 1184reboot_now("%(bcb_dev)s", "recovery"); 1185else if get_stage("%(bcb_dev)s") != "3/3" then 1186""" % bcb_dev) 1187 1188 script.Print("Verifying current system...") 1189 1190 device_specific.IncrementalOTA_VerifyBegin() 1191 1192 script.ShowProgress(0.1, 0) 1193 so_far = system_diff.EmitVerification(script) 1194 if vendor_diff: 1195 so_far += vendor_diff.EmitVerification(script) 1196 1197 if updating_boot: 1198 d = common.Difference(target_boot, source_boot) 1199 _, _, d = d.ComputePatch() 1200 print "boot target: %d source: %d diff: %d" % ( 1201 target_boot.size, source_boot.size, len(d)) 1202 1203 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 1204 1205 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 1206 1207 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 1208 (boot_type, boot_device, 1209 source_boot.size, source_boot.sha1, 1210 target_boot.size, target_boot.sha1)) 1211 so_far += source_boot.size 1212 1213 size = [] 1214 if system_diff.patch_list: size.append(system_diff.largest_source_size) 1215 if vendor_diff: 1216 if vendor_diff.patch_list: size.append(vendor_diff.largest_source_size) 1217 if size or updating_recovery or updating_boot: 1218 script.CacheFreeSpaceCheck(max(size)) 1219 1220 device_specific.IncrementalOTA_VerifyEnd() 1221 1222 if OPTIONS.two_step: 1223 script.WriteRawImage("/boot", "recovery.img") 1224 script.AppendExtra(""" 1225set_stage("%(bcb_dev)s", "2/3"); 1226reboot_now("%(bcb_dev)s", ""); 1227else 1228""" % bcb_dev) 1229 1230 script.Comment("---- start making changes here ----") 1231 1232 device_specific.IncrementalOTA_InstallBegin() 1233 1234 if OPTIONS.two_step: 1235 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 1236 script.WriteRawImage("/boot", "boot.img") 1237 print "writing full boot image (forced by two-step mode)" 1238 1239 script.Print("Removing unneeded files...") 1240 system_diff.RemoveUnneededFiles(script, ("/system/recovery.img",)) 1241 if vendor_diff: 1242 vendor_diff.RemoveUnneededFiles(script) 1243 1244 script.ShowProgress(0.8, 0) 1245 total_patch_size = 1.0 + system_diff.TotalPatchSize() 1246 if vendor_diff: 1247 total_patch_size += vendor_diff.TotalPatchSize() 1248 if updating_boot: 1249 total_patch_size += target_boot.size 1250 1251 script.Print("Patching system files...") 1252 so_far = system_diff.EmitPatches(script, total_patch_size, 0) 1253 if vendor_diff: 1254 script.Print("Patching vendor files...") 1255 so_far = vendor_diff.EmitPatches(script, total_patch_size, so_far) 1256 1257 if not OPTIONS.two_step: 1258 if updating_boot: 1259 # Produce the boot image by applying a patch to the current 1260 # contents of the boot partition, and write it back to the 1261 # partition. 1262 script.Print("Patching boot image...") 1263 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 1264 % (boot_type, boot_device, 1265 source_boot.size, source_boot.sha1, 1266 target_boot.size, target_boot.sha1), 1267 "-", 1268 target_boot.size, target_boot.sha1, 1269 source_boot.sha1, "patch/boot.img.p") 1270 so_far += target_boot.size 1271 script.SetProgress(so_far / total_patch_size) 1272 print "boot image changed; including." 1273 else: 1274 print "boot image unchanged; skipping." 1275 1276 system_items = ItemSet("system", "META/filesystem_config.txt") 1277 if vendor_diff: 1278 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 1279 1280 if updating_recovery: 1281 # Recovery is generated as a patch using both the boot image 1282 # (which contains the same linux kernel as recovery) and the file 1283 # /system/etc/recovery-resource.dat (which contains all the images 1284 # used in the recovery UI) as sources. This lets us minimize the 1285 # size of the patch, which must be included in every OTA package. 1286 # 1287 # For older builds where recovery-resource.dat is not present, we 1288 # use only the boot image as the source. 1289 1290 if not target_has_recovery_patch: 1291 def output_sink(fn, data): 1292 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 1293 system_items.Get("system/" + fn, dir=False) 1294 1295 common.MakeRecoveryPatch(OPTIONS.target_tmp, output_sink, 1296 target_recovery, target_boot) 1297 script.DeleteFiles(["/system/recovery-from-boot.p", 1298 "/system/etc/install-recovery.sh"]) 1299 print "recovery image changed; including as patch from boot." 1300 else: 1301 print "recovery image unchanged; skipping." 1302 1303 script.ShowProgress(0.1, 10) 1304 1305 target_symlinks = CopyPartitionFiles(system_items, target_zip, None) 1306 if vendor_diff: 1307 target_symlinks.extend(CopyPartitionFiles(vendor_items, target_zip, None)) 1308 1309 temp_script = script.MakeTemporary() 1310 system_items.GetMetadata(target_zip) 1311 system_items.Get("system").SetPermissions(temp_script) 1312 if vendor_diff: 1313 vendor_items.GetMetadata(target_zip) 1314 vendor_items.Get("vendor").SetPermissions(temp_script) 1315 1316 # Note that this call will mess up the trees of Items, so make sure 1317 # we're done with them. 1318 source_symlinks = CopyPartitionFiles(system_items, source_zip, None) 1319 if vendor_diff: 1320 source_symlinks.extend(CopyPartitionFiles(vendor_items, source_zip, None)) 1321 1322 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 1323 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 1324 1325 # Delete all the symlinks in source that aren't in target. This 1326 # needs to happen before verbatim files are unpacked, in case a 1327 # symlink in the source is replaced by a real file in the target. 1328 to_delete = [] 1329 for dest, link in source_symlinks: 1330 if link not in target_symlinks_d: 1331 to_delete.append(link) 1332 script.DeleteFiles(to_delete) 1333 1334 if system_diff.verbatim_targets: 1335 script.Print("Unpacking new system files...") 1336 script.UnpackPackageDir("system", "/system") 1337 if vendor_diff and vendor_diff.verbatim_targets: 1338 script.Print("Unpacking new vendor files...") 1339 script.UnpackPackageDir("vendor", "/vendor") 1340 1341 if updating_recovery and not target_has_recovery_patch: 1342 script.Print("Unpacking new recovery...") 1343 script.UnpackPackageDir("recovery", "/system") 1344 1345 system_diff.EmitRenames(script) 1346 if vendor_diff: 1347 vendor_diff.EmitRenames(script) 1348 1349 script.Print("Symlinks and permissions...") 1350 1351 # Create all the symlinks that don't already exist, or point to 1352 # somewhere different than what we want. Delete each symlink before 1353 # creating it, since the 'symlink' command won't overwrite. 1354 to_create = [] 1355 for dest, link in target_symlinks: 1356 if link in source_symlinks_d: 1357 if dest != source_symlinks_d[link]: 1358 to_create.append((dest, link)) 1359 else: 1360 to_create.append((dest, link)) 1361 script.DeleteFiles([i[1] for i in to_create]) 1362 script.MakeSymlinks(to_create) 1363 1364 # Now that the symlinks are created, we can set all the 1365 # permissions. 1366 script.AppendScript(temp_script) 1367 1368 # Do device-specific installation (eg, write radio image). 1369 device_specific.IncrementalOTA_InstallEnd() 1370 1371 if OPTIONS.extra_script is not None: 1372 script.AppendExtra(OPTIONS.extra_script) 1373 1374 # Patch the build.prop file last, so if something fails but the 1375 # device can still come up, it appears to be the old build and will 1376 # get set the OTA package again to retry. 1377 script.Print("Patching remaining system files...") 1378 system_diff.EmitDeferredPatches(script) 1379 1380 if OPTIONS.wipe_user_data: 1381 script.Print("Erasing user data...") 1382 script.FormatPartition("/data") 1383 1384 if OPTIONS.two_step: 1385 script.AppendExtra(""" 1386set_stage("%(bcb_dev)s", ""); 1387endif; 1388endif; 1389""" % bcb_dev) 1390 1391 if OPTIONS.verify and system_diff: 1392 script.Print("Remounting and verifying system partition files...") 1393 script.Unmount("/system") 1394 script.Mount("/system") 1395 system_diff.EmitExplicitTargetVerification(script) 1396 1397 if OPTIONS.verify and vendor_diff: 1398 script.Print("Remounting and verifying vendor partition files...") 1399 script.Unmount("/vendor") 1400 script.Mount("/vendor") 1401 vendor_diff.EmitExplicitTargetVerification(script) 1402 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 1403 1404 WriteMetadata(metadata, output_zip) 1405 1406 1407def main(argv): 1408 1409 def option_handler(o, a): 1410 if o == "--board_config": 1411 pass # deprecated 1412 elif o in ("-k", "--package_key"): 1413 OPTIONS.package_key = a 1414 elif o in ("-i", "--incremental_from"): 1415 OPTIONS.incremental_source = a 1416 elif o in ("-w", "--wipe_user_data"): 1417 OPTIONS.wipe_user_data = True 1418 elif o in ("-n", "--no_prereq"): 1419 OPTIONS.omit_prereq = True 1420 elif o in ("-o", "--oem_settings"): 1421 OPTIONS.oem_source = a 1422 elif o in ("-e", "--extra_script"): 1423 OPTIONS.extra_script = a 1424 elif o in ("-a", "--aslr_mode"): 1425 if a in ("on", "On", "true", "True", "yes", "Yes"): 1426 OPTIONS.aslr_mode = True 1427 else: 1428 OPTIONS.aslr_mode = False 1429 elif o in ("-t", "--worker_threads"): 1430 if a.isdigit(): 1431 OPTIONS.worker_threads = int(a) 1432 else: 1433 raise ValueError("Cannot parse value %r for option %r - only " 1434 "integers are allowed." % (a, o)) 1435 elif o in ("-2", "--two_step"): 1436 OPTIONS.two_step = True 1437 elif o == "--no_signing": 1438 OPTIONS.no_signing = True 1439 elif o in ("--verify"): 1440 OPTIONS.verify = True 1441 elif o == "--block": 1442 OPTIONS.block_based = True 1443 elif o in ("-b", "--binary"): 1444 OPTIONS.updater_binary = a 1445 elif o in ("--no_fallback_to_full",): 1446 OPTIONS.fallback_to_full = False 1447 else: 1448 return False 1449 return True 1450 1451 args = common.ParseOptions(argv, __doc__, 1452 extra_opts="b:k:i:d:wne:t:a:2o:", 1453 extra_long_opts=["board_config=", 1454 "package_key=", 1455 "incremental_from=", 1456 "wipe_user_data", 1457 "no_prereq", 1458 "extra_script=", 1459 "worker_threads=", 1460 "aslr_mode=", 1461 "two_step", 1462 "no_signing", 1463 "block", 1464 "binary=", 1465 "oem_settings=", 1466 "verify", 1467 "no_fallback_to_full", 1468 ], 1469 extra_option_handler=option_handler) 1470 1471 if len(args) != 2: 1472 common.Usage(__doc__) 1473 sys.exit(1) 1474 1475 if OPTIONS.extra_script is not None: 1476 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 1477 1478 print "unzipping target target-files..." 1479 OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0]) 1480 1481 OPTIONS.target_tmp = OPTIONS.input_tmp 1482 OPTIONS.info_dict = common.LoadInfoDict(input_zip) 1483 1484 # If this image was originally labelled with SELinux contexts, make sure we 1485 # also apply the labels in our new image. During building, the "file_contexts" 1486 # is in the out/ directory tree, but for repacking from target-files.zip it's 1487 # in the root directory of the ramdisk. 1488 if "selinux_fc" in OPTIONS.info_dict: 1489 OPTIONS.info_dict["selinux_fc"] = os.path.join(OPTIONS.input_tmp, "BOOT", "RAMDISK", 1490 "file_contexts") 1491 1492 if OPTIONS.verbose: 1493 print "--- target info ---" 1494 common.DumpInfoDict(OPTIONS.info_dict) 1495 1496 # If the caller explicitly specified the device-specific extensions 1497 # path via -s/--device_specific, use that. Otherwise, use 1498 # META/releasetools.py if it is present in the target target_files. 1499 # Otherwise, take the path of the file from 'tool_extensions' in the 1500 # info dict and look for that in the local filesystem, relative to 1501 # the current directory. 1502 1503 if OPTIONS.device_specific is None: 1504 from_input = os.path.join(OPTIONS.input_tmp, "META", "releasetools.py") 1505 if os.path.exists(from_input): 1506 print "(using device-specific extensions from target_files)" 1507 OPTIONS.device_specific = from_input 1508 else: 1509 OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None) 1510 1511 if OPTIONS.device_specific is not None: 1512 OPTIONS.device_specific = os.path.abspath(OPTIONS.device_specific) 1513 1514 while True: 1515 1516 if OPTIONS.no_signing: 1517 if os.path.exists(args[1]): os.unlink(args[1]) 1518 output_zip = zipfile.ZipFile(args[1], "w", compression=zipfile.ZIP_DEFLATED) 1519 else: 1520 temp_zip_file = tempfile.NamedTemporaryFile() 1521 output_zip = zipfile.ZipFile(temp_zip_file, "w", 1522 compression=zipfile.ZIP_DEFLATED) 1523 1524 if OPTIONS.incremental_source is None: 1525 WriteFullOTAPackage(input_zip, output_zip) 1526 if OPTIONS.package_key is None: 1527 OPTIONS.package_key = OPTIONS.info_dict.get( 1528 "default_system_dev_certificate", 1529 "build/target/product/security/testkey") 1530 break 1531 1532 else: 1533 print "unzipping source target-files..." 1534 OPTIONS.source_tmp, source_zip = common.UnzipTemp(OPTIONS.incremental_source) 1535 OPTIONS.target_info_dict = OPTIONS.info_dict 1536 OPTIONS.source_info_dict = common.LoadInfoDict(source_zip) 1537 if "selinux_fc" in OPTIONS.source_info_dict: 1538 OPTIONS.source_info_dict["selinux_fc"] = os.path.join(OPTIONS.source_tmp, "BOOT", "RAMDISK", 1539 "file_contexts") 1540 if OPTIONS.package_key is None: 1541 OPTIONS.package_key = OPTIONS.source_info_dict.get( 1542 "default_system_dev_certificate", 1543 "build/target/product/security/testkey") 1544 if OPTIONS.verbose: 1545 print "--- source info ---" 1546 common.DumpInfoDict(OPTIONS.source_info_dict) 1547 try: 1548 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) 1549 break 1550 except ValueError: 1551 if not OPTIONS.fallback_to_full: raise 1552 print "--- failed to build incremental; falling back to full ---" 1553 OPTIONS.incremental_source = None 1554 output_zip.close() 1555 1556 output_zip.close() 1557 1558 if not OPTIONS.no_signing: 1559 SignOutput(temp_zip_file.name, args[1]) 1560 temp_zip_file.close() 1561 1562 print "done." 1563 1564 1565if __name__ == '__main__': 1566 try: 1567 common.CloseInheritedPipes() 1568 main(sys.argv[1:]) 1569 except common.ExternalError, e: 1570 print 1571 print " ERROR: %s" % (e,) 1572 print 1573 sys.exit(1) 1574 finally: 1575 common.Cleanup() 1576