ota_from_target_files.py revision f83400896de5e8bc2b21a0dce98382d9ed0ab537
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 --board_config <file> 25 Deprecated. 26 27 -k (--package_key) <key> Key to use to sign the package (default is 28 the value of default_system_dev_certificate from the input 29 target-files's META/misc_info.txt, or 30 "build/target/product/security/testkey" if that value is not 31 specified). 32 33 For incremental OTAs, the default value is based on the source 34 target-file, not the target build. 35 36 -i (--incremental_from) <file> 37 Generate an incremental OTA using the given target-files zip as 38 the starting build. 39 40 -o (--oem_settings) <file> 41 Use the file to specify the expected OEM-specific properties 42 on the OEM partition of the intended device. 43 44 -w (--wipe_user_data) 45 Generate an OTA package that will wipe the user data partition 46 when installed. 47 48 -n (--no_prereq) 49 Omit the timestamp prereq check normally included at the top of 50 the build scripts (used for developer OTA packages which 51 legitimately need to go back and forth). 52 53 -e (--extra_script) <file> 54 Insert the contents of file at the end of the update script. 55 56 -a (--aslr_mode) <on|off> 57 Specify whether to turn on ASLR for the package (on by default). 58 59 -2 (--two_step) 60 Generate a 'two-step' OTA package, where recovery is updated 61 first, so that any changes made to the system partition are done 62 using the new recovery (new kernel, etc.). 63 64 --block 65 Generate a block-based OTA if possible. Will fall back to a 66 file-based OTA if the target_files is older and doesn't support 67 block-based OTAs. 68 69 -b (--binary) <file> 70 Use the given binary as the update-binary in the output package, 71 instead of the binary in the build's target_files. Use for 72 development only. 73 74 -t (--worker_threads) <int> 75 Specifies the number of worker-threads that will be used when 76 generating patches for incremental updates (defaults to 3). 77 78""" 79 80import sys 81 82if sys.hexversion < 0x02070000: 83 print >> sys.stderr, "Python 2.7 or newer is required." 84 sys.exit(1) 85 86import copy 87import errno 88import os 89import re 90import subprocess 91import tempfile 92import time 93import zipfile 94 95try: 96 from hashlib import sha1 as sha1 97except ImportError: 98 from sha import sha as sha1 99 100import common 101import edify_generator 102import build_image 103 104OPTIONS = common.OPTIONS 105OPTIONS.package_key = None 106OPTIONS.incremental_source = None 107OPTIONS.require_verbatim = set() 108OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 109OPTIONS.patch_threshold = 0.95 110OPTIONS.wipe_user_data = False 111OPTIONS.omit_prereq = False 112OPTIONS.extra_script = None 113OPTIONS.aslr_mode = True 114OPTIONS.worker_threads = 3 115OPTIONS.two_step = False 116OPTIONS.no_signing = False 117OPTIONS.block_based = False 118OPTIONS.updater_binary = None 119OPTIONS.oem_source = None 120OPTIONS.fallback_to_full = True 121 122def MostPopularKey(d, default): 123 """Given a dict, return the key corresponding to the largest 124 value. Returns 'default' if the dict is empty.""" 125 x = [(v, k) for (k, v) in d.iteritems()] 126 if not x: return default 127 x.sort() 128 return x[-1][1] 129 130 131def IsSymlink(info): 132 """Return true if the zipfile.ZipInfo object passed in represents a 133 symlink.""" 134 return (info.external_attr >> 16) == 0120777 135 136def IsRegular(info): 137 """Return true if the zipfile.ZipInfo object passed in represents a 138 symlink.""" 139 return (info.external_attr >> 28) == 010 140 141def ClosestFileMatch(src, tgtfiles, existing): 142 """Returns the closest file match between a source file and list 143 of potential matches. The exact filename match is preferred, 144 then the sha1 is searched for, and finally a file with the same 145 basename is evaluated. Rename support in the updater-binary is 146 required for the latter checks to be used.""" 147 148 result = tgtfiles.get("path:" + src.name) 149 if result is not None: 150 return result 151 152 if not OPTIONS.target_info_dict.get("update_rename_support", False): 153 return None 154 155 if src.size < 1000: 156 return None 157 158 result = tgtfiles.get("sha1:" + src.sha1) 159 if result is not None and existing.get(result.name) is None: 160 return result 161 result = tgtfiles.get("file:" + src.name.split("/")[-1]) 162 if result is not None and existing.get(result.name) is None: 163 return result 164 return None 165 166class ItemSet: 167 def __init__(self, partition, fs_config): 168 self.partition = partition 169 self.fs_config = fs_config 170 self.ITEMS = {} 171 172 def Get(self, name, dir=False): 173 if name not in self.ITEMS: 174 self.ITEMS[name] = Item(self, name, dir=dir) 175 return self.ITEMS[name] 176 177 def GetMetadata(self, input_zip): 178 # The target_files contains a record of what the uid, 179 # gid, and mode are supposed to be. 180 output = input_zip.read(self.fs_config) 181 182 for line in output.split("\n"): 183 if not line: continue 184 columns = line.split() 185 name, uid, gid, mode = columns[:4] 186 selabel = None 187 capabilities = None 188 189 # After the first 4 columns, there are a series of key=value 190 # pairs. Extract out the fields we care about. 191 for element in columns[4:]: 192 key, value = element.split("=") 193 if key == "selabel": 194 selabel = value 195 if key == "capabilities": 196 capabilities = value 197 198 i = self.ITEMS.get(name, None) 199 if i is not None: 200 i.uid = int(uid) 201 i.gid = int(gid) 202 i.mode = int(mode, 8) 203 i.selabel = selabel 204 i.capabilities = capabilities 205 if i.dir: 206 i.children.sort(key=lambda i: i.name) 207 208 # set metadata for the files generated by this script. 209 i = self.ITEMS.get("system/recovery-from-boot.p", None) 210 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0644, None, None 211 i = self.ITEMS.get("system/etc/install-recovery.sh", None) 212 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0544, None, None 213 214 215class Item: 216 """Items represent the metadata (user, group, mode) of files and 217 directories in the system image.""" 218 def __init__(self, itemset, name, dir=False): 219 self.itemset = itemset 220 self.name = name 221 self.uid = None 222 self.gid = None 223 self.mode = None 224 self.selabel = None 225 self.capabilities = None 226 self.dir = dir 227 228 if name: 229 self.parent = itemset.Get(os.path.dirname(name), dir=True) 230 self.parent.children.append(self) 231 else: 232 self.parent = None 233 if dir: 234 self.children = [] 235 236 def Dump(self, indent=0): 237 if self.uid is not None: 238 print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode) 239 else: 240 print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode) 241 if self.dir: 242 print "%s%s" % (" "*indent, self.descendants) 243 print "%s%s" % (" "*indent, self.best_subtree) 244 for i in self.children: 245 i.Dump(indent=indent+1) 246 247 def CountChildMetadata(self): 248 """Count up the (uid, gid, mode, selabel, capabilities) tuples for 249 all children and determine the best strategy for using set_perm_recursive and 250 set_perm to correctly chown/chmod all the files to their desired 251 values. Recursively calls itself for all descendants. 252 253 Returns a dict of {(uid, gid, dmode, fmode, selabel, capabilities): count} counting up 254 all descendants of this node. (dmode or fmode may be None.) Also 255 sets the best_subtree of each directory Item to the (uid, gid, 256 dmode, fmode, selabel, capabilities) tuple that will match the most 257 descendants of that Item. 258 """ 259 260 assert self.dir 261 d = self.descendants = {(self.uid, self.gid, self.mode, None, self.selabel, self.capabilities): 1} 262 for i in self.children: 263 if i.dir: 264 for k, v in i.CountChildMetadata().iteritems(): 265 d[k] = d.get(k, 0) + v 266 else: 267 k = (i.uid, i.gid, None, i.mode, i.selabel, i.capabilities) 268 d[k] = d.get(k, 0) + 1 269 270 # Find the (uid, gid, dmode, fmode, selabel, capabilities) 271 # tuple that matches the most descendants. 272 273 # First, find the (uid, gid) pair that matches the most 274 # descendants. 275 ug = {} 276 for (uid, gid, _, _, _, _), count in d.iteritems(): 277 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 278 ug = MostPopularKey(ug, (0, 0)) 279 280 # Now find the dmode, fmode, selabel, and capabilities that match 281 # the most descendants with that (uid, gid), and choose those. 282 best_dmode = (0, 0755) 283 best_fmode = (0, 0644) 284 best_selabel = (0, None) 285 best_capabilities = (0, None) 286 for k, count in d.iteritems(): 287 if k[:2] != ug: continue 288 if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2]) 289 if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3]) 290 if k[4] is not None and count >= best_selabel[0]: best_selabel = (count, k[4]) 291 if k[5] is not None and count >= best_capabilities[0]: best_capabilities = (count, k[5]) 292 self.best_subtree = ug + (best_dmode[1], best_fmode[1], best_selabel[1], best_capabilities[1]) 293 294 return d 295 296 def SetPermissions(self, script): 297 """Append set_perm/set_perm_recursive commands to 'script' to 298 set all permissions, users, and groups for the tree of files 299 rooted at 'self'.""" 300 301 self.CountChildMetadata() 302 303 def recurse(item, current): 304 # current is the (uid, gid, dmode, fmode, selabel, capabilities) tuple that the current 305 # item (and all its children) have already been set to. We only 306 # need to issue set_perm/set_perm_recursive commands if we're 307 # supposed to be something different. 308 if item.dir: 309 if current != item.best_subtree: 310 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 311 current = item.best_subtree 312 313 if item.uid != current[0] or item.gid != current[1] or \ 314 item.mode != current[2] or item.selabel != current[4] or \ 315 item.capabilities != current[5]: 316 script.SetPermissions("/"+item.name, item.uid, item.gid, 317 item.mode, item.selabel, item.capabilities) 318 319 for i in item.children: 320 recurse(i, current) 321 else: 322 if item.uid != current[0] or item.gid != current[1] or \ 323 item.mode != current[3] or item.selabel != current[4] or \ 324 item.capabilities != current[5]: 325 script.SetPermissions("/"+item.name, item.uid, item.gid, 326 item.mode, item.selabel, item.capabilities) 327 328 recurse(self, (-1, -1, -1, -1, None, None)) 329 330 331def CopyPartitionFiles(itemset, input_zip, output_zip=None, substitute=None): 332 """Copies files for the partition in the input zip to the output 333 zip. Populates the Item class with their metadata, and returns a 334 list of symlinks. output_zip may be None, in which case the copy is 335 skipped (but the other side effects still happen). substitute is an 336 optional dict of {output filename: contents} to be output instead of 337 certain input files. 338 """ 339 340 symlinks = [] 341 342 partition = itemset.partition 343 344 for info in input_zip.infolist(): 345 if info.filename.startswith(partition.upper() + "/"): 346 basefilename = info.filename[7:] 347 if IsSymlink(info): 348 symlinks.append((input_zip.read(info.filename), 349 "/" + partition + "/" + basefilename)) 350 else: 351 info2 = copy.copy(info) 352 fn = info2.filename = partition + "/" + basefilename 353 if substitute and fn in substitute and substitute[fn] is None: 354 continue 355 if output_zip is not None: 356 if substitute and fn in substitute: 357 data = substitute[fn] 358 else: 359 data = input_zip.read(info.filename) 360 output_zip.writestr(info2, data) 361 if fn.endswith("/"): 362 itemset.Get(fn[:-1], dir=True) 363 else: 364 itemset.Get(fn, dir=False) 365 366 symlinks.sort() 367 return symlinks 368 369 370def SignOutput(temp_zip_name, output_zip_name): 371 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 372 pw = key_passwords[OPTIONS.package_key] 373 374 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 375 whole_file=True) 376 377 378def AppendAssertions(script, info_dict, oem_dict = None): 379 oem_props = info_dict.get("oem_fingerprint_properties") 380 if oem_props is None: 381 device = GetBuildProp("ro.product.device", info_dict) 382 script.AssertDevice(device) 383 else: 384 if oem_dict is None: 385 raise common.ExternalError("No OEM file provided to answer expected assertions") 386 for prop in oem_props.split(): 387 if oem_dict.get(prop) is None: 388 raise common.ExternalError("The OEM file is missing the property %s" % prop) 389 script.AssertOemProperty(prop, oem_dict.get(prop)) 390 391 392def HasRecoveryPatch(target_files_zip): 393 try: 394 target_files_zip.getinfo("SYSTEM/recovery-from-boot.p") 395 return True 396 except KeyError: 397 return False 398 399def HasVendorPartition(target_files_zip): 400 try: 401 target_files_zip.getinfo("VENDOR/") 402 return True 403 except KeyError: 404 return False 405 406def GetOemProperty(name, oem_props, oem_dict, info_dict): 407 if oem_props is not None and name in oem_props: 408 return oem_dict[name] 409 return GetBuildProp(name, info_dict) 410 411 412def CalculateFingerprint(oem_props, oem_dict, info_dict): 413 if oem_props is None: 414 return GetBuildProp("ro.build.fingerprint", info_dict) 415 return "%s/%s/%s:%s" % ( 416 GetOemProperty("ro.product.brand", oem_props, oem_dict, info_dict), 417 GetOemProperty("ro.product.name", oem_props, oem_dict, info_dict), 418 GetOemProperty("ro.product.device", oem_props, oem_dict, info_dict), 419 GetBuildProp("ro.build.thumbprint", info_dict)) 420 421def GetImage(which, tmpdir, info_dict): 422 # Return (mapdata, data) for the given image. which should be 423 # "system" or "vendor". 424 425 assert which in ("system", "vendor") 426 427 path = os.path.join(tmpdir, "IMAGES", which + ".img") 428 if os.path.exists(path): 429 print "using %s.img from target-files" % (which,) 430 431 # This is a 'new' target-files, which already has the image in it. 432 # The image is a sparse image, though, so we need to unsparse it 433 # and extract the map data. 434 435 success, name = build_image.UnsparseImage(path, replace=False) 436 if not success: 437 assert False, "unsparsing " + which + ".img failed" 438 439 mmap = tempfile.NamedTemporaryFile() 440 mimg = tempfile.NamedTemporaryFile(delete=False) 441 success = build_image.MappedUnsparseImage( 442 path, name, mmap.name, mimg.name) 443 if not success: 444 assert False, "creating sparse map failed" 445 os.unlink(name) 446 name = mimg.name 447 448 with open(mmap.name) as f: 449 mapdata = f.read() 450 451 try: 452 with open(name) as f: 453 data = f.read() 454 finally: 455 os.unlink(name) 456 457 print "unsparsed data sha1 is " + sha1(data).hexdigest() 458 return mapdata, data 459 460 else: 461 print "building %s.img from target-files" % (which,) 462 463 # This is an 'old' target-files, which does not contain images 464 # already built. Build them. 465 466 import add_img_to_target_files 467 if which == "system": 468 mapdata, data = add_img_to_target_files.BuildSystem( 469 tmpdir, info_dict, sparse=False, map_file=True) 470 elif which == "vendor": 471 mapdata, data = add_img_to_target_files.BuildVendor( 472 tmpdir, info_dict, sparse=False, map_file=True) 473 474 print "built data sha1 is " + sha1(data).hexdigest() 475 return mapdata, data 476 477 478def WriteFullOTAPackage(input_zip, output_zip): 479 # TODO: how to determine this? We don't know what version it will 480 # be installed on top of. For now, we expect the API just won't 481 # change very often. 482 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 483 484 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 485 oem_dict = None 486 if oem_props is not None: 487 if OPTIONS.oem_source is None: 488 raise common.ExternalError("OEM source required for this build") 489 script.Mount("/oem") 490 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 491 492 metadata = {"post-build": CalculateFingerprint( 493 oem_props, oem_dict, OPTIONS.info_dict), 494 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 495 OPTIONS.info_dict), 496 "post-timestamp": GetBuildProp("ro.build.date.utc", 497 OPTIONS.info_dict), 498 } 499 500 device_specific = common.DeviceSpecificParams( 501 input_zip=input_zip, 502 input_version=OPTIONS.info_dict["recovery_api_version"], 503 output_zip=output_zip, 504 script=script, 505 input_tmp=OPTIONS.input_tmp, 506 metadata=metadata, 507 info_dict=OPTIONS.info_dict) 508 509 has_recovery_patch = HasRecoveryPatch(input_zip) 510 block_based = OPTIONS.block_based and has_recovery_patch 511 512 if not OPTIONS.omit_prereq: 513 ts = GetBuildProp("ro.build.date.utc", OPTIONS.info_dict) 514 ts_text = GetBuildProp("ro.build.date", OPTIONS.info_dict) 515 script.AssertOlderBuild(ts, ts_text) 516 517 AppendAssertions(script, OPTIONS.info_dict, oem_dict) 518 device_specific.FullOTA_Assertions() 519 520 # Two-step package strategy (in chronological order, which is *not* 521 # the order in which the generated script has things): 522 # 523 # if stage is not "2/3" or "3/3": 524 # write recovery image to boot partition 525 # set stage to "2/3" 526 # reboot to boot partition and restart recovery 527 # else if stage is "2/3": 528 # write recovery image to recovery partition 529 # set stage to "3/3" 530 # reboot to recovery partition and restart recovery 531 # else: 532 # (stage must be "3/3") 533 # set stage to "" 534 # do normal full package installation: 535 # wipe and install system, boot image, etc. 536 # set up system to update recovery partition on first boot 537 # complete script normally (allow recovery to mark itself finished and reboot) 538 539 recovery_img = common.GetBootableImage("recovery.img", "recovery.img", 540 OPTIONS.input_tmp, "RECOVERY") 541 if OPTIONS.two_step: 542 if not OPTIONS.info_dict.get("multistage_support", None): 543 assert False, "two-step packages not supported by this build" 544 fs = OPTIONS.info_dict["fstab"]["/misc"] 545 assert fs.fs_type.upper() == "EMMC", \ 546 "two-step packages only supported on devices with EMMC /misc partitions" 547 bcb_dev = {"bcb_dev": fs.device} 548 common.ZipWriteStr(output_zip, "recovery.img", recovery_img.data) 549 script.AppendExtra(""" 550if get_stage("%(bcb_dev)s", "stage") == "2/3" then 551""" % bcb_dev) 552 script.WriteRawImage("/recovery", "recovery.img") 553 script.AppendExtra(""" 554set_stage("%(bcb_dev)s", "3/3"); 555reboot_now("%(bcb_dev)s", "recovery"); 556else if get_stage("%(bcb_dev)s", "stage") == "3/3" then 557""" % bcb_dev) 558 559 device_specific.FullOTA_InstallBegin() 560 561 system_progress = 0.75 562 563 if OPTIONS.wipe_user_data: 564 system_progress -= 0.1 565 if HasVendorPartition(input_zip): 566 system_progress -= 0.1 567 568 if "selinux_fc" in OPTIONS.info_dict: 569 WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip) 570 571 system_items = ItemSet("system", "META/filesystem_config.txt") 572 script.ShowProgress(system_progress, 0) 573 if block_based: 574 mapdata, data = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict) 575 576 common.ZipWriteStr(output_zip, "system.map", mapdata) 577 common.ZipWriteStr(output_zip, "system.muimg", data) 578 script.WipeBlockDevice("/system") 579 script.WriteRawImage("/system", "system.muimg", mapfn="system.map") 580 else: 581 script.FormatPartition("/system") 582 script.Mount("/system") 583 if not has_recovery_patch: 584 script.UnpackPackageDir("recovery", "/system") 585 script.UnpackPackageDir("system", "/system") 586 587 symlinks = CopyPartitionFiles(system_items, input_zip, output_zip) 588 script.MakeSymlinks(symlinks) 589 590 boot_img = common.GetBootableImage("boot.img", "boot.img", 591 OPTIONS.input_tmp, "BOOT") 592 593 if not block_based: 594 def output_sink(fn, data): 595 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 596 system_items.Get("system/" + fn, dir=False) 597 598 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, 599 recovery_img, boot_img) 600 601 system_items.GetMetadata(input_zip) 602 system_items.Get("system").SetPermissions(script) 603 604 if HasVendorPartition(input_zip): 605 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 606 script.ShowProgress(0.1, 0) 607 608 if block_based: 609 mapdata, data = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict) 610 611 common.ZipWriteStr(output_zip, "vendor.map", mapdata) 612 common.ZipWriteStr(output_zip, "vendor.muimg", data) 613 script.WipeBlockDevice("/vendor") 614 script.WriteRawImage("/vendor", "vendor.muimg", mapfn="vendor.map") 615 else: 616 script.FormatPartition("/vendor") 617 script.Mount("/vendor") 618 script.UnpackPackageDir("vendor", "/vendor") 619 620 symlinks = CopyPartitionFiles(vendor_items, input_zip, output_zip) 621 script.MakeSymlinks(symlinks) 622 623 vendor_items.GetMetadata(input_zip) 624 vendor_items.Get("vendor").SetPermissions(script) 625 626 common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict) 627 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 628 629 script.ShowProgress(0.05, 5) 630 script.WriteRawImage("/boot", "boot.img") 631 632 script.ShowProgress(0.2, 10) 633 device_specific.FullOTA_InstallEnd() 634 635 if OPTIONS.extra_script is not None: 636 script.AppendExtra(OPTIONS.extra_script) 637 638 script.UnmountAll() 639 640 if OPTIONS.wipe_user_data: 641 script.ShowProgress(0.1, 10) 642 script.FormatPartition("/data") 643 644 if OPTIONS.two_step: 645 script.AppendExtra(""" 646set_stage("%(bcb_dev)s", ""); 647""" % bcb_dev) 648 script.AppendExtra("else\n") 649 script.WriteRawImage("/boot", "recovery.img") 650 script.AppendExtra(""" 651set_stage("%(bcb_dev)s", "2/3"); 652reboot_now("%(bcb_dev)s", ""); 653endif; 654endif; 655""" % bcb_dev) 656 script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary) 657 WriteMetadata(metadata, output_zip) 658 659def WritePolicyConfig(file_context, output_zip): 660 f = open(file_context, 'r'); 661 basename = os.path.basename(file_context) 662 common.ZipWriteStr(output_zip, basename, f.read()) 663 664 665def WriteMetadata(metadata, output_zip): 666 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata", 667 "".join(["%s=%s\n" % kv 668 for kv in sorted(metadata.iteritems())])) 669 670def LoadPartitionFiles(z, partition): 671 """Load all the files from the given partition in a given target-files 672 ZipFile, and return a dict of {filename: File object}.""" 673 out = {} 674 prefix = partition.upper() + "/" 675 for info in z.infolist(): 676 if info.filename.startswith(prefix) and not IsSymlink(info): 677 basefilename = info.filename[7:] 678 fn = partition + "/" + basefilename 679 data = z.read(info.filename) 680 out[fn] = common.File(fn, data) 681 return out 682 683 684def GetBuildProp(prop, info_dict): 685 """Return the fingerprint of the build of a given target-files info_dict.""" 686 try: 687 return info_dict.get("build.prop", {})[prop] 688 except KeyError: 689 raise common.ExternalError("couldn't find %s in build.prop" % (prop,)) 690 691def AddToKnownPaths(filename, known_paths): 692 if filename[-1] == "/": 693 return 694 dirs = filename.split("/")[:-1] 695 while len(dirs) > 0: 696 path = "/".join(dirs) 697 if path in known_paths: 698 break; 699 known_paths.add(path) 700 dirs.pop() 701 702class BlockDifference: 703 def __init__(self, partition, output_zip): 704 with tempfile.NamedTemporaryFile() as src_file: 705 with tempfile.NamedTemporaryFile() as tgt_file: 706 print "building source " + partition + " image..." 707 src_file = tempfile.NamedTemporaryFile() 708 src_mapdata, src_data = GetImage(partition, 709 OPTIONS.source_tmp, 710 OPTIONS.source_info_dict) 711 712 self.src_sha1 = sha1(src_data).hexdigest() 713 print "source " + partition + " sha1:", self.src_sha1 714 src_file.write(src_data) 715 716 print "building target " + partition + " image..." 717 tgt_file = tempfile.NamedTemporaryFile() 718 tgt_mapdata, tgt_data = GetImage(partition, 719 OPTIONS.target_tmp, 720 OPTIONS.target_info_dict) 721 self.tgt_sha1 = sha1(tgt_data).hexdigest() 722 print "target " + partition + " sha1:", self.tgt_sha1 723 tgt_len = len(tgt_data) 724 tgt_file.write(tgt_data) 725 726 system_type, self.device = common.GetTypeAndDevice("/" + partition, 727 OPTIONS.info_dict) 728 self.patch = common.MakePartitionPatch(src_file, tgt_file, partition) 729 730 TestBlockPatch(src_data, src_mapdata, self.patch.data, 731 tgt_mapdata, self.tgt_sha1) 732 src_data = None 733 tgt_data = None 734 735 self.patch.AddToZip(output_zip, compression=zipfile.ZIP_STORED) 736 self.src_mapfilename = self.patch.name + ".src.map" 737 common.ZipWriteStr(output_zip, self.src_mapfilename, src_mapdata) 738 self.tgt_mapfilename = self.patch.name + ".tgt.map" 739 common.ZipWriteStr(output_zip, self.tgt_mapfilename, tgt_mapdata) 740 741def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 742 source_version = OPTIONS.source_info_dict["recovery_api_version"] 743 target_version = OPTIONS.target_info_dict["recovery_api_version"] 744 745 if source_version == 0: 746 print ("WARNING: generating edify script for a source that " 747 "can't install it.") 748 script = edify_generator.EdifyGenerator(source_version, 749 OPTIONS.target_info_dict) 750 751 metadata = {"pre-device": GetBuildProp("ro.product.device", 752 OPTIONS.source_info_dict), 753 "post-timestamp": GetBuildProp("ro.build.date.utc", 754 OPTIONS.target_info_dict), 755 } 756 757 device_specific = common.DeviceSpecificParams( 758 source_zip=source_zip, 759 source_version=source_version, 760 target_zip=target_zip, 761 target_version=target_version, 762 output_zip=output_zip, 763 script=script, 764 metadata=metadata, 765 info_dict=OPTIONS.info_dict) 766 767 source_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.source_info_dict) 768 target_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.target_info_dict) 769 metadata["pre-build"] = source_fp 770 metadata["post-build"] = target_fp 771 772 source_boot = common.GetBootableImage( 773 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 774 OPTIONS.source_info_dict) 775 target_boot = common.GetBootableImage( 776 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 777 updating_boot = (not OPTIONS.two_step and 778 (source_boot.data != target_boot.data)) 779 780 source_recovery = common.GetBootableImage( 781 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 782 OPTIONS.source_info_dict) 783 target_recovery = common.GetBootableImage( 784 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 785 updating_recovery = (source_recovery.data != target_recovery.data) 786 787 system_diff = BlockDifference("system", output_zip) 788 if HasVendorPartition(target_zip): 789 if not HasVendorPartition(source_zip): 790 raise RuntimeError("can't generate incremental that adds /vendor") 791 vendor_diff = BlockDifference("vendor", output_zip) 792 793 oem_props = OPTIONS.target_info_dict.get("oem_fingerprint_properties") 794 oem_dict = None 795 if oem_props is not None: 796 if OPTIONS.oem_source is None: 797 raise common.ExternalError("OEM source required for this build") 798 script.Mount("/oem") 799 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 800 801 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 802 device_specific.IncrementalOTA_Assertions() 803 804 # Two-step incremental package strategy (in chronological order, 805 # which is *not* the order in which the generated script has 806 # things): 807 # 808 # if stage is not "2/3" or "3/3": 809 # do verification on current system 810 # write recovery image to boot partition 811 # set stage to "2/3" 812 # reboot to boot partition and restart recovery 813 # else if stage is "2/3": 814 # write recovery image to recovery partition 815 # set stage to "3/3" 816 # reboot to recovery partition and restart recovery 817 # else: 818 # (stage must be "3/3") 819 # perform update: 820 # patch system files, etc. 821 # force full install of new boot image 822 # set up system to update recovery partition on first boot 823 # complete script normally (allow recovery to mark itself finished and reboot) 824 825 if OPTIONS.two_step: 826 if not OPTIONS.info_dict.get("multistage_support", None): 827 assert False, "two-step packages not supported by this build" 828 fs = OPTIONS.info_dict["fstab"]["/misc"] 829 assert fs.fs_type.upper() == "EMMC", \ 830 "two-step packages only supported on devices with EMMC /misc partitions" 831 bcb_dev = {"bcb_dev": fs.device} 832 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 833 script.AppendExtra(""" 834if get_stage("%(bcb_dev)s", "stage") == "2/3" then 835""" % bcb_dev) 836 script.AppendExtra("sleep(20);\n"); 837 script.WriteRawImage("/recovery", "recovery.img") 838 script.AppendExtra(""" 839set_stage("%(bcb_dev)s", "3/3"); 840reboot_now("%(bcb_dev)s", "recovery"); 841else if get_stage("%(bcb_dev)s", "stage") != "3/3" then 842""" % bcb_dev) 843 844 script.Print("Verifying current system...") 845 846 device_specific.IncrementalOTA_VerifyBegin() 847 848 if oem_props is None: 849 script.AssertSomeFingerprint(source_fp, target_fp) 850 else: 851 script.AssertSomeThumbprint( 852 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 853 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 854 855 if updating_boot: 856 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 857 d = common.Difference(target_boot, source_boot) 858 _, _, d = d.ComputePatch() 859 if d is None: 860 include_full_boot = True 861 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 862 else: 863 include_full_boot = False 864 865 print "boot target: %d source: %d diff: %d" % ( 866 target_boot.size, source_boot.size, len(d)) 867 868 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 869 870 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 871 (boot_type, boot_device, 872 source_boot.size, source_boot.sha1, 873 target_boot.size, target_boot.sha1)) 874 875 device_specific.IncrementalOTA_VerifyEnd() 876 877 if OPTIONS.two_step: 878 script.WriteRawImage("/boot", "recovery.img") 879 script.AppendExtra(""" 880set_stage("%(bcb_dev)s", "2/3"); 881reboot_now("%(bcb_dev)s", ""); 882else 883""" % bcb_dev) 884 885 script.Comment("---- start making changes here ----") 886 887 device_specific.IncrementalOTA_InstallBegin() 888 889 if HasVendorPartition(target_zip): 890 script.Print("Patching vendor image...") 891 script.ShowProgress(0.1, 0) 892 script.Syspatch(vendor_diff.device, 893 vendor_diff.tgt_mapfilename, vendor_diff.tgt_sha1, 894 vendor_diff.src_mapfilename, vendor_diff.src_sha1, 895 vendor_diff.patch.name) 896 sys_progress = 0.8 897 else: 898 sys_progress = 0.9 899 900 script.Print("Patching system image...") 901 script.ShowProgress(sys_progress, 0) 902 script.Syspatch(system_diff.device, 903 system_diff.tgt_mapfilename, system_diff.tgt_sha1, 904 system_diff.src_mapfilename, system_diff.src_sha1, 905 system_diff.patch.name) 906 907 if OPTIONS.two_step: 908 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 909 script.WriteRawImage("/boot", "boot.img") 910 print "writing full boot image (forced by two-step mode)" 911 912 if not OPTIONS.two_step: 913 if updating_boot: 914 if include_full_boot: 915 print "boot image changed; including full." 916 script.Print("Installing boot image...") 917 script.WriteRawImage("/boot", "boot.img") 918 else: 919 # Produce the boot image by applying a patch to the current 920 # contents of the boot partition, and write it back to the 921 # partition. 922 print "boot image changed; including patch." 923 script.Print("Patching boot image...") 924 script.ShowProgress(0.1, 10) 925 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 926 % (boot_type, boot_device, 927 source_boot.size, source_boot.sha1, 928 target_boot.size, target_boot.sha1), 929 "-", 930 target_boot.size, target_boot.sha1, 931 source_boot.sha1, "patch/boot.img.p") 932 else: 933 print "boot image unchanged; skipping." 934 935 # Do device-specific installation (eg, write radio image). 936 device_specific.IncrementalOTA_InstallEnd() 937 938 if OPTIONS.extra_script is not None: 939 script.AppendExtra(OPTIONS.extra_script) 940 941 if OPTIONS.wipe_user_data: 942 script.Print("Erasing user data...") 943 script.FormatPartition("/data") 944 945 if OPTIONS.two_step: 946 script.AppendExtra(""" 947set_stage("%(bcb_dev)s", ""); 948endif; 949endif; 950""" % bcb_dev) 951 952 script.SetProgress(1) 953 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 954 WriteMetadata(metadata, output_zip) 955 956def ParseMap(map_str): 957 x = map_str.split() 958 assert int(x[0]) == 4096 959 assert int(x[1]) == len(x)-2 960 return int(x[0]), [int(i) for i in x[2:]] 961 962def TestBlockPatch(src_muimg, src_map, patch_data, tgt_map, tgt_sha1): 963 src_blksize, src_regions = ParseMap(src_map) 964 tgt_blksize, tgt_regions = ParseMap(tgt_map) 965 966 with tempfile.NamedTemporaryFile() as src_file,\ 967 tempfile.NamedTemporaryFile() as patch_file,\ 968 tempfile.NamedTemporaryFile() as src_map_file,\ 969 tempfile.NamedTemporaryFile() as tgt_map_file: 970 971 src_total = sum(src_regions) * src_blksize 972 src_file.truncate(src_total) 973 p = 0 974 for i in range(0, len(src_regions), 2): 975 c, dc = src_regions[i:i+2] 976 src_file.write(src_muimg[p:(p+c*src_blksize)]) 977 p += c*src_blksize 978 src_file.seek(dc*src_blksize, 1) 979 assert src_file.tell() == src_total 980 981 patch_file.write(patch_data) 982 983 src_map_file.write(src_map) 984 tgt_map_file.write(tgt_map) 985 986 src_file.flush() 987 src_map_file.flush() 988 patch_file.flush() 989 tgt_map_file.flush() 990 991 p = common.Run(["syspatch_host", src_file.name, src_map_file.name, 992 patch_file.name, src_file.name, tgt_map_file.name], 993 stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 994 stdoutdata, _ = p.communicate() 995 if p.returncode != 0: 996 print stdoutdata 997 raise ValueError("failed to reconstruct target system image from patch") 998 999 h = sha1() 1000 src_file.seek(0, 0) 1001 for i in range(0, len(tgt_regions), 2): 1002 c, dc = tgt_regions[i:i+2] 1003 h.update(src_file.read(c*tgt_blksize)) 1004 src_file.seek(dc*tgt_blksize, 1) 1005 1006 if h.hexdigest() != tgt_sha1: 1007 raise ValueError("patch reconstructed incorrect target system image") 1008 1009 print "test of system image patch succeeded" 1010 1011 1012class FileDifference: 1013 def __init__(self, partition, source_zip, target_zip, output_zip): 1014 print "Loading target..." 1015 self.target_data = target_data = LoadPartitionFiles(target_zip, partition) 1016 print "Loading source..." 1017 self.source_data = source_data = LoadPartitionFiles(source_zip, partition) 1018 1019 self.verbatim_targets = verbatim_targets = [] 1020 self.patch_list = patch_list = [] 1021 diffs = [] 1022 self.renames = renames = {} 1023 known_paths = set() 1024 largest_source_size = 0 1025 1026 matching_file_cache = {} 1027 for fn, sf in source_data.items(): 1028 assert fn == sf.name 1029 matching_file_cache["path:" + fn] = sf 1030 if fn in target_data.keys(): 1031 AddToKnownPaths(fn, known_paths) 1032 # Only allow eligibility for filename/sha matching 1033 # if there isn't a perfect path match. 1034 if target_data.get(sf.name) is None: 1035 matching_file_cache["file:" + fn.split("/")[-1]] = sf 1036 matching_file_cache["sha:" + sf.sha1] = sf 1037 1038 for fn in sorted(target_data.keys()): 1039 tf = target_data[fn] 1040 assert fn == tf.name 1041 sf = ClosestFileMatch(tf, matching_file_cache, renames) 1042 if sf is not None and sf.name != tf.name: 1043 print "File has moved from " + sf.name + " to " + tf.name 1044 renames[sf.name] = tf 1045 1046 if sf is None or fn in OPTIONS.require_verbatim: 1047 # This file should be included verbatim 1048 if fn in OPTIONS.prohibit_verbatim: 1049 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 1050 print "send", fn, "verbatim" 1051 tf.AddToZip(output_zip) 1052 verbatim_targets.append((fn, tf.size)) 1053 if fn in target_data.keys(): 1054 AddToKnownPaths(fn, known_paths) 1055 elif tf.sha1 != sf.sha1: 1056 # File is different; consider sending as a patch 1057 diffs.append(common.Difference(tf, sf)) 1058 else: 1059 # Target file data identical to source (may still be renamed) 1060 pass 1061 1062 common.ComputeDifferences(diffs) 1063 1064 for diff in diffs: 1065 tf, sf, d = diff.GetPatch() 1066 path = "/".join(tf.name.split("/")[:-1]) 1067 if d is None or len(d) > tf.size * OPTIONS.patch_threshold or \ 1068 path not in known_paths: 1069 # patch is almost as big as the file; don't bother patching 1070 # or a patch + rename cannot take place due to the target 1071 # directory not existing 1072 tf.AddToZip(output_zip) 1073 verbatim_targets.append((tf.name, tf.size)) 1074 if sf.name in renames: 1075 del renames[sf.name] 1076 AddToKnownPaths(tf.name, known_paths) 1077 else: 1078 common.ZipWriteStr(output_zip, "patch/" + sf.name + ".p", d) 1079 patch_list.append((tf, sf, tf.size, common.sha1(d).hexdigest())) 1080 largest_source_size = max(largest_source_size, sf.size) 1081 1082 self.largest_source_size = largest_source_size 1083 1084 def EmitVerification(self, script): 1085 so_far = 0 1086 for tf, sf, size, patch_sha in self.patch_list: 1087 if tf.name != sf.name: 1088 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1089 script.PatchCheck("/"+sf.name, tf.sha1, sf.sha1) 1090 so_far += sf.size 1091 return so_far 1092 1093 def RemoveUnneededFiles(self, script, extras=()): 1094 script.DeleteFiles(["/"+i[0] for i in self.verbatim_targets] + 1095 ["/"+i for i in sorted(self.source_data) 1096 if i not in self.target_data and 1097 i not in self.renames] + 1098 list(extras)) 1099 1100 def TotalPatchSize(self): 1101 return sum(i[1].size for i in self.patch_list) 1102 1103 def EmitPatches(self, script, total_patch_size, so_far): 1104 self.deferred_patch_list = deferred_patch_list = [] 1105 for item in self.patch_list: 1106 tf, sf, size, _ = item 1107 if tf.name == "system/build.prop": 1108 deferred_patch_list.append(item) 1109 continue 1110 if (sf.name != tf.name): 1111 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1112 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p") 1113 so_far += tf.size 1114 script.SetProgress(so_far / total_patch_size) 1115 return so_far 1116 1117 def EmitDeferredPatches(self, script): 1118 for item in self.deferred_patch_list: 1119 tf, sf, size, _ = item 1120 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p") 1121 script.SetPermissions("/system/build.prop", 0, 0, 0644, None, None) 1122 1123 def EmitRenames(self, script): 1124 if len(self.renames) > 0: 1125 script.Print("Renaming files...") 1126 for src, tgt in self.renames.iteritems(): 1127 print "Renaming " + src + " to " + tgt.name 1128 script.RenameFile(src, tgt.name) 1129 1130 1131 1132 1133def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): 1134 target_has_recovery_patch = HasRecoveryPatch(target_zip) 1135 source_has_recovery_patch = HasRecoveryPatch(source_zip) 1136 1137 if (OPTIONS.block_based and 1138 target_has_recovery_patch and 1139 source_has_recovery_patch): 1140 return WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip) 1141 1142 source_version = OPTIONS.source_info_dict["recovery_api_version"] 1143 target_version = OPTIONS.target_info_dict["recovery_api_version"] 1144 1145 if source_version == 0: 1146 print ("WARNING: generating edify script for a source that " 1147 "can't install it.") 1148 script = edify_generator.EdifyGenerator(source_version, 1149 OPTIONS.target_info_dict) 1150 1151 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 1152 oem_dict = None 1153 if oem_props is not None: 1154 if OPTIONS.oem_source is None: 1155 raise common.ExternalError("OEM source required for this build") 1156 script.Mount("/oem") 1157 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 1158 1159 metadata = {"pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 1160 OPTIONS.source_info_dict), 1161 "post-timestamp": GetBuildProp("ro.build.date.utc", 1162 OPTIONS.target_info_dict), 1163 } 1164 1165 device_specific = common.DeviceSpecificParams( 1166 source_zip=source_zip, 1167 source_version=source_version, 1168 target_zip=target_zip, 1169 target_version=target_version, 1170 output_zip=output_zip, 1171 script=script, 1172 metadata=metadata, 1173 info_dict=OPTIONS.info_dict) 1174 1175 system_diff = FileDifference("system", source_zip, target_zip, output_zip) 1176 script.Mount("/system") 1177 if HasVendorPartition(target_zip): 1178 vendor_diff = FileDifference("vendor", source_zip, target_zip, output_zip) 1179 script.Mount("/vendor") 1180 else: 1181 vendor_diff = None 1182 1183 target_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.target_info_dict) 1184 source_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.source_info_dict) 1185 1186 if oem_props is None: 1187 script.AssertSomeFingerprint(source_fp, target_fp) 1188 else: 1189 script.AssertSomeThumbprint( 1190 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 1191 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 1192 1193 metadata["pre-build"] = source_fp 1194 metadata["post-build"] = target_fp 1195 1196 source_boot = common.GetBootableImage( 1197 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 1198 OPTIONS.source_info_dict) 1199 target_boot = common.GetBootableImage( 1200 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 1201 updating_boot = (not OPTIONS.two_step and 1202 (source_boot.data != target_boot.data)) 1203 1204 source_recovery = common.GetBootableImage( 1205 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 1206 OPTIONS.source_info_dict) 1207 target_recovery = common.GetBootableImage( 1208 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 1209 updating_recovery = (source_recovery.data != target_recovery.data) 1210 1211 # Here's how we divide up the progress bar: 1212 # 0.1 for verifying the start state (PatchCheck calls) 1213 # 0.8 for applying patches (ApplyPatch calls) 1214 # 0.1 for unpacking verbatim files, symlinking, and doing the 1215 # device-specific commands. 1216 1217 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 1218 device_specific.IncrementalOTA_Assertions() 1219 1220 # Two-step incremental package strategy (in chronological order, 1221 # which is *not* the order in which the generated script has 1222 # things): 1223 # 1224 # if stage is not "2/3" or "3/3": 1225 # do verification on current system 1226 # write recovery image to boot partition 1227 # set stage to "2/3" 1228 # reboot to boot partition and restart recovery 1229 # else if stage is "2/3": 1230 # write recovery image to recovery partition 1231 # set stage to "3/3" 1232 # reboot to recovery partition and restart recovery 1233 # else: 1234 # (stage must be "3/3") 1235 # perform update: 1236 # patch system files, etc. 1237 # force full install of new boot image 1238 # set up system to update recovery partition on first boot 1239 # complete script normally (allow recovery to mark itself finished and reboot) 1240 1241 if OPTIONS.two_step: 1242 if not OPTIONS.info_dict.get("multistage_support", None): 1243 assert False, "two-step packages not supported by this build" 1244 fs = OPTIONS.info_dict["fstab"]["/misc"] 1245 assert fs.fs_type.upper() == "EMMC", \ 1246 "two-step packages only supported on devices with EMMC /misc partitions" 1247 bcb_dev = {"bcb_dev": fs.device} 1248 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 1249 script.AppendExtra(""" 1250if get_stage("%(bcb_dev)s", "stage") == "2/3" then 1251""" % bcb_dev) 1252 script.AppendExtra("sleep(20);\n"); 1253 script.WriteRawImage("/recovery", "recovery.img") 1254 script.AppendExtra(""" 1255set_stage("%(bcb_dev)s", "3/3"); 1256reboot_now("%(bcb_dev)s", "recovery"); 1257else if get_stage("%(bcb_dev)s", "stage") != "3/3" then 1258""" % bcb_dev) 1259 1260 script.Print("Verifying current system...") 1261 1262 device_specific.IncrementalOTA_VerifyBegin() 1263 1264 script.ShowProgress(0.1, 0) 1265 so_far = system_diff.EmitVerification(script) 1266 if vendor_diff: 1267 so_far += vendor_diff.EmitVerification(script) 1268 1269 if updating_boot: 1270 d = common.Difference(target_boot, source_boot) 1271 _, _, d = d.ComputePatch() 1272 print "boot target: %d source: %d diff: %d" % ( 1273 target_boot.size, source_boot.size, len(d)) 1274 1275 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 1276 1277 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 1278 1279 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 1280 (boot_type, boot_device, 1281 source_boot.size, source_boot.sha1, 1282 target_boot.size, target_boot.sha1)) 1283 so_far += source_boot.size 1284 1285 size = [] 1286 if system_diff.patch_list: size.append(system_diff.largest_source_size) 1287 if vendor_diff: 1288 if vendor_diff.patch_list: size.append(vendor_diff.largest_source_size) 1289 if size or updating_recovery or updating_boot: 1290 script.CacheFreeSpaceCheck(max(size)) 1291 1292 device_specific.IncrementalOTA_VerifyEnd() 1293 1294 if OPTIONS.two_step: 1295 script.WriteRawImage("/boot", "recovery.img") 1296 script.AppendExtra(""" 1297set_stage("%(bcb_dev)s", "2/3"); 1298reboot_now("%(bcb_dev)s", ""); 1299else 1300""" % bcb_dev) 1301 1302 script.Comment("---- start making changes here ----") 1303 1304 device_specific.IncrementalOTA_InstallBegin() 1305 1306 if OPTIONS.two_step: 1307 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 1308 script.WriteRawImage("/boot", "boot.img") 1309 print "writing full boot image (forced by two-step mode)" 1310 1311 script.Print("Removing unneeded files...") 1312 system_diff.RemoveUnneededFiles(script, ("/system/recovery.img",)) 1313 if vendor_diff: 1314 vendor_diff.RemoveUnneededFiles(script) 1315 1316 script.ShowProgress(0.8, 0) 1317 total_patch_size = 1.0 + system_diff.TotalPatchSize() 1318 if vendor_diff: 1319 total_patch_size += vendor_diff.TotalPatchSize() 1320 if updating_boot: 1321 total_patch_size += target_boot.size 1322 1323 script.Print("Patching system files...") 1324 so_far = system_diff.EmitPatches(script, total_patch_size, 0) 1325 if vendor_diff: 1326 script.Print("Patching vendor files...") 1327 so_far = vendor_diff.EmitPatches(script, total_patch_size, so_far) 1328 1329 if not OPTIONS.two_step: 1330 if updating_boot: 1331 # Produce the boot image by applying a patch to the current 1332 # contents of the boot partition, and write it back to the 1333 # partition. 1334 script.Print("Patching boot image...") 1335 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 1336 % (boot_type, boot_device, 1337 source_boot.size, source_boot.sha1, 1338 target_boot.size, target_boot.sha1), 1339 "-", 1340 target_boot.size, target_boot.sha1, 1341 source_boot.sha1, "patch/boot.img.p") 1342 so_far += target_boot.size 1343 script.SetProgress(so_far / total_patch_size) 1344 print "boot image changed; including." 1345 else: 1346 print "boot image unchanged; skipping." 1347 1348 system_items = ItemSet("system", "META/filesystem_config.txt") 1349 if vendor_diff: 1350 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 1351 1352 if updating_recovery: 1353 # Recovery is generated as a patch using both the boot image 1354 # (which contains the same linux kernel as recovery) and the file 1355 # /system/etc/recovery-resource.dat (which contains all the images 1356 # used in the recovery UI) as sources. This lets us minimize the 1357 # size of the patch, which must be included in every OTA package. 1358 # 1359 # For older builds where recovery-resource.dat is not present, we 1360 # use only the boot image as the source. 1361 1362 if not target_has_recovery_patch: 1363 def output_sink(fn, data): 1364 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 1365 system_items.Get("system/" + fn, dir=False) 1366 1367 common.MakeRecoveryPatch(OPTIONS.target_tmp, output_sink, 1368 target_recovery, target_boot) 1369 script.DeleteFiles(["/system/recovery-from-boot.p", 1370 "/system/etc/install-recovery.sh"]) 1371 print "recovery image changed; including as patch from boot." 1372 else: 1373 print "recovery image unchanged; skipping." 1374 1375 script.ShowProgress(0.1, 10) 1376 1377 target_symlinks = CopyPartitionFiles(system_items, target_zip, None) 1378 if vendor_diff: 1379 target_symlinks.extend(CopyPartitionFiles(vendor_items, target_zip, None)) 1380 1381 temp_script = script.MakeTemporary() 1382 system_items.GetMetadata(target_zip) 1383 system_items.Get("system").SetPermissions(temp_script) 1384 if vendor_diff: 1385 vendor_items.GetMetadata(target_zip) 1386 vendor_items.Get("vendor").SetPermissions(temp_script) 1387 1388 # Note that this call will mess up the trees of Items, so make sure 1389 # we're done with them. 1390 source_symlinks = CopyPartitionFiles(system_items, source_zip, None) 1391 if vendor_diff: 1392 source_symlinks.extend(CopyPartitionFiles(vendor_items, source_zip, None)) 1393 1394 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 1395 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 1396 1397 # Delete all the symlinks in source that aren't in target. This 1398 # needs to happen before verbatim files are unpacked, in case a 1399 # symlink in the source is replaced by a real file in the target. 1400 to_delete = [] 1401 for dest, link in source_symlinks: 1402 if link not in target_symlinks_d: 1403 to_delete.append(link) 1404 script.DeleteFiles(to_delete) 1405 1406 if system_diff.verbatim_targets: 1407 script.Print("Unpacking new system files...") 1408 script.UnpackPackageDir("system", "/system") 1409 if vendor_diff and vendor_diff.verbatim_targets: 1410 script.Print("Unpacking new vendor files...") 1411 script.UnpackPackageDir("vendor", "/vendor") 1412 1413 if updating_recovery and not target_has_recovery_patch: 1414 script.Print("Unpacking new recovery...") 1415 script.UnpackPackageDir("recovery", "/system") 1416 1417 system_diff.EmitRenames(script) 1418 if vendor_diff: 1419 vendor_diff.EmitRenames(script) 1420 1421 script.Print("Symlinks and permissions...") 1422 1423 # Create all the symlinks that don't already exist, or point to 1424 # somewhere different than what we want. Delete each symlink before 1425 # creating it, since the 'symlink' command won't overwrite. 1426 to_create = [] 1427 for dest, link in target_symlinks: 1428 if link in source_symlinks_d: 1429 if dest != source_symlinks_d[link]: 1430 to_create.append((dest, link)) 1431 else: 1432 to_create.append((dest, link)) 1433 script.DeleteFiles([i[1] for i in to_create]) 1434 script.MakeSymlinks(to_create) 1435 1436 # Now that the symlinks are created, we can set all the 1437 # permissions. 1438 script.AppendScript(temp_script) 1439 1440 # Do device-specific installation (eg, write radio image). 1441 device_specific.IncrementalOTA_InstallEnd() 1442 1443 if OPTIONS.extra_script is not None: 1444 script.AppendExtra(OPTIONS.extra_script) 1445 1446 # Patch the build.prop file last, so if something fails but the 1447 # device can still come up, it appears to be the old build and will 1448 # get set the OTA package again to retry. 1449 script.Print("Patching remaining system files...") 1450 system_diff.EmitDeferredPatches(script) 1451 1452 if OPTIONS.wipe_user_data: 1453 script.Print("Erasing user data...") 1454 script.FormatPartition("/data") 1455 1456 if OPTIONS.two_step: 1457 script.AppendExtra(""" 1458set_stage("%(bcb_dev)s", ""); 1459endif; 1460endif; 1461""" % bcb_dev) 1462 1463 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 1464 WriteMetadata(metadata, output_zip) 1465 1466 1467def main(argv): 1468 1469 def option_handler(o, a): 1470 if o == "--board_config": 1471 pass # deprecated 1472 elif o in ("-k", "--package_key"): 1473 OPTIONS.package_key = a 1474 elif o in ("-i", "--incremental_from"): 1475 OPTIONS.incremental_source = a 1476 elif o in ("-w", "--wipe_user_data"): 1477 OPTIONS.wipe_user_data = True 1478 elif o in ("-n", "--no_prereq"): 1479 OPTIONS.omit_prereq = True 1480 elif o in ("-o", "--oem_settings"): 1481 OPTIONS.oem_source = a 1482 elif o in ("-e", "--extra_script"): 1483 OPTIONS.extra_script = a 1484 elif o in ("-a", "--aslr_mode"): 1485 if a in ("on", "On", "true", "True", "yes", "Yes"): 1486 OPTIONS.aslr_mode = True 1487 else: 1488 OPTIONS.aslr_mode = False 1489 elif o in ("-t", "--worker_threads"): 1490 if a.isdigit(): 1491 OPTIONS.worker_threads = int(a) 1492 else: 1493 raise ValueError("Cannot parse value %r for option %r - only " 1494 "integers are allowed." % (a, o)) 1495 elif o in ("-2", "--two_step"): 1496 OPTIONS.two_step = True 1497 elif o == "--no_signing": 1498 OPTIONS.no_signing = True 1499 elif o == "--block": 1500 OPTIONS.block_based = True 1501 elif o in ("-b", "--binary"): 1502 OPTIONS.updater_binary = a 1503 elif o in ("--no_fallback_to_full",): 1504 OPTIONS.fallback_to_full = False 1505 else: 1506 return False 1507 return True 1508 1509 args = common.ParseOptions(argv, __doc__, 1510 extra_opts="b:k:i:d:wne:t:a:2o:", 1511 extra_long_opts=["board_config=", 1512 "package_key=", 1513 "incremental_from=", 1514 "wipe_user_data", 1515 "no_prereq", 1516 "extra_script=", 1517 "worker_threads=", 1518 "aslr_mode=", 1519 "two_step", 1520 "no_signing", 1521 "block", 1522 "binary=", 1523 "oem_settings=", 1524 "no_fallback_to_full", 1525 ], 1526 extra_option_handler=option_handler) 1527 1528 if len(args) != 2: 1529 common.Usage(__doc__) 1530 sys.exit(1) 1531 1532 if OPTIONS.extra_script is not None: 1533 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 1534 1535 print "unzipping target target-files..." 1536 OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0]) 1537 1538 OPTIONS.target_tmp = OPTIONS.input_tmp 1539 OPTIONS.info_dict = common.LoadInfoDict(input_zip) 1540 1541 # If this image was originally labelled with SELinux contexts, make sure we 1542 # also apply the labels in our new image. During building, the "file_contexts" 1543 # is in the out/ directory tree, but for repacking from target-files.zip it's 1544 # in the root directory of the ramdisk. 1545 if "selinux_fc" in OPTIONS.info_dict: 1546 OPTIONS.info_dict["selinux_fc"] = os.path.join(OPTIONS.input_tmp, "BOOT", "RAMDISK", 1547 "file_contexts") 1548 1549 if OPTIONS.verbose: 1550 print "--- target info ---" 1551 common.DumpInfoDict(OPTIONS.info_dict) 1552 1553 # If the caller explicitly specified the device-specific extensions 1554 # path via -s/--device_specific, use that. Otherwise, use 1555 # META/releasetools.py if it is present in the target target_files. 1556 # Otherwise, take the path of the file from 'tool_extensions' in the 1557 # info dict and look for that in the local filesystem, relative to 1558 # the current directory. 1559 1560 if OPTIONS.device_specific is None: 1561 from_input = os.path.join(OPTIONS.input_tmp, "META", "releasetools.py") 1562 if os.path.exists(from_input): 1563 print "(using device-specific extensions from target_files)" 1564 OPTIONS.device_specific = from_input 1565 else: 1566 OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None) 1567 1568 if OPTIONS.device_specific is not None: 1569 OPTIONS.device_specific = os.path.abspath(OPTIONS.device_specific) 1570 1571 while True: 1572 1573 if OPTIONS.no_signing: 1574 if os.path.exists(args[1]): os.unlink(args[1]) 1575 output_zip = zipfile.ZipFile(args[1], "w", compression=zipfile.ZIP_DEFLATED) 1576 else: 1577 temp_zip_file = tempfile.NamedTemporaryFile() 1578 output_zip = zipfile.ZipFile(temp_zip_file, "w", 1579 compression=zipfile.ZIP_DEFLATED) 1580 1581 if OPTIONS.incremental_source is None: 1582 WriteFullOTAPackage(input_zip, output_zip) 1583 if OPTIONS.package_key is None: 1584 OPTIONS.package_key = OPTIONS.info_dict.get( 1585 "default_system_dev_certificate", 1586 "build/target/product/security/testkey") 1587 break 1588 1589 else: 1590 print "unzipping source target-files..." 1591 OPTIONS.source_tmp, source_zip = common.UnzipTemp(OPTIONS.incremental_source) 1592 OPTIONS.target_info_dict = OPTIONS.info_dict 1593 OPTIONS.source_info_dict = common.LoadInfoDict(source_zip) 1594 if "selinux_fc" in OPTIONS.source_info_dict: 1595 OPTIONS.source_info_dict["selinux_fc"] = os.path.join(OPTIONS.source_tmp, "BOOT", "RAMDISK", 1596 "file_contexts") 1597 if OPTIONS.package_key is None: 1598 OPTIONS.package_key = OPTIONS.source_info_dict.get( 1599 "default_system_dev_certificate", 1600 "build/target/product/security/testkey") 1601 if OPTIONS.verbose: 1602 print "--- source info ---" 1603 common.DumpInfoDict(OPTIONS.source_info_dict) 1604 try: 1605 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) 1606 break 1607 except ValueError: 1608 if not OPTIONS.fallback_to_full: raise 1609 print "--- failed to build incremental; falling back to full ---" 1610 OPTIONS.incremental_source = None 1611 output_zip.close() 1612 1613 output_zip.close() 1614 1615 if not OPTIONS.no_signing: 1616 SignOutput(temp_zip_file.name, args[1]) 1617 temp_zip_file.close() 1618 1619 common.Cleanup() 1620 1621 print "done." 1622 1623 1624if __name__ == '__main__': 1625 try: 1626 common.CloseInheritedPipes() 1627 main(sys.argv[1:]) 1628 except common.ExternalError, e: 1629 print 1630 print " ERROR: %s" % (e,) 1631 print 1632 sys.exit(1) 1633