ota_from_target_files.py revision 3c84f569487c4e59baa332be33b5430fdefb76b3
1#!/usr/bin/env python 2# 3# Copyright (C) 2008 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile, produces an OTA package that installs 19that build. An incremental OTA is produced if -i is given, otherwise 20a full OTA is produced. 21 22Usage: ota_from_target_files [flags] input_target_files output_ota_package 23 24 --board_config <file> 25 Deprecated. 26 27 -k (--package_key) <key> Key to use to sign the package (default is 28 the value of default_system_dev_certificate from the input 29 target-files's META/misc_info.txt, or 30 "build/target/product/security/testkey" if that value is not 31 specified). 32 33 For incremental OTAs, the default value is based on the source 34 target-file, not the target build. 35 36 -i (--incremental_from) <file> 37 Generate an incremental OTA using the given target-files zip as 38 the starting build. 39 40 -o (--oem_settings) <file> 41 Use the file to specify the expected OEM-specific properties 42 on the OEM partition of the intended device. 43 44 -w (--wipe_user_data) 45 Generate an OTA package that will wipe the user data partition 46 when installed. 47 48 -n (--no_prereq) 49 Omit the timestamp prereq check normally included at the top of 50 the build scripts (used for developer OTA packages which 51 legitimately need to go back and forth). 52 53 -e (--extra_script) <file> 54 Insert the contents of file at the end of the update script. 55 56 -a (--aslr_mode) <on|off> 57 Specify whether to turn on ASLR for the package (on by default). 58 59 -2 (--two_step) 60 Generate a 'two-step' OTA package, where recovery is updated 61 first, so that any changes made to the system partition are done 62 using the new recovery (new kernel, etc.). 63 64 --block 65 Generate a block-based OTA if possible. Will fall back to a 66 file-based OTA if the target_files is older and doesn't support 67 block-based OTAs. 68 69 -b (--binary) <file> 70 Use the given binary as the update-binary in the output package, 71 instead of the binary in the build's target_files. Use for 72 development only. 73 74 -t (--worker_threads) <int> 75 Specifies the number of worker-threads that will be used when 76 generating patches for incremental updates (defaults to 3). 77 78""" 79 80import sys 81 82if sys.hexversion < 0x02070000: 83 print >> sys.stderr, "Python 2.7 or newer is required." 84 sys.exit(1) 85 86import copy 87import errno 88import os 89import re 90import subprocess 91import tempfile 92import time 93import zipfile 94 95try: 96 from hashlib import sha1 as sha1 97except ImportError: 98 from sha import sha as sha1 99 100import common 101import edify_generator 102import build_image 103 104OPTIONS = common.OPTIONS 105OPTIONS.package_key = None 106OPTIONS.incremental_source = None 107OPTIONS.require_verbatim = set() 108OPTIONS.prohibit_verbatim = set(("system/build.prop",)) 109OPTIONS.patch_threshold = 0.95 110OPTIONS.wipe_user_data = False 111OPTIONS.omit_prereq = False 112OPTIONS.extra_script = None 113OPTIONS.aslr_mode = True 114OPTIONS.worker_threads = 3 115OPTIONS.two_step = False 116OPTIONS.no_signing = False 117OPTIONS.block_based = False 118OPTIONS.updater_binary = None 119OPTIONS.oem_source = None 120 121def MostPopularKey(d, default): 122 """Given a dict, return the key corresponding to the largest 123 value. Returns 'default' if the dict is empty.""" 124 x = [(v, k) for (k, v) in d.iteritems()] 125 if not x: return default 126 x.sort() 127 return x[-1][1] 128 129 130def IsSymlink(info): 131 """Return true if the zipfile.ZipInfo object passed in represents a 132 symlink.""" 133 return (info.external_attr >> 16) == 0120777 134 135def IsRegular(info): 136 """Return true if the zipfile.ZipInfo object passed in represents a 137 symlink.""" 138 return (info.external_attr >> 28) == 010 139 140def ClosestFileMatch(src, tgtfiles, existing): 141 """Returns the closest file match between a source file and list 142 of potential matches. The exact filename match is preferred, 143 then the sha1 is searched for, and finally a file with the same 144 basename is evaluated. Rename support in the updater-binary is 145 required for the latter checks to be used.""" 146 147 result = tgtfiles.get("path:" + src.name) 148 if result is not None: 149 return result 150 151 if not OPTIONS.target_info_dict.get("update_rename_support", False): 152 return None 153 154 if src.size < 1000: 155 return None 156 157 result = tgtfiles.get("sha1:" + src.sha1) 158 if result is not None and existing.get(result.name) is None: 159 return result 160 result = tgtfiles.get("file:" + src.name.split("/")[-1]) 161 if result is not None and existing.get(result.name) is None: 162 return result 163 return None 164 165class ItemSet: 166 def __init__(self, partition, fs_config): 167 self.partition = partition 168 self.fs_config = fs_config 169 self.ITEMS = {} 170 171 def Get(self, name, dir=False): 172 if name not in self.ITEMS: 173 self.ITEMS[name] = Item(self, name, dir=dir) 174 return self.ITEMS[name] 175 176 def GetMetadata(self, input_zip): 177 # The target_files contains a record of what the uid, 178 # gid, and mode are supposed to be. 179 output = input_zip.read(self.fs_config) 180 181 for line in output.split("\n"): 182 if not line: continue 183 columns = line.split() 184 name, uid, gid, mode = columns[:4] 185 selabel = None 186 capabilities = None 187 188 # After the first 4 columns, there are a series of key=value 189 # pairs. Extract out the fields we care about. 190 for element in columns[4:]: 191 key, value = element.split("=") 192 if key == "selabel": 193 selabel = value 194 if key == "capabilities": 195 capabilities = value 196 197 i = self.ITEMS.get(name, None) 198 if i is not None: 199 i.uid = int(uid) 200 i.gid = int(gid) 201 i.mode = int(mode, 8) 202 i.selabel = selabel 203 i.capabilities = capabilities 204 if i.dir: 205 i.children.sort(key=lambda i: i.name) 206 207 # set metadata for the files generated by this script. 208 i = self.ITEMS.get("system/recovery-from-boot.p", None) 209 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0644, None, None 210 i = self.ITEMS.get("system/etc/install-recovery.sh", None) 211 if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0544, None, None 212 213 214class Item: 215 """Items represent the metadata (user, group, mode) of files and 216 directories in the system image.""" 217 def __init__(self, itemset, name, dir=False): 218 self.itemset = itemset 219 self.name = name 220 self.uid = None 221 self.gid = None 222 self.mode = None 223 self.selabel = None 224 self.capabilities = None 225 self.dir = dir 226 227 if name: 228 self.parent = itemset.Get(os.path.dirname(name), dir=True) 229 self.parent.children.append(self) 230 else: 231 self.parent = None 232 if dir: 233 self.children = [] 234 235 def Dump(self, indent=0): 236 if self.uid is not None: 237 print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode) 238 else: 239 print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode) 240 if self.dir: 241 print "%s%s" % (" "*indent, self.descendants) 242 print "%s%s" % (" "*indent, self.best_subtree) 243 for i in self.children: 244 i.Dump(indent=indent+1) 245 246 def CountChildMetadata(self): 247 """Count up the (uid, gid, mode, selabel, capabilities) tuples for 248 all children and determine the best strategy for using set_perm_recursive and 249 set_perm to correctly chown/chmod all the files to their desired 250 values. Recursively calls itself for all descendants. 251 252 Returns a dict of {(uid, gid, dmode, fmode, selabel, capabilities): count} counting up 253 all descendants of this node. (dmode or fmode may be None.) Also 254 sets the best_subtree of each directory Item to the (uid, gid, 255 dmode, fmode, selabel, capabilities) tuple that will match the most 256 descendants of that Item. 257 """ 258 259 assert self.dir 260 d = self.descendants = {(self.uid, self.gid, self.mode, None, self.selabel, self.capabilities): 1} 261 for i in self.children: 262 if i.dir: 263 for k, v in i.CountChildMetadata().iteritems(): 264 d[k] = d.get(k, 0) + v 265 else: 266 k = (i.uid, i.gid, None, i.mode, i.selabel, i.capabilities) 267 d[k] = d.get(k, 0) + 1 268 269 # Find the (uid, gid, dmode, fmode, selabel, capabilities) 270 # tuple that matches the most descendants. 271 272 # First, find the (uid, gid) pair that matches the most 273 # descendants. 274 ug = {} 275 for (uid, gid, _, _, _, _), count in d.iteritems(): 276 ug[(uid, gid)] = ug.get((uid, gid), 0) + count 277 ug = MostPopularKey(ug, (0, 0)) 278 279 # Now find the dmode, fmode, selabel, and capabilities that match 280 # the most descendants with that (uid, gid), and choose those. 281 best_dmode = (0, 0755) 282 best_fmode = (0, 0644) 283 best_selabel = (0, None) 284 best_capabilities = (0, None) 285 for k, count in d.iteritems(): 286 if k[:2] != ug: continue 287 if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2]) 288 if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3]) 289 if k[4] is not None and count >= best_selabel[0]: best_selabel = (count, k[4]) 290 if k[5] is not None and count >= best_capabilities[0]: best_capabilities = (count, k[5]) 291 self.best_subtree = ug + (best_dmode[1], best_fmode[1], best_selabel[1], best_capabilities[1]) 292 293 return d 294 295 def SetPermissions(self, script): 296 """Append set_perm/set_perm_recursive commands to 'script' to 297 set all permissions, users, and groups for the tree of files 298 rooted at 'self'.""" 299 300 self.CountChildMetadata() 301 302 def recurse(item, current): 303 # current is the (uid, gid, dmode, fmode, selabel, capabilities) tuple that the current 304 # item (and all its children) have already been set to. We only 305 # need to issue set_perm/set_perm_recursive commands if we're 306 # supposed to be something different. 307 if item.dir: 308 if current != item.best_subtree: 309 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree) 310 current = item.best_subtree 311 312 if item.uid != current[0] or item.gid != current[1] or \ 313 item.mode != current[2] or item.selabel != current[4] or \ 314 item.capabilities != current[5]: 315 script.SetPermissions("/"+item.name, item.uid, item.gid, 316 item.mode, item.selabel, item.capabilities) 317 318 for i in item.children: 319 recurse(i, current) 320 else: 321 if item.uid != current[0] or item.gid != current[1] or \ 322 item.mode != current[3] or item.selabel != current[4] or \ 323 item.capabilities != current[5]: 324 script.SetPermissions("/"+item.name, item.uid, item.gid, 325 item.mode, item.selabel, item.capabilities) 326 327 recurse(self, (-1, -1, -1, -1, None, None)) 328 329 330def CopyPartitionFiles(itemset, input_zip, output_zip=None, substitute=None): 331 """Copies files for the partition in the input zip to the output 332 zip. Populates the Item class with their metadata, and returns a 333 list of symlinks. output_zip may be None, in which case the copy is 334 skipped (but the other side effects still happen). substitute is an 335 optional dict of {output filename: contents} to be output instead of 336 certain input files. 337 """ 338 339 symlinks = [] 340 341 partition = itemset.partition 342 343 for info in input_zip.infolist(): 344 if info.filename.startswith(partition.upper() + "/"): 345 basefilename = info.filename[7:] 346 if IsSymlink(info): 347 symlinks.append((input_zip.read(info.filename), 348 "/" + partition + "/" + basefilename)) 349 else: 350 info2 = copy.copy(info) 351 fn = info2.filename = partition + "/" + basefilename 352 if substitute and fn in substitute and substitute[fn] is None: 353 continue 354 if output_zip is not None: 355 if substitute and fn in substitute: 356 data = substitute[fn] 357 else: 358 data = input_zip.read(info.filename) 359 output_zip.writestr(info2, data) 360 if fn.endswith("/"): 361 itemset.Get(fn[:-1], dir=True) 362 else: 363 itemset.Get(fn, dir=False) 364 365 symlinks.sort() 366 return symlinks 367 368 369def SignOutput(temp_zip_name, output_zip_name): 370 key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) 371 pw = key_passwords[OPTIONS.package_key] 372 373 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw, 374 whole_file=True) 375 376 377def AppendAssertions(script, info_dict, oem_dict = None): 378 oem_props = info_dict.get("oem_fingerprint_properties") 379 if oem_props is None: 380 device = GetBuildProp("ro.product.device", info_dict) 381 script.AssertDevice(device) 382 else: 383 if oem_dict is None: 384 raise common.ExternalError("No OEM file provided to answer expected assertions") 385 for prop in oem_props.split(): 386 if oem_dict.get(prop) is None: 387 raise common.ExternalError("The OEM file is missing the property %s" % prop) 388 script.AssertOemProperty(prop, oem_dict.get(prop)) 389 390 391def HasRecoveryPatch(target_files_zip): 392 try: 393 target_files_zip.getinfo("SYSTEM/recovery-from-boot.p") 394 return True 395 except KeyError: 396 return False 397 398def HasVendorPartition(target_files_zip): 399 try: 400 target_files_zip.getinfo("VENDOR/") 401 return True 402 except KeyError: 403 return False 404 405def GetOemProperty(name, oem_props, oem_dict, info_dict): 406 if oem_props is not None and name in oem_props: 407 return oem_dict[name] 408 return GetBuildProp(name, info_dict) 409 410 411def CalculateFingerprint(oem_props, oem_dict, info_dict): 412 if oem_props is None: 413 return GetBuildProp("ro.build.fingerprint", info_dict) 414 return "%s/%s/%s:%s" % ( 415 GetOemProperty("ro.product.brand", oem_props, oem_dict, info_dict), 416 GetOemProperty("ro.product.name", oem_props, oem_dict, info_dict), 417 GetOemProperty("ro.product.device", oem_props, oem_dict, info_dict), 418 GetBuildProp("ro.build.thumbprint", info_dict)) 419 420def GetImage(which, tmpdir, info_dict): 421 # Return (mapdata, data) for the given image. which should be 422 # "system" or "vendor". 423 424 assert which in ("system", "vendor") 425 426 path = os.path.join(tmpdir, "IMAGES", which + ".img") 427 if os.path.exists(path): 428 print "using %s.img from target-files" % (which,) 429 430 # This is a 'new' target-files, which already has the image in it. 431 # The image is a sparse image, though, so we need to unsparse it 432 # and extract the map data. 433 434 success, name = build_image.UnsparseImage(path, replace=False) 435 if not success: 436 assert False, "unsparsing " + which + ".img failed" 437 438 mmap = tempfile.NamedTemporaryFile() 439 mimg = tempfile.NamedTemporaryFile(delete=False) 440 success = build_image.MappedUnsparseImage( 441 path, name, mmap.name, mimg.name) 442 if not success: 443 assert False, "creating sparse map failed" 444 os.unlink(name) 445 name = mimg.name 446 447 with open(mmap.name) as f: 448 mapdata = f.read() 449 450 try: 451 with open(name) as f: 452 data = f.read() 453 finally: 454 os.unlink(name) 455 456 print "unsparsed data sha1 is " + sha1(data).hexdigest() 457 return mapdata, data 458 459 else: 460 print "building %s.img from target-files" % (which,) 461 462 # This is an 'old' target-files, which does not contain images 463 # already built. Build them. 464 465 import add_img_to_target_files 466 if which == "system": 467 mapdata, data = add_img_to_target_files.BuildSystem( 468 tmpdir, info_dict, sparse=False, map_file=True) 469 elif which == "vendor": 470 mapdata, data = add_img_to_target_files.BuildVendor( 471 tmpdir, info_dict, sparse=False, map_file=True) 472 473 print "built data sha1 is " + sha1(data).hexdigest() 474 return mapdata, data 475 476 477def WriteFullOTAPackage(input_zip, output_zip): 478 # TODO: how to determine this? We don't know what version it will 479 # be installed on top of. For now, we expect the API just won't 480 # change very often. 481 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict) 482 483 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 484 oem_dict = None 485 if oem_props is not None: 486 if OPTIONS.oem_source is None: 487 raise common.ExternalError("OEM source required for this build") 488 script.Mount("/oem") 489 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 490 491 metadata = {"post-build": CalculateFingerprint( 492 oem_props, oem_dict, OPTIONS.info_dict), 493 "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 494 OPTIONS.info_dict), 495 "post-timestamp": GetBuildProp("ro.build.date.utc", 496 OPTIONS.info_dict), 497 } 498 499 device_specific = common.DeviceSpecificParams( 500 input_zip=input_zip, 501 input_version=OPTIONS.info_dict["recovery_api_version"], 502 output_zip=output_zip, 503 script=script, 504 input_tmp=OPTIONS.input_tmp, 505 metadata=metadata, 506 info_dict=OPTIONS.info_dict) 507 508 has_recovery_patch = HasRecoveryPatch(input_zip) 509 block_based = OPTIONS.block_based and has_recovery_patch 510 511 if not OPTIONS.omit_prereq: 512 ts = GetBuildProp("ro.build.date.utc", OPTIONS.info_dict) 513 ts_text = GetBuildProp("ro.build.date", OPTIONS.info_dict) 514 script.AssertOlderBuild(ts, ts_text) 515 516 AppendAssertions(script, OPTIONS.info_dict, oem_dict) 517 device_specific.FullOTA_Assertions() 518 519 # Two-step package strategy (in chronological order, which is *not* 520 # the order in which the generated script has things): 521 # 522 # if stage is not "2/3" or "3/3": 523 # write recovery image to boot partition 524 # set stage to "2/3" 525 # reboot to boot partition and restart recovery 526 # else if stage is "2/3": 527 # write recovery image to recovery partition 528 # set stage to "3/3" 529 # reboot to recovery partition and restart recovery 530 # else: 531 # (stage must be "3/3") 532 # set stage to "" 533 # do normal full package installation: 534 # wipe and install system, boot image, etc. 535 # set up system to update recovery partition on first boot 536 # complete script normally (allow recovery to mark itself finished and reboot) 537 538 recovery_img = common.GetBootableImage("recovery.img", "recovery.img", 539 OPTIONS.input_tmp, "RECOVERY") 540 if OPTIONS.two_step: 541 if not OPTIONS.info_dict.get("multistage_support", None): 542 assert False, "two-step packages not supported by this build" 543 fs = OPTIONS.info_dict["fstab"]["/misc"] 544 assert fs.fs_type.upper() == "EMMC", \ 545 "two-step packages only supported on devices with EMMC /misc partitions" 546 bcb_dev = {"bcb_dev": fs.device} 547 common.ZipWriteStr(output_zip, "recovery.img", recovery_img.data) 548 script.AppendExtra(""" 549if get_stage("%(bcb_dev)s", "stage") == "2/3" then 550""" % bcb_dev) 551 script.WriteRawImage("/recovery", "recovery.img") 552 script.AppendExtra(""" 553set_stage("%(bcb_dev)s", "3/3"); 554reboot_now("%(bcb_dev)s", "recovery"); 555else if get_stage("%(bcb_dev)s", "stage") == "3/3" then 556""" % bcb_dev) 557 558 device_specific.FullOTA_InstallBegin() 559 560 system_progress = 0.75 561 562 if OPTIONS.wipe_user_data: 563 system_progress -= 0.1 564 if HasVendorPartition(input_zip): 565 system_progress -= 0.1 566 567 if "selinux_fc" in OPTIONS.info_dict: 568 WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip) 569 570 system_items = ItemSet("system", "META/filesystem_config.txt") 571 script.ShowProgress(system_progress, 0) 572 if block_based: 573 mapdata, data = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict) 574 575 common.ZipWriteStr(output_zip, "system.map", mapdata) 576 common.ZipWriteStr(output_zip, "system.muimg", data) 577 script.WipeBlockDevice("/system") 578 script.WriteRawImage("/system", "system.muimg", mapfn="system.map") 579 else: 580 script.FormatPartition("/system") 581 script.Mount("/system") 582 if not has_recovery_patch: 583 script.UnpackPackageDir("recovery", "/system") 584 script.UnpackPackageDir("system", "/system") 585 586 symlinks = CopyPartitionFiles(system_items, input_zip, output_zip) 587 script.MakeSymlinks(symlinks) 588 589 boot_img = common.GetBootableImage("boot.img", "boot.img", 590 OPTIONS.input_tmp, "BOOT") 591 592 if not block_based: 593 def output_sink(fn, data): 594 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 595 system_items.Get("system/" + fn, dir=False) 596 597 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, 598 recovery_img, boot_img) 599 600 system_items.GetMetadata(input_zip) 601 system_items.Get("system").SetPermissions(script) 602 603 if HasVendorPartition(input_zip): 604 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 605 script.ShowProgress(0.1, 0) 606 607 if block_based: 608 mapdata, data = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict) 609 610 common.ZipWriteStr(output_zip, "vendor.map", mapdata) 611 common.ZipWriteStr(output_zip, "vendor.muimg", data) 612 script.WipeBlockDevice("/vendor") 613 script.WriteRawImage("/vendor", "vendor.muimg", mapfn="vendor.map") 614 else: 615 script.FormatPartition("/vendor") 616 script.Mount("/vendor") 617 script.UnpackPackageDir("vendor", "/vendor") 618 619 symlinks = CopyPartitionFiles(vendor_items, input_zip, output_zip) 620 script.MakeSymlinks(symlinks) 621 622 vendor_items.GetMetadata(input_zip) 623 vendor_items.Get("vendor").SetPermissions(script) 624 625 common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict) 626 common.ZipWriteStr(output_zip, "boot.img", boot_img.data) 627 628 script.ShowProgress(0.05, 5) 629 script.WriteRawImage("/boot", "boot.img") 630 631 script.ShowProgress(0.2, 10) 632 device_specific.FullOTA_InstallEnd() 633 634 if OPTIONS.extra_script is not None: 635 script.AppendExtra(OPTIONS.extra_script) 636 637 script.UnmountAll() 638 639 if OPTIONS.wipe_user_data: 640 script.ShowProgress(0.1, 10) 641 script.FormatPartition("/data") 642 643 if OPTIONS.two_step: 644 script.AppendExtra(""" 645set_stage("%(bcb_dev)s", ""); 646""" % bcb_dev) 647 script.AppendExtra("else\n") 648 script.WriteRawImage("/boot", "recovery.img") 649 script.AppendExtra(""" 650set_stage("%(bcb_dev)s", "2/3"); 651reboot_now("%(bcb_dev)s", ""); 652endif; 653endif; 654""" % bcb_dev) 655 script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary) 656 WriteMetadata(metadata, output_zip) 657 658def WritePolicyConfig(file_context, output_zip): 659 f = open(file_context, 'r'); 660 basename = os.path.basename(file_context) 661 common.ZipWriteStr(output_zip, basename, f.read()) 662 663 664def WriteMetadata(metadata, output_zip): 665 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata", 666 "".join(["%s=%s\n" % kv 667 for kv in sorted(metadata.iteritems())])) 668 669def LoadPartitionFiles(z, partition): 670 """Load all the files from the given partition in a given target-files 671 ZipFile, and return a dict of {filename: File object}.""" 672 out = {} 673 prefix = partition.upper() + "/" 674 for info in z.infolist(): 675 if info.filename.startswith(prefix) and not IsSymlink(info): 676 basefilename = info.filename[7:] 677 fn = partition + "/" + basefilename 678 data = z.read(info.filename) 679 out[fn] = common.File(fn, data) 680 return out 681 682 683def GetBuildProp(prop, info_dict): 684 """Return the fingerprint of the build of a given target-files info_dict.""" 685 try: 686 return info_dict.get("build.prop", {})[prop] 687 except KeyError: 688 raise common.ExternalError("couldn't find %s in build.prop" % (prop,)) 689 690def AddToKnownPaths(filename, known_paths): 691 if filename[-1] == "/": 692 return 693 dirs = filename.split("/")[:-1] 694 while len(dirs) > 0: 695 path = "/".join(dirs) 696 if path in known_paths: 697 break; 698 known_paths.add(path) 699 dirs.pop() 700 701class BlockDifference: 702 def __init__(self, partition, output_zip): 703 with tempfile.NamedTemporaryFile() as src_file: 704 with tempfile.NamedTemporaryFile() as tgt_file: 705 print "building source " + partition + " image..." 706 src_file = tempfile.NamedTemporaryFile() 707 src_mapdata, src_data = GetImage(partition, 708 OPTIONS.source_tmp, 709 OPTIONS.source_info_dict) 710 711 self.src_sha1 = sha1(src_data).hexdigest() 712 print "source " + partition + " sha1:", self.src_sha1 713 src_file.write(src_data) 714 715 print "building target " + partition + " image..." 716 tgt_file = tempfile.NamedTemporaryFile() 717 tgt_mapdata, tgt_data = GetImage(partition, 718 OPTIONS.target_tmp, 719 OPTIONS.target_info_dict) 720 self.tgt_sha1 = sha1(tgt_data).hexdigest() 721 print "target " + partition + " sha1:", self.tgt_sha1 722 tgt_len = len(tgt_data) 723 tgt_file.write(tgt_data) 724 725 system_type, self.device = common.GetTypeAndDevice("/" + partition, 726 OPTIONS.info_dict) 727 self.patch = common.MakePartitionPatch(src_file, tgt_file, partition) 728 729 TestBlockPatch(src_data, src_mapdata, self.patch.data, 730 tgt_mapdata, self.tgt_sha1) 731 src_data = None 732 tgt_data = None 733 734 self.patch.AddToZip(output_zip, compression=zipfile.ZIP_STORED) 735 self.src_mapfilename = self.patch.name + ".src.map" 736 common.ZipWriteStr(output_zip, self.src_mapfilename, src_mapdata) 737 self.tgt_mapfilename = self.patch.name + ".tgt.map" 738 common.ZipWriteStr(output_zip, self.tgt_mapfilename, tgt_mapdata) 739 740def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 741 source_version = OPTIONS.source_info_dict["recovery_api_version"] 742 target_version = OPTIONS.target_info_dict["recovery_api_version"] 743 744 if source_version == 0: 745 print ("WARNING: generating edify script for a source that " 746 "can't install it.") 747 script = edify_generator.EdifyGenerator(source_version, 748 OPTIONS.target_info_dict) 749 750 metadata = {"pre-device": GetBuildProp("ro.product.device", 751 OPTIONS.source_info_dict), 752 "post-timestamp": GetBuildProp("ro.build.date.utc", 753 OPTIONS.target_info_dict), 754 } 755 756 device_specific = common.DeviceSpecificParams( 757 source_zip=source_zip, 758 source_version=source_version, 759 target_zip=target_zip, 760 target_version=target_version, 761 output_zip=output_zip, 762 script=script, 763 metadata=metadata, 764 info_dict=OPTIONS.info_dict) 765 766 source_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.source_info_dict) 767 target_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.target_info_dict) 768 metadata["pre-build"] = source_fp 769 metadata["post-build"] = target_fp 770 771 source_boot = common.GetBootableImage( 772 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 773 OPTIONS.source_info_dict) 774 target_boot = common.GetBootableImage( 775 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 776 updating_boot = (not OPTIONS.two_step and 777 (source_boot.data != target_boot.data)) 778 779 source_recovery = common.GetBootableImage( 780 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 781 OPTIONS.source_info_dict) 782 target_recovery = common.GetBootableImage( 783 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 784 updating_recovery = (source_recovery.data != target_recovery.data) 785 786 system_diff = BlockDifference("system", output_zip) 787 if HasVendorPartition(target_zip): 788 if not HasVendorPartition(source_zip): 789 raise RuntimeError("can't generate incremental that adds /vendor") 790 vendor_diff = BlockDifference("vendor", output_zip) 791 792 oem_props = OPTIONS.target_info_dict.get("oem_fingerprint_properties") 793 oem_dict = None 794 if oem_props is not None: 795 if OPTIONS.oem_source is None: 796 raise common.ExternalError("OEM source required for this build") 797 script.Mount("/oem") 798 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 799 800 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 801 device_specific.IncrementalOTA_Assertions() 802 803 # Two-step incremental package strategy (in chronological order, 804 # which is *not* the order in which the generated script has 805 # things): 806 # 807 # if stage is not "2/3" or "3/3": 808 # do verification on current system 809 # write recovery image to boot partition 810 # set stage to "2/3" 811 # reboot to boot partition and restart recovery 812 # else if stage is "2/3": 813 # write recovery image to recovery partition 814 # set stage to "3/3" 815 # reboot to recovery partition and restart recovery 816 # else: 817 # (stage must be "3/3") 818 # perform update: 819 # patch system files, etc. 820 # force full install of new boot image 821 # set up system to update recovery partition on first boot 822 # complete script normally (allow recovery to mark itself finished and reboot) 823 824 if OPTIONS.two_step: 825 if not OPTIONS.info_dict.get("multistage_support", None): 826 assert False, "two-step packages not supported by this build" 827 fs = OPTIONS.info_dict["fstab"]["/misc"] 828 assert fs.fs_type.upper() == "EMMC", \ 829 "two-step packages only supported on devices with EMMC /misc partitions" 830 bcb_dev = {"bcb_dev": fs.device} 831 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 832 script.AppendExtra(""" 833if get_stage("%(bcb_dev)s", "stage") == "2/3" then 834""" % bcb_dev) 835 script.AppendExtra("sleep(20);\n"); 836 script.WriteRawImage("/recovery", "recovery.img") 837 script.AppendExtra(""" 838set_stage("%(bcb_dev)s", "3/3"); 839reboot_now("%(bcb_dev)s", "recovery"); 840else if get_stage("%(bcb_dev)s", "stage") != "3/3" then 841""" % bcb_dev) 842 843 script.Print("Verifying current system...") 844 845 device_specific.IncrementalOTA_VerifyBegin() 846 847 if oem_props is None: 848 script.AssertSomeFingerprint(source_fp, target_fp) 849 else: 850 script.AssertSomeThumbprint( 851 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 852 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 853 854 if updating_boot: 855 d = common.Difference(target_boot, source_boot) 856 _, _, d = d.ComputePatch() 857 print "boot target: %d source: %d diff: %d" % ( 858 target_boot.size, source_boot.size, len(d)) 859 860 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 861 862 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 863 864 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 865 (boot_type, boot_device, 866 source_boot.size, source_boot.sha1, 867 target_boot.size, target_boot.sha1)) 868 869 device_specific.IncrementalOTA_VerifyEnd() 870 871 if OPTIONS.two_step: 872 script.WriteRawImage("/boot", "recovery.img") 873 script.AppendExtra(""" 874set_stage("%(bcb_dev)s", "2/3"); 875reboot_now("%(bcb_dev)s", ""); 876else 877""" % bcb_dev) 878 879 script.Comment("---- start making changes here ----") 880 881 device_specific.IncrementalOTA_InstallBegin() 882 883 if HasVendorPartition(target_zip): 884 script.Print("Patching vendor image...") 885 script.ShowProgress(0.1, 0) 886 script.Syspatch(vendor_diff.device, 887 vendor_diff.tgt_mapfilename, vendor_diff.tgt_sha1, 888 vendor_diff.src_mapfilename, vendor_diff.src_sha1, 889 vendor_diff.patch.name) 890 sys_progress = 0.8 891 else: 892 sys_progress = 0.9 893 894 script.Print("Patching system image...") 895 script.ShowProgress(sys_progress, 0) 896 script.Syspatch(system_diff.device, 897 system_diff.tgt_mapfilename, system_diff.tgt_sha1, 898 system_diff.src_mapfilename, system_diff.src_sha1, 899 system_diff.patch.name) 900 901 if OPTIONS.two_step: 902 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 903 script.WriteRawImage("/boot", "boot.img") 904 print "writing full boot image (forced by two-step mode)" 905 906 if not OPTIONS.two_step: 907 if updating_boot: 908 # Produce the boot image by applying a patch to the current 909 # contents of the boot partition, and write it back to the 910 # partition. 911 script.Print("Patching boot image...") 912 script.ShowProgress(0.1, 10) 913 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 914 % (boot_type, boot_device, 915 source_boot.size, source_boot.sha1, 916 target_boot.size, target_boot.sha1), 917 "-", 918 target_boot.size, target_boot.sha1, 919 source_boot.sha1, "patch/boot.img.p") 920 print "boot image changed; including." 921 else: 922 print "boot image unchanged; skipping." 923 924 # Do device-specific installation (eg, write radio image). 925 device_specific.IncrementalOTA_InstallEnd() 926 927 if OPTIONS.extra_script is not None: 928 script.AppendExtra(OPTIONS.extra_script) 929 930 if OPTIONS.wipe_user_data: 931 script.Print("Erasing user data...") 932 script.FormatPartition("/data") 933 934 if OPTIONS.two_step: 935 script.AppendExtra(""" 936set_stage("%(bcb_dev)s", ""); 937endif; 938endif; 939""" % bcb_dev) 940 941 script.SetProgress(1) 942 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 943 WriteMetadata(metadata, output_zip) 944 945def ParseMap(map_str): 946 x = map_str.split() 947 assert int(x[0]) == 4096 948 assert int(x[1]) == len(x)-2 949 return int(x[0]), [int(i) for i in x[2:]] 950 951def TestBlockPatch(src_muimg, src_map, patch_data, tgt_map, tgt_sha1): 952 src_blksize, src_regions = ParseMap(src_map) 953 tgt_blksize, tgt_regions = ParseMap(tgt_map) 954 955 with tempfile.NamedTemporaryFile() as src_file,\ 956 tempfile.NamedTemporaryFile() as patch_file,\ 957 tempfile.NamedTemporaryFile() as src_map_file,\ 958 tempfile.NamedTemporaryFile() as tgt_map_file: 959 960 src_total = sum(src_regions) * src_blksize 961 src_file.truncate(src_total) 962 p = 0 963 for i in range(0, len(src_regions), 2): 964 c, dc = src_regions[i:i+2] 965 src_file.write(src_muimg[p:(p+c*src_blksize)]) 966 p += c*src_blksize 967 src_file.seek(dc*src_blksize, 1) 968 assert src_file.tell() == src_total 969 970 patch_file.write(patch_data) 971 972 src_map_file.write(src_map) 973 tgt_map_file.write(tgt_map) 974 975 src_file.flush() 976 src_map_file.flush() 977 patch_file.flush() 978 tgt_map_file.flush() 979 980 p = common.Run(["syspatch_host", src_file.name, src_map_file.name, 981 patch_file.name, src_file.name, tgt_map_file.name], 982 stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 983 stdoutdata, _ = p.communicate() 984 if p.returncode != 0: 985 print stdoutdata 986 raise ValueError("failed to reconstruct target system image from patch") 987 988 h = sha1() 989 src_file.seek(0, 0) 990 for i in range(0, len(tgt_regions), 2): 991 c, dc = tgt_regions[i:i+2] 992 h.update(src_file.read(c*tgt_blksize)) 993 src_file.seek(dc*tgt_blksize, 1) 994 995 if h.hexdigest() != tgt_sha1: 996 raise ValueError("patch reconstructed incorrect target system image") 997 998 print "test of system image patch succeeded" 999 1000 1001class FileDifference: 1002 def __init__(self, partition, source_zip, target_zip, output_zip): 1003 print "Loading target..." 1004 self.target_data = target_data = LoadPartitionFiles(target_zip, partition) 1005 print "Loading source..." 1006 self.source_data = source_data = LoadPartitionFiles(source_zip, partition) 1007 1008 self.verbatim_targets = verbatim_targets = [] 1009 self.patch_list = patch_list = [] 1010 diffs = [] 1011 self.renames = renames = {} 1012 known_paths = set() 1013 largest_source_size = 0 1014 1015 matching_file_cache = {} 1016 for fn, sf in source_data.items(): 1017 assert fn == sf.name 1018 matching_file_cache["path:" + fn] = sf 1019 if fn in target_data.keys(): 1020 AddToKnownPaths(fn, known_paths) 1021 # Only allow eligibility for filename/sha matching 1022 # if there isn't a perfect path match. 1023 if target_data.get(sf.name) is None: 1024 matching_file_cache["file:" + fn.split("/")[-1]] = sf 1025 matching_file_cache["sha:" + sf.sha1] = sf 1026 1027 for fn in sorted(target_data.keys()): 1028 tf = target_data[fn] 1029 assert fn == tf.name 1030 sf = ClosestFileMatch(tf, matching_file_cache, renames) 1031 if sf is not None and sf.name != tf.name: 1032 print "File has moved from " + sf.name + " to " + tf.name 1033 renames[sf.name] = tf 1034 1035 if sf is None or fn in OPTIONS.require_verbatim: 1036 # This file should be included verbatim 1037 if fn in OPTIONS.prohibit_verbatim: 1038 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,)) 1039 print "send", fn, "verbatim" 1040 tf.AddToZip(output_zip) 1041 verbatim_targets.append((fn, tf.size)) 1042 if fn in target_data.keys(): 1043 AddToKnownPaths(fn, known_paths) 1044 elif tf.sha1 != sf.sha1: 1045 # File is different; consider sending as a patch 1046 diffs.append(common.Difference(tf, sf)) 1047 else: 1048 # Target file data identical to source (may still be renamed) 1049 pass 1050 1051 common.ComputeDifferences(diffs) 1052 1053 for diff in diffs: 1054 tf, sf, d = diff.GetPatch() 1055 path = "/".join(tf.name.split("/")[:-1]) 1056 if d is None or len(d) > tf.size * OPTIONS.patch_threshold or \ 1057 path not in known_paths: 1058 # patch is almost as big as the file; don't bother patching 1059 # or a patch + rename cannot take place due to the target 1060 # directory not existing 1061 tf.AddToZip(output_zip) 1062 verbatim_targets.append((tf.name, tf.size)) 1063 if sf.name in renames: 1064 del renames[sf.name] 1065 AddToKnownPaths(tf.name, known_paths) 1066 else: 1067 common.ZipWriteStr(output_zip, "patch/" + sf.name + ".p", d) 1068 patch_list.append((tf, sf, tf.size, common.sha1(d).hexdigest())) 1069 largest_source_size = max(largest_source_size, sf.size) 1070 1071 self.largest_source_size = largest_source_size 1072 1073 def EmitVerification(self, script): 1074 so_far = 0 1075 for tf, sf, size, patch_sha in self.patch_list: 1076 if tf.name != sf.name: 1077 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1078 script.PatchCheck("/"+sf.name, tf.sha1, sf.sha1) 1079 so_far += sf.size 1080 return so_far 1081 1082 def RemoveUnneededFiles(self, script, extras=()): 1083 script.DeleteFiles(["/"+i[0] for i in self.verbatim_targets] + 1084 ["/"+i for i in sorted(self.source_data) 1085 if i not in self.target_data and 1086 i not in self.renames] + 1087 list(extras)) 1088 1089 def TotalPatchSize(self): 1090 return sum(i[1].size for i in self.patch_list) 1091 1092 def EmitPatches(self, script, total_patch_size, so_far): 1093 self.deferred_patch_list = deferred_patch_list = [] 1094 for item in self.patch_list: 1095 tf, sf, size, _ = item 1096 if tf.name == "system/build.prop": 1097 deferred_patch_list.append(item) 1098 continue 1099 if (sf.name != tf.name): 1100 script.SkipNextActionIfTargetExists(tf.name, tf.sha1) 1101 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p") 1102 so_far += tf.size 1103 script.SetProgress(so_far / total_patch_size) 1104 return so_far 1105 1106 def EmitDeferredPatches(self, script): 1107 for item in self.deferred_patch_list: 1108 tf, sf, size, _ = item 1109 script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p") 1110 script.SetPermissions("/system/build.prop", 0, 0, 0644, None, None) 1111 1112 def EmitRenames(self, script): 1113 if len(self.renames) > 0: 1114 script.Print("Renaming files...") 1115 for src, tgt in self.renames.iteritems(): 1116 print "Renaming " + src + " to " + tgt.name 1117 script.RenameFile(src, tgt.name) 1118 1119 1120 1121 1122def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): 1123 target_has_recovery_patch = HasRecoveryPatch(target_zip) 1124 source_has_recovery_patch = HasRecoveryPatch(source_zip) 1125 1126 if (OPTIONS.block_based and 1127 target_has_recovery_patch and 1128 source_has_recovery_patch): 1129 return WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip) 1130 1131 source_version = OPTIONS.source_info_dict["recovery_api_version"] 1132 target_version = OPTIONS.target_info_dict["recovery_api_version"] 1133 1134 if source_version == 0: 1135 print ("WARNING: generating edify script for a source that " 1136 "can't install it.") 1137 script = edify_generator.EdifyGenerator(source_version, 1138 OPTIONS.target_info_dict) 1139 1140 oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties") 1141 oem_dict = None 1142 if oem_props is not None: 1143 if OPTIONS.oem_source is None: 1144 raise common.ExternalError("OEM source required for this build") 1145 script.Mount("/oem") 1146 oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines()) 1147 1148 metadata = {"pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict, 1149 OPTIONS.source_info_dict), 1150 "post-timestamp": GetBuildProp("ro.build.date.utc", 1151 OPTIONS.target_info_dict), 1152 } 1153 1154 device_specific = common.DeviceSpecificParams( 1155 source_zip=source_zip, 1156 source_version=source_version, 1157 target_zip=target_zip, 1158 target_version=target_version, 1159 output_zip=output_zip, 1160 script=script, 1161 metadata=metadata, 1162 info_dict=OPTIONS.info_dict) 1163 1164 system_diff = FileDifference("system", source_zip, target_zip, output_zip) 1165 script.Mount("/system") 1166 if HasVendorPartition(target_zip): 1167 vendor_diff = FileDifference("vendor", source_zip, target_zip, output_zip) 1168 script.Mount("/vendor") 1169 else: 1170 vendor_diff = None 1171 1172 target_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.target_info_dict) 1173 source_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.source_info_dict) 1174 1175 if oem_props is None: 1176 script.AssertSomeFingerprint(source_fp, target_fp) 1177 else: 1178 script.AssertSomeThumbprint( 1179 GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict), 1180 GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict)) 1181 1182 metadata["pre-build"] = source_fp 1183 metadata["post-build"] = target_fp 1184 1185 source_boot = common.GetBootableImage( 1186 "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT", 1187 OPTIONS.source_info_dict) 1188 target_boot = common.GetBootableImage( 1189 "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT") 1190 updating_boot = (not OPTIONS.two_step and 1191 (source_boot.data != target_boot.data)) 1192 1193 source_recovery = common.GetBootableImage( 1194 "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY", 1195 OPTIONS.source_info_dict) 1196 target_recovery = common.GetBootableImage( 1197 "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") 1198 updating_recovery = (source_recovery.data != target_recovery.data) 1199 1200 # Here's how we divide up the progress bar: 1201 # 0.1 for verifying the start state (PatchCheck calls) 1202 # 0.8 for applying patches (ApplyPatch calls) 1203 # 0.1 for unpacking verbatim files, symlinking, and doing the 1204 # device-specific commands. 1205 1206 AppendAssertions(script, OPTIONS.target_info_dict, oem_dict) 1207 device_specific.IncrementalOTA_Assertions() 1208 1209 # Two-step incremental package strategy (in chronological order, 1210 # which is *not* the order in which the generated script has 1211 # things): 1212 # 1213 # if stage is not "2/3" or "3/3": 1214 # do verification on current system 1215 # write recovery image to boot partition 1216 # set stage to "2/3" 1217 # reboot to boot partition and restart recovery 1218 # else if stage is "2/3": 1219 # write recovery image to recovery partition 1220 # set stage to "3/3" 1221 # reboot to recovery partition and restart recovery 1222 # else: 1223 # (stage must be "3/3") 1224 # perform update: 1225 # patch system files, etc. 1226 # force full install of new boot image 1227 # set up system to update recovery partition on first boot 1228 # complete script normally (allow recovery to mark itself finished and reboot) 1229 1230 if OPTIONS.two_step: 1231 if not OPTIONS.info_dict.get("multistage_support", None): 1232 assert False, "two-step packages not supported by this build" 1233 fs = OPTIONS.info_dict["fstab"]["/misc"] 1234 assert fs.fs_type.upper() == "EMMC", \ 1235 "two-step packages only supported on devices with EMMC /misc partitions" 1236 bcb_dev = {"bcb_dev": fs.device} 1237 common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data) 1238 script.AppendExtra(""" 1239if get_stage("%(bcb_dev)s", "stage") == "2/3" then 1240""" % bcb_dev) 1241 script.AppendExtra("sleep(20);\n"); 1242 script.WriteRawImage("/recovery", "recovery.img") 1243 script.AppendExtra(""" 1244set_stage("%(bcb_dev)s", "3/3"); 1245reboot_now("%(bcb_dev)s", "recovery"); 1246else if get_stage("%(bcb_dev)s", "stage") != "3/3" then 1247""" % bcb_dev) 1248 1249 script.Print("Verifying current system...") 1250 1251 device_specific.IncrementalOTA_VerifyBegin() 1252 1253 script.ShowProgress(0.1, 0) 1254 so_far = system_diff.EmitVerification(script) 1255 if vendor_diff: 1256 so_far += vendor_diff.EmitVerification(script) 1257 1258 if updating_boot: 1259 d = common.Difference(target_boot, source_boot) 1260 _, _, d = d.ComputePatch() 1261 print "boot target: %d source: %d diff: %d" % ( 1262 target_boot.size, source_boot.size, len(d)) 1263 1264 common.ZipWriteStr(output_zip, "patch/boot.img.p", d) 1265 1266 boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict) 1267 1268 script.PatchCheck("%s:%s:%d:%s:%d:%s" % 1269 (boot_type, boot_device, 1270 source_boot.size, source_boot.sha1, 1271 target_boot.size, target_boot.sha1)) 1272 so_far += source_boot.size 1273 1274 size = [] 1275 if system_diff.patch_list: size.append(system_diff.largest_source_size) 1276 if vendor_diff: 1277 if vendor_diff.patch_list: size.append(vendor_diff.largest_source_size) 1278 if size or updating_recovery or updating_boot: 1279 script.CacheFreeSpaceCheck(max(size)) 1280 1281 device_specific.IncrementalOTA_VerifyEnd() 1282 1283 if OPTIONS.two_step: 1284 script.WriteRawImage("/boot", "recovery.img") 1285 script.AppendExtra(""" 1286set_stage("%(bcb_dev)s", "2/3"); 1287reboot_now("%(bcb_dev)s", ""); 1288else 1289""" % bcb_dev) 1290 1291 script.Comment("---- start making changes here ----") 1292 1293 device_specific.IncrementalOTA_InstallBegin() 1294 1295 if OPTIONS.two_step: 1296 common.ZipWriteStr(output_zip, "boot.img", target_boot.data) 1297 script.WriteRawImage("/boot", "boot.img") 1298 print "writing full boot image (forced by two-step mode)" 1299 1300 script.Print("Removing unneeded files...") 1301 system_diff.RemoveUnneededFiles(script, ("/system/recovery.img",)) 1302 if vendor_diff: 1303 vendor_diff.RemoveUnneededFiles(script) 1304 1305 script.ShowProgress(0.8, 0) 1306 total_patch_size = 1.0 + system_diff.TotalPatchSize() 1307 if vendor_diff: 1308 total_patch_size += vendor_diff.TotalPatchSize() 1309 if updating_boot: 1310 total_patch_size += target_boot.size 1311 1312 script.Print("Patching system files...") 1313 so_far = system_diff.EmitPatches(script, total_patch_size, 0) 1314 if vendor_diff: 1315 script.Print("Patching vendor files...") 1316 so_far = vendor_diff.EmitPatches(script, total_patch_size, so_far) 1317 1318 if not OPTIONS.two_step: 1319 if updating_boot: 1320 # Produce the boot image by applying a patch to the current 1321 # contents of the boot partition, and write it back to the 1322 # partition. 1323 script.Print("Patching boot image...") 1324 script.ApplyPatch("%s:%s:%d:%s:%d:%s" 1325 % (boot_type, boot_device, 1326 source_boot.size, source_boot.sha1, 1327 target_boot.size, target_boot.sha1), 1328 "-", 1329 target_boot.size, target_boot.sha1, 1330 source_boot.sha1, "patch/boot.img.p") 1331 so_far += target_boot.size 1332 script.SetProgress(so_far / total_patch_size) 1333 print "boot image changed; including." 1334 else: 1335 print "boot image unchanged; skipping." 1336 1337 system_items = ItemSet("system", "META/filesystem_config.txt") 1338 if vendor_diff: 1339 vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt") 1340 1341 if updating_recovery: 1342 # Recovery is generated as a patch using both the boot image 1343 # (which contains the same linux kernel as recovery) and the file 1344 # /system/etc/recovery-resource.dat (which contains all the images 1345 # used in the recovery UI) as sources. This lets us minimize the 1346 # size of the patch, which must be included in every OTA package. 1347 # 1348 # For older builds where recovery-resource.dat is not present, we 1349 # use only the boot image as the source. 1350 1351 if not target_has_recovery_patch: 1352 def output_sink(fn, data): 1353 common.ZipWriteStr(output_zip, "recovery/" + fn, data) 1354 system_items.Get("system/" + fn, dir=False) 1355 1356 common.MakeRecoveryPatch(OPTIONS.target_tmp, output_sink, 1357 target_recovery, target_boot) 1358 script.DeleteFiles(["/system/recovery-from-boot.p", 1359 "/system/etc/install-recovery.sh"]) 1360 print "recovery image changed; including as patch from boot." 1361 else: 1362 print "recovery image unchanged; skipping." 1363 1364 script.ShowProgress(0.1, 10) 1365 1366 target_symlinks = CopyPartitionFiles(system_items, target_zip, None) 1367 if vendor_diff: 1368 target_symlinks.extend(CopyPartitionFiles(vendor_items, target_zip, None)) 1369 1370 temp_script = script.MakeTemporary() 1371 system_items.GetMetadata(target_zip) 1372 system_items.Get("system").SetPermissions(temp_script) 1373 if vendor_diff: 1374 vendor_items.GetMetadata(target_zip) 1375 vendor_items.Get("vendor").SetPermissions(temp_script) 1376 1377 # Note that this call will mess up the trees of Items, so make sure 1378 # we're done with them. 1379 source_symlinks = CopyPartitionFiles(system_items, source_zip, None) 1380 if vendor_diff: 1381 source_symlinks.extend(CopyPartitionFiles(vendor_items, source_zip, None)) 1382 1383 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) 1384 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) 1385 1386 # Delete all the symlinks in source that aren't in target. This 1387 # needs to happen before verbatim files are unpacked, in case a 1388 # symlink in the source is replaced by a real file in the target. 1389 to_delete = [] 1390 for dest, link in source_symlinks: 1391 if link not in target_symlinks_d: 1392 to_delete.append(link) 1393 script.DeleteFiles(to_delete) 1394 1395 if system_diff.verbatim_targets: 1396 script.Print("Unpacking new system files...") 1397 script.UnpackPackageDir("system", "/system") 1398 if vendor_diff and vendor_diff.verbatim_targets: 1399 script.Print("Unpacking new vendor files...") 1400 script.UnpackPackageDir("vendor", "/vendor") 1401 1402 if updating_recovery and not target_has_recovery_patch: 1403 script.Print("Unpacking new recovery...") 1404 script.UnpackPackageDir("recovery", "/system") 1405 1406 system_diff.EmitRenames(script) 1407 if vendor_diff: 1408 vendor_diff.EmitRenames(script) 1409 1410 script.Print("Symlinks and permissions...") 1411 1412 # Create all the symlinks that don't already exist, or point to 1413 # somewhere different than what we want. Delete each symlink before 1414 # creating it, since the 'symlink' command won't overwrite. 1415 to_create = [] 1416 for dest, link in target_symlinks: 1417 if link in source_symlinks_d: 1418 if dest != source_symlinks_d[link]: 1419 to_create.append((dest, link)) 1420 else: 1421 to_create.append((dest, link)) 1422 script.DeleteFiles([i[1] for i in to_create]) 1423 script.MakeSymlinks(to_create) 1424 1425 # Now that the symlinks are created, we can set all the 1426 # permissions. 1427 script.AppendScript(temp_script) 1428 1429 # Do device-specific installation (eg, write radio image). 1430 device_specific.IncrementalOTA_InstallEnd() 1431 1432 if OPTIONS.extra_script is not None: 1433 script.AppendExtra(OPTIONS.extra_script) 1434 1435 # Patch the build.prop file last, so if something fails but the 1436 # device can still come up, it appears to be the old build and will 1437 # get set the OTA package again to retry. 1438 script.Print("Patching remaining system files...") 1439 system_diff.EmitDeferredPatches(script) 1440 1441 if OPTIONS.wipe_user_data: 1442 script.Print("Erasing user data...") 1443 script.FormatPartition("/data") 1444 1445 if OPTIONS.two_step: 1446 script.AppendExtra(""" 1447set_stage("%(bcb_dev)s", ""); 1448endif; 1449endif; 1450""" % bcb_dev) 1451 1452 script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary) 1453 WriteMetadata(metadata, output_zip) 1454 1455 1456def main(argv): 1457 1458 def option_handler(o, a): 1459 if o == "--board_config": 1460 pass # deprecated 1461 elif o in ("-k", "--package_key"): 1462 OPTIONS.package_key = a 1463 elif o in ("-i", "--incremental_from"): 1464 OPTIONS.incremental_source = a 1465 elif o in ("-w", "--wipe_user_data"): 1466 OPTIONS.wipe_user_data = True 1467 elif o in ("-n", "--no_prereq"): 1468 OPTIONS.omit_prereq = True 1469 elif o in ("-o", "--oem_settings"): 1470 OPTIONS.oem_source = a 1471 elif o in ("-e", "--extra_script"): 1472 OPTIONS.extra_script = a 1473 elif o in ("-a", "--aslr_mode"): 1474 if a in ("on", "On", "true", "True", "yes", "Yes"): 1475 OPTIONS.aslr_mode = True 1476 else: 1477 OPTIONS.aslr_mode = False 1478 elif o in ("-t", "--worker_threads"): 1479 if a.isdigit(): 1480 OPTIONS.worker_threads = int(a) 1481 else: 1482 raise ValueError("Cannot parse value %r for option %r - only " 1483 "integers are allowed." % (a, o)) 1484 elif o in ("-2", "--two_step"): 1485 OPTIONS.two_step = True 1486 elif o == "--no_signing": 1487 OPTIONS.no_signing = True 1488 elif o == "--block": 1489 OPTIONS.block_based = True 1490 elif o in ("-b", "--binary"): 1491 OPTIONS.updater_binary = a 1492 else: 1493 return False 1494 return True 1495 1496 args = common.ParseOptions(argv, __doc__, 1497 extra_opts="b:k:i:d:wne:t:a:2o:", 1498 extra_long_opts=["board_config=", 1499 "package_key=", 1500 "incremental_from=", 1501 "wipe_user_data", 1502 "no_prereq", 1503 "extra_script=", 1504 "worker_threads=", 1505 "aslr_mode=", 1506 "two_step", 1507 "no_signing", 1508 "block", 1509 "binary=", 1510 "oem_settings=", 1511 ], 1512 extra_option_handler=option_handler) 1513 1514 if len(args) != 2: 1515 common.Usage(__doc__) 1516 sys.exit(1) 1517 1518 if OPTIONS.extra_script is not None: 1519 OPTIONS.extra_script = open(OPTIONS.extra_script).read() 1520 1521 print "unzipping target target-files..." 1522 OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0]) 1523 1524 OPTIONS.target_tmp = OPTIONS.input_tmp 1525 OPTIONS.info_dict = common.LoadInfoDict(input_zip) 1526 1527 # If this image was originally labelled with SELinux contexts, make sure we 1528 # also apply the labels in our new image. During building, the "file_contexts" 1529 # is in the out/ directory tree, but for repacking from target-files.zip it's 1530 # in the root directory of the ramdisk. 1531 if "selinux_fc" in OPTIONS.info_dict: 1532 OPTIONS.info_dict["selinux_fc"] = os.path.join(OPTIONS.input_tmp, "BOOT", "RAMDISK", 1533 "file_contexts") 1534 1535 if OPTIONS.verbose: 1536 print "--- target info ---" 1537 common.DumpInfoDict(OPTIONS.info_dict) 1538 1539 # If the caller explicitly specified the device-specific extensions 1540 # path via -s/--device_specific, use that. Otherwise, use 1541 # META/releasetools.py if it is present in the target target_files. 1542 # Otherwise, take the path of the file from 'tool_extensions' in the 1543 # info dict and look for that in the local filesystem, relative to 1544 # the current directory. 1545 1546 if OPTIONS.device_specific is None: 1547 from_input = os.path.join(OPTIONS.input_tmp, "META", "releasetools.py") 1548 if os.path.exists(from_input): 1549 print "(using device-specific extensions from target_files)" 1550 OPTIONS.device_specific = from_input 1551 else: 1552 OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None) 1553 1554 if OPTIONS.device_specific is not None: 1555 OPTIONS.device_specific = os.path.abspath(OPTIONS.device_specific) 1556 1557 if OPTIONS.no_signing: 1558 output_zip = zipfile.ZipFile(args[1], "w", compression=zipfile.ZIP_DEFLATED) 1559 else: 1560 temp_zip_file = tempfile.NamedTemporaryFile() 1561 output_zip = zipfile.ZipFile(temp_zip_file, "w", 1562 compression=zipfile.ZIP_DEFLATED) 1563 1564 if OPTIONS.incremental_source is None: 1565 WriteFullOTAPackage(input_zip, output_zip) 1566 if OPTIONS.package_key is None: 1567 OPTIONS.package_key = OPTIONS.info_dict.get( 1568 "default_system_dev_certificate", 1569 "build/target/product/security/testkey") 1570 else: 1571 print "unzipping source target-files..." 1572 OPTIONS.source_tmp, source_zip = common.UnzipTemp(OPTIONS.incremental_source) 1573 OPTIONS.target_info_dict = OPTIONS.info_dict 1574 OPTIONS.source_info_dict = common.LoadInfoDict(source_zip) 1575 if "selinux_fc" in OPTIONS.source_info_dict: 1576 OPTIONS.source_info_dict["selinux_fc"] = os.path.join(OPTIONS.source_tmp, "BOOT", "RAMDISK", 1577 "file_contexts") 1578 if OPTIONS.package_key is None: 1579 OPTIONS.package_key = OPTIONS.source_info_dict.get( 1580 "default_system_dev_certificate", 1581 "build/target/product/security/testkey") 1582 if OPTIONS.verbose: 1583 print "--- source info ---" 1584 common.DumpInfoDict(OPTIONS.source_info_dict) 1585 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) 1586 1587 output_zip.close() 1588 1589 if not OPTIONS.no_signing: 1590 SignOutput(temp_zip_file.name, args[1]) 1591 temp_zip_file.close() 1592 1593 common.Cleanup() 1594 1595 print "done." 1596 1597 1598if __name__ == '__main__': 1599 try: 1600 common.CloseInheritedPipes() 1601 main(sys.argv[1:]) 1602 except common.ExternalError, e: 1603 print 1604 print " ERROR: %s" % (e,) 1605 print 1606 sys.exit(1) 1607