common.py revision 40ef35b9e916163bc408d6d9945f92f012885d33
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32import blockimgdiff 33import rangelib 34 35from hashlib import sha1 as sha1 36 37 38class Options(object): 39 def __init__(self): 40 platform_search_path = { 41 "linux2": "out/host/linux-x86", 42 "darwin": "out/host/darwin-x86", 43 } 44 45 self.search_path = platform_search_path.get(sys.platform, None) 46 self.signapk_path = "framework/signapk.jar" # Relative to search_path 47 self.signapk_shared_library_path = "lib64" # Relative to search_path 48 self.extra_signapk_args = [] 49 self.java_path = "java" # Use the one on the path by default. 50 self.java_args = "-Xmx2048m" # JVM Args 51 self.public_key_suffix = ".x509.pem" 52 self.private_key_suffix = ".pk8" 53 # use otatools built boot_signer by default 54 self.boot_signer_path = "boot_signer" 55 self.boot_signer_args = [] 56 self.verity_signer_path = None 57 self.verity_signer_args = [] 58 self.verbose = False 59 self.tempfiles = [] 60 self.device_specific = None 61 self.extras = {} 62 self.info_dict = None 63 self.source_info_dict = None 64 self.target_info_dict = None 65 self.worker_threads = None 66 # Stash size cannot exceed cache_size * threshold. 67 self.cache_size = None 68 self.stash_threshold = 0.8 69 70 71OPTIONS = Options() 72 73 74# Values for "certificate" in apkcerts that mean special things. 75SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 76 77 78class ExternalError(RuntimeError): 79 pass 80 81 82def Run(args, **kwargs): 83 """Create and return a subprocess.Popen object, printing the command 84 line on the terminal if -v was specified.""" 85 if OPTIONS.verbose: 86 print " running: ", " ".join(args) 87 return subprocess.Popen(args, **kwargs) 88 89 90def CloseInheritedPipes(): 91 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 92 before doing other work.""" 93 if platform.system() != "Darwin": 94 return 95 for d in range(3, 1025): 96 try: 97 stat = os.fstat(d) 98 if stat is not None: 99 pipebit = stat[0] & 0x1000 100 if pipebit != 0: 101 os.close(d) 102 except OSError: 103 pass 104 105 106def LoadInfoDict(input_file, input_dir=None): 107 """Read and parse the META/misc_info.txt key/value pairs from the 108 input target files and return a dict.""" 109 110 def read_helper(fn): 111 if isinstance(input_file, zipfile.ZipFile): 112 return input_file.read(fn) 113 else: 114 path = os.path.join(input_file, *fn.split("/")) 115 try: 116 with open(path) as f: 117 return f.read() 118 except IOError as e: 119 if e.errno == errno.ENOENT: 120 raise KeyError(fn) 121 d = {} 122 try: 123 d = LoadDictionaryFromLines(read_helper("META/misc_info.txt").split("\n")) 124 except KeyError: 125 # ok if misc_info.txt doesn't exist 126 pass 127 128 # backwards compatibility: These values used to be in their own 129 # files. Look for them, in case we're processing an old 130 # target_files zip. 131 132 if "mkyaffs2_extra_flags" not in d: 133 try: 134 d["mkyaffs2_extra_flags"] = read_helper( 135 "META/mkyaffs2-extra-flags.txt").strip() 136 except KeyError: 137 # ok if flags don't exist 138 pass 139 140 if "recovery_api_version" not in d: 141 try: 142 d["recovery_api_version"] = read_helper( 143 "META/recovery-api-version.txt").strip() 144 except KeyError: 145 raise ValueError("can't find recovery API version in input target-files") 146 147 if "tool_extensions" not in d: 148 try: 149 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip() 150 except KeyError: 151 # ok if extensions don't exist 152 pass 153 154 if "fstab_version" not in d: 155 d["fstab_version"] = "1" 156 157 # A few properties are stored as links to the files in the out/ directory. 158 # It works fine with the build system. However, they are no longer available 159 # when (re)generating from target_files zip. If input_dir is not None, we 160 # are doing repacking. Redirect those properties to the actual files in the 161 # unzipped directory. 162 if input_dir is not None: 163 # We carry a copy of file_contexts.bin under META/. If not available, 164 # search BOOT/RAMDISK/. Note that sometimes we may need a different file 165 # to build images than the one running on device, such as when enabling 166 # system_root_image. In that case, we must have the one for image 167 # generation copied to META/. 168 fc_basename = os.path.basename(d.get("selinux_fc", "file_contexts")) 169 fc_config = os.path.join(input_dir, "META", fc_basename) 170 if d.get("system_root_image") == "true": 171 assert os.path.exists(fc_config) 172 if not os.path.exists(fc_config): 173 fc_config = os.path.join(input_dir, "BOOT", "RAMDISK", fc_basename) 174 if not os.path.exists(fc_config): 175 fc_config = None 176 177 if fc_config: 178 d["selinux_fc"] = fc_config 179 180 # Similarly we need to redirect "ramdisk_dir" and "ramdisk_fs_config". 181 if d.get("system_root_image") == "true": 182 d["ramdisk_dir"] = os.path.join(input_dir, "ROOT") 183 d["ramdisk_fs_config"] = os.path.join( 184 input_dir, "META", "root_filesystem_config.txt") 185 186 try: 187 data = read_helper("META/imagesizes.txt") 188 for line in data.split("\n"): 189 if not line: 190 continue 191 name, value = line.split(" ", 1) 192 if not value: 193 continue 194 if name == "blocksize": 195 d[name] = value 196 else: 197 d[name + "_size"] = value 198 except KeyError: 199 pass 200 201 def makeint(key): 202 if key in d: 203 d[key] = int(d[key], 0) 204 205 makeint("recovery_api_version") 206 makeint("blocksize") 207 makeint("system_size") 208 makeint("vendor_size") 209 makeint("userdata_size") 210 makeint("cache_size") 211 makeint("recovery_size") 212 makeint("boot_size") 213 makeint("fstab_version") 214 215 if d.get("no_recovery", False) == "true": 216 d["fstab"] = None 217 else: 218 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"], 219 d.get("system_root_image", False)) 220 d["build.prop"] = LoadBuildProp(read_helper) 221 return d 222 223def LoadBuildProp(read_helper): 224 try: 225 data = read_helper("SYSTEM/build.prop") 226 except KeyError: 227 print "Warning: could not find SYSTEM/build.prop in %s" % zip 228 data = "" 229 return LoadDictionaryFromLines(data.split("\n")) 230 231def LoadDictionaryFromLines(lines): 232 d = {} 233 for line in lines: 234 line = line.strip() 235 if not line or line.startswith("#"): 236 continue 237 if "=" in line: 238 name, value = line.split("=", 1) 239 d[name] = value 240 return d 241 242def LoadRecoveryFSTab(read_helper, fstab_version, system_root_image=False): 243 class Partition(object): 244 def __init__(self, mount_point, fs_type, device, length, device2, context): 245 self.mount_point = mount_point 246 self.fs_type = fs_type 247 self.device = device 248 self.length = length 249 self.device2 = device2 250 self.context = context 251 252 try: 253 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab") 254 except KeyError: 255 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab" 256 data = "" 257 258 if fstab_version == 1: 259 d = {} 260 for line in data.split("\n"): 261 line = line.strip() 262 if not line or line.startswith("#"): 263 continue 264 pieces = line.split() 265 if not 3 <= len(pieces) <= 4: 266 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 267 options = None 268 if len(pieces) >= 4: 269 if pieces[3].startswith("/"): 270 device2 = pieces[3] 271 if len(pieces) >= 5: 272 options = pieces[4] 273 else: 274 device2 = None 275 options = pieces[3] 276 else: 277 device2 = None 278 279 mount_point = pieces[0] 280 length = 0 281 if options: 282 options = options.split(",") 283 for i in options: 284 if i.startswith("length="): 285 length = int(i[7:]) 286 else: 287 print "%s: unknown option \"%s\"" % (mount_point, i) 288 289 d[mount_point] = Partition(mount_point=mount_point, fs_type=pieces[1], 290 device=pieces[2], length=length, 291 device2=device2) 292 293 elif fstab_version == 2: 294 d = {} 295 for line in data.split("\n"): 296 line = line.strip() 297 if not line or line.startswith("#"): 298 continue 299 # <src> <mnt_point> <type> <mnt_flags and options> <fs_mgr_flags> 300 pieces = line.split() 301 if len(pieces) != 5: 302 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 303 304 # Ignore entries that are managed by vold 305 options = pieces[4] 306 if "voldmanaged=" in options: 307 continue 308 309 # It's a good line, parse it 310 length = 0 311 options = options.split(",") 312 for i in options: 313 if i.startswith("length="): 314 length = int(i[7:]) 315 else: 316 # Ignore all unknown options in the unified fstab 317 continue 318 319 mount_flags = pieces[3] 320 # Honor the SELinux context if present. 321 context = None 322 for i in mount_flags.split(","): 323 if i.startswith("context="): 324 context = i 325 326 mount_point = pieces[1] 327 d[mount_point] = Partition(mount_point=mount_point, fs_type=pieces[2], 328 device=pieces[0], length=length, 329 device2=None, context=context) 330 331 else: 332 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 333 334 # / is used for the system mount point when the root directory is included in 335 # system. Other areas assume system is always at "/system" so point /system 336 # at /. 337 if system_root_image: 338 assert not d.has_key("/system") and d.has_key("/") 339 d["/system"] = d["/"] 340 return d 341 342 343def DumpInfoDict(d): 344 for k, v in sorted(d.items()): 345 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 346 347 348def _BuildBootableImage(sourcedir, fs_config_file, info_dict=None, 349 has_ramdisk=False): 350 """Build a bootable image from the specified sourcedir. 351 352 Take a kernel, cmdline, and optionally a ramdisk directory from the input (in 353 'sourcedir'), and turn them into a boot image. Return the image data, or 354 None if sourcedir does not appear to contains files for building the 355 requested image.""" 356 357 def make_ramdisk(): 358 ramdisk_img = tempfile.NamedTemporaryFile() 359 360 if os.access(fs_config_file, os.F_OK): 361 cmd = ["mkbootfs", "-f", fs_config_file, 362 os.path.join(sourcedir, "RAMDISK")] 363 else: 364 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 365 p1 = Run(cmd, stdout=subprocess.PIPE) 366 p2 = Run(["minigzip"], stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 367 368 p2.wait() 369 p1.wait() 370 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (sourcedir,) 371 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (sourcedir,) 372 373 return ramdisk_img 374 375 if not os.access(os.path.join(sourcedir, "kernel"), os.F_OK): 376 return None 377 378 if has_ramdisk and not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK): 379 return None 380 381 if info_dict is None: 382 info_dict = OPTIONS.info_dict 383 384 img = tempfile.NamedTemporaryFile() 385 386 if has_ramdisk: 387 ramdisk_img = make_ramdisk() 388 389 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 390 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 391 392 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 393 394 fn = os.path.join(sourcedir, "second") 395 if os.access(fn, os.F_OK): 396 cmd.append("--second") 397 cmd.append(fn) 398 399 fn = os.path.join(sourcedir, "cmdline") 400 if os.access(fn, os.F_OK): 401 cmd.append("--cmdline") 402 cmd.append(open(fn).read().rstrip("\n")) 403 404 fn = os.path.join(sourcedir, "base") 405 if os.access(fn, os.F_OK): 406 cmd.append("--base") 407 cmd.append(open(fn).read().rstrip("\n")) 408 409 fn = os.path.join(sourcedir, "pagesize") 410 if os.access(fn, os.F_OK): 411 cmd.append("--pagesize") 412 cmd.append(open(fn).read().rstrip("\n")) 413 414 args = info_dict.get("mkbootimg_args", None) 415 if args and args.strip(): 416 cmd.extend(shlex.split(args)) 417 418 if has_ramdisk: 419 cmd.extend(["--ramdisk", ramdisk_img.name]) 420 421 img_unsigned = None 422 if info_dict.get("vboot", None): 423 img_unsigned = tempfile.NamedTemporaryFile() 424 cmd.extend(["--output", img_unsigned.name]) 425 else: 426 cmd.extend(["--output", img.name]) 427 428 p = Run(cmd, stdout=subprocess.PIPE) 429 p.communicate() 430 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 431 os.path.basename(sourcedir),) 432 433 if (info_dict.get("boot_signer", None) == "true" and 434 info_dict.get("verity_key", None)): 435 path = "/" + os.path.basename(sourcedir).lower() 436 cmd = [OPTIONS.boot_signer_path] 437 cmd.extend(OPTIONS.boot_signer_args) 438 cmd.extend([path, img.name, 439 info_dict["verity_key"] + ".pk8", 440 info_dict["verity_key"] + ".x509.pem", img.name]) 441 p = Run(cmd, stdout=subprocess.PIPE) 442 p.communicate() 443 assert p.returncode == 0, "boot_signer of %s image failed" % path 444 445 # Sign the image if vboot is non-empty. 446 elif info_dict.get("vboot", None): 447 path = "/" + os.path.basename(sourcedir).lower() 448 img_keyblock = tempfile.NamedTemporaryFile() 449 cmd = [info_dict["vboot_signer_cmd"], info_dict["futility"], 450 img_unsigned.name, info_dict["vboot_key"] + ".vbpubk", 451 info_dict["vboot_key"] + ".vbprivk", 452 info_dict["vboot_subkey"] + ".vbprivk", 453 img_keyblock.name, 454 img.name] 455 p = Run(cmd, stdout=subprocess.PIPE) 456 p.communicate() 457 assert p.returncode == 0, "vboot_signer of %s image failed" % path 458 459 # Clean up the temp files. 460 img_unsigned.close() 461 img_keyblock.close() 462 463 img.seek(os.SEEK_SET, 0) 464 data = img.read() 465 466 if has_ramdisk: 467 ramdisk_img.close() 468 img.close() 469 470 return data 471 472 473def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 474 info_dict=None): 475 """Return a File object with the desired bootable image. 476 477 Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 'prebuilt_name', 478 otherwise look for it under 'unpack_dir'/IMAGES, otherwise construct it from 479 the source files in 'unpack_dir'/'tree_subdir'.""" 480 481 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 482 if os.path.exists(prebuilt_path): 483 print "using prebuilt %s from BOOTABLE_IMAGES..." % (prebuilt_name,) 484 return File.FromLocalFile(name, prebuilt_path) 485 486 prebuilt_path = os.path.join(unpack_dir, "IMAGES", prebuilt_name) 487 if os.path.exists(prebuilt_path): 488 print "using prebuilt %s from IMAGES..." % (prebuilt_name,) 489 return File.FromLocalFile(name, prebuilt_path) 490 491 print "building image from target_files %s..." % (tree_subdir,) 492 493 if info_dict is None: 494 info_dict = OPTIONS.info_dict 495 496 # With system_root_image == "true", we don't pack ramdisk into the boot image. 497 # Unless "recovery_as_boot" is specified, in which case we carry the ramdisk 498 # for recovery. 499 has_ramdisk = (info_dict.get("system_root_image") != "true" or 500 prebuilt_name != "boot.img" or 501 info_dict.get("recovery_as_boot") == "true") 502 503 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 504 data = _BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 505 os.path.join(unpack_dir, fs_config), 506 info_dict, has_ramdisk) 507 if data: 508 return File(name, data) 509 return None 510 511 512def UnzipTemp(filename, pattern=None): 513 """Unzip the given archive into a temporary directory and return the name. 514 515 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 516 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 517 518 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 519 main file), open for reading. 520 """ 521 522 tmp = tempfile.mkdtemp(prefix="targetfiles-") 523 OPTIONS.tempfiles.append(tmp) 524 525 def unzip_to_dir(filename, dirname): 526 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 527 if pattern is not None: 528 cmd.append(pattern) 529 p = Run(cmd, stdout=subprocess.PIPE) 530 p.communicate() 531 if p.returncode != 0: 532 raise ExternalError("failed to unzip input target-files \"%s\"" % 533 (filename,)) 534 535 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 536 if m: 537 unzip_to_dir(m.group(1), tmp) 538 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 539 filename = m.group(1) 540 else: 541 unzip_to_dir(filename, tmp) 542 543 return tmp, zipfile.ZipFile(filename, "r") 544 545 546def GetKeyPasswords(keylist): 547 """Given a list of keys, prompt the user to enter passwords for 548 those which require them. Return a {key: password} dict. password 549 will be None if the key has no password.""" 550 551 no_passwords = [] 552 need_passwords = [] 553 key_passwords = {} 554 devnull = open("/dev/null", "w+b") 555 for k in sorted(keylist): 556 # We don't need a password for things that aren't really keys. 557 if k in SPECIAL_CERT_STRINGS: 558 no_passwords.append(k) 559 continue 560 561 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 562 "-inform", "DER", "-nocrypt"], 563 stdin=devnull.fileno(), 564 stdout=devnull.fileno(), 565 stderr=subprocess.STDOUT) 566 p.communicate() 567 if p.returncode == 0: 568 # Definitely an unencrypted key. 569 no_passwords.append(k) 570 else: 571 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 572 "-inform", "DER", "-passin", "pass:"], 573 stdin=devnull.fileno(), 574 stdout=devnull.fileno(), 575 stderr=subprocess.PIPE) 576 _, stderr = p.communicate() 577 if p.returncode == 0: 578 # Encrypted key with empty string as password. 579 key_passwords[k] = '' 580 elif stderr.startswith('Error decrypting key'): 581 # Definitely encrypted key. 582 # It would have said "Error reading key" if it didn't parse correctly. 583 need_passwords.append(k) 584 else: 585 # Potentially, a type of key that openssl doesn't understand. 586 # We'll let the routines in signapk.jar handle it. 587 no_passwords.append(k) 588 devnull.close() 589 590 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 591 key_passwords.update(dict.fromkeys(no_passwords, None)) 592 return key_passwords 593 594 595def SignFile(input_name, output_name, key, password, whole_file=False): 596 """Sign the input_name zip/jar/apk, producing output_name. Use the 597 given key and password (the latter may be None if the key does not 598 have a password. 599 600 If whole_file is true, use the "-w" option to SignApk to embed a 601 signature that covers the whole file in the archive comment of the 602 zip file. 603 """ 604 605 java_library_path = os.path.join( 606 OPTIONS.search_path, OPTIONS.signapk_shared_library_path) 607 608 cmd = [OPTIONS.java_path, OPTIONS.java_args, 609 "-Djava.library.path=" + java_library_path, 610 "-jar", 611 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 612 cmd.extend(OPTIONS.extra_signapk_args) 613 if whole_file: 614 cmd.append("-w") 615 cmd.extend([key + OPTIONS.public_key_suffix, 616 key + OPTIONS.private_key_suffix, 617 input_name, output_name]) 618 619 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 620 if password is not None: 621 password += "\n" 622 p.communicate(password) 623 if p.returncode != 0: 624 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 625 626 627def CheckSize(data, target, info_dict): 628 """Check the data string passed against the max size limit, if 629 any, for the given target. Raise exception if the data is too big. 630 Print a warning if the data is nearing the maximum size.""" 631 632 if target.endswith(".img"): 633 target = target[:-4] 634 mount_point = "/" + target 635 636 fs_type = None 637 limit = None 638 if info_dict["fstab"]: 639 if mount_point == "/userdata": 640 mount_point = "/data" 641 p = info_dict["fstab"][mount_point] 642 fs_type = p.fs_type 643 device = p.device 644 if "/" in device: 645 device = device[device.rfind("/")+1:] 646 limit = info_dict.get(device + "_size", None) 647 if not fs_type or not limit: 648 return 649 650 if fs_type == "yaffs2": 651 # image size should be increased by 1/64th to account for the 652 # spare area (64 bytes per 2k page) 653 limit = limit / 2048 * (2048+64) 654 size = len(data) 655 pct = float(size) * 100.0 / limit 656 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 657 if pct >= 99.0: 658 raise ExternalError(msg) 659 elif pct >= 95.0: 660 print 661 print " WARNING: ", msg 662 print 663 elif OPTIONS.verbose: 664 print " ", msg 665 666 667def ReadApkCerts(tf_zip): 668 """Given a target_files ZipFile, parse the META/apkcerts.txt file 669 and return a {package: cert} dict.""" 670 certmap = {} 671 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 672 line = line.strip() 673 if not line: 674 continue 675 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 676 r'private_key="(.*)"$', line) 677 if m: 678 name, cert, privkey = m.groups() 679 public_key_suffix_len = len(OPTIONS.public_key_suffix) 680 private_key_suffix_len = len(OPTIONS.private_key_suffix) 681 if cert in SPECIAL_CERT_STRINGS and not privkey: 682 certmap[name] = cert 683 elif (cert.endswith(OPTIONS.public_key_suffix) and 684 privkey.endswith(OPTIONS.private_key_suffix) and 685 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 686 certmap[name] = cert[:-public_key_suffix_len] 687 else: 688 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 689 return certmap 690 691 692COMMON_DOCSTRING = """ 693 -p (--path) <dir> 694 Prepend <dir>/bin to the list of places to search for binaries 695 run by this script, and expect to find jars in <dir>/framework. 696 697 -s (--device_specific) <file> 698 Path to the python module containing device-specific 699 releasetools code. 700 701 -x (--extra) <key=value> 702 Add a key/value pair to the 'extras' dict, which device-specific 703 extension code may look at. 704 705 -v (--verbose) 706 Show command lines being executed. 707 708 -h (--help) 709 Display this usage message and exit. 710""" 711 712def Usage(docstring): 713 print docstring.rstrip("\n") 714 print COMMON_DOCSTRING 715 716 717def ParseOptions(argv, 718 docstring, 719 extra_opts="", extra_long_opts=(), 720 extra_option_handler=None): 721 """Parse the options in argv and return any arguments that aren't 722 flags. docstring is the calling module's docstring, to be displayed 723 for errors and -h. extra_opts and extra_long_opts are for flags 724 defined by the caller, which are processed by passing them to 725 extra_option_handler.""" 726 727 try: 728 opts, args = getopt.getopt( 729 argv, "hvp:s:x:" + extra_opts, 730 ["help", "verbose", "path=", "signapk_path=", 731 "signapk_shared_library_path=", "extra_signapk_args=", 732 "java_path=", "java_args=", "public_key_suffix=", 733 "private_key_suffix=", "boot_signer_path=", "boot_signer_args=", 734 "verity_signer_path=", "verity_signer_args=", "device_specific=", 735 "extra="] + 736 list(extra_long_opts)) 737 except getopt.GetoptError as err: 738 Usage(docstring) 739 print "**", str(err), "**" 740 sys.exit(2) 741 742 for o, a in opts: 743 if o in ("-h", "--help"): 744 Usage(docstring) 745 sys.exit() 746 elif o in ("-v", "--verbose"): 747 OPTIONS.verbose = True 748 elif o in ("-p", "--path"): 749 OPTIONS.search_path = a 750 elif o in ("--signapk_path",): 751 OPTIONS.signapk_path = a 752 elif o in ("--signapk_shared_library_path",): 753 OPTIONS.signapk_shared_library_path = a 754 elif o in ("--extra_signapk_args",): 755 OPTIONS.extra_signapk_args = shlex.split(a) 756 elif o in ("--java_path",): 757 OPTIONS.java_path = a 758 elif o in ("--java_args",): 759 OPTIONS.java_args = a 760 elif o in ("--public_key_suffix",): 761 OPTIONS.public_key_suffix = a 762 elif o in ("--private_key_suffix",): 763 OPTIONS.private_key_suffix = a 764 elif o in ("--boot_signer_path",): 765 OPTIONS.boot_signer_path = a 766 elif o in ("--boot_signer_args",): 767 OPTIONS.boot_signer_args = shlex.split(a) 768 elif o in ("--verity_signer_path",): 769 OPTIONS.verity_signer_path = a 770 elif o in ("--verity_signer_args",): 771 OPTIONS.verity_signer_args = shlex.split(a) 772 elif o in ("-s", "--device_specific"): 773 OPTIONS.device_specific = a 774 elif o in ("-x", "--extra"): 775 key, value = a.split("=", 1) 776 OPTIONS.extras[key] = value 777 else: 778 if extra_option_handler is None or not extra_option_handler(o, a): 779 assert False, "unknown option \"%s\"" % (o,) 780 781 if OPTIONS.search_path: 782 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 783 os.pathsep + os.environ["PATH"]) 784 785 return args 786 787 788def MakeTempFile(prefix=None, suffix=None): 789 """Make a temp file and add it to the list of things to be deleted 790 when Cleanup() is called. Return the filename.""" 791 fd, fn = tempfile.mkstemp(prefix=prefix, suffix=suffix) 792 os.close(fd) 793 OPTIONS.tempfiles.append(fn) 794 return fn 795 796 797def Cleanup(): 798 for i in OPTIONS.tempfiles: 799 if os.path.isdir(i): 800 shutil.rmtree(i) 801 else: 802 os.remove(i) 803 804 805class PasswordManager(object): 806 def __init__(self): 807 self.editor = os.getenv("EDITOR", None) 808 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 809 810 def GetPasswords(self, items): 811 """Get passwords corresponding to each string in 'items', 812 returning a dict. (The dict may have keys in addition to the 813 values in 'items'.) 814 815 Uses the passwords in $ANDROID_PW_FILE if available, letting the 816 user edit that file to add more needed passwords. If no editor is 817 available, or $ANDROID_PW_FILE isn't define, prompts the user 818 interactively in the ordinary way. 819 """ 820 821 current = self.ReadFile() 822 823 first = True 824 while True: 825 missing = [] 826 for i in items: 827 if i not in current or not current[i]: 828 missing.append(i) 829 # Are all the passwords already in the file? 830 if not missing: 831 return current 832 833 for i in missing: 834 current[i] = "" 835 836 if not first: 837 print "key file %s still missing some passwords." % (self.pwfile,) 838 answer = raw_input("try to edit again? [y]> ").strip() 839 if answer and answer[0] not in 'yY': 840 raise RuntimeError("key passwords unavailable") 841 first = False 842 843 current = self.UpdateAndReadFile(current) 844 845 def PromptResult(self, current): # pylint: disable=no-self-use 846 """Prompt the user to enter a value (password) for each key in 847 'current' whose value is fales. Returns a new dict with all the 848 values. 849 """ 850 result = {} 851 for k, v in sorted(current.iteritems()): 852 if v: 853 result[k] = v 854 else: 855 while True: 856 result[k] = getpass.getpass( 857 "Enter password for %s key> " % k).strip() 858 if result[k]: 859 break 860 return result 861 862 def UpdateAndReadFile(self, current): 863 if not self.editor or not self.pwfile: 864 return self.PromptResult(current) 865 866 f = open(self.pwfile, "w") 867 os.chmod(self.pwfile, 0o600) 868 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 869 f.write("# (Additional spaces are harmless.)\n\n") 870 871 first_line = None 872 sorted_list = sorted([(not v, k, v) for (k, v) in current.iteritems()]) 873 for i, (_, k, v) in enumerate(sorted_list): 874 f.write("[[[ %s ]]] %s\n" % (v, k)) 875 if not v and first_line is None: 876 # position cursor on first line with no password. 877 first_line = i + 4 878 f.close() 879 880 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 881 _, _ = p.communicate() 882 883 return self.ReadFile() 884 885 def ReadFile(self): 886 result = {} 887 if self.pwfile is None: 888 return result 889 try: 890 f = open(self.pwfile, "r") 891 for line in f: 892 line = line.strip() 893 if not line or line[0] == '#': 894 continue 895 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 896 if not m: 897 print "failed to parse password file: ", line 898 else: 899 result[m.group(2)] = m.group(1) 900 f.close() 901 except IOError as e: 902 if e.errno != errno.ENOENT: 903 print "error reading password file: ", str(e) 904 return result 905 906 907def ZipWrite(zip_file, filename, arcname=None, perms=0o644, 908 compress_type=None): 909 import datetime 910 911 # http://b/18015246 912 # Python 2.7's zipfile implementation wrongly thinks that zip64 is required 913 # for files larger than 2GiB. We can work around this by adjusting their 914 # limit. Note that `zipfile.writestr()` will not work for strings larger than 915 # 2GiB. The Python interpreter sometimes rejects strings that large (though 916 # it isn't clear to me exactly what circumstances cause this). 917 # `zipfile.write()` must be used directly to work around this. 918 # 919 # This mess can be avoided if we port to python3. 920 saved_zip64_limit = zipfile.ZIP64_LIMIT 921 zipfile.ZIP64_LIMIT = (1 << 32) - 1 922 923 if compress_type is None: 924 compress_type = zip_file.compression 925 if arcname is None: 926 arcname = filename 927 928 saved_stat = os.stat(filename) 929 930 try: 931 # `zipfile.write()` doesn't allow us to pass ZipInfo, so just modify the 932 # file to be zipped and reset it when we're done. 933 os.chmod(filename, perms) 934 935 # Use a fixed timestamp so the output is repeatable. 936 epoch = datetime.datetime.fromtimestamp(0) 937 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 938 os.utime(filename, (timestamp, timestamp)) 939 940 zip_file.write(filename, arcname=arcname, compress_type=compress_type) 941 finally: 942 os.chmod(filename, saved_stat.st_mode) 943 os.utime(filename, (saved_stat.st_atime, saved_stat.st_mtime)) 944 zipfile.ZIP64_LIMIT = saved_zip64_limit 945 946 947def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=None, 948 compress_type=None): 949 """Wrap zipfile.writestr() function to work around the zip64 limit. 950 951 Even with the ZIP64_LIMIT workaround, it won't allow writing a string 952 longer than 2GiB. It gives 'OverflowError: size does not fit in an int' 953 when calling crc32(bytes). 954 955 But it still works fine to write a shorter string into a large zip file. 956 We should use ZipWrite() whenever possible, and only use ZipWriteStr() 957 when we know the string won't be too long. 958 """ 959 960 saved_zip64_limit = zipfile.ZIP64_LIMIT 961 zipfile.ZIP64_LIMIT = (1 << 32) - 1 962 963 if not isinstance(zinfo_or_arcname, zipfile.ZipInfo): 964 zinfo = zipfile.ZipInfo(filename=zinfo_or_arcname) 965 zinfo.compress_type = zip_file.compression 966 if perms is None: 967 perms = 0o100644 968 else: 969 zinfo = zinfo_or_arcname 970 971 # If compress_type is given, it overrides the value in zinfo. 972 if compress_type is not None: 973 zinfo.compress_type = compress_type 974 975 # If perms is given, it has a priority. 976 if perms is not None: 977 # If perms doesn't set the file type, mark it as a regular file. 978 if perms & 0o770000 == 0: 979 perms |= 0o100000 980 zinfo.external_attr = perms << 16 981 982 # Use a fixed timestamp so the output is repeatable. 983 zinfo.date_time = (2009, 1, 1, 0, 0, 0) 984 985 zip_file.writestr(zinfo, data) 986 zipfile.ZIP64_LIMIT = saved_zip64_limit 987 988 989def ZipClose(zip_file): 990 # http://b/18015246 991 # zipfile also refers to ZIP64_LIMIT during close() when it writes out the 992 # central directory. 993 saved_zip64_limit = zipfile.ZIP64_LIMIT 994 zipfile.ZIP64_LIMIT = (1 << 32) - 1 995 996 zip_file.close() 997 998 zipfile.ZIP64_LIMIT = saved_zip64_limit 999 1000 1001class DeviceSpecificParams(object): 1002 module = None 1003 def __init__(self, **kwargs): 1004 """Keyword arguments to the constructor become attributes of this 1005 object, which is passed to all functions in the device-specific 1006 module.""" 1007 for k, v in kwargs.iteritems(): 1008 setattr(self, k, v) 1009 self.extras = OPTIONS.extras 1010 1011 if self.module is None: 1012 path = OPTIONS.device_specific 1013 if not path: 1014 return 1015 try: 1016 if os.path.isdir(path): 1017 info = imp.find_module("releasetools", [path]) 1018 else: 1019 d, f = os.path.split(path) 1020 b, x = os.path.splitext(f) 1021 if x == ".py": 1022 f = b 1023 info = imp.find_module(f, [d]) 1024 print "loaded device-specific extensions from", path 1025 self.module = imp.load_module("device_specific", *info) 1026 except ImportError: 1027 print "unable to load device-specific module; assuming none" 1028 1029 def _DoCall(self, function_name, *args, **kwargs): 1030 """Call the named function in the device-specific module, passing 1031 the given args and kwargs. The first argument to the call will be 1032 the DeviceSpecific object itself. If there is no module, or the 1033 module does not define the function, return the value of the 1034 'default' kwarg (which itself defaults to None).""" 1035 if self.module is None or not hasattr(self.module, function_name): 1036 return kwargs.get("default", None) 1037 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 1038 1039 def FullOTA_Assertions(self): 1040 """Called after emitting the block of assertions at the top of a 1041 full OTA package. Implementations can add whatever additional 1042 assertions they like.""" 1043 return self._DoCall("FullOTA_Assertions") 1044 1045 def FullOTA_InstallBegin(self): 1046 """Called at the start of full OTA installation.""" 1047 return self._DoCall("FullOTA_InstallBegin") 1048 1049 def FullOTA_InstallEnd(self): 1050 """Called at the end of full OTA installation; typically this is 1051 used to install the image for the device's baseband processor.""" 1052 return self._DoCall("FullOTA_InstallEnd") 1053 1054 def IncrementalOTA_Assertions(self): 1055 """Called after emitting the block of assertions at the top of an 1056 incremental OTA package. Implementations can add whatever 1057 additional assertions they like.""" 1058 return self._DoCall("IncrementalOTA_Assertions") 1059 1060 def IncrementalOTA_VerifyBegin(self): 1061 """Called at the start of the verification phase of incremental 1062 OTA installation; additional checks can be placed here to abort 1063 the script before any changes are made.""" 1064 return self._DoCall("IncrementalOTA_VerifyBegin") 1065 1066 def IncrementalOTA_VerifyEnd(self): 1067 """Called at the end of the verification phase of incremental OTA 1068 installation; additional checks can be placed here to abort the 1069 script before any changes are made.""" 1070 return self._DoCall("IncrementalOTA_VerifyEnd") 1071 1072 def IncrementalOTA_InstallBegin(self): 1073 """Called at the start of incremental OTA installation (after 1074 verification is complete).""" 1075 return self._DoCall("IncrementalOTA_InstallBegin") 1076 1077 def IncrementalOTA_InstallEnd(self): 1078 """Called at the end of incremental OTA installation; typically 1079 this is used to install the image for the device's baseband 1080 processor.""" 1081 return self._DoCall("IncrementalOTA_InstallEnd") 1082 1083 def VerifyOTA_Assertions(self): 1084 return self._DoCall("VerifyOTA_Assertions") 1085 1086class File(object): 1087 def __init__(self, name, data): 1088 self.name = name 1089 self.data = data 1090 self.size = len(data) 1091 self.sha1 = sha1(data).hexdigest() 1092 1093 @classmethod 1094 def FromLocalFile(cls, name, diskname): 1095 f = open(diskname, "rb") 1096 data = f.read() 1097 f.close() 1098 return File(name, data) 1099 1100 def WriteToTemp(self): 1101 t = tempfile.NamedTemporaryFile() 1102 t.write(self.data) 1103 t.flush() 1104 return t 1105 1106 def AddToZip(self, z, compression=None): 1107 ZipWriteStr(z, self.name, self.data, compress_type=compression) 1108 1109DIFF_PROGRAM_BY_EXT = { 1110 ".gz" : "imgdiff", 1111 ".zip" : ["imgdiff", "-z"], 1112 ".jar" : ["imgdiff", "-z"], 1113 ".apk" : ["imgdiff", "-z"], 1114 ".img" : "imgdiff", 1115 } 1116 1117class Difference(object): 1118 def __init__(self, tf, sf, diff_program=None): 1119 self.tf = tf 1120 self.sf = sf 1121 self.patch = None 1122 self.diff_program = diff_program 1123 1124 def ComputePatch(self): 1125 """Compute the patch (as a string of data) needed to turn sf into 1126 tf. Returns the same tuple as GetPatch().""" 1127 1128 tf = self.tf 1129 sf = self.sf 1130 1131 if self.diff_program: 1132 diff_program = self.diff_program 1133 else: 1134 ext = os.path.splitext(tf.name)[1] 1135 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 1136 1137 ttemp = tf.WriteToTemp() 1138 stemp = sf.WriteToTemp() 1139 1140 ext = os.path.splitext(tf.name)[1] 1141 1142 try: 1143 ptemp = tempfile.NamedTemporaryFile() 1144 if isinstance(diff_program, list): 1145 cmd = copy.copy(diff_program) 1146 else: 1147 cmd = [diff_program] 1148 cmd.append(stemp.name) 1149 cmd.append(ttemp.name) 1150 cmd.append(ptemp.name) 1151 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1152 err = [] 1153 def run(): 1154 _, e = p.communicate() 1155 if e: 1156 err.append(e) 1157 th = threading.Thread(target=run) 1158 th.start() 1159 th.join(timeout=300) # 5 mins 1160 if th.is_alive(): 1161 print "WARNING: diff command timed out" 1162 p.terminate() 1163 th.join(5) 1164 if th.is_alive(): 1165 p.kill() 1166 th.join() 1167 1168 if err or p.returncode != 0: 1169 print "WARNING: failure running %s:\n%s\n" % ( 1170 diff_program, "".join(err)) 1171 self.patch = None 1172 return None, None, None 1173 diff = ptemp.read() 1174 finally: 1175 ptemp.close() 1176 stemp.close() 1177 ttemp.close() 1178 1179 self.patch = diff 1180 return self.tf, self.sf, self.patch 1181 1182 1183 def GetPatch(self): 1184 """Return a tuple (target_file, source_file, patch_data). 1185 patch_data may be None if ComputePatch hasn't been called, or if 1186 computing the patch failed.""" 1187 return self.tf, self.sf, self.patch 1188 1189 1190def ComputeDifferences(diffs): 1191 """Call ComputePatch on all the Difference objects in 'diffs'.""" 1192 print len(diffs), "diffs to compute" 1193 1194 # Do the largest files first, to try and reduce the long-pole effect. 1195 by_size = [(i.tf.size, i) for i in diffs] 1196 by_size.sort(reverse=True) 1197 by_size = [i[1] for i in by_size] 1198 1199 lock = threading.Lock() 1200 diff_iter = iter(by_size) # accessed under lock 1201 1202 def worker(): 1203 try: 1204 lock.acquire() 1205 for d in diff_iter: 1206 lock.release() 1207 start = time.time() 1208 d.ComputePatch() 1209 dur = time.time() - start 1210 lock.acquire() 1211 1212 tf, sf, patch = d.GetPatch() 1213 if sf.name == tf.name: 1214 name = tf.name 1215 else: 1216 name = "%s (%s)" % (tf.name, sf.name) 1217 if patch is None: 1218 print "patching failed! %s" % (name,) 1219 else: 1220 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 1221 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 1222 lock.release() 1223 except Exception as e: 1224 print e 1225 raise 1226 1227 # start worker threads; wait for them all to finish. 1228 threads = [threading.Thread(target=worker) 1229 for i in range(OPTIONS.worker_threads)] 1230 for th in threads: 1231 th.start() 1232 while threads: 1233 threads.pop().join() 1234 1235 1236class BlockDifference(object): 1237 def __init__(self, partition, tgt, src=None, check_first_block=False, 1238 version=None): 1239 self.tgt = tgt 1240 self.src = src 1241 self.partition = partition 1242 self.check_first_block = check_first_block 1243 1244 # Due to http://b/20939131, check_first_block is disabled temporarily. 1245 assert not self.check_first_block 1246 1247 if version is None: 1248 version = 1 1249 if OPTIONS.info_dict: 1250 version = max( 1251 int(i) for i in 1252 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(",")) 1253 self.version = version 1254 1255 b = blockimgdiff.BlockImageDiff(tgt, src, threads=OPTIONS.worker_threads, 1256 version=self.version) 1257 tmpdir = tempfile.mkdtemp() 1258 OPTIONS.tempfiles.append(tmpdir) 1259 self.path = os.path.join(tmpdir, partition) 1260 b.Compute(self.path) 1261 1262 if src is None: 1263 _, self.device = GetTypeAndDevice("/" + partition, OPTIONS.info_dict) 1264 else: 1265 _, self.device = GetTypeAndDevice("/" + partition, 1266 OPTIONS.source_info_dict) 1267 1268 def WriteScript(self, script, output_zip, progress=None): 1269 if not self.src: 1270 # write the output unconditionally 1271 script.Print("Patching %s image unconditionally..." % (self.partition,)) 1272 else: 1273 script.Print("Patching %s image after verification." % (self.partition,)) 1274 1275 if progress: 1276 script.ShowProgress(progress, 0) 1277 self._WriteUpdate(script, output_zip) 1278 self._WritePostInstallVerifyScript(script) 1279 1280 def WriteStrictVerifyScript(self, script): 1281 """Verify all the blocks in the care_map, including clobbered blocks. 1282 1283 This differs from the WriteVerifyScript() function: a) it prints different 1284 error messages; b) it doesn't allow half-way updated images to pass the 1285 verification.""" 1286 1287 partition = self.partition 1288 script.Print("Verifying %s..." % (partition,)) 1289 ranges = self.tgt.care_map 1290 ranges_str = ranges.to_string_raw() 1291 script.AppendExtra('range_sha1("%s", "%s") == "%s" && ' 1292 'ui_print(" Verified.") || ' 1293 'ui_print("\\"%s\\" has unexpected contents.");' % ( 1294 self.device, ranges_str, 1295 self.tgt.TotalSha1(include_clobbered_blocks=True), 1296 self.device)) 1297 script.AppendExtra("") 1298 1299 def WriteVerifyScript(self, script): 1300 partition = self.partition 1301 if not self.src: 1302 script.Print("Image %s will be patched unconditionally." % (partition,)) 1303 else: 1304 ranges = self.src.care_map.subtract(self.src.clobbered_blocks) 1305 ranges_str = ranges.to_string_raw() 1306 if self.version >= 4: 1307 script.AppendExtra(('if (range_sha1("%s", "%s") == "%s" || ' 1308 'block_image_verify("%s", ' 1309 'package_extract_file("%s.transfer.list"), ' 1310 '"%s.new.dat", "%s.patch.dat") || ' 1311 '(block_image_recover("%s", "%s") && ' 1312 'block_image_verify("%s", ' 1313 'package_extract_file("%s.transfer.list"), ' 1314 '"%s.new.dat", "%s.patch.dat"))) then') % ( 1315 self.device, ranges_str, self.src.TotalSha1(), 1316 self.device, partition, partition, partition, 1317 self.device, ranges_str, 1318 self.device, partition, partition, partition)) 1319 elif self.version == 3: 1320 script.AppendExtra(('if (range_sha1("%s", "%s") == "%s" || ' 1321 'block_image_verify("%s", ' 1322 'package_extract_file("%s.transfer.list"), ' 1323 '"%s.new.dat", "%s.patch.dat")) then') % ( 1324 self.device, ranges_str, self.src.TotalSha1(), 1325 self.device, partition, partition, partition)) 1326 else: 1327 script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' % ( 1328 self.device, ranges_str, self.src.TotalSha1())) 1329 script.Print('Verified %s image...' % (partition,)) 1330 script.AppendExtra('else') 1331 1332 # When generating incrementals for the system and vendor partitions, 1333 # explicitly check the first block (which contains the superblock) of 1334 # the partition to see if it's what we expect. If this check fails, 1335 # give an explicit log message about the partition having been 1336 # remounted R/W (the most likely explanation) and the need to flash to 1337 # get OTAs working again. 1338 if self.check_first_block: 1339 self._CheckFirstBlock(script) 1340 1341 # Abort the OTA update. Note that the incremental OTA cannot be applied 1342 # even if it may match the checksum of the target partition. 1343 # a) If version < 3, operations like move and erase will make changes 1344 # unconditionally and damage the partition. 1345 # b) If version >= 3, it won't even reach here. 1346 script.AppendExtra(('abort("%s partition has unexpected contents");\n' 1347 'endif;') % (partition,)) 1348 1349 def _WritePostInstallVerifyScript(self, script): 1350 partition = self.partition 1351 script.Print('Verifying the updated %s image...' % (partition,)) 1352 # Unlike pre-install verification, clobbered_blocks should not be ignored. 1353 ranges = self.tgt.care_map 1354 ranges_str = ranges.to_string_raw() 1355 script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' % ( 1356 self.device, ranges_str, 1357 self.tgt.TotalSha1(include_clobbered_blocks=True))) 1358 1359 # Bug: 20881595 1360 # Verify that extended blocks are really zeroed out. 1361 if self.tgt.extended: 1362 ranges_str = self.tgt.extended.to_string_raw() 1363 script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' % ( 1364 self.device, ranges_str, 1365 self._HashZeroBlocks(self.tgt.extended.size()))) 1366 script.Print('Verified the updated %s image.' % (partition,)) 1367 script.AppendExtra( 1368 'else\n' 1369 ' abort("%s partition has unexpected non-zero contents after OTA ' 1370 'update");\n' 1371 'endif;' % (partition,)) 1372 else: 1373 script.Print('Verified the updated %s image.' % (partition,)) 1374 1375 script.AppendExtra( 1376 'else\n' 1377 ' abort("%s partition has unexpected contents after OTA update");\n' 1378 'endif;' % (partition,)) 1379 1380 def _WriteUpdate(self, script, output_zip): 1381 ZipWrite(output_zip, 1382 '{}.transfer.list'.format(self.path), 1383 '{}.transfer.list'.format(self.partition)) 1384 ZipWrite(output_zip, 1385 '{}.new.dat'.format(self.path), 1386 '{}.new.dat'.format(self.partition)) 1387 ZipWrite(output_zip, 1388 '{}.patch.dat'.format(self.path), 1389 '{}.patch.dat'.format(self.partition), 1390 compress_type=zipfile.ZIP_STORED) 1391 1392 call = ('block_image_update("{device}", ' 1393 'package_extract_file("{partition}.transfer.list"), ' 1394 '"{partition}.new.dat", "{partition}.patch.dat");\n'.format( 1395 device=self.device, partition=self.partition)) 1396 script.AppendExtra(script.WordWrap(call)) 1397 1398 def _HashBlocks(self, source, ranges): # pylint: disable=no-self-use 1399 data = source.ReadRangeSet(ranges) 1400 ctx = sha1() 1401 1402 for p in data: 1403 ctx.update(p) 1404 1405 return ctx.hexdigest() 1406 1407 def _HashZeroBlocks(self, num_blocks): # pylint: disable=no-self-use 1408 """Return the hash value for all zero blocks.""" 1409 zero_block = '\x00' * 4096 1410 ctx = sha1() 1411 for _ in range(num_blocks): 1412 ctx.update(zero_block) 1413 1414 return ctx.hexdigest() 1415 1416 # TODO(tbao): Due to http://b/20939131, block 0 may be changed without 1417 # remounting R/W. Will change the checking to a finer-grained way to 1418 # mask off those bits. 1419 def _CheckFirstBlock(self, script): 1420 r = rangelib.RangeSet((0, 1)) 1421 srchash = self._HashBlocks(self.src, r) 1422 1423 script.AppendExtra(('(range_sha1("%s", "%s") == "%s") || ' 1424 'abort("%s has been remounted R/W; ' 1425 'reflash device to reenable OTA updates");') 1426 % (self.device, r.to_string_raw(), srchash, 1427 self.device)) 1428 1429DataImage = blockimgdiff.DataImage 1430 1431 1432# map recovery.fstab's fs_types to mount/format "partition types" 1433PARTITION_TYPES = { 1434 "yaffs2": "MTD", 1435 "mtd": "MTD", 1436 "ext4": "EMMC", 1437 "emmc": "EMMC", 1438 "f2fs": "EMMC", 1439 "squashfs": "EMMC" 1440} 1441 1442def GetTypeAndDevice(mount_point, info): 1443 fstab = info["fstab"] 1444 if fstab: 1445 return (PARTITION_TYPES[fstab[mount_point].fs_type], 1446 fstab[mount_point].device) 1447 else: 1448 raise KeyError 1449 1450 1451def ParseCertificate(data): 1452 """Parse a PEM-format certificate.""" 1453 cert = [] 1454 save = False 1455 for line in data.split("\n"): 1456 if "--END CERTIFICATE--" in line: 1457 break 1458 if save: 1459 cert.append(line) 1460 if "--BEGIN CERTIFICATE--" in line: 1461 save = True 1462 cert = "".join(cert).decode('base64') 1463 return cert 1464 1465def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img, 1466 info_dict=None): 1467 """Generate a binary patch that creates the recovery image starting 1468 with the boot image. (Most of the space in these images is just the 1469 kernel, which is identical for the two, so the resulting patch 1470 should be efficient.) Add it to the output zip, along with a shell 1471 script that is run from init.rc on first boot to actually do the 1472 patching and install the new recovery image. 1473 1474 recovery_img and boot_img should be File objects for the 1475 corresponding images. info should be the dictionary returned by 1476 common.LoadInfoDict() on the input target_files. 1477 """ 1478 1479 if info_dict is None: 1480 info_dict = OPTIONS.info_dict 1481 1482 full_recovery_image = info_dict.get("full_recovery_image", None) == "true" 1483 system_root_image = info_dict.get("system_root_image", None) == "true" 1484 1485 if full_recovery_image: 1486 output_sink("etc/recovery.img", recovery_img.data) 1487 1488 else: 1489 diff_program = ["imgdiff"] 1490 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat") 1491 if os.path.exists(path): 1492 diff_program.append("-b") 1493 diff_program.append(path) 1494 bonus_args = "-b /system/etc/recovery-resource.dat" 1495 else: 1496 bonus_args = "" 1497 1498 d = Difference(recovery_img, boot_img, diff_program=diff_program) 1499 _, _, patch = d.ComputePatch() 1500 output_sink("recovery-from-boot.p", patch) 1501 1502 try: 1503 # The following GetTypeAndDevice()s need to use the path in the target 1504 # info_dict instead of source_info_dict. 1505 boot_type, boot_device = GetTypeAndDevice("/boot", info_dict) 1506 recovery_type, recovery_device = GetTypeAndDevice("/recovery", info_dict) 1507 except KeyError: 1508 return 1509 1510 if full_recovery_image: 1511 sh = """#!/system/bin/sh 1512if ! applypatch -c %(type)s:%(device)s:%(size)d:%(sha1)s; then 1513 applypatch /system/etc/recovery.img %(type)s:%(device)s %(sha1)s %(size)d && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1514else 1515 log -t recovery "Recovery image already installed" 1516fi 1517""" % {'type': recovery_type, 1518 'device': recovery_device, 1519 'sha1': recovery_img.sha1, 1520 'size': recovery_img.size} 1521 else: 1522 sh = """#!/system/bin/sh 1523if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 1524 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1525else 1526 log -t recovery "Recovery image already installed" 1527fi 1528""" % {'boot_size': boot_img.size, 1529 'boot_sha1': boot_img.sha1, 1530 'recovery_size': recovery_img.size, 1531 'recovery_sha1': recovery_img.sha1, 1532 'boot_type': boot_type, 1533 'boot_device': boot_device, 1534 'recovery_type': recovery_type, 1535 'recovery_device': recovery_device, 1536 'bonus_args': bonus_args} 1537 1538 # The install script location moved from /system/etc to /system/bin 1539 # in the L release. Parse init.*.rc files to find out where the 1540 # target-files expects it to be, and put it there. 1541 sh_location = "etc/install-recovery.sh" 1542 found = False 1543 if system_root_image: 1544 init_rc_dir = os.path.join(input_dir, "ROOT") 1545 else: 1546 init_rc_dir = os.path.join(input_dir, "BOOT", "RAMDISK") 1547 init_rc_files = os.listdir(init_rc_dir) 1548 for init_rc_file in init_rc_files: 1549 if (not init_rc_file.startswith('init.') or 1550 not init_rc_file.endswith('.rc')): 1551 continue 1552 1553 with open(os.path.join(init_rc_dir, init_rc_file)) as f: 1554 for line in f: 1555 m = re.match(r"^service flash_recovery /system/(\S+)\s*$", line) 1556 if m: 1557 sh_location = m.group(1) 1558 found = True 1559 break 1560 1561 if found: 1562 break 1563 1564 print "putting script in", sh_location 1565 1566 output_sink(sh_location, sh) 1567