common.py revision 852a5b531cba16b51108399d993ec2706e35096b
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32import blockimgdiff 33import rangelib 34 35from hashlib import sha1 as sha1 36 37 38class Options(object): 39 def __init__(self): 40 platform_search_path = { 41 "linux2": "out/host/linux-x86", 42 "darwin": "out/host/darwin-x86", 43 } 44 45 self.search_path = platform_search_path.get(sys.platform, None) 46 self.signapk_path = "framework/signapk.jar" # Relative to search_path 47 self.extra_signapk_args = [] 48 self.java_path = "java" # Use the one on the path by default. 49 self.java_args = "-Xmx2048m" # JVM Args 50 self.public_key_suffix = ".x509.pem" 51 self.private_key_suffix = ".pk8" 52 # use otatools built boot_signer by default 53 self.boot_signer_path = "boot_signer" 54 self.verbose = False 55 self.tempfiles = [] 56 self.device_specific = None 57 self.extras = {} 58 self.info_dict = None 59 self.worker_threads = None 60 61 62OPTIONS = Options() 63 64 65# Values for "certificate" in apkcerts that mean special things. 66SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 67 68 69class ExternalError(RuntimeError): 70 pass 71 72 73def Run(args, **kwargs): 74 """Create and return a subprocess.Popen object, printing the command 75 line on the terminal if -v was specified.""" 76 if OPTIONS.verbose: 77 print " running: ", " ".join(args) 78 return subprocess.Popen(args, **kwargs) 79 80 81def CloseInheritedPipes(): 82 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 83 before doing other work.""" 84 if platform.system() != "Darwin": 85 return 86 for d in range(3, 1025): 87 try: 88 stat = os.fstat(d) 89 if stat is not None: 90 pipebit = stat[0] & 0x1000 91 if pipebit != 0: 92 os.close(d) 93 except OSError: 94 pass 95 96 97def LoadInfoDict(input_file): 98 """Read and parse the META/misc_info.txt key/value pairs from the 99 input target files and return a dict.""" 100 101 def read_helper(fn): 102 if isinstance(input_file, zipfile.ZipFile): 103 return input_file.read(fn) 104 else: 105 path = os.path.join(input_file, *fn.split("/")) 106 try: 107 with open(path) as f: 108 return f.read() 109 except IOError as e: 110 if e.errno == errno.ENOENT: 111 raise KeyError(fn) 112 d = {} 113 try: 114 d = LoadDictionaryFromLines(read_helper("META/misc_info.txt").split("\n")) 115 except KeyError: 116 # ok if misc_info.txt doesn't exist 117 pass 118 119 # backwards compatibility: These values used to be in their own 120 # files. Look for them, in case we're processing an old 121 # target_files zip. 122 123 if "mkyaffs2_extra_flags" not in d: 124 try: 125 d["mkyaffs2_extra_flags"] = read_helper( 126 "META/mkyaffs2-extra-flags.txt").strip() 127 except KeyError: 128 # ok if flags don't exist 129 pass 130 131 if "recovery_api_version" not in d: 132 try: 133 d["recovery_api_version"] = read_helper( 134 "META/recovery-api-version.txt").strip() 135 except KeyError: 136 raise ValueError("can't find recovery API version in input target-files") 137 138 if "tool_extensions" not in d: 139 try: 140 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip() 141 except KeyError: 142 # ok if extensions don't exist 143 pass 144 145 if "fstab_version" not in d: 146 d["fstab_version"] = "1" 147 148 try: 149 data = read_helper("META/imagesizes.txt") 150 for line in data.split("\n"): 151 if not line: 152 continue 153 name, value = line.split(" ", 1) 154 if not value: 155 continue 156 if name == "blocksize": 157 d[name] = value 158 else: 159 d[name + "_size"] = value 160 except KeyError: 161 pass 162 163 def makeint(key): 164 if key in d: 165 d[key] = int(d[key], 0) 166 167 makeint("recovery_api_version") 168 makeint("blocksize") 169 makeint("system_size") 170 makeint("vendor_size") 171 makeint("userdata_size") 172 makeint("cache_size") 173 makeint("recovery_size") 174 makeint("boot_size") 175 makeint("fstab_version") 176 177 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"]) 178 d["build.prop"] = LoadBuildProp(read_helper) 179 return d 180 181def LoadBuildProp(read_helper): 182 try: 183 data = read_helper("SYSTEM/build.prop") 184 except KeyError: 185 print "Warning: could not find SYSTEM/build.prop in %s" % zip 186 data = "" 187 return LoadDictionaryFromLines(data.split("\n")) 188 189def LoadDictionaryFromLines(lines): 190 d = {} 191 for line in lines: 192 line = line.strip() 193 if not line or line.startswith("#"): 194 continue 195 if "=" in line: 196 name, value = line.split("=", 1) 197 d[name] = value 198 return d 199 200def LoadRecoveryFSTab(read_helper, fstab_version): 201 class Partition(object): 202 def __init__(self, mount_point, fs_type, device, length, device2): 203 self.mount_point = mount_point 204 self.fs_type = fs_type 205 self.device = device 206 self.length = length 207 self.device2 = device2 208 209 try: 210 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab") 211 except KeyError: 212 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab" 213 data = "" 214 215 if fstab_version == 1: 216 d = {} 217 for line in data.split("\n"): 218 line = line.strip() 219 if not line or line.startswith("#"): 220 continue 221 pieces = line.split() 222 if not 3 <= len(pieces) <= 4: 223 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 224 options = None 225 if len(pieces) >= 4: 226 if pieces[3].startswith("/"): 227 device2 = pieces[3] 228 if len(pieces) >= 5: 229 options = pieces[4] 230 else: 231 device2 = None 232 options = pieces[3] 233 else: 234 device2 = None 235 236 mount_point = pieces[0] 237 length = 0 238 if options: 239 options = options.split(",") 240 for i in options: 241 if i.startswith("length="): 242 length = int(i[7:]) 243 else: 244 print "%s: unknown option \"%s\"" % (mount_point, i) 245 246 d[mount_point] = Partition(mount_point=mount_point, fs_type=pieces[1], 247 device=pieces[2], length=length, 248 device2=device2) 249 250 elif fstab_version == 2: 251 d = {} 252 for line in data.split("\n"): 253 line = line.strip() 254 if not line or line.startswith("#"): 255 continue 256 pieces = line.split() 257 if len(pieces) != 5: 258 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 259 260 # Ignore entries that are managed by vold 261 options = pieces[4] 262 if "voldmanaged=" in options: 263 continue 264 265 # It's a good line, parse it 266 length = 0 267 options = options.split(",") 268 for i in options: 269 if i.startswith("length="): 270 length = int(i[7:]) 271 else: 272 # Ignore all unknown options in the unified fstab 273 continue 274 275 mount_point = pieces[1] 276 d[mount_point] = Partition(mount_point=mount_point, fs_type=pieces[2], 277 device=pieces[0], length=length, device2=None) 278 279 else: 280 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 281 282 return d 283 284 285def DumpInfoDict(d): 286 for k, v in sorted(d.items()): 287 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 288 289 290def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 291 """Take a kernel, cmdline, and ramdisk directory from the input (in 292 'sourcedir'), and turn them into a boot image. Return the image 293 data, or None if sourcedir does not appear to contains files for 294 building the requested image.""" 295 296 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 297 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 298 return None 299 300 if info_dict is None: 301 info_dict = OPTIONS.info_dict 302 303 ramdisk_img = tempfile.NamedTemporaryFile() 304 img = tempfile.NamedTemporaryFile() 305 306 if os.access(fs_config_file, os.F_OK): 307 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 308 else: 309 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 310 p1 = Run(cmd, stdout=subprocess.PIPE) 311 p2 = Run(["minigzip"], 312 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 313 314 p2.wait() 315 p1.wait() 316 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (sourcedir,) 317 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (sourcedir,) 318 319 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 320 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 321 322 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 323 324 fn = os.path.join(sourcedir, "second") 325 if os.access(fn, os.F_OK): 326 cmd.append("--second") 327 cmd.append(fn) 328 329 fn = os.path.join(sourcedir, "cmdline") 330 if os.access(fn, os.F_OK): 331 cmd.append("--cmdline") 332 cmd.append(open(fn).read().rstrip("\n")) 333 334 fn = os.path.join(sourcedir, "base") 335 if os.access(fn, os.F_OK): 336 cmd.append("--base") 337 cmd.append(open(fn).read().rstrip("\n")) 338 339 fn = os.path.join(sourcedir, "pagesize") 340 if os.access(fn, os.F_OK): 341 cmd.append("--pagesize") 342 cmd.append(open(fn).read().rstrip("\n")) 343 344 args = info_dict.get("mkbootimg_args", None) 345 if args and args.strip(): 346 cmd.extend(shlex.split(args)) 347 348 img_unsigned = None 349 if info_dict.get("vboot", None): 350 img_unsigned = tempfile.NamedTemporaryFile() 351 cmd.extend(["--ramdisk", ramdisk_img.name, 352 "--output", img_unsigned.name]) 353 else: 354 cmd.extend(["--ramdisk", ramdisk_img.name, 355 "--output", img.name]) 356 357 p = Run(cmd, stdout=subprocess.PIPE) 358 p.communicate() 359 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 360 os.path.basename(sourcedir),) 361 362 if info_dict.get("verity_key", None): 363 path = "/" + os.path.basename(sourcedir).lower() 364 cmd = [OPTIONS.boot_signer_path, path, img.name, 365 info_dict["verity_key"] + ".pk8", 366 info_dict["verity_key"] + ".x509.pem", img.name] 367 p = Run(cmd, stdout=subprocess.PIPE) 368 p.communicate() 369 assert p.returncode == 0, "boot_signer of %s image failed" % path 370 371 # Sign the image if vboot is non-empty. 372 elif info_dict.get("vboot", None): 373 path = "/" + os.path.basename(sourcedir).lower() 374 img_keyblock = tempfile.NamedTemporaryFile() 375 cmd = [info_dict["vboot_signer_cmd"], info_dict["futility"], 376 img_unsigned.name, info_dict["vboot_key"] + ".vbpubk", 377 info_dict["vboot_key"] + ".vbprivk", img_keyblock.name, 378 img.name] 379 p = Run(cmd, stdout=subprocess.PIPE) 380 p.communicate() 381 assert p.returncode == 0, "vboot_signer of %s image failed" % path 382 383 # Clean up the temp files. 384 img_unsigned.close() 385 img_keyblock.close() 386 387 img.seek(os.SEEK_SET, 0) 388 data = img.read() 389 390 ramdisk_img.close() 391 img.close() 392 393 return data 394 395 396def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 397 info_dict=None): 398 """Return a File object (with name 'name') with the desired bootable 399 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 400 'prebuilt_name', otherwise look for it under 'unpack_dir'/IMAGES, 401 otherwise construct it from the source files in 402 'unpack_dir'/'tree_subdir'.""" 403 404 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 405 if os.path.exists(prebuilt_path): 406 print "using prebuilt %s from BOOTABLE_IMAGES..." % (prebuilt_name,) 407 return File.FromLocalFile(name, prebuilt_path) 408 409 prebuilt_path = os.path.join(unpack_dir, "IMAGES", prebuilt_name) 410 if os.path.exists(prebuilt_path): 411 print "using prebuilt %s from IMAGES..." % (prebuilt_name,) 412 return File.FromLocalFile(name, prebuilt_path) 413 414 print "building image from target_files %s..." % (tree_subdir,) 415 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 416 data = BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 417 os.path.join(unpack_dir, fs_config), 418 info_dict) 419 if data: 420 return File(name, data) 421 return None 422 423 424def UnzipTemp(filename, pattern=None): 425 """Unzip the given archive into a temporary directory and return the name. 426 427 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 428 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 429 430 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 431 main file), open for reading. 432 """ 433 434 tmp = tempfile.mkdtemp(prefix="targetfiles-") 435 OPTIONS.tempfiles.append(tmp) 436 437 def unzip_to_dir(filename, dirname): 438 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 439 if pattern is not None: 440 cmd.append(pattern) 441 p = Run(cmd, stdout=subprocess.PIPE) 442 p.communicate() 443 if p.returncode != 0: 444 raise ExternalError("failed to unzip input target-files \"%s\"" % 445 (filename,)) 446 447 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 448 if m: 449 unzip_to_dir(m.group(1), tmp) 450 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 451 filename = m.group(1) 452 else: 453 unzip_to_dir(filename, tmp) 454 455 return tmp, zipfile.ZipFile(filename, "r") 456 457 458def GetKeyPasswords(keylist): 459 """Given a list of keys, prompt the user to enter passwords for 460 those which require them. Return a {key: password} dict. password 461 will be None if the key has no password.""" 462 463 no_passwords = [] 464 need_passwords = [] 465 key_passwords = {} 466 devnull = open("/dev/null", "w+b") 467 for k in sorted(keylist): 468 # We don't need a password for things that aren't really keys. 469 if k in SPECIAL_CERT_STRINGS: 470 no_passwords.append(k) 471 continue 472 473 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 474 "-inform", "DER", "-nocrypt"], 475 stdin=devnull.fileno(), 476 stdout=devnull.fileno(), 477 stderr=subprocess.STDOUT) 478 p.communicate() 479 if p.returncode == 0: 480 # Definitely an unencrypted key. 481 no_passwords.append(k) 482 else: 483 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 484 "-inform", "DER", "-passin", "pass:"], 485 stdin=devnull.fileno(), 486 stdout=devnull.fileno(), 487 stderr=subprocess.PIPE) 488 _, stderr = p.communicate() 489 if p.returncode == 0: 490 # Encrypted key with empty string as password. 491 key_passwords[k] = '' 492 elif stderr.startswith('Error decrypting key'): 493 # Definitely encrypted key. 494 # It would have said "Error reading key" if it didn't parse correctly. 495 need_passwords.append(k) 496 else: 497 # Potentially, a type of key that openssl doesn't understand. 498 # We'll let the routines in signapk.jar handle it. 499 no_passwords.append(k) 500 devnull.close() 501 502 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 503 key_passwords.update(dict.fromkeys(no_passwords, None)) 504 return key_passwords 505 506 507def SignFile(input_name, output_name, key, password, align=None, 508 whole_file=False): 509 """Sign the input_name zip/jar/apk, producing output_name. Use the 510 given key and password (the latter may be None if the key does not 511 have a password. 512 513 If align is an integer > 1, zipalign is run to align stored files in 514 the output zip on 'align'-byte boundaries. 515 516 If whole_file is true, use the "-w" option to SignApk to embed a 517 signature that covers the whole file in the archive comment of the 518 zip file. 519 """ 520 521 if align == 0 or align == 1: 522 align = None 523 524 if align: 525 temp = tempfile.NamedTemporaryFile() 526 sign_name = temp.name 527 else: 528 sign_name = output_name 529 530 cmd = [OPTIONS.java_path, OPTIONS.java_args, "-jar", 531 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 532 cmd.extend(OPTIONS.extra_signapk_args) 533 if whole_file: 534 cmd.append("-w") 535 cmd.extend([key + OPTIONS.public_key_suffix, 536 key + OPTIONS.private_key_suffix, 537 input_name, sign_name]) 538 539 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 540 if password is not None: 541 password += "\n" 542 p.communicate(password) 543 if p.returncode != 0: 544 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 545 546 if align: 547 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 548 p.communicate() 549 if p.returncode != 0: 550 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 551 temp.close() 552 553 554def CheckSize(data, target, info_dict): 555 """Check the data string passed against the max size limit, if 556 any, for the given target. Raise exception if the data is too big. 557 Print a warning if the data is nearing the maximum size.""" 558 559 if target.endswith(".img"): 560 target = target[:-4] 561 mount_point = "/" + target 562 563 fs_type = None 564 limit = None 565 if info_dict["fstab"]: 566 if mount_point == "/userdata": 567 mount_point = "/data" 568 p = info_dict["fstab"][mount_point] 569 fs_type = p.fs_type 570 device = p.device 571 if "/" in device: 572 device = device[device.rfind("/")+1:] 573 limit = info_dict.get(device + "_size", None) 574 if not fs_type or not limit: 575 return 576 577 if fs_type == "yaffs2": 578 # image size should be increased by 1/64th to account for the 579 # spare area (64 bytes per 2k page) 580 limit = limit / 2048 * (2048+64) 581 size = len(data) 582 pct = float(size) * 100.0 / limit 583 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 584 if pct >= 99.0: 585 raise ExternalError(msg) 586 elif pct >= 95.0: 587 print 588 print " WARNING: ", msg 589 print 590 elif OPTIONS.verbose: 591 print " ", msg 592 593 594def ReadApkCerts(tf_zip): 595 """Given a target_files ZipFile, parse the META/apkcerts.txt file 596 and return a {package: cert} dict.""" 597 certmap = {} 598 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 599 line = line.strip() 600 if not line: 601 continue 602 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 603 r'private_key="(.*)"$', line) 604 if m: 605 name, cert, privkey = m.groups() 606 public_key_suffix_len = len(OPTIONS.public_key_suffix) 607 private_key_suffix_len = len(OPTIONS.private_key_suffix) 608 if cert in SPECIAL_CERT_STRINGS and not privkey: 609 certmap[name] = cert 610 elif (cert.endswith(OPTIONS.public_key_suffix) and 611 privkey.endswith(OPTIONS.private_key_suffix) and 612 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 613 certmap[name] = cert[:-public_key_suffix_len] 614 else: 615 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 616 return certmap 617 618 619COMMON_DOCSTRING = """ 620 -p (--path) <dir> 621 Prepend <dir>/bin to the list of places to search for binaries 622 run by this script, and expect to find jars in <dir>/framework. 623 624 -s (--device_specific) <file> 625 Path to the python module containing device-specific 626 releasetools code. 627 628 -x (--extra) <key=value> 629 Add a key/value pair to the 'extras' dict, which device-specific 630 extension code may look at. 631 632 -v (--verbose) 633 Show command lines being executed. 634 635 -h (--help) 636 Display this usage message and exit. 637""" 638 639def Usage(docstring): 640 print docstring.rstrip("\n") 641 print COMMON_DOCSTRING 642 643 644def ParseOptions(argv, 645 docstring, 646 extra_opts="", extra_long_opts=(), 647 extra_option_handler=None): 648 """Parse the options in argv and return any arguments that aren't 649 flags. docstring is the calling module's docstring, to be displayed 650 for errors and -h. extra_opts and extra_long_opts are for flags 651 defined by the caller, which are processed by passing them to 652 extra_option_handler.""" 653 654 try: 655 opts, args = getopt.getopt( 656 argv, "hvp:s:x:" + extra_opts, 657 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 658 "java_path=", "java_args=", "public_key_suffix=", 659 "private_key_suffix=", "boot_signer_path=", "device_specific=", 660 "extra="] + 661 list(extra_long_opts)) 662 except getopt.GetoptError as err: 663 Usage(docstring) 664 print "**", str(err), "**" 665 sys.exit(2) 666 667 for o, a in opts: 668 if o in ("-h", "--help"): 669 Usage(docstring) 670 sys.exit() 671 elif o in ("-v", "--verbose"): 672 OPTIONS.verbose = True 673 elif o in ("-p", "--path"): 674 OPTIONS.search_path = a 675 elif o in ("--signapk_path",): 676 OPTIONS.signapk_path = a 677 elif o in ("--extra_signapk_args",): 678 OPTIONS.extra_signapk_args = shlex.split(a) 679 elif o in ("--java_path",): 680 OPTIONS.java_path = a 681 elif o in ("--java_args",): 682 OPTIONS.java_args = a 683 elif o in ("--public_key_suffix",): 684 OPTIONS.public_key_suffix = a 685 elif o in ("--private_key_suffix",): 686 OPTIONS.private_key_suffix = a 687 elif o in ("--boot_signer_path",): 688 OPTIONS.boot_signer_path = a 689 elif o in ("-s", "--device_specific"): 690 OPTIONS.device_specific = a 691 elif o in ("-x", "--extra"): 692 key, value = a.split("=", 1) 693 OPTIONS.extras[key] = value 694 else: 695 if extra_option_handler is None or not extra_option_handler(o, a): 696 assert False, "unknown option \"%s\"" % (o,) 697 698 if OPTIONS.search_path: 699 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 700 os.pathsep + os.environ["PATH"]) 701 702 return args 703 704 705def MakeTempFile(prefix=None, suffix=None): 706 """Make a temp file and add it to the list of things to be deleted 707 when Cleanup() is called. Return the filename.""" 708 fd, fn = tempfile.mkstemp(prefix=prefix, suffix=suffix) 709 os.close(fd) 710 OPTIONS.tempfiles.append(fn) 711 return fn 712 713 714def Cleanup(): 715 for i in OPTIONS.tempfiles: 716 if os.path.isdir(i): 717 shutil.rmtree(i) 718 else: 719 os.remove(i) 720 721 722class PasswordManager(object): 723 def __init__(self): 724 self.editor = os.getenv("EDITOR", None) 725 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 726 727 def GetPasswords(self, items): 728 """Get passwords corresponding to each string in 'items', 729 returning a dict. (The dict may have keys in addition to the 730 values in 'items'.) 731 732 Uses the passwords in $ANDROID_PW_FILE if available, letting the 733 user edit that file to add more needed passwords. If no editor is 734 available, or $ANDROID_PW_FILE isn't define, prompts the user 735 interactively in the ordinary way. 736 """ 737 738 current = self.ReadFile() 739 740 first = True 741 while True: 742 missing = [] 743 for i in items: 744 if i not in current or not current[i]: 745 missing.append(i) 746 # Are all the passwords already in the file? 747 if not missing: 748 return current 749 750 for i in missing: 751 current[i] = "" 752 753 if not first: 754 print "key file %s still missing some passwords." % (self.pwfile,) 755 answer = raw_input("try to edit again? [y]> ").strip() 756 if answer and answer[0] not in 'yY': 757 raise RuntimeError("key passwords unavailable") 758 first = False 759 760 current = self.UpdateAndReadFile(current) 761 762 def PromptResult(self, current): # pylint: disable=no-self-use 763 """Prompt the user to enter a value (password) for each key in 764 'current' whose value is fales. Returns a new dict with all the 765 values. 766 """ 767 result = {} 768 for k, v in sorted(current.iteritems()): 769 if v: 770 result[k] = v 771 else: 772 while True: 773 result[k] = getpass.getpass( 774 "Enter password for %s key> " % k).strip() 775 if result[k]: 776 break 777 return result 778 779 def UpdateAndReadFile(self, current): 780 if not self.editor or not self.pwfile: 781 return self.PromptResult(current) 782 783 f = open(self.pwfile, "w") 784 os.chmod(self.pwfile, 0o600) 785 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 786 f.write("# (Additional spaces are harmless.)\n\n") 787 788 first_line = None 789 sorted_list = sorted([(not v, k, v) for (k, v) in current.iteritems()]) 790 for i, (_, k, v) in enumerate(sorted_list): 791 f.write("[[[ %s ]]] %s\n" % (v, k)) 792 if not v and first_line is None: 793 # position cursor on first line with no password. 794 first_line = i + 4 795 f.close() 796 797 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 798 _, _ = p.communicate() 799 800 return self.ReadFile() 801 802 def ReadFile(self): 803 result = {} 804 if self.pwfile is None: 805 return result 806 try: 807 f = open(self.pwfile, "r") 808 for line in f: 809 line = line.strip() 810 if not line or line[0] == '#': 811 continue 812 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 813 if not m: 814 print "failed to parse password file: ", line 815 else: 816 result[m.group(2)] = m.group(1) 817 f.close() 818 except IOError as e: 819 if e.errno != errno.ENOENT: 820 print "error reading password file: ", str(e) 821 return result 822 823 824def ZipWrite(zip_file, filename, arcname=None, perms=0o644, 825 compress_type=None): 826 import datetime 827 828 # http://b/18015246 829 # Python 2.7's zipfile implementation wrongly thinks that zip64 is required 830 # for files larger than 2GiB. We can work around this by adjusting their 831 # limit. Note that `zipfile.writestr()` will not work for strings larger than 832 # 2GiB. The Python interpreter sometimes rejects strings that large (though 833 # it isn't clear to me exactly what circumstances cause this). 834 # `zipfile.write()` must be used directly to work around this. 835 # 836 # This mess can be avoided if we port to python3. 837 saved_zip64_limit = zipfile.ZIP64_LIMIT 838 zipfile.ZIP64_LIMIT = (1 << 32) - 1 839 840 if compress_type is None: 841 compress_type = zip_file.compression 842 if arcname is None: 843 arcname = filename 844 845 saved_stat = os.stat(filename) 846 847 try: 848 # `zipfile.write()` doesn't allow us to pass ZipInfo, so just modify the 849 # file to be zipped and reset it when we're done. 850 os.chmod(filename, perms) 851 852 # Use a fixed timestamp so the output is repeatable. 853 epoch = datetime.datetime.fromtimestamp(0) 854 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 855 os.utime(filename, (timestamp, timestamp)) 856 857 zip_file.write(filename, arcname=arcname, compress_type=compress_type) 858 finally: 859 os.chmod(filename, saved_stat.st_mode) 860 os.utime(filename, (saved_stat.st_atime, saved_stat.st_mtime)) 861 zipfile.ZIP64_LIMIT = saved_zip64_limit 862 863 864def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=0o644, 865 compress_type=None): 866 """Wrap zipfile.writestr() function to work around the zip64 limit. 867 868 Even with the ZIP64_LIMIT workaround, it won't allow writing a string 869 longer than 2GiB. It gives 'OverflowError: size does not fit in an int' 870 when calling crc32(bytes). 871 872 But it still works fine to write a shorter string into a large zip file. 873 We should use ZipWrite() whenever possible, and only use ZipWriteStr() 874 when we know the string won't be too long. 875 """ 876 877 saved_zip64_limit = zipfile.ZIP64_LIMIT 878 zipfile.ZIP64_LIMIT = (1 << 32) - 1 879 880 if not isinstance(zinfo_or_arcname, zipfile.ZipInfo): 881 zinfo = zipfile.ZipInfo(filename=zinfo_or_arcname) 882 zinfo.compress_type = zip_file.compression 883 else: 884 zinfo = zinfo_or_arcname 885 886 # If compress_type is given, it overrides the value in zinfo. 887 if compress_type is not None: 888 zinfo.compress_type = compress_type 889 890 # Use a fixed timestamp so the output is repeatable. 891 zinfo.external_attr = perms << 16 892 zinfo.date_time = (2009, 1, 1, 0, 0, 0) 893 894 zip_file.writestr(zinfo, data) 895 zipfile.ZIP64_LIMIT = saved_zip64_limit 896 897 898def ZipClose(zip_file): 899 # http://b/18015246 900 # zipfile also refers to ZIP64_LIMIT during close() when it writes out the 901 # central directory. 902 saved_zip64_limit = zipfile.ZIP64_LIMIT 903 zipfile.ZIP64_LIMIT = (1 << 32) - 1 904 905 zip_file.close() 906 907 zipfile.ZIP64_LIMIT = saved_zip64_limit 908 909 910class DeviceSpecificParams(object): 911 module = None 912 def __init__(self, **kwargs): 913 """Keyword arguments to the constructor become attributes of this 914 object, which is passed to all functions in the device-specific 915 module.""" 916 for k, v in kwargs.iteritems(): 917 setattr(self, k, v) 918 self.extras = OPTIONS.extras 919 920 if self.module is None: 921 path = OPTIONS.device_specific 922 if not path: 923 return 924 try: 925 if os.path.isdir(path): 926 info = imp.find_module("releasetools", [path]) 927 else: 928 d, f = os.path.split(path) 929 b, x = os.path.splitext(f) 930 if x == ".py": 931 f = b 932 info = imp.find_module(f, [d]) 933 print "loaded device-specific extensions from", path 934 self.module = imp.load_module("device_specific", *info) 935 except ImportError: 936 print "unable to load device-specific module; assuming none" 937 938 def _DoCall(self, function_name, *args, **kwargs): 939 """Call the named function in the device-specific module, passing 940 the given args and kwargs. The first argument to the call will be 941 the DeviceSpecific object itself. If there is no module, or the 942 module does not define the function, return the value of the 943 'default' kwarg (which itself defaults to None).""" 944 if self.module is None or not hasattr(self.module, function_name): 945 return kwargs.get("default", None) 946 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 947 948 def FullOTA_Assertions(self): 949 """Called after emitting the block of assertions at the top of a 950 full OTA package. Implementations can add whatever additional 951 assertions they like.""" 952 return self._DoCall("FullOTA_Assertions") 953 954 def FullOTA_InstallBegin(self): 955 """Called at the start of full OTA installation.""" 956 return self._DoCall("FullOTA_InstallBegin") 957 958 def FullOTA_InstallEnd(self): 959 """Called at the end of full OTA installation; typically this is 960 used to install the image for the device's baseband processor.""" 961 return self._DoCall("FullOTA_InstallEnd") 962 963 def IncrementalOTA_Assertions(self): 964 """Called after emitting the block of assertions at the top of an 965 incremental OTA package. Implementations can add whatever 966 additional assertions they like.""" 967 return self._DoCall("IncrementalOTA_Assertions") 968 969 def IncrementalOTA_VerifyBegin(self): 970 """Called at the start of the verification phase of incremental 971 OTA installation; additional checks can be placed here to abort 972 the script before any changes are made.""" 973 return self._DoCall("IncrementalOTA_VerifyBegin") 974 975 def IncrementalOTA_VerifyEnd(self): 976 """Called at the end of the verification phase of incremental OTA 977 installation; additional checks can be placed here to abort the 978 script before any changes are made.""" 979 return self._DoCall("IncrementalOTA_VerifyEnd") 980 981 def IncrementalOTA_InstallBegin(self): 982 """Called at the start of incremental OTA installation (after 983 verification is complete).""" 984 return self._DoCall("IncrementalOTA_InstallBegin") 985 986 def IncrementalOTA_InstallEnd(self): 987 """Called at the end of incremental OTA installation; typically 988 this is used to install the image for the device's baseband 989 processor.""" 990 return self._DoCall("IncrementalOTA_InstallEnd") 991 992class File(object): 993 def __init__(self, name, data): 994 self.name = name 995 self.data = data 996 self.size = len(data) 997 self.sha1 = sha1(data).hexdigest() 998 999 @classmethod 1000 def FromLocalFile(cls, name, diskname): 1001 f = open(diskname, "rb") 1002 data = f.read() 1003 f.close() 1004 return File(name, data) 1005 1006 def WriteToTemp(self): 1007 t = tempfile.NamedTemporaryFile() 1008 t.write(self.data) 1009 t.flush() 1010 return t 1011 1012 def AddToZip(self, z, compression=None): 1013 ZipWriteStr(z, self.name, self.data, compress_type=compression) 1014 1015DIFF_PROGRAM_BY_EXT = { 1016 ".gz" : "imgdiff", 1017 ".zip" : ["imgdiff", "-z"], 1018 ".jar" : ["imgdiff", "-z"], 1019 ".apk" : ["imgdiff", "-z"], 1020 ".img" : "imgdiff", 1021 } 1022 1023class Difference(object): 1024 def __init__(self, tf, sf, diff_program=None): 1025 self.tf = tf 1026 self.sf = sf 1027 self.patch = None 1028 self.diff_program = diff_program 1029 1030 def ComputePatch(self): 1031 """Compute the patch (as a string of data) needed to turn sf into 1032 tf. Returns the same tuple as GetPatch().""" 1033 1034 tf = self.tf 1035 sf = self.sf 1036 1037 if self.diff_program: 1038 diff_program = self.diff_program 1039 else: 1040 ext = os.path.splitext(tf.name)[1] 1041 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 1042 1043 ttemp = tf.WriteToTemp() 1044 stemp = sf.WriteToTemp() 1045 1046 ext = os.path.splitext(tf.name)[1] 1047 1048 try: 1049 ptemp = tempfile.NamedTemporaryFile() 1050 if isinstance(diff_program, list): 1051 cmd = copy.copy(diff_program) 1052 else: 1053 cmd = [diff_program] 1054 cmd.append(stemp.name) 1055 cmd.append(ttemp.name) 1056 cmd.append(ptemp.name) 1057 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1058 err = [] 1059 def run(): 1060 _, e = p.communicate() 1061 if e: 1062 err.append(e) 1063 th = threading.Thread(target=run) 1064 th.start() 1065 th.join(timeout=300) # 5 mins 1066 if th.is_alive(): 1067 print "WARNING: diff command timed out" 1068 p.terminate() 1069 th.join(5) 1070 if th.is_alive(): 1071 p.kill() 1072 th.join() 1073 1074 if err or p.returncode != 0: 1075 print "WARNING: failure running %s:\n%s\n" % ( 1076 diff_program, "".join(err)) 1077 self.patch = None 1078 return None, None, None 1079 diff = ptemp.read() 1080 finally: 1081 ptemp.close() 1082 stemp.close() 1083 ttemp.close() 1084 1085 self.patch = diff 1086 return self.tf, self.sf, self.patch 1087 1088 1089 def GetPatch(self): 1090 """Return a tuple (target_file, source_file, patch_data). 1091 patch_data may be None if ComputePatch hasn't been called, or if 1092 computing the patch failed.""" 1093 return self.tf, self.sf, self.patch 1094 1095 1096def ComputeDifferences(diffs): 1097 """Call ComputePatch on all the Difference objects in 'diffs'.""" 1098 print len(diffs), "diffs to compute" 1099 1100 # Do the largest files first, to try and reduce the long-pole effect. 1101 by_size = [(i.tf.size, i) for i in diffs] 1102 by_size.sort(reverse=True) 1103 by_size = [i[1] for i in by_size] 1104 1105 lock = threading.Lock() 1106 diff_iter = iter(by_size) # accessed under lock 1107 1108 def worker(): 1109 try: 1110 lock.acquire() 1111 for d in diff_iter: 1112 lock.release() 1113 start = time.time() 1114 d.ComputePatch() 1115 dur = time.time() - start 1116 lock.acquire() 1117 1118 tf, sf, patch = d.GetPatch() 1119 if sf.name == tf.name: 1120 name = tf.name 1121 else: 1122 name = "%s (%s)" % (tf.name, sf.name) 1123 if patch is None: 1124 print "patching failed! %s" % (name,) 1125 else: 1126 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 1127 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 1128 lock.release() 1129 except Exception as e: 1130 print e 1131 raise 1132 1133 # start worker threads; wait for them all to finish. 1134 threads = [threading.Thread(target=worker) 1135 for i in range(OPTIONS.worker_threads)] 1136 for th in threads: 1137 th.start() 1138 while threads: 1139 threads.pop().join() 1140 1141 1142class BlockDifference(object): 1143 def __init__(self, partition, tgt, src=None, check_first_block=False, 1144 version=None): 1145 self.tgt = tgt 1146 self.src = src 1147 self.partition = partition 1148 self.check_first_block = check_first_block 1149 1150 if version is None: 1151 version = 1 1152 if OPTIONS.info_dict: 1153 version = max( 1154 int(i) for i in 1155 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(",")) 1156 self.version = version 1157 1158 b = blockimgdiff.BlockImageDiff(tgt, src, threads=OPTIONS.worker_threads, 1159 version=self.version) 1160 tmpdir = tempfile.mkdtemp() 1161 OPTIONS.tempfiles.append(tmpdir) 1162 self.path = os.path.join(tmpdir, partition) 1163 b.Compute(self.path) 1164 1165 _, self.device = GetTypeAndDevice("/" + partition, OPTIONS.info_dict) 1166 1167 def WriteScript(self, script, output_zip, progress=None): 1168 if not self.src: 1169 # write the output unconditionally 1170 script.Print("Patching %s image unconditionally..." % (self.partition,)) 1171 else: 1172 script.Print("Patching %s image after verification." % (self.partition,)) 1173 1174 if progress: 1175 script.ShowProgress(progress, 0) 1176 self._WriteUpdate(script, output_zip) 1177 1178 def WriteVerifyScript(self, script): 1179 partition = self.partition 1180 if not self.src: 1181 script.Print("Image %s will be patched unconditionally." % (partition,)) 1182 else: 1183 if self.version >= 3: 1184 script.AppendExtra(('if (range_sha1("%s", "%s") == "%s" || ' 1185 'block_image_verify("%s", ' 1186 'package_extract_file("%s.transfer.list"), ' 1187 '"%s.new.dat", "%s.patch.dat")) then') % ( 1188 self.device, self.src.care_map.to_string_raw(), 1189 self.src.TotalSha1(), 1190 self.device, partition, partition, partition)) 1191 else: 1192 script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' % ( 1193 self.device, self.src.care_map.to_string_raw(), 1194 self.src.TotalSha1())) 1195 script.Print('Verified %s image...' % (partition,)) 1196 script.AppendExtra('else') 1197 1198 # When generating incrementals for the system and vendor partitions, 1199 # explicitly check the first block (which contains the superblock) of 1200 # the partition to see if it's what we expect. If this check fails, 1201 # give an explicit log message about the partition having been 1202 # remounted R/W (the most likely explanation) and the need to flash to 1203 # get OTAs working again. 1204 if self.check_first_block: 1205 self._CheckFirstBlock(script) 1206 1207 # Abort the OTA update. Note that the incremental OTA cannot be applied 1208 # even if it may match the checksum of the target partition. 1209 # a) If version < 3, operations like move and erase will make changes 1210 # unconditionally and damage the partition. 1211 # b) If version >= 3, it won't even reach here. 1212 script.AppendExtra(('abort("%s partition has unexpected contents");\n' 1213 'endif;') % (partition,)) 1214 1215 def _WriteUpdate(self, script, output_zip): 1216 ZipWrite(output_zip, 1217 '{}.transfer.list'.format(self.path), 1218 '{}.transfer.list'.format(self.partition)) 1219 ZipWrite(output_zip, 1220 '{}.new.dat'.format(self.path), 1221 '{}.new.dat'.format(self.partition)) 1222 ZipWrite(output_zip, 1223 '{}.patch.dat'.format(self.path), 1224 '{}.patch.dat'.format(self.partition), 1225 compress_type=zipfile.ZIP_STORED) 1226 1227 call = ('block_image_update("{device}", ' 1228 'package_extract_file("{partition}.transfer.list"), ' 1229 '"{partition}.new.dat", "{partition}.patch.dat");\n'.format( 1230 device=self.device, partition=self.partition)) 1231 script.AppendExtra(script.WordWrap(call)) 1232 1233 def _HashBlocks(self, source, ranges): # pylint: disable=no-self-use 1234 data = source.ReadRangeSet(ranges) 1235 ctx = sha1() 1236 1237 for p in data: 1238 ctx.update(p) 1239 1240 return ctx.hexdigest() 1241 1242 def _CheckFirstBlock(self, script): 1243 r = rangelib.RangeSet((0, 1)) 1244 srchash = self._HashBlocks(self.src, r) 1245 1246 script.AppendExtra(('(range_sha1("%s", "%s") == "%s") || ' 1247 'abort("%s has been remounted R/W; ' 1248 'reflash device to reenable OTA updates");') 1249 % (self.device, r.to_string_raw(), srchash, 1250 self.device)) 1251 1252DataImage = blockimgdiff.DataImage 1253 1254 1255# map recovery.fstab's fs_types to mount/format "partition types" 1256PARTITION_TYPES = { 1257 "yaffs2": "MTD", 1258 "mtd": "MTD", 1259 "ext4": "EMMC", 1260 "emmc": "EMMC", 1261 "f2fs": "EMMC", 1262 "squashfs": "EMMC" 1263} 1264 1265def GetTypeAndDevice(mount_point, info): 1266 fstab = info["fstab"] 1267 if fstab: 1268 return (PARTITION_TYPES[fstab[mount_point].fs_type], 1269 fstab[mount_point].device) 1270 else: 1271 raise KeyError 1272 1273 1274def ParseCertificate(data): 1275 """Parse a PEM-format certificate.""" 1276 cert = [] 1277 save = False 1278 for line in data.split("\n"): 1279 if "--END CERTIFICATE--" in line: 1280 break 1281 if save: 1282 cert.append(line) 1283 if "--BEGIN CERTIFICATE--" in line: 1284 save = True 1285 cert = "".join(cert).decode('base64') 1286 return cert 1287 1288def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img, 1289 info_dict=None): 1290 """Generate a binary patch that creates the recovery image starting 1291 with the boot image. (Most of the space in these images is just the 1292 kernel, which is identical for the two, so the resulting patch 1293 should be efficient.) Add it to the output zip, along with a shell 1294 script that is run from init.rc on first boot to actually do the 1295 patching and install the new recovery image. 1296 1297 recovery_img and boot_img should be File objects for the 1298 corresponding images. info should be the dictionary returned by 1299 common.LoadInfoDict() on the input target_files. 1300 """ 1301 1302 if info_dict is None: 1303 info_dict = OPTIONS.info_dict 1304 1305 diff_program = ["imgdiff"] 1306 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat") 1307 if os.path.exists(path): 1308 diff_program.append("-b") 1309 diff_program.append(path) 1310 bonus_args = "-b /system/etc/recovery-resource.dat" 1311 else: 1312 bonus_args = "" 1313 1314 d = Difference(recovery_img, boot_img, diff_program=diff_program) 1315 _, _, patch = d.ComputePatch() 1316 output_sink("recovery-from-boot.p", patch) 1317 1318 try: 1319 boot_type, boot_device = GetTypeAndDevice("/boot", info_dict) 1320 recovery_type, recovery_device = GetTypeAndDevice("/recovery", info_dict) 1321 except KeyError: 1322 return 1323 1324 sh = """#!/system/bin/sh 1325if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 1326 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1327else 1328 log -t recovery "Recovery image already installed" 1329fi 1330""" % {'boot_size': boot_img.size, 1331 'boot_sha1': boot_img.sha1, 1332 'recovery_size': recovery_img.size, 1333 'recovery_sha1': recovery_img.sha1, 1334 'boot_type': boot_type, 1335 'boot_device': boot_device, 1336 'recovery_type': recovery_type, 1337 'recovery_device': recovery_device, 1338 'bonus_args': bonus_args} 1339 1340 # The install script location moved from /system/etc to /system/bin 1341 # in the L release. Parse the init.rc file to find out where the 1342 # target-files expects it to be, and put it there. 1343 sh_location = "etc/install-recovery.sh" 1344 try: 1345 with open(os.path.join(input_dir, "BOOT", "RAMDISK", "init.rc")) as f: 1346 for line in f: 1347 m = re.match(r"^service flash_recovery /system/(\S+)\s*$", line) 1348 if m: 1349 sh_location = m.group(1) 1350 print "putting script in", sh_location 1351 break 1352 except (OSError, IOError) as e: 1353 print "failed to read init.rc: %s" % (e,) 1354 1355 output_sink(sh_location, sh) 1356