common.py revision 95e74c1424f9380976b1e557815949ed65738c62
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32import blockimgdiff 33import rangelib 34 35try: 36 from hashlib import sha1 as sha1 37except ImportError: 38 from sha import sha as sha1 39 40 41class Options(object): 42 def __init__(self): 43 platform_search_path = { 44 "linux2": "out/host/linux-x86", 45 "darwin": "out/host/darwin-x86", 46 } 47 48 self.search_path = platform_search_path.get(sys.platform, None) 49 self.signapk_path = "framework/signapk.jar" # Relative to search_path 50 self.extra_signapk_args = [] 51 self.java_path = "java" # Use the one on the path by default. 52 self.java_args = "-Xmx2048m" # JVM Args 53 self.public_key_suffix = ".x509.pem" 54 self.private_key_suffix = ".pk8" 55 # use otatools built boot_signer by default 56 self.boot_signer_path = "boot_signer" 57 self.verbose = False 58 self.tempfiles = [] 59 self.device_specific = None 60 self.extras = {} 61 self.info_dict = None 62 self.worker_threads = None 63 64 65OPTIONS = Options() 66 67 68# Values for "certificate" in apkcerts that mean special things. 69SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 70 71 72class ExternalError(RuntimeError): 73 pass 74 75 76def Run(args, **kwargs): 77 """Create and return a subprocess.Popen object, printing the command 78 line on the terminal if -v was specified.""" 79 if OPTIONS.verbose: 80 print " running: ", " ".join(args) 81 return subprocess.Popen(args, **kwargs) 82 83 84def CloseInheritedPipes(): 85 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 86 before doing other work.""" 87 if platform.system() != "Darwin": 88 return 89 for d in range(3, 1025): 90 try: 91 stat = os.fstat(d) 92 if stat is not None: 93 pipebit = stat[0] & 0x1000 94 if pipebit != 0: 95 os.close(d) 96 except OSError: 97 pass 98 99 100def LoadInfoDict(input_file): 101 """Read and parse the META/misc_info.txt key/value pairs from the 102 input target files and return a dict.""" 103 104 def read_helper(fn): 105 if isinstance(input_file, zipfile.ZipFile): 106 return input_file.read(fn) 107 else: 108 path = os.path.join(input_file, *fn.split("/")) 109 try: 110 with open(path) as f: 111 return f.read() 112 except IOError as e: 113 if e.errno == errno.ENOENT: 114 raise KeyError(fn) 115 d = {} 116 try: 117 d = LoadDictionaryFromLines(read_helper("META/misc_info.txt").split("\n")) 118 except KeyError: 119 # ok if misc_info.txt doesn't exist 120 pass 121 122 # backwards compatibility: These values used to be in their own 123 # files. Look for them, in case we're processing an old 124 # target_files zip. 125 126 if "mkyaffs2_extra_flags" not in d: 127 try: 128 d["mkyaffs2_extra_flags"] = read_helper( 129 "META/mkyaffs2-extra-flags.txt").strip() 130 except KeyError: 131 # ok if flags don't exist 132 pass 133 134 if "recovery_api_version" not in d: 135 try: 136 d["recovery_api_version"] = read_helper( 137 "META/recovery-api-version.txt").strip() 138 except KeyError: 139 raise ValueError("can't find recovery API version in input target-files") 140 141 if "tool_extensions" not in d: 142 try: 143 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip() 144 except KeyError: 145 # ok if extensions don't exist 146 pass 147 148 if "fstab_version" not in d: 149 d["fstab_version"] = "1" 150 151 try: 152 data = read_helper("META/imagesizes.txt") 153 for line in data.split("\n"): 154 if not line: 155 continue 156 name, value = line.split(" ", 1) 157 if not value: 158 continue 159 if name == "blocksize": 160 d[name] = value 161 else: 162 d[name + "_size"] = value 163 except KeyError: 164 pass 165 166 def makeint(key): 167 if key in d: 168 d[key] = int(d[key], 0) 169 170 makeint("recovery_api_version") 171 makeint("blocksize") 172 makeint("system_size") 173 makeint("vendor_size") 174 makeint("userdata_size") 175 makeint("cache_size") 176 makeint("recovery_size") 177 makeint("boot_size") 178 makeint("fstab_version") 179 180 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"]) 181 d["build.prop"] = LoadBuildProp(read_helper) 182 return d 183 184def LoadBuildProp(read_helper): 185 try: 186 data = read_helper("SYSTEM/build.prop") 187 except KeyError: 188 print "Warning: could not find SYSTEM/build.prop in %s" % zip 189 data = "" 190 return LoadDictionaryFromLines(data.split("\n")) 191 192def LoadDictionaryFromLines(lines): 193 d = {} 194 for line in lines: 195 line = line.strip() 196 if not line or line.startswith("#"): 197 continue 198 if "=" in line: 199 name, value = line.split("=", 1) 200 d[name] = value 201 return d 202 203def LoadRecoveryFSTab(read_helper, fstab_version): 204 class Partition(object): 205 def __init__(self, mount_point, fs_type, device, length, device2): 206 self.mount_point = mount_point 207 self.fs_type = fs_type 208 self.device = device 209 self.length = length 210 self.device2 = device2 211 212 try: 213 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab") 214 except KeyError: 215 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab" 216 data = "" 217 218 if fstab_version == 1: 219 d = {} 220 for line in data.split("\n"): 221 line = line.strip() 222 if not line or line.startswith("#"): 223 continue 224 pieces = line.split() 225 if not 3 <= len(pieces) <= 4: 226 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 227 options = None 228 if len(pieces) >= 4: 229 if pieces[3].startswith("/"): 230 device2 = pieces[3] 231 if len(pieces) >= 5: 232 options = pieces[4] 233 else: 234 device2 = None 235 options = pieces[3] 236 else: 237 device2 = None 238 239 mount_point = pieces[0] 240 length = 0 241 if options: 242 options = options.split(",") 243 for i in options: 244 if i.startswith("length="): 245 length = int(i[7:]) 246 else: 247 print "%s: unknown option \"%s\"" % (mount_point, i) 248 249 d[mount_point] = Partition(mount_point=mount_point, fs_type=pieces[1], 250 device=pieces[2], length=length, 251 device2=device2) 252 253 elif fstab_version == 2: 254 d = {} 255 for line in data.split("\n"): 256 line = line.strip() 257 if not line or line.startswith("#"): 258 continue 259 pieces = line.split() 260 if len(pieces) != 5: 261 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 262 263 # Ignore entries that are managed by vold 264 options = pieces[4] 265 if "voldmanaged=" in options: 266 continue 267 268 # It's a good line, parse it 269 length = 0 270 options = options.split(",") 271 for i in options: 272 if i.startswith("length="): 273 length = int(i[7:]) 274 else: 275 # Ignore all unknown options in the unified fstab 276 continue 277 278 mount_point = pieces[1] 279 d[mount_point] = Partition(mount_point=mount_point, fs_type=pieces[2], 280 device=pieces[0], length=length, device2=None) 281 282 else: 283 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 284 285 return d 286 287 288def DumpInfoDict(d): 289 for k, v in sorted(d.items()): 290 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 291 292 293def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 294 """Take a kernel, cmdline, and ramdisk directory from the input (in 295 'sourcedir'), and turn them into a boot image. Return the image 296 data, or None if sourcedir does not appear to contains files for 297 building the requested image.""" 298 299 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 300 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 301 return None 302 303 if info_dict is None: 304 info_dict = OPTIONS.info_dict 305 306 ramdisk_img = tempfile.NamedTemporaryFile() 307 img = tempfile.NamedTemporaryFile() 308 309 if os.access(fs_config_file, os.F_OK): 310 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 311 else: 312 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 313 p1 = Run(cmd, stdout=subprocess.PIPE) 314 p2 = Run(["minigzip"], 315 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 316 317 p2.wait() 318 p1.wait() 319 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (sourcedir,) 320 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (sourcedir,) 321 322 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 323 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 324 325 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 326 327 fn = os.path.join(sourcedir, "second") 328 if os.access(fn, os.F_OK): 329 cmd.append("--second") 330 cmd.append(fn) 331 332 fn = os.path.join(sourcedir, "cmdline") 333 if os.access(fn, os.F_OK): 334 cmd.append("--cmdline") 335 cmd.append(open(fn).read().rstrip("\n")) 336 337 fn = os.path.join(sourcedir, "base") 338 if os.access(fn, os.F_OK): 339 cmd.append("--base") 340 cmd.append(open(fn).read().rstrip("\n")) 341 342 fn = os.path.join(sourcedir, "pagesize") 343 if os.access(fn, os.F_OK): 344 cmd.append("--pagesize") 345 cmd.append(open(fn).read().rstrip("\n")) 346 347 args = info_dict.get("mkbootimg_args", None) 348 if args and args.strip(): 349 cmd.extend(shlex.split(args)) 350 351 img_unsigned = None 352 if info_dict.get("vboot", None): 353 img_unsigned = tempfile.NamedTemporaryFile() 354 cmd.extend(["--ramdisk", ramdisk_img.name, 355 "--output", img_unsigned.name]) 356 else: 357 cmd.extend(["--ramdisk", ramdisk_img.name, 358 "--output", img.name]) 359 360 p = Run(cmd, stdout=subprocess.PIPE) 361 p.communicate() 362 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 363 os.path.basename(sourcedir),) 364 365 if (info_dict.get("boot_signer", None) == "true" and 366 info_dict.get("verity_key", None)): 367 path = "/" + os.path.basename(sourcedir).lower() 368 cmd = [OPTIONS.boot_signer_path, path, img.name, 369 info_dict["verity_key"] + ".pk8", 370 info_dict["verity_key"] + ".x509.pem", img.name] 371 p = Run(cmd, stdout=subprocess.PIPE) 372 p.communicate() 373 assert p.returncode == 0, "boot_signer of %s image failed" % path 374 375 # Sign the image if vboot is non-empty. 376 elif info_dict.get("vboot", None): 377 path = "/" + os.path.basename(sourcedir).lower() 378 img_keyblock = tempfile.NamedTemporaryFile() 379 cmd = [info_dict["vboot_signer_cmd"], info_dict["futility"], 380 img_unsigned.name, info_dict["vboot_key"] + ".vbpubk", 381 info_dict["vboot_key"] + ".vbprivk", img_keyblock.name, 382 img.name] 383 p = Run(cmd, stdout=subprocess.PIPE) 384 p.communicate() 385 assert p.returncode == 0, "vboot_signer of %s image failed" % path 386 387 img.seek(os.SEEK_SET, 0) 388 data = img.read() 389 390 ramdisk_img.close() 391 img.close() 392 393 return data 394 395 396def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 397 info_dict=None): 398 """Return a File object (with name 'name') with the desired bootable 399 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 400 'prebuilt_name', otherwise look for it under 'unpack_dir'/IMAGES, 401 otherwise construct it from the source files in 402 'unpack_dir'/'tree_subdir'.""" 403 404 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 405 if os.path.exists(prebuilt_path): 406 print "using prebuilt %s from BOOTABLE_IMAGES..." % (prebuilt_name,) 407 return File.FromLocalFile(name, prebuilt_path) 408 409 prebuilt_path = os.path.join(unpack_dir, "IMAGES", prebuilt_name) 410 if os.path.exists(prebuilt_path): 411 print "using prebuilt %s from IMAGES..." % (prebuilt_name,) 412 return File.FromLocalFile(name, prebuilt_path) 413 414 print "building image from target_files %s..." % (tree_subdir,) 415 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 416 data = BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 417 os.path.join(unpack_dir, fs_config), 418 info_dict) 419 if data: 420 return File(name, data) 421 return None 422 423 424def UnzipTemp(filename, pattern=None): 425 """Unzip the given archive into a temporary directory and return the name. 426 427 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 428 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 429 430 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 431 main file), open for reading. 432 """ 433 434 tmp = tempfile.mkdtemp(prefix="targetfiles-") 435 OPTIONS.tempfiles.append(tmp) 436 437 def unzip_to_dir(filename, dirname): 438 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 439 if pattern is not None: 440 cmd.append(pattern) 441 p = Run(cmd, stdout=subprocess.PIPE) 442 p.communicate() 443 if p.returncode != 0: 444 raise ExternalError("failed to unzip input target-files \"%s\"" % 445 (filename,)) 446 447 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 448 if m: 449 unzip_to_dir(m.group(1), tmp) 450 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 451 filename = m.group(1) 452 else: 453 unzip_to_dir(filename, tmp) 454 455 return tmp, zipfile.ZipFile(filename, "r") 456 457 458def GetKeyPasswords(keylist): 459 """Given a list of keys, prompt the user to enter passwords for 460 those which require them. Return a {key: password} dict. password 461 will be None if the key has no password.""" 462 463 no_passwords = [] 464 need_passwords = [] 465 key_passwords = {} 466 devnull = open("/dev/null", "w+b") 467 for k in sorted(keylist): 468 # We don't need a password for things that aren't really keys. 469 if k in SPECIAL_CERT_STRINGS: 470 no_passwords.append(k) 471 continue 472 473 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 474 "-inform", "DER", "-nocrypt"], 475 stdin=devnull.fileno(), 476 stdout=devnull.fileno(), 477 stderr=subprocess.STDOUT) 478 p.communicate() 479 if p.returncode == 0: 480 # Definitely an unencrypted key. 481 no_passwords.append(k) 482 else: 483 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 484 "-inform", "DER", "-passin", "pass:"], 485 stdin=devnull.fileno(), 486 stdout=devnull.fileno(), 487 stderr=subprocess.PIPE) 488 _, stderr = p.communicate() 489 if p.returncode == 0: 490 # Encrypted key with empty string as password. 491 key_passwords[k] = '' 492 elif stderr.startswith('Error decrypting key'): 493 # Definitely encrypted key. 494 # It would have said "Error reading key" if it didn't parse correctly. 495 need_passwords.append(k) 496 else: 497 # Potentially, a type of key that openssl doesn't understand. 498 # We'll let the routines in signapk.jar handle it. 499 no_passwords.append(k) 500 devnull.close() 501 502 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 503 key_passwords.update(dict.fromkeys(no_passwords, None)) 504 return key_passwords 505 506 507def SignFile(input_name, output_name, key, password, align=None, 508 whole_file=False): 509 """Sign the input_name zip/jar/apk, producing output_name. Use the 510 given key and password (the latter may be None if the key does not 511 have a password. 512 513 If align is an integer > 1, zipalign is run to align stored files in 514 the output zip on 'align'-byte boundaries. 515 516 If whole_file is true, use the "-w" option to SignApk to embed a 517 signature that covers the whole file in the archive comment of the 518 zip file. 519 """ 520 521 if align == 0 or align == 1: 522 align = None 523 524 if align: 525 temp = tempfile.NamedTemporaryFile() 526 sign_name = temp.name 527 else: 528 sign_name = output_name 529 530 cmd = [OPTIONS.java_path, OPTIONS.java_args, "-jar", 531 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 532 cmd.extend(OPTIONS.extra_signapk_args) 533 if whole_file: 534 cmd.append("-w") 535 cmd.extend([key + OPTIONS.public_key_suffix, 536 key + OPTIONS.private_key_suffix, 537 input_name, sign_name]) 538 539 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 540 if password is not None: 541 password += "\n" 542 p.communicate(password) 543 if p.returncode != 0: 544 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 545 546 if align: 547 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 548 p.communicate() 549 if p.returncode != 0: 550 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 551 temp.close() 552 553 554def CheckSize(data, target, info_dict): 555 """Check the data string passed against the max size limit, if 556 any, for the given target. Raise exception if the data is too big. 557 Print a warning if the data is nearing the maximum size.""" 558 559 if target.endswith(".img"): 560 target = target[:-4] 561 mount_point = "/" + target 562 563 fs_type = None 564 limit = None 565 if info_dict["fstab"]: 566 if mount_point == "/userdata": 567 mount_point = "/data" 568 p = info_dict["fstab"][mount_point] 569 fs_type = p.fs_type 570 device = p.device 571 if "/" in device: 572 device = device[device.rfind("/")+1:] 573 limit = info_dict.get(device + "_size", None) 574 if not fs_type or not limit: 575 return 576 577 if fs_type == "yaffs2": 578 # image size should be increased by 1/64th to account for the 579 # spare area (64 bytes per 2k page) 580 limit = limit / 2048 * (2048+64) 581 size = len(data) 582 pct = float(size) * 100.0 / limit 583 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 584 if pct >= 99.0: 585 raise ExternalError(msg) 586 elif pct >= 95.0: 587 print 588 print " WARNING: ", msg 589 print 590 elif OPTIONS.verbose: 591 print " ", msg 592 593 594def ReadApkCerts(tf_zip): 595 """Given a target_files ZipFile, parse the META/apkcerts.txt file 596 and return a {package: cert} dict.""" 597 certmap = {} 598 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 599 line = line.strip() 600 if not line: 601 continue 602 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 603 r'private_key="(.*)"$', line) 604 if m: 605 name, cert, privkey = m.groups() 606 public_key_suffix_len = len(OPTIONS.public_key_suffix) 607 private_key_suffix_len = len(OPTIONS.private_key_suffix) 608 if cert in SPECIAL_CERT_STRINGS and not privkey: 609 certmap[name] = cert 610 elif (cert.endswith(OPTIONS.public_key_suffix) and 611 privkey.endswith(OPTIONS.private_key_suffix) and 612 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 613 certmap[name] = cert[:-public_key_suffix_len] 614 else: 615 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 616 return certmap 617 618 619COMMON_DOCSTRING = """ 620 -p (--path) <dir> 621 Prepend <dir>/bin to the list of places to search for binaries 622 run by this script, and expect to find jars in <dir>/framework. 623 624 -s (--device_specific) <file> 625 Path to the python module containing device-specific 626 releasetools code. 627 628 -x (--extra) <key=value> 629 Add a key/value pair to the 'extras' dict, which device-specific 630 extension code may look at. 631 632 -v (--verbose) 633 Show command lines being executed. 634 635 -h (--help) 636 Display this usage message and exit. 637""" 638 639def Usage(docstring): 640 print docstring.rstrip("\n") 641 print COMMON_DOCSTRING 642 643 644def ParseOptions(argv, 645 docstring, 646 extra_opts="", extra_long_opts=(), 647 extra_option_handler=None): 648 """Parse the options in argv and return any arguments that aren't 649 flags. docstring is the calling module's docstring, to be displayed 650 for errors and -h. extra_opts and extra_long_opts are for flags 651 defined by the caller, which are processed by passing them to 652 extra_option_handler.""" 653 654 try: 655 opts, args = getopt.getopt( 656 argv, "hvp:s:x:" + extra_opts, 657 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 658 "java_path=", "java_args=", "public_key_suffix=", 659 "private_key_suffix=", "boot_signer_path=", "device_specific=", 660 "extra="] + 661 list(extra_long_opts)) 662 except getopt.GetoptError as err: 663 Usage(docstring) 664 print "**", str(err), "**" 665 sys.exit(2) 666 667 for o, a in opts: 668 if o in ("-h", "--help"): 669 Usage(docstring) 670 sys.exit() 671 elif o in ("-v", "--verbose"): 672 OPTIONS.verbose = True 673 elif o in ("-p", "--path"): 674 OPTIONS.search_path = a 675 elif o in ("--signapk_path",): 676 OPTIONS.signapk_path = a 677 elif o in ("--extra_signapk_args",): 678 OPTIONS.extra_signapk_args = shlex.split(a) 679 elif o in ("--java_path",): 680 OPTIONS.java_path = a 681 elif o in ("--java_args",): 682 OPTIONS.java_args = a 683 elif o in ("--public_key_suffix",): 684 OPTIONS.public_key_suffix = a 685 elif o in ("--private_key_suffix",): 686 OPTIONS.private_key_suffix = a 687 elif o in ("--boot_signer_path",): 688 OPTIONS.boot_signer_path = a 689 elif o in ("-s", "--device_specific"): 690 OPTIONS.device_specific = a 691 elif o in ("-x", "--extra"): 692 key, value = a.split("=", 1) 693 OPTIONS.extras[key] = value 694 else: 695 if extra_option_handler is None or not extra_option_handler(o, a): 696 assert False, "unknown option \"%s\"" % (o,) 697 698 if OPTIONS.search_path: 699 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 700 os.pathsep + os.environ["PATH"]) 701 702 return args 703 704 705def MakeTempFile(prefix=None, suffix=None): 706 """Make a temp file and add it to the list of things to be deleted 707 when Cleanup() is called. Return the filename.""" 708 fd, fn = tempfile.mkstemp(prefix=prefix, suffix=suffix) 709 os.close(fd) 710 OPTIONS.tempfiles.append(fn) 711 return fn 712 713 714def Cleanup(): 715 for i in OPTIONS.tempfiles: 716 if os.path.isdir(i): 717 shutil.rmtree(i) 718 else: 719 os.remove(i) 720 721 722class PasswordManager(object): 723 def __init__(self): 724 self.editor = os.getenv("EDITOR", None) 725 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 726 727 def GetPasswords(self, items): 728 """Get passwords corresponding to each string in 'items', 729 returning a dict. (The dict may have keys in addition to the 730 values in 'items'.) 731 732 Uses the passwords in $ANDROID_PW_FILE if available, letting the 733 user edit that file to add more needed passwords. If no editor is 734 available, or $ANDROID_PW_FILE isn't define, prompts the user 735 interactively in the ordinary way. 736 """ 737 738 current = self.ReadFile() 739 740 first = True 741 while True: 742 missing = [] 743 for i in items: 744 if i not in current or not current[i]: 745 missing.append(i) 746 # Are all the passwords already in the file? 747 if not missing: 748 return current 749 750 for i in missing: 751 current[i] = "" 752 753 if not first: 754 print "key file %s still missing some passwords." % (self.pwfile,) 755 answer = raw_input("try to edit again? [y]> ").strip() 756 if answer and answer[0] not in 'yY': 757 raise RuntimeError("key passwords unavailable") 758 first = False 759 760 current = self.UpdateAndReadFile(current) 761 762 def PromptResult(self, current): # pylint: disable=no-self-use 763 """Prompt the user to enter a value (password) for each key in 764 'current' whose value is fales. Returns a new dict with all the 765 values. 766 """ 767 result = {} 768 for k, v in sorted(current.iteritems()): 769 if v: 770 result[k] = v 771 else: 772 while True: 773 result[k] = getpass.getpass( 774 "Enter password for %s key> " % k).strip() 775 if result[k]: 776 break 777 return result 778 779 def UpdateAndReadFile(self, current): 780 if not self.editor or not self.pwfile: 781 return self.PromptResult(current) 782 783 f = open(self.pwfile, "w") 784 os.chmod(self.pwfile, 0o600) 785 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 786 f.write("# (Additional spaces are harmless.)\n\n") 787 788 first_line = None 789 sorted_list = sorted([(not v, k, v) for (k, v) in current.iteritems()]) 790 for i, (_, k, v) in enumerate(sorted_list): 791 f.write("[[[ %s ]]] %s\n" % (v, k)) 792 if not v and first_line is None: 793 # position cursor on first line with no password. 794 first_line = i + 4 795 f.close() 796 797 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 798 _, _ = p.communicate() 799 800 return self.ReadFile() 801 802 def ReadFile(self): 803 result = {} 804 if self.pwfile is None: 805 return result 806 try: 807 f = open(self.pwfile, "r") 808 for line in f: 809 line = line.strip() 810 if not line or line[0] == '#': 811 continue 812 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 813 if not m: 814 print "failed to parse password file: ", line 815 else: 816 result[m.group(2)] = m.group(1) 817 f.close() 818 except IOError as e: 819 if e.errno != errno.ENOENT: 820 print "error reading password file: ", str(e) 821 return result 822 823 824def ZipWrite(zip_file, filename, arcname=None, perms=0o644, 825 compress_type=None): 826 import datetime 827 828 # http://b/18015246 829 # Python 2.7's zipfile implementation wrongly thinks that zip64 is required 830 # for files larger than 2GiB. We can work around this by adjusting their 831 # limit. Note that `zipfile.writestr()` will not work for strings larger than 832 # 2GiB. The Python interpreter sometimes rejects strings that large (though 833 # it isn't clear to me exactly what circumstances cause this). 834 # `zipfile.write()` must be used directly to work around this. 835 # 836 # This mess can be avoided if we port to python3. 837 saved_zip64_limit = zipfile.ZIP64_LIMIT 838 zipfile.ZIP64_LIMIT = (1 << 32) - 1 839 840 if compress_type is None: 841 compress_type = zip_file.compression 842 if arcname is None: 843 arcname = filename 844 845 saved_stat = os.stat(filename) 846 847 try: 848 # `zipfile.write()` doesn't allow us to pass ZipInfo, so just modify the 849 # file to be zipped and reset it when we're done. 850 os.chmod(filename, perms) 851 852 # Use a fixed timestamp so the output is repeatable. 853 epoch = datetime.datetime.fromtimestamp(0) 854 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 855 os.utime(filename, (timestamp, timestamp)) 856 857 zip_file.write(filename, arcname=arcname, compress_type=compress_type) 858 finally: 859 os.chmod(filename, saved_stat.st_mode) 860 os.utime(filename, (saved_stat.st_atime, saved_stat.st_mtime)) 861 zipfile.ZIP64_LIMIT = saved_zip64_limit 862 863 864def ZipWriteStr(zip_file, filename, data, perms=0o644, compression=None): 865 # use a fixed timestamp so the output is repeatable. 866 zinfo = zipfile.ZipInfo(filename=filename, 867 date_time=(2009, 1, 1, 0, 0, 0)) 868 if compression is None: 869 zinfo.compress_type = zip_file.compression 870 else: 871 zinfo.compress_type = compression 872 zinfo.external_attr = perms << 16 873 zip_file.writestr(zinfo, data) 874 875 876class DeviceSpecificParams(object): 877 module = None 878 def __init__(self, **kwargs): 879 """Keyword arguments to the constructor become attributes of this 880 object, which is passed to all functions in the device-specific 881 module.""" 882 for k, v in kwargs.iteritems(): 883 setattr(self, k, v) 884 self.extras = OPTIONS.extras 885 886 if self.module is None: 887 path = OPTIONS.device_specific 888 if not path: 889 return 890 try: 891 if os.path.isdir(path): 892 info = imp.find_module("releasetools", [path]) 893 else: 894 d, f = os.path.split(path) 895 b, x = os.path.splitext(f) 896 if x == ".py": 897 f = b 898 info = imp.find_module(f, [d]) 899 print "loaded device-specific extensions from", path 900 self.module = imp.load_module("device_specific", *info) 901 except ImportError: 902 print "unable to load device-specific module; assuming none" 903 904 def _DoCall(self, function_name, *args, **kwargs): 905 """Call the named function in the device-specific module, passing 906 the given args and kwargs. The first argument to the call will be 907 the DeviceSpecific object itself. If there is no module, or the 908 module does not define the function, return the value of the 909 'default' kwarg (which itself defaults to None).""" 910 if self.module is None or not hasattr(self.module, function_name): 911 return kwargs.get("default", None) 912 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 913 914 def FullOTA_Assertions(self): 915 """Called after emitting the block of assertions at the top of a 916 full OTA package. Implementations can add whatever additional 917 assertions they like.""" 918 return self._DoCall("FullOTA_Assertions") 919 920 def FullOTA_InstallBegin(self): 921 """Called at the start of full OTA installation.""" 922 return self._DoCall("FullOTA_InstallBegin") 923 924 def FullOTA_InstallEnd(self): 925 """Called at the end of full OTA installation; typically this is 926 used to install the image for the device's baseband processor.""" 927 return self._DoCall("FullOTA_InstallEnd") 928 929 def IncrementalOTA_Assertions(self): 930 """Called after emitting the block of assertions at the top of an 931 incremental OTA package. Implementations can add whatever 932 additional assertions they like.""" 933 return self._DoCall("IncrementalOTA_Assertions") 934 935 def IncrementalOTA_VerifyBegin(self): 936 """Called at the start of the verification phase of incremental 937 OTA installation; additional checks can be placed here to abort 938 the script before any changes are made.""" 939 return self._DoCall("IncrementalOTA_VerifyBegin") 940 941 def IncrementalOTA_VerifyEnd(self): 942 """Called at the end of the verification phase of incremental OTA 943 installation; additional checks can be placed here to abort the 944 script before any changes are made.""" 945 return self._DoCall("IncrementalOTA_VerifyEnd") 946 947 def IncrementalOTA_InstallBegin(self): 948 """Called at the start of incremental OTA installation (after 949 verification is complete).""" 950 return self._DoCall("IncrementalOTA_InstallBegin") 951 952 def IncrementalOTA_InstallEnd(self): 953 """Called at the end of incremental OTA installation; typically 954 this is used to install the image for the device's baseband 955 processor.""" 956 return self._DoCall("IncrementalOTA_InstallEnd") 957 958class File(object): 959 def __init__(self, name, data): 960 self.name = name 961 self.data = data 962 self.size = len(data) 963 self.sha1 = sha1(data).hexdigest() 964 965 @classmethod 966 def FromLocalFile(cls, name, diskname): 967 f = open(diskname, "rb") 968 data = f.read() 969 f.close() 970 return File(name, data) 971 972 def WriteToTemp(self): 973 t = tempfile.NamedTemporaryFile() 974 t.write(self.data) 975 t.flush() 976 return t 977 978 def AddToZip(self, z, compression=None): 979 ZipWriteStr(z, self.name, self.data, compression=compression) 980 981DIFF_PROGRAM_BY_EXT = { 982 ".gz" : "imgdiff", 983 ".zip" : ["imgdiff", "-z"], 984 ".jar" : ["imgdiff", "-z"], 985 ".apk" : ["imgdiff", "-z"], 986 ".img" : "imgdiff", 987 } 988 989class Difference(object): 990 def __init__(self, tf, sf, diff_program=None): 991 self.tf = tf 992 self.sf = sf 993 self.patch = None 994 self.diff_program = diff_program 995 996 def ComputePatch(self): 997 """Compute the patch (as a string of data) needed to turn sf into 998 tf. Returns the same tuple as GetPatch().""" 999 1000 tf = self.tf 1001 sf = self.sf 1002 1003 if self.diff_program: 1004 diff_program = self.diff_program 1005 else: 1006 ext = os.path.splitext(tf.name)[1] 1007 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 1008 1009 ttemp = tf.WriteToTemp() 1010 stemp = sf.WriteToTemp() 1011 1012 ext = os.path.splitext(tf.name)[1] 1013 1014 try: 1015 ptemp = tempfile.NamedTemporaryFile() 1016 if isinstance(diff_program, list): 1017 cmd = copy.copy(diff_program) 1018 else: 1019 cmd = [diff_program] 1020 cmd.append(stemp.name) 1021 cmd.append(ttemp.name) 1022 cmd.append(ptemp.name) 1023 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1024 err = [] 1025 def run(): 1026 _, e = p.communicate() 1027 if e: 1028 err.append(e) 1029 th = threading.Thread(target=run) 1030 th.start() 1031 th.join(timeout=300) # 5 mins 1032 if th.is_alive(): 1033 print "WARNING: diff command timed out" 1034 p.terminate() 1035 th.join(5) 1036 if th.is_alive(): 1037 p.kill() 1038 th.join() 1039 1040 if err or p.returncode != 0: 1041 print "WARNING: failure running %s:\n%s\n" % ( 1042 diff_program, "".join(err)) 1043 self.patch = None 1044 return None, None, None 1045 diff = ptemp.read() 1046 finally: 1047 ptemp.close() 1048 stemp.close() 1049 ttemp.close() 1050 1051 self.patch = diff 1052 return self.tf, self.sf, self.patch 1053 1054 1055 def GetPatch(self): 1056 """Return a tuple (target_file, source_file, patch_data). 1057 patch_data may be None if ComputePatch hasn't been called, or if 1058 computing the patch failed.""" 1059 return self.tf, self.sf, self.patch 1060 1061 1062def ComputeDifferences(diffs): 1063 """Call ComputePatch on all the Difference objects in 'diffs'.""" 1064 print len(diffs), "diffs to compute" 1065 1066 # Do the largest files first, to try and reduce the long-pole effect. 1067 by_size = [(i.tf.size, i) for i in diffs] 1068 by_size.sort(reverse=True) 1069 by_size = [i[1] for i in by_size] 1070 1071 lock = threading.Lock() 1072 diff_iter = iter(by_size) # accessed under lock 1073 1074 def worker(): 1075 try: 1076 lock.acquire() 1077 for d in diff_iter: 1078 lock.release() 1079 start = time.time() 1080 d.ComputePatch() 1081 dur = time.time() - start 1082 lock.acquire() 1083 1084 tf, sf, patch = d.GetPatch() 1085 if sf.name == tf.name: 1086 name = tf.name 1087 else: 1088 name = "%s (%s)" % (tf.name, sf.name) 1089 if patch is None: 1090 print "patching failed! %s" % (name,) 1091 else: 1092 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 1093 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 1094 lock.release() 1095 except Exception as e: 1096 print e 1097 raise 1098 1099 # start worker threads; wait for them all to finish. 1100 threads = [threading.Thread(target=worker) 1101 for i in range(OPTIONS.worker_threads)] 1102 for th in threads: 1103 th.start() 1104 while threads: 1105 threads.pop().join() 1106 1107 1108class BlockDifference(object): 1109 def __init__(self, partition, tgt, src=None, check_first_block=False, 1110 version=None): 1111 self.tgt = tgt 1112 self.src = src 1113 self.partition = partition 1114 self.check_first_block = check_first_block 1115 1116 if version is None: 1117 version = 1 1118 if OPTIONS.info_dict: 1119 version = max( 1120 int(i) for i in 1121 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(",")) 1122 self.version = version 1123 1124 b = blockimgdiff.BlockImageDiff(tgt, src, threads=OPTIONS.worker_threads, 1125 version=self.version) 1126 tmpdir = tempfile.mkdtemp() 1127 OPTIONS.tempfiles.append(tmpdir) 1128 self.path = os.path.join(tmpdir, partition) 1129 b.Compute(self.path) 1130 1131 _, self.device = GetTypeAndDevice("/" + partition, OPTIONS.info_dict) 1132 1133 def WriteScript(self, script, output_zip, progress=None): 1134 if not self.src: 1135 # write the output unconditionally 1136 script.Print("Patching %s image unconditionally..." % (self.partition,)) 1137 else: 1138 script.Print("Patching %s image after verification." % (self.partition,)) 1139 1140 if progress: 1141 script.ShowProgress(progress, 0) 1142 self._WriteUpdate(script, output_zip) 1143 1144 def WriteVerifyScript(self, script): 1145 partition = self.partition 1146 if not self.src: 1147 script.Print("Image %s will be patched unconditionally." % (partition,)) 1148 else: 1149 if self.version >= 3: 1150 script.AppendExtra(('if (range_sha1("%s", "%s") == "%s" || ' 1151 'block_image_verify("%s", ' 1152 'package_extract_file("%s.transfer.list"), ' 1153 '"%s.new.dat", "%s.patch.dat")) then') % ( 1154 self.device, self.src.care_map.to_string_raw(), 1155 self.src.TotalSha1(), 1156 self.device, partition, partition, partition)) 1157 else: 1158 script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' % ( 1159 self.device, self.src.care_map.to_string_raw(), 1160 self.src.TotalSha1())) 1161 script.Print('Verified %s image...' % (partition,)) 1162 script.AppendExtra('else') 1163 1164 # When generating incrementals for the system and vendor partitions, 1165 # explicitly check the first block (which contains the superblock) of 1166 # the partition to see if it's what we expect. If this check fails, 1167 # give an explicit log message about the partition having been 1168 # remounted R/W (the most likely explanation) and the need to flash to 1169 # get OTAs working again. 1170 if self.check_first_block: 1171 self._CheckFirstBlock(script) 1172 1173 # Abort the OTA update. Note that the incremental OTA cannot be applied 1174 # even if it may match the checksum of the target partition. 1175 # a) If version < 3, operations like move and erase will make changes 1176 # unconditionally and damage the partition. 1177 # b) If version >= 3, it won't even reach here. 1178 script.AppendExtra(('abort("%s partition has unexpected contents");\n' 1179 'endif;') % (partition,)) 1180 1181 def _WriteUpdate(self, script, output_zip): 1182 ZipWrite(output_zip, 1183 '{}.transfer.list'.format(self.path), 1184 '{}.transfer.list'.format(self.partition)) 1185 ZipWrite(output_zip, 1186 '{}.new.dat'.format(self.path), 1187 '{}.new.dat'.format(self.partition)) 1188 ZipWrite(output_zip, 1189 '{}.patch.dat'.format(self.path), 1190 '{}.patch.dat'.format(self.partition), 1191 compress_type=zipfile.ZIP_STORED) 1192 1193 call = ('block_image_update("{device}", ' 1194 'package_extract_file("{partition}.transfer.list"), ' 1195 '"{partition}.new.dat", "{partition}.patch.dat");\n'.format( 1196 device=self.device, partition=self.partition)) 1197 script.AppendExtra(script.WordWrap(call)) 1198 1199 def _HashBlocks(self, source, ranges): # pylint: disable=no-self-use 1200 data = source.ReadRangeSet(ranges) 1201 ctx = sha1() 1202 1203 for p in data: 1204 ctx.update(p) 1205 1206 return ctx.hexdigest() 1207 1208 def _CheckFirstBlock(self, script): 1209 r = rangelib.RangeSet((0, 1)) 1210 srchash = self._HashBlocks(self.src, r) 1211 1212 script.AppendExtra(('(range_sha1("%s", "%s") == "%s") || ' 1213 'abort("%s has been remounted R/W; ' 1214 'reflash device to reenable OTA updates");') 1215 % (self.device, r.to_string_raw(), srchash, 1216 self.device)) 1217 1218DataImage = blockimgdiff.DataImage 1219 1220 1221# map recovery.fstab's fs_types to mount/format "partition types" 1222PARTITION_TYPES = { 1223 "yaffs2": "MTD", 1224 "mtd": "MTD", 1225 "ext4": "EMMC", 1226 "emmc": "EMMC", 1227 "f2fs": "EMMC", 1228 "squashfs": "EMMC" 1229} 1230 1231def GetTypeAndDevice(mount_point, info): 1232 fstab = info["fstab"] 1233 if fstab: 1234 return (PARTITION_TYPES[fstab[mount_point].fs_type], 1235 fstab[mount_point].device) 1236 else: 1237 raise KeyError 1238 1239 1240def ParseCertificate(data): 1241 """Parse a PEM-format certificate.""" 1242 cert = [] 1243 save = False 1244 for line in data.split("\n"): 1245 if "--END CERTIFICATE--" in line: 1246 break 1247 if save: 1248 cert.append(line) 1249 if "--BEGIN CERTIFICATE--" in line: 1250 save = True 1251 cert = "".join(cert).decode('base64') 1252 return cert 1253 1254def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img, 1255 info_dict=None): 1256 """Generate a binary patch that creates the recovery image starting 1257 with the boot image. (Most of the space in these images is just the 1258 kernel, which is identical for the two, so the resulting patch 1259 should be efficient.) Add it to the output zip, along with a shell 1260 script that is run from init.rc on first boot to actually do the 1261 patching and install the new recovery image. 1262 1263 recovery_img and boot_img should be File objects for the 1264 corresponding images. info should be the dictionary returned by 1265 common.LoadInfoDict() on the input target_files. 1266 """ 1267 1268 if info_dict is None: 1269 info_dict = OPTIONS.info_dict 1270 1271 diff_program = ["imgdiff"] 1272 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat") 1273 if os.path.exists(path): 1274 diff_program.append("-b") 1275 diff_program.append(path) 1276 bonus_args = "-b /system/etc/recovery-resource.dat" 1277 else: 1278 bonus_args = "" 1279 1280 d = Difference(recovery_img, boot_img, diff_program=diff_program) 1281 _, _, patch = d.ComputePatch() 1282 output_sink("recovery-from-boot.p", patch) 1283 1284 try: 1285 boot_type, boot_device = GetTypeAndDevice("/boot", info_dict) 1286 recovery_type, recovery_device = GetTypeAndDevice("/recovery", info_dict) 1287 except KeyError: 1288 return 1289 1290 sh = """#!/system/bin/sh 1291if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 1292 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1293else 1294 log -t recovery "Recovery image already installed" 1295fi 1296""" % {'boot_size': boot_img.size, 1297 'boot_sha1': boot_img.sha1, 1298 'recovery_size': recovery_img.size, 1299 'recovery_sha1': recovery_img.sha1, 1300 'boot_type': boot_type, 1301 'boot_device': boot_device, 1302 'recovery_type': recovery_type, 1303 'recovery_device': recovery_device, 1304 'bonus_args': bonus_args} 1305 1306 # The install script location moved from /system/etc to /system/bin 1307 # in the L release. Parse the init.rc file to find out where the 1308 # target-files expects it to be, and put it there. 1309 sh_location = "etc/install-recovery.sh" 1310 try: 1311 with open(os.path.join(input_dir, "BOOT", "RAMDISK", "init.rc")) as f: 1312 for line in f: 1313 m = re.match(r"^service flash_recovery /system/(\S+)\s*$", line) 1314 if m: 1315 sh_location = m.group(1) 1316 print "putting script in", sh_location 1317 break 1318 except (OSError, IOError) as e: 1319 print "failed to read init.rc: %s" % (e,) 1320 1321 output_sink(sh_location, sh) 1322