common.py revision e204868f1f2bf2c6a85cf09edb52c06d593584c0
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32import blockimgdiff 33from rangelib import * 34 35try: 36 from hashlib import sha1 as sha1 37except ImportError: 38 from sha import sha as sha1 39 40# missing in Python 2.4 and before 41if not hasattr(os, "SEEK_SET"): 42 os.SEEK_SET = 0 43 44class Options(object): pass 45OPTIONS = Options() 46 47DEFAULT_SEARCH_PATH_BY_PLATFORM = { 48 "linux2": "out/host/linux-x86", 49 "darwin": "out/host/darwin-x86", 50 } 51OPTIONS.search_path = DEFAULT_SEARCH_PATH_BY_PLATFORM.get(sys.platform, None) 52 53OPTIONS.signapk_path = "framework/signapk.jar" # Relative to search_path 54OPTIONS.extra_signapk_args = [] 55OPTIONS.java_path = "java" # Use the one on the path by default. 56OPTIONS.java_args = "-Xmx2048m" # JVM Args 57OPTIONS.public_key_suffix = ".x509.pem" 58OPTIONS.private_key_suffix = ".pk8" 59OPTIONS.boot_signer_path = "boot_signer" # use otatools built boot_signer by default 60OPTIONS.verbose = False 61OPTIONS.tempfiles = [] 62OPTIONS.device_specific = None 63OPTIONS.extras = {} 64OPTIONS.info_dict = None 65 66 67# Values for "certificate" in apkcerts that mean special things. 68SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 69 70 71class ExternalError(RuntimeError): pass 72 73 74def Run(args, **kwargs): 75 """Create and return a subprocess.Popen object, printing the command 76 line on the terminal if -v was specified.""" 77 if OPTIONS.verbose: 78 print " running: ", " ".join(args) 79 return subprocess.Popen(args, **kwargs) 80 81 82def CloseInheritedPipes(): 83 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 84 before doing other work.""" 85 if platform.system() != "Darwin": 86 return 87 for d in range(3, 1025): 88 try: 89 stat = os.fstat(d) 90 if stat is not None: 91 pipebit = stat[0] & 0x1000 92 if pipebit != 0: 93 os.close(d) 94 except OSError: 95 pass 96 97 98def LoadInfoDict(input): 99 """Read and parse the META/misc_info.txt key/value pairs from the 100 input target files and return a dict.""" 101 102 def read_helper(fn): 103 if isinstance(input, zipfile.ZipFile): 104 return input.read(fn) 105 else: 106 path = os.path.join(input, *fn.split("/")) 107 try: 108 with open(path) as f: 109 return f.read() 110 except IOError, e: 111 if e.errno == errno.ENOENT: 112 raise KeyError(fn) 113 d = {} 114 try: 115 d = LoadDictionaryFromLines(read_helper("META/misc_info.txt").split("\n")) 116 except KeyError: 117 # ok if misc_info.txt doesn't exist 118 pass 119 120 # backwards compatibility: These values used to be in their own 121 # files. Look for them, in case we're processing an old 122 # target_files zip. 123 124 if "mkyaffs2_extra_flags" not in d: 125 try: 126 d["mkyaffs2_extra_flags"] = read_helper("META/mkyaffs2-extra-flags.txt").strip() 127 except KeyError: 128 # ok if flags don't exist 129 pass 130 131 if "recovery_api_version" not in d: 132 try: 133 d["recovery_api_version"] = read_helper("META/recovery-api-version.txt").strip() 134 except KeyError: 135 raise ValueError("can't find recovery API version in input target-files") 136 137 if "tool_extensions" not in d: 138 try: 139 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip() 140 except KeyError: 141 # ok if extensions don't exist 142 pass 143 144 if "fstab_version" not in d: 145 d["fstab_version"] = "1" 146 147 try: 148 data = read_helper("META/imagesizes.txt") 149 for line in data.split("\n"): 150 if not line: continue 151 name, value = line.split(" ", 1) 152 if not value: continue 153 if name == "blocksize": 154 d[name] = value 155 else: 156 d[name + "_size"] = value 157 except KeyError: 158 pass 159 160 def makeint(key): 161 if key in d: 162 d[key] = int(d[key], 0) 163 164 makeint("recovery_api_version") 165 makeint("blocksize") 166 makeint("system_size") 167 makeint("vendor_size") 168 makeint("userdata_size") 169 makeint("cache_size") 170 makeint("recovery_size") 171 makeint("boot_size") 172 makeint("fstab_version") 173 174 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"]) 175 d["build.prop"] = LoadBuildProp(read_helper) 176 return d 177 178def LoadBuildProp(read_helper): 179 try: 180 data = read_helper("SYSTEM/build.prop") 181 except KeyError: 182 print "Warning: could not find SYSTEM/build.prop in %s" % zip 183 data = "" 184 return LoadDictionaryFromLines(data.split("\n")) 185 186def LoadDictionaryFromLines(lines): 187 d = {} 188 for line in lines: 189 line = line.strip() 190 if not line or line.startswith("#"): continue 191 if "=" in line: 192 name, value = line.split("=", 1) 193 d[name] = value 194 return d 195 196def LoadRecoveryFSTab(read_helper, fstab_version): 197 class Partition(object): 198 pass 199 200 try: 201 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab") 202 except KeyError: 203 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab" 204 data = "" 205 206 if fstab_version == 1: 207 d = {} 208 for line in data.split("\n"): 209 line = line.strip() 210 if not line or line.startswith("#"): continue 211 pieces = line.split() 212 if not (3 <= len(pieces) <= 4): 213 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 214 215 p = Partition() 216 p.mount_point = pieces[0] 217 p.fs_type = pieces[1] 218 p.device = pieces[2] 219 p.length = 0 220 options = None 221 if len(pieces) >= 4: 222 if pieces[3].startswith("/"): 223 p.device2 = pieces[3] 224 if len(pieces) >= 5: 225 options = pieces[4] 226 else: 227 p.device2 = None 228 options = pieces[3] 229 else: 230 p.device2 = None 231 232 if options: 233 options = options.split(",") 234 for i in options: 235 if i.startswith("length="): 236 p.length = int(i[7:]) 237 else: 238 print "%s: unknown option \"%s\"" % (p.mount_point, i) 239 240 d[p.mount_point] = p 241 242 elif fstab_version == 2: 243 d = {} 244 for line in data.split("\n"): 245 line = line.strip() 246 if not line or line.startswith("#"): continue 247 pieces = line.split() 248 if len(pieces) != 5: 249 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 250 251 # Ignore entries that are managed by vold 252 options = pieces[4] 253 if "voldmanaged=" in options: continue 254 255 # It's a good line, parse it 256 p = Partition() 257 p.device = pieces[0] 258 p.mount_point = pieces[1] 259 p.fs_type = pieces[2] 260 p.device2 = None 261 p.length = 0 262 263 options = options.split(",") 264 for i in options: 265 if i.startswith("length="): 266 p.length = int(i[7:]) 267 else: 268 # Ignore all unknown options in the unified fstab 269 continue 270 271 d[p.mount_point] = p 272 273 else: 274 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 275 276 return d 277 278 279def DumpInfoDict(d): 280 for k, v in sorted(d.items()): 281 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 282 283def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 284 """Take a kernel, cmdline, and ramdisk directory from the input (in 285 'sourcedir'), and turn them into a boot image. Return the image 286 data, or None if sourcedir does not appear to contains files for 287 building the requested image.""" 288 289 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 290 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 291 return None 292 293 if info_dict is None: 294 info_dict = OPTIONS.info_dict 295 296 ramdisk_img = tempfile.NamedTemporaryFile() 297 img = tempfile.NamedTemporaryFile() 298 299 if os.access(fs_config_file, os.F_OK): 300 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 301 else: 302 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 303 p1 = Run(cmd, stdout=subprocess.PIPE) 304 p2 = Run(["minigzip"], 305 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 306 307 p2.wait() 308 p1.wait() 309 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 310 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 311 312 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 313 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 314 315 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 316 317 fn = os.path.join(sourcedir, "second") 318 if os.access(fn, os.F_OK): 319 cmd.append("--second") 320 cmd.append(fn) 321 322 fn = os.path.join(sourcedir, "cmdline") 323 if os.access(fn, os.F_OK): 324 cmd.append("--cmdline") 325 cmd.append(open(fn).read().rstrip("\n")) 326 327 fn = os.path.join(sourcedir, "base") 328 if os.access(fn, os.F_OK): 329 cmd.append("--base") 330 cmd.append(open(fn).read().rstrip("\n")) 331 332 fn = os.path.join(sourcedir, "pagesize") 333 if os.access(fn, os.F_OK): 334 cmd.append("--pagesize") 335 cmd.append(open(fn).read().rstrip("\n")) 336 337 args = info_dict.get("mkbootimg_args", None) 338 if args and args.strip(): 339 cmd.extend(shlex.split(args)) 340 341 cmd.extend(["--ramdisk", ramdisk_img.name, 342 "--output", img.name]) 343 344 p = Run(cmd, stdout=subprocess.PIPE) 345 p.communicate() 346 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 347 os.path.basename(sourcedir),) 348 349 if info_dict.get("verity_key", None): 350 path = "/" + os.path.basename(sourcedir).lower() 351 cmd = [OPTIONS.boot_signer_path, path, img.name, info_dict["verity_key"] + ".pk8", info_dict["verity_key"] + ".x509.pem", img.name] 352 p = Run(cmd, stdout=subprocess.PIPE) 353 p.communicate() 354 assert p.returncode == 0, "boot_signer of %s image failed" % path 355 356 img.seek(os.SEEK_SET, 0) 357 data = img.read() 358 359 ramdisk_img.close() 360 img.close() 361 362 return data 363 364 365def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 366 info_dict=None): 367 """Return a File object (with name 'name') with the desired bootable 368 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 369 'prebuilt_name', otherwise look for it under 'unpack_dir'/IMAGES, 370 otherwise construct it from the source files in 371 'unpack_dir'/'tree_subdir'.""" 372 373 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 374 if os.path.exists(prebuilt_path): 375 print "using prebuilt %s from BOOTABLE_IMAGES..." % (prebuilt_name,) 376 return File.FromLocalFile(name, prebuilt_path) 377 378 prebuilt_path = os.path.join(unpack_dir, "IMAGES", prebuilt_name) 379 if os.path.exists(prebuilt_path): 380 print "using prebuilt %s from IMAGES..." % (prebuilt_name,) 381 return File.FromLocalFile(name, prebuilt_path) 382 383 print "building image from target_files %s..." % (tree_subdir,) 384 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 385 data = BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 386 os.path.join(unpack_dir, fs_config), 387 info_dict) 388 if data: 389 return File(name, data) 390 return None 391 392 393def UnzipTemp(filename, pattern=None): 394 """Unzip the given archive into a temporary directory and return the name. 395 396 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 397 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 398 399 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 400 main file), open for reading. 401 """ 402 403 tmp = tempfile.mkdtemp(prefix="targetfiles-") 404 OPTIONS.tempfiles.append(tmp) 405 406 def unzip_to_dir(filename, dirname): 407 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 408 if pattern is not None: 409 cmd.append(pattern) 410 p = Run(cmd, stdout=subprocess.PIPE) 411 p.communicate() 412 if p.returncode != 0: 413 raise ExternalError("failed to unzip input target-files \"%s\"" % 414 (filename,)) 415 416 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 417 if m: 418 unzip_to_dir(m.group(1), tmp) 419 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 420 filename = m.group(1) 421 else: 422 unzip_to_dir(filename, tmp) 423 424 return tmp, zipfile.ZipFile(filename, "r") 425 426 427def GetKeyPasswords(keylist): 428 """Given a list of keys, prompt the user to enter passwords for 429 those which require them. Return a {key: password} dict. password 430 will be None if the key has no password.""" 431 432 no_passwords = [] 433 need_passwords = [] 434 key_passwords = {} 435 devnull = open("/dev/null", "w+b") 436 for k in sorted(keylist): 437 # We don't need a password for things that aren't really keys. 438 if k in SPECIAL_CERT_STRINGS: 439 no_passwords.append(k) 440 continue 441 442 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 443 "-inform", "DER", "-nocrypt"], 444 stdin=devnull.fileno(), 445 stdout=devnull.fileno(), 446 stderr=subprocess.STDOUT) 447 p.communicate() 448 if p.returncode == 0: 449 # Definitely an unencrypted key. 450 no_passwords.append(k) 451 else: 452 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 453 "-inform", "DER", "-passin", "pass:"], 454 stdin=devnull.fileno(), 455 stdout=devnull.fileno(), 456 stderr=subprocess.PIPE) 457 stdout, stderr = p.communicate() 458 if p.returncode == 0: 459 # Encrypted key with empty string as password. 460 key_passwords[k] = '' 461 elif stderr.startswith('Error decrypting key'): 462 # Definitely encrypted key. 463 # It would have said "Error reading key" if it didn't parse correctly. 464 need_passwords.append(k) 465 else: 466 # Potentially, a type of key that openssl doesn't understand. 467 # We'll let the routines in signapk.jar handle it. 468 no_passwords.append(k) 469 devnull.close() 470 471 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 472 key_passwords.update(dict.fromkeys(no_passwords, None)) 473 return key_passwords 474 475 476def SignFile(input_name, output_name, key, password, align=None, 477 whole_file=False): 478 """Sign the input_name zip/jar/apk, producing output_name. Use the 479 given key and password (the latter may be None if the key does not 480 have a password. 481 482 If align is an integer > 1, zipalign is run to align stored files in 483 the output zip on 'align'-byte boundaries. 484 485 If whole_file is true, use the "-w" option to SignApk to embed a 486 signature that covers the whole file in the archive comment of the 487 zip file. 488 """ 489 490 if align == 0 or align == 1: 491 align = None 492 493 if align: 494 temp = tempfile.NamedTemporaryFile() 495 sign_name = temp.name 496 else: 497 sign_name = output_name 498 499 cmd = [OPTIONS.java_path, OPTIONS.java_args, "-jar", 500 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 501 cmd.extend(OPTIONS.extra_signapk_args) 502 if whole_file: 503 cmd.append("-w") 504 cmd.extend([key + OPTIONS.public_key_suffix, 505 key + OPTIONS.private_key_suffix, 506 input_name, sign_name]) 507 508 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 509 if password is not None: 510 password += "\n" 511 p.communicate(password) 512 if p.returncode != 0: 513 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 514 515 if align: 516 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 517 p.communicate() 518 if p.returncode != 0: 519 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 520 temp.close() 521 522 523def CheckSize(data, target, info_dict): 524 """Check the data string passed against the max size limit, if 525 any, for the given target. Raise exception if the data is too big. 526 Print a warning if the data is nearing the maximum size.""" 527 528 if target.endswith(".img"): target = target[:-4] 529 mount_point = "/" + target 530 531 fs_type = None 532 limit = None 533 if info_dict["fstab"]: 534 if mount_point == "/userdata": mount_point = "/data" 535 p = info_dict["fstab"][mount_point] 536 fs_type = p.fs_type 537 device = p.device 538 if "/" in device: 539 device = device[device.rfind("/")+1:] 540 limit = info_dict.get(device + "_size", None) 541 if not fs_type or not limit: return 542 543 if fs_type == "yaffs2": 544 # image size should be increased by 1/64th to account for the 545 # spare area (64 bytes per 2k page) 546 limit = limit / 2048 * (2048+64) 547 size = len(data) 548 pct = float(size) * 100.0 / limit 549 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 550 if pct >= 99.0: 551 raise ExternalError(msg) 552 elif pct >= 95.0: 553 print 554 print " WARNING: ", msg 555 print 556 elif OPTIONS.verbose: 557 print " ", msg 558 559 560def ReadApkCerts(tf_zip): 561 """Given a target_files ZipFile, parse the META/apkcerts.txt file 562 and return a {package: cert} dict.""" 563 certmap = {} 564 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 565 line = line.strip() 566 if not line: continue 567 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 568 r'private_key="(.*)"$', line) 569 if m: 570 name, cert, privkey = m.groups() 571 public_key_suffix_len = len(OPTIONS.public_key_suffix) 572 private_key_suffix_len = len(OPTIONS.private_key_suffix) 573 if cert in SPECIAL_CERT_STRINGS and not privkey: 574 certmap[name] = cert 575 elif (cert.endswith(OPTIONS.public_key_suffix) and 576 privkey.endswith(OPTIONS.private_key_suffix) and 577 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 578 certmap[name] = cert[:-public_key_suffix_len] 579 else: 580 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 581 return certmap 582 583 584COMMON_DOCSTRING = """ 585 -p (--path) <dir> 586 Prepend <dir>/bin to the list of places to search for binaries 587 run by this script, and expect to find jars in <dir>/framework. 588 589 -s (--device_specific) <file> 590 Path to the python module containing device-specific 591 releasetools code. 592 593 -x (--extra) <key=value> 594 Add a key/value pair to the 'extras' dict, which device-specific 595 extension code may look at. 596 597 -v (--verbose) 598 Show command lines being executed. 599 600 -h (--help) 601 Display this usage message and exit. 602""" 603 604def Usage(docstring): 605 print docstring.rstrip("\n") 606 print COMMON_DOCSTRING 607 608 609def ParseOptions(argv, 610 docstring, 611 extra_opts="", extra_long_opts=(), 612 extra_option_handler=None): 613 """Parse the options in argv and return any arguments that aren't 614 flags. docstring is the calling module's docstring, to be displayed 615 for errors and -h. extra_opts and extra_long_opts are for flags 616 defined by the caller, which are processed by passing them to 617 extra_option_handler.""" 618 619 try: 620 opts, args = getopt.getopt( 621 argv, "hvp:s:x:" + extra_opts, 622 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 623 "java_path=", "java_args=", "public_key_suffix=", 624 "private_key_suffix=", "boot_signer_path=", "device_specific=", 625 "extra="] + 626 list(extra_long_opts)) 627 except getopt.GetoptError, err: 628 Usage(docstring) 629 print "**", str(err), "**" 630 sys.exit(2) 631 632 path_specified = False 633 634 for o, a in opts: 635 if o in ("-h", "--help"): 636 Usage(docstring) 637 sys.exit() 638 elif o in ("-v", "--verbose"): 639 OPTIONS.verbose = True 640 elif o in ("-p", "--path"): 641 OPTIONS.search_path = a 642 elif o in ("--signapk_path",): 643 OPTIONS.signapk_path = a 644 elif o in ("--extra_signapk_args",): 645 OPTIONS.extra_signapk_args = shlex.split(a) 646 elif o in ("--java_path",): 647 OPTIONS.java_path = a 648 elif o in ("--java_args",): 649 OPTIONS.java_args = a 650 elif o in ("--public_key_suffix",): 651 OPTIONS.public_key_suffix = a 652 elif o in ("--private_key_suffix",): 653 OPTIONS.private_key_suffix = a 654 elif o in ("--boot_signer_path",): 655 OPTIONS.boot_signer_path = a 656 elif o in ("-s", "--device_specific"): 657 OPTIONS.device_specific = a 658 elif o in ("-x", "--extra"): 659 key, value = a.split("=", 1) 660 OPTIONS.extras[key] = value 661 else: 662 if extra_option_handler is None or not extra_option_handler(o, a): 663 assert False, "unknown option \"%s\"" % (o,) 664 665 if OPTIONS.search_path: 666 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 667 os.pathsep + os.environ["PATH"]) 668 669 return args 670 671 672def MakeTempFile(prefix=None, suffix=None): 673 """Make a temp file and add it to the list of things to be deleted 674 when Cleanup() is called. Return the filename.""" 675 fd, fn = tempfile.mkstemp(prefix=prefix, suffix=suffix) 676 os.close(fd) 677 OPTIONS.tempfiles.append(fn) 678 return fn 679 680 681def Cleanup(): 682 for i in OPTIONS.tempfiles: 683 if os.path.isdir(i): 684 shutil.rmtree(i) 685 else: 686 os.remove(i) 687 688 689class PasswordManager(object): 690 def __init__(self): 691 self.editor = os.getenv("EDITOR", None) 692 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 693 694 def GetPasswords(self, items): 695 """Get passwords corresponding to each string in 'items', 696 returning a dict. (The dict may have keys in addition to the 697 values in 'items'.) 698 699 Uses the passwords in $ANDROID_PW_FILE if available, letting the 700 user edit that file to add more needed passwords. If no editor is 701 available, or $ANDROID_PW_FILE isn't define, prompts the user 702 interactively in the ordinary way. 703 """ 704 705 current = self.ReadFile() 706 707 first = True 708 while True: 709 missing = [] 710 for i in items: 711 if i not in current or not current[i]: 712 missing.append(i) 713 # Are all the passwords already in the file? 714 if not missing: return current 715 716 for i in missing: 717 current[i] = "" 718 719 if not first: 720 print "key file %s still missing some passwords." % (self.pwfile,) 721 answer = raw_input("try to edit again? [y]> ").strip() 722 if answer and answer[0] not in 'yY': 723 raise RuntimeError("key passwords unavailable") 724 first = False 725 726 current = self.UpdateAndReadFile(current) 727 728 def PromptResult(self, current): 729 """Prompt the user to enter a value (password) for each key in 730 'current' whose value is fales. Returns a new dict with all the 731 values. 732 """ 733 result = {} 734 for k, v in sorted(current.iteritems()): 735 if v: 736 result[k] = v 737 else: 738 while True: 739 result[k] = getpass.getpass("Enter password for %s key> " 740 % (k,)).strip() 741 if result[k]: break 742 return result 743 744 def UpdateAndReadFile(self, current): 745 if not self.editor or not self.pwfile: 746 return self.PromptResult(current) 747 748 f = open(self.pwfile, "w") 749 os.chmod(self.pwfile, 0600) 750 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 751 f.write("# (Additional spaces are harmless.)\n\n") 752 753 first_line = None 754 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 755 sorted.sort() 756 for i, (_, k, v) in enumerate(sorted): 757 f.write("[[[ %s ]]] %s\n" % (v, k)) 758 if not v and first_line is None: 759 # position cursor on first line with no password. 760 first_line = i + 4 761 f.close() 762 763 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 764 _, _ = p.communicate() 765 766 return self.ReadFile() 767 768 def ReadFile(self): 769 result = {} 770 if self.pwfile is None: return result 771 try: 772 f = open(self.pwfile, "r") 773 for line in f: 774 line = line.strip() 775 if not line or line[0] == '#': continue 776 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 777 if not m: 778 print "failed to parse password file: ", line 779 else: 780 result[m.group(2)] = m.group(1) 781 f.close() 782 except IOError, e: 783 if e.errno != errno.ENOENT: 784 print "error reading password file: ", str(e) 785 return result 786 787 788def ZipWrite(zip_file, filename, arcname=None, perms=0o644, 789 compress_type=None): 790 import datetime 791 792 # http://b/18015246 793 # Python 2.7's zipfile implementation wrongly thinks that zip64 is required 794 # for files larger than 2GiB. We can work around this by adjusting their 795 # limit. Note that `zipfile.writestr()` will not work for strings larger than 796 # 2GiB. The Python interpreter sometimes rejects strings that large (though 797 # it isn't clear to me exactly what circumstances cause this). 798 # `zipfile.write()` must be used directly to work around this. 799 # 800 # This mess can be avoided if we port to python3. 801 saved_zip64_limit = zipfile.ZIP64_LIMIT 802 zipfile.ZIP64_LIMIT = (1 << 32) - 1 803 804 compress_type = compress_type or zip_file.compression 805 arcname = arcname or filename 806 807 saved_stat = os.stat(filename) 808 809 try: 810 # `zipfile.write()` doesn't allow us to pass ZipInfo, so just modify the 811 # file to be zipped and reset it when we're done. 812 os.chmod(filename, perms) 813 814 # Use a fixed timestamp so the output is repeatable. 815 epoch = datetime.datetime.fromtimestamp(0) 816 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 817 os.utime(filename, (timestamp, timestamp)) 818 819 zip_file.write(filename, arcname=arcname, compress_type=compress_type) 820 finally: 821 os.chmod(filename, saved_stat.st_mode) 822 os.utime(filename, (saved_stat.st_atime, saved_stat.st_mtime)) 823 zipfile.ZIP64_LIMIT = saved_zip64_limit 824 825 826def ZipWriteStr(zip, filename, data, perms=0644, compression=None): 827 # use a fixed timestamp so the output is repeatable. 828 zinfo = zipfile.ZipInfo(filename=filename, 829 date_time=(2009, 1, 1, 0, 0, 0)) 830 if compression is None: 831 zinfo.compress_type = zip.compression 832 else: 833 zinfo.compress_type = compression 834 zinfo.external_attr = perms << 16 835 zip.writestr(zinfo, data) 836 837 838class DeviceSpecificParams(object): 839 module = None 840 def __init__(self, **kwargs): 841 """Keyword arguments to the constructor become attributes of this 842 object, which is passed to all functions in the device-specific 843 module.""" 844 for k, v in kwargs.iteritems(): 845 setattr(self, k, v) 846 self.extras = OPTIONS.extras 847 848 if self.module is None: 849 path = OPTIONS.device_specific 850 if not path: return 851 try: 852 if os.path.isdir(path): 853 info = imp.find_module("releasetools", [path]) 854 else: 855 d, f = os.path.split(path) 856 b, x = os.path.splitext(f) 857 if x == ".py": 858 f = b 859 info = imp.find_module(f, [d]) 860 print "loaded device-specific extensions from", path 861 self.module = imp.load_module("device_specific", *info) 862 except ImportError: 863 print "unable to load device-specific module; assuming none" 864 865 def _DoCall(self, function_name, *args, **kwargs): 866 """Call the named function in the device-specific module, passing 867 the given args and kwargs. The first argument to the call will be 868 the DeviceSpecific object itself. If there is no module, or the 869 module does not define the function, return the value of the 870 'default' kwarg (which itself defaults to None).""" 871 if self.module is None or not hasattr(self.module, function_name): 872 return kwargs.get("default", None) 873 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 874 875 def FullOTA_Assertions(self): 876 """Called after emitting the block of assertions at the top of a 877 full OTA package. Implementations can add whatever additional 878 assertions they like.""" 879 return self._DoCall("FullOTA_Assertions") 880 881 def FullOTA_InstallBegin(self): 882 """Called at the start of full OTA installation.""" 883 return self._DoCall("FullOTA_InstallBegin") 884 885 def FullOTA_InstallEnd(self): 886 """Called at the end of full OTA installation; typically this is 887 used to install the image for the device's baseband processor.""" 888 return self._DoCall("FullOTA_InstallEnd") 889 890 def IncrementalOTA_Assertions(self): 891 """Called after emitting the block of assertions at the top of an 892 incremental OTA package. Implementations can add whatever 893 additional assertions they like.""" 894 return self._DoCall("IncrementalOTA_Assertions") 895 896 def IncrementalOTA_VerifyBegin(self): 897 """Called at the start of the verification phase of incremental 898 OTA installation; additional checks can be placed here to abort 899 the script before any changes are made.""" 900 return self._DoCall("IncrementalOTA_VerifyBegin") 901 902 def IncrementalOTA_VerifyEnd(self): 903 """Called at the end of the verification phase of incremental OTA 904 installation; additional checks can be placed here to abort the 905 script before any changes are made.""" 906 return self._DoCall("IncrementalOTA_VerifyEnd") 907 908 def IncrementalOTA_InstallBegin(self): 909 """Called at the start of incremental OTA installation (after 910 verification is complete).""" 911 return self._DoCall("IncrementalOTA_InstallBegin") 912 913 def IncrementalOTA_InstallEnd(self): 914 """Called at the end of incremental OTA installation; typically 915 this is used to install the image for the device's baseband 916 processor.""" 917 return self._DoCall("IncrementalOTA_InstallEnd") 918 919class File(object): 920 def __init__(self, name, data): 921 self.name = name 922 self.data = data 923 self.size = len(data) 924 self.sha1 = sha1(data).hexdigest() 925 926 @classmethod 927 def FromLocalFile(cls, name, diskname): 928 f = open(diskname, "rb") 929 data = f.read() 930 f.close() 931 return File(name, data) 932 933 def WriteToTemp(self): 934 t = tempfile.NamedTemporaryFile() 935 t.write(self.data) 936 t.flush() 937 return t 938 939 def AddToZip(self, z, compression=None): 940 ZipWriteStr(z, self.name, self.data, compression=compression) 941 942DIFF_PROGRAM_BY_EXT = { 943 ".gz" : "imgdiff", 944 ".zip" : ["imgdiff", "-z"], 945 ".jar" : ["imgdiff", "-z"], 946 ".apk" : ["imgdiff", "-z"], 947 ".img" : "imgdiff", 948 } 949 950class Difference(object): 951 def __init__(self, tf, sf, diff_program=None): 952 self.tf = tf 953 self.sf = sf 954 self.patch = None 955 self.diff_program = diff_program 956 957 def ComputePatch(self): 958 """Compute the patch (as a string of data) needed to turn sf into 959 tf. Returns the same tuple as GetPatch().""" 960 961 tf = self.tf 962 sf = self.sf 963 964 if self.diff_program: 965 diff_program = self.diff_program 966 else: 967 ext = os.path.splitext(tf.name)[1] 968 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 969 970 ttemp = tf.WriteToTemp() 971 stemp = sf.WriteToTemp() 972 973 ext = os.path.splitext(tf.name)[1] 974 975 try: 976 ptemp = tempfile.NamedTemporaryFile() 977 if isinstance(diff_program, list): 978 cmd = copy.copy(diff_program) 979 else: 980 cmd = [diff_program] 981 cmd.append(stemp.name) 982 cmd.append(ttemp.name) 983 cmd.append(ptemp.name) 984 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 985 err = [] 986 def run(): 987 _, e = p.communicate() 988 if e: err.append(e) 989 th = threading.Thread(target=run) 990 th.start() 991 th.join(timeout=300) # 5 mins 992 if th.is_alive(): 993 print "WARNING: diff command timed out" 994 p.terminate() 995 th.join(5) 996 if th.is_alive(): 997 p.kill() 998 th.join() 999 1000 if err or p.returncode != 0: 1001 print "WARNING: failure running %s:\n%s\n" % ( 1002 diff_program, "".join(err)) 1003 self.patch = None 1004 return None, None, None 1005 diff = ptemp.read() 1006 finally: 1007 ptemp.close() 1008 stemp.close() 1009 ttemp.close() 1010 1011 self.patch = diff 1012 return self.tf, self.sf, self.patch 1013 1014 1015 def GetPatch(self): 1016 """Return a tuple (target_file, source_file, patch_data). 1017 patch_data may be None if ComputePatch hasn't been called, or if 1018 computing the patch failed.""" 1019 return self.tf, self.sf, self.patch 1020 1021 1022def ComputeDifferences(diffs): 1023 """Call ComputePatch on all the Difference objects in 'diffs'.""" 1024 print len(diffs), "diffs to compute" 1025 1026 # Do the largest files first, to try and reduce the long-pole effect. 1027 by_size = [(i.tf.size, i) for i in diffs] 1028 by_size.sort(reverse=True) 1029 by_size = [i[1] for i in by_size] 1030 1031 lock = threading.Lock() 1032 diff_iter = iter(by_size) # accessed under lock 1033 1034 def worker(): 1035 try: 1036 lock.acquire() 1037 for d in diff_iter: 1038 lock.release() 1039 start = time.time() 1040 d.ComputePatch() 1041 dur = time.time() - start 1042 lock.acquire() 1043 1044 tf, sf, patch = d.GetPatch() 1045 if sf.name == tf.name: 1046 name = tf.name 1047 else: 1048 name = "%s (%s)" % (tf.name, sf.name) 1049 if patch is None: 1050 print "patching failed! %s" % (name,) 1051 else: 1052 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 1053 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 1054 lock.release() 1055 except Exception, e: 1056 print e 1057 raise 1058 1059 # start worker threads; wait for them all to finish. 1060 threads = [threading.Thread(target=worker) 1061 for i in range(OPTIONS.worker_threads)] 1062 for th in threads: 1063 th.start() 1064 while threads: 1065 threads.pop().join() 1066 1067 1068class BlockDifference: 1069 def __init__(self, partition, tgt, src=None, check_first_block=False): 1070 self.tgt = tgt 1071 self.src = src 1072 self.partition = partition 1073 self.check_first_block = check_first_block 1074 1075 version = 1 1076 if OPTIONS.info_dict: 1077 version = max( 1078 int(i) for i in 1079 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(",")) 1080 1081 b = blockimgdiff.BlockImageDiff(tgt, src, threads=OPTIONS.worker_threads, 1082 version=version) 1083 tmpdir = tempfile.mkdtemp() 1084 OPTIONS.tempfiles.append(tmpdir) 1085 self.path = os.path.join(tmpdir, partition) 1086 b.Compute(self.path) 1087 1088 _, self.device = GetTypeAndDevice("/" + partition, OPTIONS.info_dict) 1089 1090 def WriteScript(self, script, output_zip, progress=None): 1091 if not self.src: 1092 # write the output unconditionally 1093 script.Print("Patching %s image unconditionally..." % (self.partition,)) 1094 else: 1095 script.Print("Patching %s image after verification." % (self.partition,)) 1096 1097 if progress: script.ShowProgress(progress, 0) 1098 self._WriteUpdate(script, output_zip) 1099 1100 def WriteVerifyScript(self, script): 1101 partition = self.partition 1102 if not self.src: 1103 script.Print("Image %s will be patched unconditionally." % (partition,)) 1104 else: 1105 script.AppendExtra(('if block_image_verify("%s", ' 1106 'package_extract_file("%s.transfer.list"), ' 1107 '"%s.new.dat", "%s.patch.dat") then') % 1108 (self.device, partition, partition, partition)) 1109 script.Print("Verified %s image..." % (partition,)) 1110 script.AppendExtra('else'); 1111 1112 if self.check_first_block: 1113 self._CheckFirstBlock(script) 1114 1115 script.AppendExtra(('(range_sha1("%s", "%s") == "%s") ||\n' 1116 ' abort("%s partition has unexpected contents");\n' 1117 'endif;') % 1118 (self.device, self.tgt.care_map.to_string_raw(), 1119 self.tgt.TotalSha1(), self.partition)) 1120 1121 def _WriteUpdate(self, script, output_zip): 1122 ZipWrite(output_zip, 1123 '{}.transfer.list'.format(self.path), 1124 '{}.transfer.list'.format(self.partition)) 1125 ZipWrite(output_zip, 1126 '{}.new.dat'.format(self.path), 1127 '{}.new.dat'.format(self.partition)) 1128 ZipWrite(output_zip, 1129 '{}.patch.dat'.format(self.path), 1130 '{}.patch.dat'.format(self.partition), 1131 compress_type=zipfile.ZIP_STORED) 1132 1133 call = ('block_image_update("{device}", ' 1134 'package_extract_file("{partition}.transfer.list"), ' 1135 '"{partition}.new.dat", "{partition}.patch.dat");\n'.format( 1136 device=self.device, partition=self.partition)) 1137 script.AppendExtra(script._WordWrap(call)) 1138 1139 def _HashBlocks(self, source, ranges): 1140 data = source.ReadRangeSet(ranges) 1141 ctx = sha1() 1142 1143 for p in data: 1144 ctx.update(p) 1145 1146 return ctx.hexdigest() 1147 1148 def _CheckFirstBlock(self, script): 1149 r = RangeSet((0, 1)) 1150 srchash = self._HashBlocks(self.src, r); 1151 tgthash = self._HashBlocks(self.tgt, r); 1152 1153 script.AppendExtra(('(range_sha1("%s", "%s") == "%s") || ' 1154 '(range_sha1("%s", "%s") == "%s") || ' 1155 'abort("%s has been remounted R/W; ' 1156 'reflash device to reenable OTA updates");') 1157 % (self.device, r.to_string_raw(), srchash, 1158 self.device, r.to_string_raw(), tgthash, 1159 self.device)) 1160 1161DataImage = blockimgdiff.DataImage 1162 1163 1164# map recovery.fstab's fs_types to mount/format "partition types" 1165PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 1166 "ext4": "EMMC", "emmc": "EMMC", 1167 "f2fs": "EMMC" } 1168 1169def GetTypeAndDevice(mount_point, info): 1170 fstab = info["fstab"] 1171 if fstab: 1172 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 1173 else: 1174 return None 1175 1176 1177def ParseCertificate(data): 1178 """Parse a PEM-format certificate.""" 1179 cert = [] 1180 save = False 1181 for line in data.split("\n"): 1182 if "--END CERTIFICATE--" in line: 1183 break 1184 if save: 1185 cert.append(line) 1186 if "--BEGIN CERTIFICATE--" in line: 1187 save = True 1188 cert = "".join(cert).decode('base64') 1189 return cert 1190 1191def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img, 1192 info_dict=None): 1193 """Generate a binary patch that creates the recovery image starting 1194 with the boot image. (Most of the space in these images is just the 1195 kernel, which is identical for the two, so the resulting patch 1196 should be efficient.) Add it to the output zip, along with a shell 1197 script that is run from init.rc on first boot to actually do the 1198 patching and install the new recovery image. 1199 1200 recovery_img and boot_img should be File objects for the 1201 corresponding images. info should be the dictionary returned by 1202 common.LoadInfoDict() on the input target_files. 1203 """ 1204 1205 if info_dict is None: 1206 info_dict = OPTIONS.info_dict 1207 1208 diff_program = ["imgdiff"] 1209 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat") 1210 if os.path.exists(path): 1211 diff_program.append("-b") 1212 diff_program.append(path) 1213 bonus_args = "-b /system/etc/recovery-resource.dat" 1214 else: 1215 bonus_args = "" 1216 1217 d = Difference(recovery_img, boot_img, diff_program=diff_program) 1218 _, _, patch = d.ComputePatch() 1219 output_sink("recovery-from-boot.p", patch) 1220 1221 td_pair = GetTypeAndDevice("/boot", info_dict) 1222 if not td_pair: 1223 return 1224 boot_type, boot_device = td_pair 1225 td_pair = GetTypeAndDevice("/recovery", info_dict) 1226 if not td_pair: 1227 return 1228 recovery_type, recovery_device = td_pair 1229 1230 sh = """#!/system/bin/sh 1231if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 1232 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1233else 1234 log -t recovery "Recovery image already installed" 1235fi 1236""" % { 'boot_size': boot_img.size, 1237 'boot_sha1': boot_img.sha1, 1238 'recovery_size': recovery_img.size, 1239 'recovery_sha1': recovery_img.sha1, 1240 'boot_type': boot_type, 1241 'boot_device': boot_device, 1242 'recovery_type': recovery_type, 1243 'recovery_device': recovery_device, 1244 'bonus_args': bonus_args, 1245 } 1246 1247 # The install script location moved from /system/etc to /system/bin 1248 # in the L release. Parse the init.rc file to find out where the 1249 # target-files expects it to be, and put it there. 1250 sh_location = "etc/install-recovery.sh" 1251 try: 1252 with open(os.path.join(input_dir, "BOOT", "RAMDISK", "init.rc")) as f: 1253 for line in f: 1254 m = re.match("^service flash_recovery /system/(\S+)\s*$", line) 1255 if m: 1256 sh_location = m.group(1) 1257 print "putting script in", sh_location 1258 break 1259 except (OSError, IOError), e: 1260 print "failed to read init.rc: %s" % (e,) 1261 1262 output_sink(sh_location, sh) 1263