common.py revision 854487739933f7ff33980df0c65d2bcd03b049f9
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32import blockimgdiff 33 34try: 35 from hashlib import sha1 as sha1 36except ImportError: 37 from sha import sha as sha1 38 39# missing in Python 2.4 and before 40if not hasattr(os, "SEEK_SET"): 41 os.SEEK_SET = 0 42 43class Options(object): pass 44OPTIONS = Options() 45 46DEFAULT_SEARCH_PATH_BY_PLATFORM = { 47 "linux2": "out/host/linux-x86", 48 "darwin": "out/host/darwin-x86", 49 } 50OPTIONS.search_path = DEFAULT_SEARCH_PATH_BY_PLATFORM.get(sys.platform, None) 51 52OPTIONS.signapk_path = "framework/signapk.jar" # Relative to search_path 53OPTIONS.extra_signapk_args = [] 54OPTIONS.java_path = "java" # Use the one on the path by default. 55OPTIONS.java_args = "-Xmx2048m" # JVM Args 56OPTIONS.public_key_suffix = ".x509.pem" 57OPTIONS.private_key_suffix = ".pk8" 58OPTIONS.verbose = False 59OPTIONS.tempfiles = [] 60OPTIONS.device_specific = None 61OPTIONS.extras = {} 62OPTIONS.info_dict = None 63 64 65# Values for "certificate" in apkcerts that mean special things. 66SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 67 68 69class ExternalError(RuntimeError): pass 70 71 72def Run(args, **kwargs): 73 """Create and return a subprocess.Popen object, printing the command 74 line on the terminal if -v was specified.""" 75 if OPTIONS.verbose: 76 print " running: ", " ".join(args) 77 return subprocess.Popen(args, **kwargs) 78 79 80def CloseInheritedPipes(): 81 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 82 before doing other work.""" 83 if platform.system() != "Darwin": 84 return 85 for d in range(3, 1025): 86 try: 87 stat = os.fstat(d) 88 if stat is not None: 89 pipebit = stat[0] & 0x1000 90 if pipebit != 0: 91 os.close(d) 92 except OSError: 93 pass 94 95 96def LoadInfoDict(input): 97 """Read and parse the META/misc_info.txt key/value pairs from the 98 input target files and return a dict.""" 99 100 def read_helper(fn): 101 if isinstance(input, zipfile.ZipFile): 102 return input.read(fn) 103 else: 104 path = os.path.join(input, *fn.split("/")) 105 try: 106 with open(path) as f: 107 return f.read() 108 except IOError, e: 109 if e.errno == errno.ENOENT: 110 raise KeyError(fn) 111 d = {} 112 try: 113 d = LoadDictionaryFromLines(read_helper("META/misc_info.txt").split("\n")) 114 except KeyError: 115 # ok if misc_info.txt doesn't exist 116 pass 117 118 # backwards compatibility: These values used to be in their own 119 # files. Look for them, in case we're processing an old 120 # target_files zip. 121 122 if "mkyaffs2_extra_flags" not in d: 123 try: 124 d["mkyaffs2_extra_flags"] = read_helper("META/mkyaffs2-extra-flags.txt").strip() 125 except KeyError: 126 # ok if flags don't exist 127 pass 128 129 if "recovery_api_version" not in d: 130 try: 131 d["recovery_api_version"] = read_helper("META/recovery-api-version.txt").strip() 132 except KeyError: 133 raise ValueError("can't find recovery API version in input target-files") 134 135 if "tool_extensions" not in d: 136 try: 137 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip() 138 except KeyError: 139 # ok if extensions don't exist 140 pass 141 142 if "fstab_version" not in d: 143 d["fstab_version"] = "1" 144 145 try: 146 data = read_helper("META/imagesizes.txt") 147 for line in data.split("\n"): 148 if not line: continue 149 name, value = line.split(" ", 1) 150 if not value: continue 151 if name == "blocksize": 152 d[name] = value 153 else: 154 d[name + "_size"] = value 155 except KeyError: 156 pass 157 158 def makeint(key): 159 if key in d: 160 d[key] = int(d[key], 0) 161 162 makeint("recovery_api_version") 163 makeint("blocksize") 164 makeint("system_size") 165 makeint("vendor_size") 166 makeint("userdata_size") 167 makeint("cache_size") 168 makeint("recovery_size") 169 makeint("boot_size") 170 makeint("fstab_version") 171 172 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"]) 173 d["build.prop"] = LoadBuildProp(read_helper) 174 return d 175 176def LoadBuildProp(read_helper): 177 try: 178 data = read_helper("SYSTEM/build.prop") 179 except KeyError: 180 print "Warning: could not find SYSTEM/build.prop in %s" % zip 181 data = "" 182 return LoadDictionaryFromLines(data.split("\n")) 183 184def LoadDictionaryFromLines(lines): 185 d = {} 186 for line in lines: 187 line = line.strip() 188 if not line or line.startswith("#"): continue 189 if "=" in line: 190 name, value = line.split("=", 1) 191 d[name] = value 192 return d 193 194def LoadRecoveryFSTab(read_helper, fstab_version): 195 class Partition(object): 196 pass 197 198 try: 199 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab") 200 except KeyError: 201 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab" 202 data = "" 203 204 if fstab_version == 1: 205 d = {} 206 for line in data.split("\n"): 207 line = line.strip() 208 if not line or line.startswith("#"): continue 209 pieces = line.split() 210 if not (3 <= len(pieces) <= 4): 211 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 212 213 p = Partition() 214 p.mount_point = pieces[0] 215 p.fs_type = pieces[1] 216 p.device = pieces[2] 217 p.length = 0 218 options = None 219 if len(pieces) >= 4: 220 if pieces[3].startswith("/"): 221 p.device2 = pieces[3] 222 if len(pieces) >= 5: 223 options = pieces[4] 224 else: 225 p.device2 = None 226 options = pieces[3] 227 else: 228 p.device2 = None 229 230 if options: 231 options = options.split(",") 232 for i in options: 233 if i.startswith("length="): 234 p.length = int(i[7:]) 235 else: 236 print "%s: unknown option \"%s\"" % (p.mount_point, i) 237 238 d[p.mount_point] = p 239 240 elif fstab_version == 2: 241 d = {} 242 for line in data.split("\n"): 243 line = line.strip() 244 if not line or line.startswith("#"): continue 245 pieces = line.split() 246 if len(pieces) != 5: 247 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 248 249 # Ignore entries that are managed by vold 250 options = pieces[4] 251 if "voldmanaged=" in options: continue 252 253 # It's a good line, parse it 254 p = Partition() 255 p.device = pieces[0] 256 p.mount_point = pieces[1] 257 p.fs_type = pieces[2] 258 p.device2 = None 259 p.length = 0 260 261 options = options.split(",") 262 for i in options: 263 if i.startswith("length="): 264 p.length = int(i[7:]) 265 else: 266 # Ignore all unknown options in the unified fstab 267 continue 268 269 d[p.mount_point] = p 270 271 else: 272 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 273 274 return d 275 276 277def DumpInfoDict(d): 278 for k, v in sorted(d.items()): 279 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 280 281def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 282 """Take a kernel, cmdline, and ramdisk directory from the input (in 283 'sourcedir'), and turn them into a boot image. Return the image 284 data, or None if sourcedir does not appear to contains files for 285 building the requested image.""" 286 287 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 288 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 289 return None 290 291 if info_dict is None: 292 info_dict = OPTIONS.info_dict 293 294 ramdisk_img = tempfile.NamedTemporaryFile() 295 img = tempfile.NamedTemporaryFile() 296 297 if os.access(fs_config_file, os.F_OK): 298 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 299 else: 300 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 301 p1 = Run(cmd, stdout=subprocess.PIPE) 302 p2 = Run(["minigzip"], 303 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 304 305 p2.wait() 306 p1.wait() 307 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 308 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 309 310 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 311 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 312 313 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 314 315 fn = os.path.join(sourcedir, "second") 316 if os.access(fn, os.F_OK): 317 cmd.append("--second") 318 cmd.append(fn) 319 320 fn = os.path.join(sourcedir, "cmdline") 321 if os.access(fn, os.F_OK): 322 cmd.append("--cmdline") 323 cmd.append(open(fn).read().rstrip("\n")) 324 325 fn = os.path.join(sourcedir, "base") 326 if os.access(fn, os.F_OK): 327 cmd.append("--base") 328 cmd.append(open(fn).read().rstrip("\n")) 329 330 fn = os.path.join(sourcedir, "pagesize") 331 if os.access(fn, os.F_OK): 332 cmd.append("--pagesize") 333 cmd.append(open(fn).read().rstrip("\n")) 334 335 args = info_dict.get("mkbootimg_args", None) 336 if args and args.strip(): 337 cmd.extend(shlex.split(args)) 338 339 cmd.extend(["--ramdisk", ramdisk_img.name, 340 "--output", img.name]) 341 342 p = Run(cmd, stdout=subprocess.PIPE) 343 p.communicate() 344 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 345 os.path.basename(sourcedir),) 346 347 if info_dict.get("verity_key", None): 348 path = "/" + os.path.basename(sourcedir).lower() 349 cmd = ["boot_signer", path, img.name, info_dict["verity_key"], img.name] 350 p = Run(cmd, stdout=subprocess.PIPE) 351 p.communicate() 352 assert p.returncode == 0, "boot_signer of %s image failed" % path 353 354 img.seek(os.SEEK_SET, 0) 355 data = img.read() 356 357 ramdisk_img.close() 358 img.close() 359 360 return data 361 362 363def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 364 info_dict=None): 365 """Return a File object (with name 'name') with the desired bootable 366 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 367 'prebuilt_name', otherwise look for it under 'unpack_dir'/IMAGES, 368 otherwise construct it from the source files in 369 'unpack_dir'/'tree_subdir'.""" 370 371 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 372 if os.path.exists(prebuilt_path): 373 print "using prebuilt %s from BOOTABLE_IMAGES..." % (prebuilt_name,) 374 return File.FromLocalFile(name, prebuilt_path) 375 376 prebuilt_path = os.path.join(unpack_dir, "IMAGES", prebuilt_name) 377 if os.path.exists(prebuilt_path): 378 print "using prebuilt %s from IMAGES..." % (prebuilt_name,) 379 return File.FromLocalFile(name, prebuilt_path) 380 381 print "building image from target_files %s..." % (tree_subdir,) 382 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 383 data = BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 384 os.path.join(unpack_dir, fs_config), 385 info_dict) 386 if data: 387 return File(name, data) 388 return None 389 390 391def UnzipTemp(filename, pattern=None): 392 """Unzip the given archive into a temporary directory and return the name. 393 394 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 395 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 396 397 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 398 main file), open for reading. 399 """ 400 401 tmp = tempfile.mkdtemp(prefix="targetfiles-") 402 OPTIONS.tempfiles.append(tmp) 403 404 def unzip_to_dir(filename, dirname): 405 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 406 if pattern is not None: 407 cmd.append(pattern) 408 p = Run(cmd, stdout=subprocess.PIPE) 409 p.communicate() 410 if p.returncode != 0: 411 raise ExternalError("failed to unzip input target-files \"%s\"" % 412 (filename,)) 413 414 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 415 if m: 416 unzip_to_dir(m.group(1), tmp) 417 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 418 filename = m.group(1) 419 else: 420 unzip_to_dir(filename, tmp) 421 422 return tmp, zipfile.ZipFile(filename, "r") 423 424 425def GetKeyPasswords(keylist): 426 """Given a list of keys, prompt the user to enter passwords for 427 those which require them. Return a {key: password} dict. password 428 will be None if the key has no password.""" 429 430 no_passwords = [] 431 need_passwords = [] 432 key_passwords = {} 433 devnull = open("/dev/null", "w+b") 434 for k in sorted(keylist): 435 # We don't need a password for things that aren't really keys. 436 if k in SPECIAL_CERT_STRINGS: 437 no_passwords.append(k) 438 continue 439 440 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 441 "-inform", "DER", "-nocrypt"], 442 stdin=devnull.fileno(), 443 stdout=devnull.fileno(), 444 stderr=subprocess.STDOUT) 445 p.communicate() 446 if p.returncode == 0: 447 # Definitely an unencrypted key. 448 no_passwords.append(k) 449 else: 450 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 451 "-inform", "DER", "-passin", "pass:"], 452 stdin=devnull.fileno(), 453 stdout=devnull.fileno(), 454 stderr=subprocess.PIPE) 455 stdout, stderr = p.communicate() 456 if p.returncode == 0: 457 # Encrypted key with empty string as password. 458 key_passwords[k] = '' 459 elif stderr.startswith('Error decrypting key'): 460 # Definitely encrypted key. 461 # It would have said "Error reading key" if it didn't parse correctly. 462 need_passwords.append(k) 463 else: 464 # Potentially, a type of key that openssl doesn't understand. 465 # We'll let the routines in signapk.jar handle it. 466 no_passwords.append(k) 467 devnull.close() 468 469 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 470 key_passwords.update(dict.fromkeys(no_passwords, None)) 471 return key_passwords 472 473 474def SignFile(input_name, output_name, key, password, align=None, 475 whole_file=False): 476 """Sign the input_name zip/jar/apk, producing output_name. Use the 477 given key and password (the latter may be None if the key does not 478 have a password. 479 480 If align is an integer > 1, zipalign is run to align stored files in 481 the output zip on 'align'-byte boundaries. 482 483 If whole_file is true, use the "-w" option to SignApk to embed a 484 signature that covers the whole file in the archive comment of the 485 zip file. 486 """ 487 488 if align == 0 or align == 1: 489 align = None 490 491 if align: 492 temp = tempfile.NamedTemporaryFile() 493 sign_name = temp.name 494 else: 495 sign_name = output_name 496 497 cmd = [OPTIONS.java_path, OPTIONS.java_args, "-jar", 498 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 499 cmd.extend(OPTIONS.extra_signapk_args) 500 if whole_file: 501 cmd.append("-w") 502 cmd.extend([key + OPTIONS.public_key_suffix, 503 key + OPTIONS.private_key_suffix, 504 input_name, sign_name]) 505 506 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 507 if password is not None: 508 password += "\n" 509 p.communicate(password) 510 if p.returncode != 0: 511 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 512 513 if align: 514 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 515 p.communicate() 516 if p.returncode != 0: 517 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 518 temp.close() 519 520 521def CheckSize(data, target, info_dict): 522 """Check the data string passed against the max size limit, if 523 any, for the given target. Raise exception if the data is too big. 524 Print a warning if the data is nearing the maximum size.""" 525 526 if target.endswith(".img"): target = target[:-4] 527 mount_point = "/" + target 528 529 fs_type = None 530 limit = None 531 if info_dict["fstab"]: 532 if mount_point == "/userdata": mount_point = "/data" 533 p = info_dict["fstab"][mount_point] 534 fs_type = p.fs_type 535 device = p.device 536 if "/" in device: 537 device = device[device.rfind("/")+1:] 538 limit = info_dict.get(device + "_size", None) 539 if not fs_type or not limit: return 540 541 if fs_type == "yaffs2": 542 # image size should be increased by 1/64th to account for the 543 # spare area (64 bytes per 2k page) 544 limit = limit / 2048 * (2048+64) 545 size = len(data) 546 pct = float(size) * 100.0 / limit 547 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 548 if pct >= 99.0: 549 raise ExternalError(msg) 550 elif pct >= 95.0: 551 print 552 print " WARNING: ", msg 553 print 554 elif OPTIONS.verbose: 555 print " ", msg 556 557 558def ReadApkCerts(tf_zip): 559 """Given a target_files ZipFile, parse the META/apkcerts.txt file 560 and return a {package: cert} dict.""" 561 certmap = {} 562 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 563 line = line.strip() 564 if not line: continue 565 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 566 r'private_key="(.*)"$', line) 567 if m: 568 name, cert, privkey = m.groups() 569 public_key_suffix_len = len(OPTIONS.public_key_suffix) 570 private_key_suffix_len = len(OPTIONS.private_key_suffix) 571 if cert in SPECIAL_CERT_STRINGS and not privkey: 572 certmap[name] = cert 573 elif (cert.endswith(OPTIONS.public_key_suffix) and 574 privkey.endswith(OPTIONS.private_key_suffix) and 575 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 576 certmap[name] = cert[:-public_key_suffix_len] 577 else: 578 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 579 return certmap 580 581 582COMMON_DOCSTRING = """ 583 -p (--path) <dir> 584 Prepend <dir>/bin to the list of places to search for binaries 585 run by this script, and expect to find jars in <dir>/framework. 586 587 -s (--device_specific) <file> 588 Path to the python module containing device-specific 589 releasetools code. 590 591 -x (--extra) <key=value> 592 Add a key/value pair to the 'extras' dict, which device-specific 593 extension code may look at. 594 595 -v (--verbose) 596 Show command lines being executed. 597 598 -h (--help) 599 Display this usage message and exit. 600""" 601 602def Usage(docstring): 603 print docstring.rstrip("\n") 604 print COMMON_DOCSTRING 605 606 607def ParseOptions(argv, 608 docstring, 609 extra_opts="", extra_long_opts=(), 610 extra_option_handler=None): 611 """Parse the options in argv and return any arguments that aren't 612 flags. docstring is the calling module's docstring, to be displayed 613 for errors and -h. extra_opts and extra_long_opts are for flags 614 defined by the caller, which are processed by passing them to 615 extra_option_handler.""" 616 617 try: 618 opts, args = getopt.getopt( 619 argv, "hvp:s:x:" + extra_opts, 620 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 621 "java_path=", "java_args=", "public_key_suffix=", 622 "private_key_suffix=", "device_specific=", "extra="] + 623 list(extra_long_opts)) 624 except getopt.GetoptError, err: 625 Usage(docstring) 626 print "**", str(err), "**" 627 sys.exit(2) 628 629 path_specified = False 630 631 for o, a in opts: 632 if o in ("-h", "--help"): 633 Usage(docstring) 634 sys.exit() 635 elif o in ("-v", "--verbose"): 636 OPTIONS.verbose = True 637 elif o in ("-p", "--path"): 638 OPTIONS.search_path = a 639 elif o in ("--signapk_path",): 640 OPTIONS.signapk_path = a 641 elif o in ("--extra_signapk_args",): 642 OPTIONS.extra_signapk_args = shlex.split(a) 643 elif o in ("--java_path",): 644 OPTIONS.java_path = a 645 elif o in ("--java_args",): 646 OPTIONS.java_args = a 647 elif o in ("--public_key_suffix",): 648 OPTIONS.public_key_suffix = a 649 elif o in ("--private_key_suffix",): 650 OPTIONS.private_key_suffix = a 651 elif o in ("-s", "--device_specific"): 652 OPTIONS.device_specific = a 653 elif o in ("-x", "--extra"): 654 key, value = a.split("=", 1) 655 OPTIONS.extras[key] = value 656 else: 657 if extra_option_handler is None or not extra_option_handler(o, a): 658 assert False, "unknown option \"%s\"" % (o,) 659 660 if OPTIONS.search_path: 661 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 662 os.pathsep + os.environ["PATH"]) 663 664 return args 665 666 667def MakeTempFile(prefix=None, suffix=None): 668 """Make a temp file and add it to the list of things to be deleted 669 when Cleanup() is called. Return the filename.""" 670 fd, fn = tempfile.mkstemp(prefix=prefix, suffix=suffix) 671 os.close(fd) 672 OPTIONS.tempfiles.append(fn) 673 return fn 674 675 676def Cleanup(): 677 for i in OPTIONS.tempfiles: 678 if os.path.isdir(i): 679 shutil.rmtree(i) 680 else: 681 os.remove(i) 682 683 684class PasswordManager(object): 685 def __init__(self): 686 self.editor = os.getenv("EDITOR", None) 687 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 688 689 def GetPasswords(self, items): 690 """Get passwords corresponding to each string in 'items', 691 returning a dict. (The dict may have keys in addition to the 692 values in 'items'.) 693 694 Uses the passwords in $ANDROID_PW_FILE if available, letting the 695 user edit that file to add more needed passwords. If no editor is 696 available, or $ANDROID_PW_FILE isn't define, prompts the user 697 interactively in the ordinary way. 698 """ 699 700 current = self.ReadFile() 701 702 first = True 703 while True: 704 missing = [] 705 for i in items: 706 if i not in current or not current[i]: 707 missing.append(i) 708 # Are all the passwords already in the file? 709 if not missing: return current 710 711 for i in missing: 712 current[i] = "" 713 714 if not first: 715 print "key file %s still missing some passwords." % (self.pwfile,) 716 answer = raw_input("try to edit again? [y]> ").strip() 717 if answer and answer[0] not in 'yY': 718 raise RuntimeError("key passwords unavailable") 719 first = False 720 721 current = self.UpdateAndReadFile(current) 722 723 def PromptResult(self, current): 724 """Prompt the user to enter a value (password) for each key in 725 'current' whose value is fales. Returns a new dict with all the 726 values. 727 """ 728 result = {} 729 for k, v in sorted(current.iteritems()): 730 if v: 731 result[k] = v 732 else: 733 while True: 734 result[k] = getpass.getpass("Enter password for %s key> " 735 % (k,)).strip() 736 if result[k]: break 737 return result 738 739 def UpdateAndReadFile(self, current): 740 if not self.editor or not self.pwfile: 741 return self.PromptResult(current) 742 743 f = open(self.pwfile, "w") 744 os.chmod(self.pwfile, 0600) 745 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 746 f.write("# (Additional spaces are harmless.)\n\n") 747 748 first_line = None 749 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 750 sorted.sort() 751 for i, (_, k, v) in enumerate(sorted): 752 f.write("[[[ %s ]]] %s\n" % (v, k)) 753 if not v and first_line is None: 754 # position cursor on first line with no password. 755 first_line = i + 4 756 f.close() 757 758 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 759 _, _ = p.communicate() 760 761 return self.ReadFile() 762 763 def ReadFile(self): 764 result = {} 765 if self.pwfile is None: return result 766 try: 767 f = open(self.pwfile, "r") 768 for line in f: 769 line = line.strip() 770 if not line or line[0] == '#': continue 771 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 772 if not m: 773 print "failed to parse password file: ", line 774 else: 775 result[m.group(2)] = m.group(1) 776 f.close() 777 except IOError, e: 778 if e.errno != errno.ENOENT: 779 print "error reading password file: ", str(e) 780 return result 781 782 783def ZipWriteStr(zip, filename, data, perms=0644, compression=None): 784 # use a fixed timestamp so the output is repeatable. 785 zinfo = zipfile.ZipInfo(filename=filename, 786 date_time=(2009, 1, 1, 0, 0, 0)) 787 if compression is None: 788 zinfo.compress_type = zip.compression 789 else: 790 zinfo.compress_type = compression 791 zinfo.external_attr = perms << 16 792 zip.writestr(zinfo, data) 793 794 795class DeviceSpecificParams(object): 796 module = None 797 def __init__(self, **kwargs): 798 """Keyword arguments to the constructor become attributes of this 799 object, which is passed to all functions in the device-specific 800 module.""" 801 for k, v in kwargs.iteritems(): 802 setattr(self, k, v) 803 self.extras = OPTIONS.extras 804 805 if self.module is None: 806 path = OPTIONS.device_specific 807 if not path: return 808 try: 809 if os.path.isdir(path): 810 info = imp.find_module("releasetools", [path]) 811 else: 812 d, f = os.path.split(path) 813 b, x = os.path.splitext(f) 814 if x == ".py": 815 f = b 816 info = imp.find_module(f, [d]) 817 print "loaded device-specific extensions from", path 818 self.module = imp.load_module("device_specific", *info) 819 except ImportError: 820 print "unable to load device-specific module; assuming none" 821 822 def _DoCall(self, function_name, *args, **kwargs): 823 """Call the named function in the device-specific module, passing 824 the given args and kwargs. The first argument to the call will be 825 the DeviceSpecific object itself. If there is no module, or the 826 module does not define the function, return the value of the 827 'default' kwarg (which itself defaults to None).""" 828 if self.module is None or not hasattr(self.module, function_name): 829 return kwargs.get("default", None) 830 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 831 832 def FullOTA_Assertions(self): 833 """Called after emitting the block of assertions at the top of a 834 full OTA package. Implementations can add whatever additional 835 assertions they like.""" 836 return self._DoCall("FullOTA_Assertions") 837 838 def FullOTA_InstallBegin(self): 839 """Called at the start of full OTA installation.""" 840 return self._DoCall("FullOTA_InstallBegin") 841 842 def FullOTA_InstallEnd(self): 843 """Called at the end of full OTA installation; typically this is 844 used to install the image for the device's baseband processor.""" 845 return self._DoCall("FullOTA_InstallEnd") 846 847 def IncrementalOTA_Assertions(self): 848 """Called after emitting the block of assertions at the top of an 849 incremental OTA package. Implementations can add whatever 850 additional assertions they like.""" 851 return self._DoCall("IncrementalOTA_Assertions") 852 853 def IncrementalOTA_VerifyBegin(self): 854 """Called at the start of the verification phase of incremental 855 OTA installation; additional checks can be placed here to abort 856 the script before any changes are made.""" 857 return self._DoCall("IncrementalOTA_VerifyBegin") 858 859 def IncrementalOTA_VerifyEnd(self): 860 """Called at the end of the verification phase of incremental OTA 861 installation; additional checks can be placed here to abort the 862 script before any changes are made.""" 863 return self._DoCall("IncrementalOTA_VerifyEnd") 864 865 def IncrementalOTA_InstallBegin(self): 866 """Called at the start of incremental OTA installation (after 867 verification is complete).""" 868 return self._DoCall("IncrementalOTA_InstallBegin") 869 870 def IncrementalOTA_InstallEnd(self): 871 """Called at the end of incremental OTA installation; typically 872 this is used to install the image for the device's baseband 873 processor.""" 874 return self._DoCall("IncrementalOTA_InstallEnd") 875 876class File(object): 877 def __init__(self, name, data): 878 self.name = name 879 self.data = data 880 self.size = len(data) 881 self.sha1 = sha1(data).hexdigest() 882 883 @classmethod 884 def FromLocalFile(cls, name, diskname): 885 f = open(diskname, "rb") 886 data = f.read() 887 f.close() 888 return File(name, data) 889 890 def WriteToTemp(self): 891 t = tempfile.NamedTemporaryFile() 892 t.write(self.data) 893 t.flush() 894 return t 895 896 def AddToZip(self, z, compression=None): 897 ZipWriteStr(z, self.name, self.data, compression=compression) 898 899DIFF_PROGRAM_BY_EXT = { 900 ".gz" : "imgdiff", 901 ".zip" : ["imgdiff", "-z"], 902 ".jar" : ["imgdiff", "-z"], 903 ".apk" : ["imgdiff", "-z"], 904 ".img" : "imgdiff", 905 } 906 907class Difference(object): 908 def __init__(self, tf, sf, diff_program=None): 909 self.tf = tf 910 self.sf = sf 911 self.patch = None 912 self.diff_program = diff_program 913 914 def ComputePatch(self): 915 """Compute the patch (as a string of data) needed to turn sf into 916 tf. Returns the same tuple as GetPatch().""" 917 918 tf = self.tf 919 sf = self.sf 920 921 if self.diff_program: 922 diff_program = self.diff_program 923 else: 924 ext = os.path.splitext(tf.name)[1] 925 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 926 927 ttemp = tf.WriteToTemp() 928 stemp = sf.WriteToTemp() 929 930 ext = os.path.splitext(tf.name)[1] 931 932 try: 933 ptemp = tempfile.NamedTemporaryFile() 934 if isinstance(diff_program, list): 935 cmd = copy.copy(diff_program) 936 else: 937 cmd = [diff_program] 938 cmd.append(stemp.name) 939 cmd.append(ttemp.name) 940 cmd.append(ptemp.name) 941 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 942 err = [] 943 def run(): 944 _, e = p.communicate() 945 if e: err.append(e) 946 th = threading.Thread(target=run) 947 th.start() 948 th.join(timeout=300) # 5 mins 949 if th.is_alive(): 950 print "WARNING: diff command timed out" 951 p.terminate() 952 th.join(5) 953 if th.is_alive(): 954 p.kill() 955 th.join() 956 957 if err or p.returncode != 0: 958 print "WARNING: failure running %s:\n%s\n" % ( 959 diff_program, "".join(err)) 960 self.patch = None 961 return None, None, None 962 diff = ptemp.read() 963 finally: 964 ptemp.close() 965 stemp.close() 966 ttemp.close() 967 968 self.patch = diff 969 return self.tf, self.sf, self.patch 970 971 972 def GetPatch(self): 973 """Return a tuple (target_file, source_file, patch_data). 974 patch_data may be None if ComputePatch hasn't been called, or if 975 computing the patch failed.""" 976 return self.tf, self.sf, self.patch 977 978 979def ComputeDifferences(diffs): 980 """Call ComputePatch on all the Difference objects in 'diffs'.""" 981 print len(diffs), "diffs to compute" 982 983 # Do the largest files first, to try and reduce the long-pole effect. 984 by_size = [(i.tf.size, i) for i in diffs] 985 by_size.sort(reverse=True) 986 by_size = [i[1] for i in by_size] 987 988 lock = threading.Lock() 989 diff_iter = iter(by_size) # accessed under lock 990 991 def worker(): 992 try: 993 lock.acquire() 994 for d in diff_iter: 995 lock.release() 996 start = time.time() 997 d.ComputePatch() 998 dur = time.time() - start 999 lock.acquire() 1000 1001 tf, sf, patch = d.GetPatch() 1002 if sf.name == tf.name: 1003 name = tf.name 1004 else: 1005 name = "%s (%s)" % (tf.name, sf.name) 1006 if patch is None: 1007 print "patching failed! %s" % (name,) 1008 else: 1009 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 1010 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 1011 lock.release() 1012 except Exception, e: 1013 print e 1014 raise 1015 1016 # start worker threads; wait for them all to finish. 1017 threads = [threading.Thread(target=worker) 1018 for i in range(OPTIONS.worker_threads)] 1019 for th in threads: 1020 th.start() 1021 while threads: 1022 threads.pop().join() 1023 1024 1025class BlockDifference: 1026 def __init__(self, partition, tgt, src=None): 1027 self.tgt = tgt 1028 self.src = src 1029 self.partition = partition 1030 1031 b = blockimgdiff.BlockImageDiff(tgt, src, threads=OPTIONS.worker_threads) 1032 tmpdir = tempfile.mkdtemp() 1033 OPTIONS.tempfiles.append(tmpdir) 1034 self.path = os.path.join(tmpdir, partition) 1035 b.Compute(self.path) 1036 1037 _, self.device = GetTypeAndDevice("/" + partition, OPTIONS.info_dict) 1038 1039 def WriteScript(self, script, output_zip, progress=None): 1040 if not self.src: 1041 # write the output unconditionally 1042 if progress: script.ShowProgress(progress, 0) 1043 self._WriteUpdate(script, output_zip) 1044 1045 else: 1046 script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' % 1047 (self.device, self.src.care_map.to_string_raw(), 1048 self.src.TotalSha1())) 1049 script.Print("Patching %s image..." % (self.partition,)) 1050 if progress: script.ShowProgress(progress, 0) 1051 self._WriteUpdate(script, output_zip) 1052 script.AppendExtra(('else\n' 1053 ' (range_sha1("%s", "%s") == "%s") ||\n' 1054 ' abort("%s partition has unexpected contents");\n' 1055 'endif;') % 1056 (self.device, self.tgt.care_map.to_string_raw(), 1057 self.tgt.TotalSha1(), self.partition)) 1058 1059 def _WriteUpdate(self, script, output_zip): 1060 partition = self.partition 1061 with open(self.path + ".transfer.list", "rb") as f: 1062 ZipWriteStr(output_zip, partition + ".transfer.list", f.read()) 1063 with open(self.path + ".new.dat", "rb") as f: 1064 ZipWriteStr(output_zip, partition + ".new.dat", f.read()) 1065 with open(self.path + ".patch.dat", "rb") as f: 1066 ZipWriteStr(output_zip, partition + ".patch.dat", f.read(), 1067 compression=zipfile.ZIP_STORED) 1068 1069 call = (('block_image_update("%s", ' 1070 'package_extract_file("%s.transfer.list"), ' 1071 '"%s.new.dat", "%s.patch.dat");\n') % 1072 (self.device, partition, partition, partition)) 1073 script.AppendExtra(script._WordWrap(call)) 1074 1075 1076DataImage = blockimgdiff.DataImage 1077 1078 1079# map recovery.fstab's fs_types to mount/format "partition types" 1080PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 1081 "ext4": "EMMC", "emmc": "EMMC", 1082 "f2fs": "EMMC" } 1083 1084def GetTypeAndDevice(mount_point, info): 1085 fstab = info["fstab"] 1086 if fstab: 1087 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 1088 else: 1089 return None 1090 1091 1092def ParseCertificate(data): 1093 """Parse a PEM-format certificate.""" 1094 cert = [] 1095 save = False 1096 for line in data.split("\n"): 1097 if "--END CERTIFICATE--" in line: 1098 break 1099 if save: 1100 cert.append(line) 1101 if "--BEGIN CERTIFICATE--" in line: 1102 save = True 1103 cert = "".join(cert).decode('base64') 1104 return cert 1105 1106def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img, 1107 info_dict=None): 1108 """Generate a binary patch that creates the recovery image starting 1109 with the boot image. (Most of the space in these images is just the 1110 kernel, which is identical for the two, so the resulting patch 1111 should be efficient.) Add it to the output zip, along with a shell 1112 script that is run from init.rc on first boot to actually do the 1113 patching and install the new recovery image. 1114 1115 recovery_img and boot_img should be File objects for the 1116 corresponding images. info should be the dictionary returned by 1117 common.LoadInfoDict() on the input target_files. 1118 """ 1119 1120 if info_dict is None: 1121 info_dict = OPTIONS.info_dict 1122 1123 diff_program = ["imgdiff"] 1124 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat") 1125 if os.path.exists(path): 1126 diff_program.append("-b") 1127 diff_program.append(path) 1128 bonus_args = "-b /system/etc/recovery-resource.dat" 1129 else: 1130 bonus_args = "" 1131 1132 d = Difference(recovery_img, boot_img, diff_program=diff_program) 1133 _, _, patch = d.ComputePatch() 1134 output_sink("recovery-from-boot.p", patch) 1135 1136 td_pair = GetTypeAndDevice("/boot", info_dict) 1137 if not td_pair: 1138 return 1139 boot_type, boot_device = td_pair 1140 td_pair = GetTypeAndDevice("/recovery", info_dict) 1141 if not td_pair: 1142 return 1143 recovery_type, recovery_device = td_pair 1144 1145 sh = """#!/system/bin/sh 1146if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 1147 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1148else 1149 log -t recovery "Recovery image already installed" 1150fi 1151""" % { 'boot_size': boot_img.size, 1152 'boot_sha1': boot_img.sha1, 1153 'recovery_size': recovery_img.size, 1154 'recovery_sha1': recovery_img.sha1, 1155 'boot_type': boot_type, 1156 'boot_device': boot_device, 1157 'recovery_type': recovery_type, 1158 'recovery_device': recovery_device, 1159 'bonus_args': bonus_args, 1160 } 1161 1162 # The install script location moved from /system/etc to /system/bin 1163 # in the L release. Parse the init.rc file to find out where the 1164 # target-files expects it to be, and put it there. 1165 sh_location = "etc/install-recovery.sh" 1166 try: 1167 with open(os.path.join(input_dir, "BOOT", "RAMDISK", "init.rc")) as f: 1168 for line in f: 1169 m = re.match("^service flash_recovery /system/(\S+)\s*$", line) 1170 if m: 1171 sh_location = m.group(1) 1172 print "putting script in", sh_location 1173 break 1174 except (OSError, IOError), e: 1175 print "failed to read init.rc: %s" % (e,) 1176 1177 output_sink(sh_location, sh) 1178