common.py revision 95ebe7a09b61931fc9e84b3ce43439c1eda40308
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32try: 33 from hashlib import sha1 as sha1 34except ImportError: 35 from sha import sha as sha1 36 37# missing in Python 2.4 and before 38if not hasattr(os, "SEEK_SET"): 39 os.SEEK_SET = 0 40 41class Options(object): pass 42OPTIONS = Options() 43OPTIONS.search_path = "out/host/linux-x86" 44OPTIONS.signapk_path = "framework/signapk.jar" # Relative to search_path 45OPTIONS.extra_signapk_args = [] 46OPTIONS.java_path = "java" # Use the one on the path by default. 47OPTIONS.public_key_suffix = ".x509.pem" 48OPTIONS.private_key_suffix = ".pk8" 49OPTIONS.verbose = False 50OPTIONS.tempfiles = [] 51OPTIONS.device_specific = None 52OPTIONS.extras = {} 53OPTIONS.info_dict = None 54 55 56# Values for "certificate" in apkcerts that mean special things. 57SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 58 59 60class ExternalError(RuntimeError): pass 61 62 63def Run(args, **kwargs): 64 """Create and return a subprocess.Popen object, printing the command 65 line on the terminal if -v was specified.""" 66 if OPTIONS.verbose: 67 print " running: ", " ".join(args) 68 return subprocess.Popen(args, **kwargs) 69 70 71def CloseInheritedPipes(): 72 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 73 before doing other work.""" 74 if platform.system() != "Darwin": 75 return 76 for d in range(3, 1025): 77 try: 78 stat = os.fstat(d) 79 if stat is not None: 80 pipebit = stat[0] & 0x1000 81 if pipebit != 0: 82 os.close(d) 83 except OSError: 84 pass 85 86 87def LoadInfoDict(input): 88 """Read and parse the META/misc_info.txt key/value pairs from the 89 input target files and return a dict.""" 90 91 def read_helper(fn): 92 if isinstance(input, zipfile.ZipFile): 93 return input.read(fn) 94 else: 95 path = os.path.join(input, *fn.split("/")) 96 try: 97 with open(path) as f: 98 return f.read() 99 except IOError, e: 100 if e.errno == errno.ENOENT: 101 raise KeyError(fn) 102 d = {} 103 try: 104 d = LoadDictionaryFromLines(read_helper("META/misc_info.txt").split("\n")) 105 except KeyError: 106 # ok if misc_info.txt doesn't exist 107 pass 108 109 # backwards compatibility: These values used to be in their own 110 # files. Look for them, in case we're processing an old 111 # target_files zip. 112 113 if "mkyaffs2_extra_flags" not in d: 114 try: 115 d["mkyaffs2_extra_flags"] = read_helper("META/mkyaffs2-extra-flags.txt").strip() 116 except KeyError: 117 # ok if flags don't exist 118 pass 119 120 if "recovery_api_version" not in d: 121 try: 122 d["recovery_api_version"] = read_helper("META/recovery-api-version.txt").strip() 123 except KeyError: 124 raise ValueError("can't find recovery API version in input target-files") 125 126 if "tool_extensions" not in d: 127 try: 128 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip() 129 except KeyError: 130 # ok if extensions don't exist 131 pass 132 133 if "fstab_version" not in d: 134 d["fstab_version"] = "1" 135 136 try: 137 data = read_helper("META/imagesizes.txt") 138 for line in data.split("\n"): 139 if not line: continue 140 name, value = line.split(" ", 1) 141 if not value: continue 142 if name == "blocksize": 143 d[name] = value 144 else: 145 d[name + "_size"] = value 146 except KeyError: 147 pass 148 149 def makeint(key): 150 if key in d: 151 d[key] = int(d[key], 0) 152 153 makeint("recovery_api_version") 154 makeint("blocksize") 155 makeint("system_size") 156 makeint("vendor_size") 157 makeint("userdata_size") 158 makeint("cache_size") 159 makeint("recovery_size") 160 makeint("boot_size") 161 makeint("fstab_version") 162 163 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"]) 164 d["build.prop"] = LoadBuildProp(read_helper) 165 return d 166 167def LoadBuildProp(read_helper): 168 try: 169 data = read_helper("SYSTEM/build.prop") 170 except KeyError: 171 print "Warning: could not find SYSTEM/build.prop in %s" % zip 172 data = "" 173 return LoadDictionaryFromLines(data.split("\n")) 174 175def LoadDictionaryFromLines(lines): 176 d = {} 177 for line in lines: 178 line = line.strip() 179 if not line or line.startswith("#"): continue 180 if "=" in line: 181 name, value = line.split("=", 1) 182 d[name] = value 183 return d 184 185def LoadRecoveryFSTab(read_helper, fstab_version): 186 class Partition(object): 187 pass 188 189 try: 190 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab") 191 except KeyError: 192 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab" 193 data = "" 194 195 if fstab_version == 1: 196 d = {} 197 for line in data.split("\n"): 198 line = line.strip() 199 if not line or line.startswith("#"): continue 200 pieces = line.split() 201 if not (3 <= len(pieces) <= 4): 202 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 203 204 p = Partition() 205 p.mount_point = pieces[0] 206 p.fs_type = pieces[1] 207 p.device = pieces[2] 208 p.length = 0 209 options = None 210 if len(pieces) >= 4: 211 if pieces[3].startswith("/"): 212 p.device2 = pieces[3] 213 if len(pieces) >= 5: 214 options = pieces[4] 215 else: 216 p.device2 = None 217 options = pieces[3] 218 else: 219 p.device2 = None 220 221 if options: 222 options = options.split(",") 223 for i in options: 224 if i.startswith("length="): 225 p.length = int(i[7:]) 226 else: 227 print "%s: unknown option \"%s\"" % (p.mount_point, i) 228 229 d[p.mount_point] = p 230 231 elif fstab_version == 2: 232 d = {} 233 for line in data.split("\n"): 234 line = line.strip() 235 if not line or line.startswith("#"): continue 236 pieces = line.split() 237 if len(pieces) != 5: 238 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 239 240 # Ignore entries that are managed by vold 241 options = pieces[4] 242 if "voldmanaged=" in options: continue 243 244 # It's a good line, parse it 245 p = Partition() 246 p.device = pieces[0] 247 p.mount_point = pieces[1] 248 p.fs_type = pieces[2] 249 p.device2 = None 250 p.length = 0 251 252 options = options.split(",") 253 for i in options: 254 if i.startswith("length="): 255 p.length = int(i[7:]) 256 else: 257 # Ignore all unknown options in the unified fstab 258 continue 259 260 d[p.mount_point] = p 261 262 else: 263 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 264 265 return d 266 267 268def DumpInfoDict(d): 269 for k, v in sorted(d.items()): 270 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 271 272def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 273 """Take a kernel, cmdline, and ramdisk directory from the input (in 274 'sourcedir'), and turn them into a boot image. Return the image 275 data, or None if sourcedir does not appear to contains files for 276 building the requested image.""" 277 278 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 279 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 280 return None 281 282 if info_dict is None: 283 info_dict = OPTIONS.info_dict 284 285 ramdisk_img = tempfile.NamedTemporaryFile() 286 img = tempfile.NamedTemporaryFile() 287 288 if os.access(fs_config_file, os.F_OK): 289 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 290 else: 291 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 292 p1 = Run(cmd, stdout=subprocess.PIPE) 293 p2 = Run(["minigzip"], 294 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 295 296 p2.wait() 297 p1.wait() 298 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 299 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 300 301 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 302 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 303 304 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 305 306 fn = os.path.join(sourcedir, "second") 307 if os.access(fn, os.F_OK): 308 cmd.append("--second") 309 cmd.append(fn) 310 311 fn = os.path.join(sourcedir, "cmdline") 312 if os.access(fn, os.F_OK): 313 cmd.append("--cmdline") 314 cmd.append(open(fn).read().rstrip("\n")) 315 316 fn = os.path.join(sourcedir, "base") 317 if os.access(fn, os.F_OK): 318 cmd.append("--base") 319 cmd.append(open(fn).read().rstrip("\n")) 320 321 fn = os.path.join(sourcedir, "pagesize") 322 if os.access(fn, os.F_OK): 323 cmd.append("--pagesize") 324 cmd.append(open(fn).read().rstrip("\n")) 325 326 args = info_dict.get("mkbootimg_args", None) 327 if args and args.strip(): 328 cmd.extend(shlex.split(args)) 329 330 cmd.extend(["--ramdisk", ramdisk_img.name, 331 "--output", img.name]) 332 333 p = Run(cmd, stdout=subprocess.PIPE) 334 p.communicate() 335 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 336 os.path.basename(sourcedir),) 337 338 if info_dict.get("verity_key", None): 339 path = "/" + os.path.basename(sourcedir).lower() 340 cmd = ["boot_signer", path, img.name, info_dict["verity_key"], img.name] 341 p = Run(cmd, stdout=subprocess.PIPE) 342 p.communicate() 343 assert p.returncode == 0, "boot_signer of %s image failed" % path 344 345 img.seek(os.SEEK_SET, 0) 346 data = img.read() 347 348 ramdisk_img.close() 349 img.close() 350 351 return data 352 353 354def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 355 info_dict=None): 356 """Return a File object (with name 'name') with the desired bootable 357 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 358 'prebuilt_name', otherwise construct it from the source files in 359 'unpack_dir'/'tree_subdir'.""" 360 361 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 362 if os.path.exists(prebuilt_path): 363 print "using prebuilt %s..." % (prebuilt_name,) 364 return File.FromLocalFile(name, prebuilt_path) 365 else: 366 print "building image from target_files %s..." % (tree_subdir,) 367 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 368 data = BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 369 os.path.join(unpack_dir, fs_config), 370 info_dict) 371 if data: 372 return File(name, data) 373 return None 374 375 376def UnzipTemp(filename, pattern=None): 377 """Unzip the given archive into a temporary directory and return the name. 378 379 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 380 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 381 382 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 383 main file), open for reading. 384 """ 385 386 tmp = tempfile.mkdtemp(prefix="targetfiles-") 387 OPTIONS.tempfiles.append(tmp) 388 389 def unzip_to_dir(filename, dirname): 390 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 391 if pattern is not None: 392 cmd.append(pattern) 393 p = Run(cmd, stdout=subprocess.PIPE) 394 p.communicate() 395 if p.returncode != 0: 396 raise ExternalError("failed to unzip input target-files \"%s\"" % 397 (filename,)) 398 399 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 400 if m: 401 unzip_to_dir(m.group(1), tmp) 402 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 403 filename = m.group(1) 404 else: 405 unzip_to_dir(filename, tmp) 406 407 return tmp, zipfile.ZipFile(filename, "r") 408 409 410def GetKeyPasswords(keylist): 411 """Given a list of keys, prompt the user to enter passwords for 412 those which require them. Return a {key: password} dict. password 413 will be None if the key has no password.""" 414 415 no_passwords = [] 416 need_passwords = [] 417 key_passwords = {} 418 devnull = open("/dev/null", "w+b") 419 for k in sorted(keylist): 420 # We don't need a password for things that aren't really keys. 421 if k in SPECIAL_CERT_STRINGS: 422 no_passwords.append(k) 423 continue 424 425 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 426 "-inform", "DER", "-nocrypt"], 427 stdin=devnull.fileno(), 428 stdout=devnull.fileno(), 429 stderr=subprocess.STDOUT) 430 p.communicate() 431 if p.returncode == 0: 432 # Definitely an unencrypted key. 433 no_passwords.append(k) 434 else: 435 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 436 "-inform", "DER", "-passin", "pass:"], 437 stdin=devnull.fileno(), 438 stdout=devnull.fileno(), 439 stderr=subprocess.PIPE) 440 stdout, stderr = p.communicate() 441 if p.returncode == 0: 442 # Encrypted key with empty string as password. 443 key_passwords[k] = '' 444 elif stderr.startswith('Error decrypting key'): 445 # Definitely encrypted key. 446 # It would have said "Error reading key" if it didn't parse correctly. 447 need_passwords.append(k) 448 else: 449 # Potentially, a type of key that openssl doesn't understand. 450 # We'll let the routines in signapk.jar handle it. 451 no_passwords.append(k) 452 devnull.close() 453 454 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 455 key_passwords.update(dict.fromkeys(no_passwords, None)) 456 return key_passwords 457 458 459def SignFile(input_name, output_name, key, password, align=None, 460 whole_file=False): 461 """Sign the input_name zip/jar/apk, producing output_name. Use the 462 given key and password (the latter may be None if the key does not 463 have a password. 464 465 If align is an integer > 1, zipalign is run to align stored files in 466 the output zip on 'align'-byte boundaries. 467 468 If whole_file is true, use the "-w" option to SignApk to embed a 469 signature that covers the whole file in the archive comment of the 470 zip file. 471 """ 472 473 if align == 0 or align == 1: 474 align = None 475 476 if align: 477 temp = tempfile.NamedTemporaryFile() 478 sign_name = temp.name 479 else: 480 sign_name = output_name 481 482 cmd = [OPTIONS.java_path, "-Xmx2048m", "-jar", 483 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 484 cmd.extend(OPTIONS.extra_signapk_args) 485 if whole_file: 486 cmd.append("-w") 487 cmd.extend([key + OPTIONS.public_key_suffix, 488 key + OPTIONS.private_key_suffix, 489 input_name, sign_name]) 490 491 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 492 if password is not None: 493 password += "\n" 494 p.communicate(password) 495 if p.returncode != 0: 496 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 497 498 if align: 499 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 500 p.communicate() 501 if p.returncode != 0: 502 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 503 temp.close() 504 505 506def CheckSize(data, target, info_dict): 507 """Check the data string passed against the max size limit, if 508 any, for the given target. Raise exception if the data is too big. 509 Print a warning if the data is nearing the maximum size.""" 510 511 if target.endswith(".img"): target = target[:-4] 512 mount_point = "/" + target 513 514 fs_type = None 515 limit = None 516 if info_dict["fstab"]: 517 if mount_point == "/userdata": mount_point = "/data" 518 p = info_dict["fstab"][mount_point] 519 fs_type = p.fs_type 520 device = p.device 521 if "/" in device: 522 device = device[device.rfind("/")+1:] 523 limit = info_dict.get(device + "_size", None) 524 if not fs_type or not limit: return 525 526 if fs_type == "yaffs2": 527 # image size should be increased by 1/64th to account for the 528 # spare area (64 bytes per 2k page) 529 limit = limit / 2048 * (2048+64) 530 size = len(data) 531 pct = float(size) * 100.0 / limit 532 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 533 if pct >= 99.0: 534 raise ExternalError(msg) 535 elif pct >= 95.0: 536 print 537 print " WARNING: ", msg 538 print 539 elif OPTIONS.verbose: 540 print " ", msg 541 542 543def ReadApkCerts(tf_zip): 544 """Given a target_files ZipFile, parse the META/apkcerts.txt file 545 and return a {package: cert} dict.""" 546 certmap = {} 547 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 548 line = line.strip() 549 if not line: continue 550 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 551 r'private_key="(.*)"$', line) 552 if m: 553 name, cert, privkey = m.groups() 554 public_key_suffix_len = len(OPTIONS.public_key_suffix) 555 private_key_suffix_len = len(OPTIONS.private_key_suffix) 556 if cert in SPECIAL_CERT_STRINGS and not privkey: 557 certmap[name] = cert 558 elif (cert.endswith(OPTIONS.public_key_suffix) and 559 privkey.endswith(OPTIONS.private_key_suffix) and 560 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 561 certmap[name] = cert[:-public_key_suffix_len] 562 else: 563 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 564 return certmap 565 566 567COMMON_DOCSTRING = """ 568 -p (--path) <dir> 569 Prepend <dir>/bin to the list of places to search for binaries 570 run by this script, and expect to find jars in <dir>/framework. 571 572 -s (--device_specific) <file> 573 Path to the python module containing device-specific 574 releasetools code. 575 576 -x (--extra) <key=value> 577 Add a key/value pair to the 'extras' dict, which device-specific 578 extension code may look at. 579 580 -v (--verbose) 581 Show command lines being executed. 582 583 -h (--help) 584 Display this usage message and exit. 585""" 586 587def Usage(docstring): 588 print docstring.rstrip("\n") 589 print COMMON_DOCSTRING 590 591 592def ParseOptions(argv, 593 docstring, 594 extra_opts="", extra_long_opts=(), 595 extra_option_handler=None): 596 """Parse the options in argv and return any arguments that aren't 597 flags. docstring is the calling module's docstring, to be displayed 598 for errors and -h. extra_opts and extra_long_opts are for flags 599 defined by the caller, which are processed by passing them to 600 extra_option_handler.""" 601 602 try: 603 opts, args = getopt.getopt( 604 argv, "hvp:s:x:" + extra_opts, 605 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 606 "java_path=", "public_key_suffix=", "private_key_suffix=", 607 "device_specific=", "extra="] + 608 list(extra_long_opts)) 609 except getopt.GetoptError, err: 610 Usage(docstring) 611 print "**", str(err), "**" 612 sys.exit(2) 613 614 path_specified = False 615 616 for o, a in opts: 617 if o in ("-h", "--help"): 618 Usage(docstring) 619 sys.exit() 620 elif o in ("-v", "--verbose"): 621 OPTIONS.verbose = True 622 elif o in ("-p", "--path"): 623 OPTIONS.search_path = a 624 elif o in ("--signapk_path",): 625 OPTIONS.signapk_path = a 626 elif o in ("--extra_signapk_args",): 627 OPTIONS.extra_signapk_args = shlex.split(a) 628 elif o in ("--java_path",): 629 OPTIONS.java_path = a 630 elif o in ("--public_key_suffix",): 631 OPTIONS.public_key_suffix = a 632 elif o in ("--private_key_suffix",): 633 OPTIONS.private_key_suffix = a 634 elif o in ("-s", "--device_specific"): 635 OPTIONS.device_specific = a 636 elif o in ("-x", "--extra"): 637 key, value = a.split("=", 1) 638 OPTIONS.extras[key] = value 639 else: 640 if extra_option_handler is None or not extra_option_handler(o, a): 641 assert False, "unknown option \"%s\"" % (o,) 642 643 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 644 os.pathsep + os.environ["PATH"]) 645 646 return args 647 648 649def Cleanup(): 650 for i in OPTIONS.tempfiles: 651 if os.path.isdir(i): 652 shutil.rmtree(i) 653 else: 654 os.remove(i) 655 656 657class PasswordManager(object): 658 def __init__(self): 659 self.editor = os.getenv("EDITOR", None) 660 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 661 662 def GetPasswords(self, items): 663 """Get passwords corresponding to each string in 'items', 664 returning a dict. (The dict may have keys in addition to the 665 values in 'items'.) 666 667 Uses the passwords in $ANDROID_PW_FILE if available, letting the 668 user edit that file to add more needed passwords. If no editor is 669 available, or $ANDROID_PW_FILE isn't define, prompts the user 670 interactively in the ordinary way. 671 """ 672 673 current = self.ReadFile() 674 675 first = True 676 while True: 677 missing = [] 678 for i in items: 679 if i not in current or not current[i]: 680 missing.append(i) 681 # Are all the passwords already in the file? 682 if not missing: return current 683 684 for i in missing: 685 current[i] = "" 686 687 if not first: 688 print "key file %s still missing some passwords." % (self.pwfile,) 689 answer = raw_input("try to edit again? [y]> ").strip() 690 if answer and answer[0] not in 'yY': 691 raise RuntimeError("key passwords unavailable") 692 first = False 693 694 current = self.UpdateAndReadFile(current) 695 696 def PromptResult(self, current): 697 """Prompt the user to enter a value (password) for each key in 698 'current' whose value is fales. Returns a new dict with all the 699 values. 700 """ 701 result = {} 702 for k, v in sorted(current.iteritems()): 703 if v: 704 result[k] = v 705 else: 706 while True: 707 result[k] = getpass.getpass("Enter password for %s key> " 708 % (k,)).strip() 709 if result[k]: break 710 return result 711 712 def UpdateAndReadFile(self, current): 713 if not self.editor or not self.pwfile: 714 return self.PromptResult(current) 715 716 f = open(self.pwfile, "w") 717 os.chmod(self.pwfile, 0600) 718 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 719 f.write("# (Additional spaces are harmless.)\n\n") 720 721 first_line = None 722 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 723 sorted.sort() 724 for i, (_, k, v) in enumerate(sorted): 725 f.write("[[[ %s ]]] %s\n" % (v, k)) 726 if not v and first_line is None: 727 # position cursor on first line with no password. 728 first_line = i + 4 729 f.close() 730 731 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 732 _, _ = p.communicate() 733 734 return self.ReadFile() 735 736 def ReadFile(self): 737 result = {} 738 if self.pwfile is None: return result 739 try: 740 f = open(self.pwfile, "r") 741 for line in f: 742 line = line.strip() 743 if not line or line[0] == '#': continue 744 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 745 if not m: 746 print "failed to parse password file: ", line 747 else: 748 result[m.group(2)] = m.group(1) 749 f.close() 750 except IOError, e: 751 if e.errno != errno.ENOENT: 752 print "error reading password file: ", str(e) 753 return result 754 755 756def ZipWriteStr(zip, filename, data, perms=0644, compression=None): 757 # use a fixed timestamp so the output is repeatable. 758 zinfo = zipfile.ZipInfo(filename=filename, 759 date_time=(2009, 1, 1, 0, 0, 0)) 760 if compression is None: 761 zinfo.compress_type = zip.compression 762 else: 763 zinfo.compress_type = compression 764 zinfo.external_attr = perms << 16 765 zip.writestr(zinfo, data) 766 767 768class DeviceSpecificParams(object): 769 module = None 770 def __init__(self, **kwargs): 771 """Keyword arguments to the constructor become attributes of this 772 object, which is passed to all functions in the device-specific 773 module.""" 774 for k, v in kwargs.iteritems(): 775 setattr(self, k, v) 776 self.extras = OPTIONS.extras 777 778 if self.module is None: 779 path = OPTIONS.device_specific 780 if not path: return 781 try: 782 if os.path.isdir(path): 783 info = imp.find_module("releasetools", [path]) 784 else: 785 d, f = os.path.split(path) 786 b, x = os.path.splitext(f) 787 if x == ".py": 788 f = b 789 info = imp.find_module(f, [d]) 790 print "loaded device-specific extensions from", path 791 self.module = imp.load_module("device_specific", *info) 792 except ImportError: 793 print "unable to load device-specific module; assuming none" 794 795 def _DoCall(self, function_name, *args, **kwargs): 796 """Call the named function in the device-specific module, passing 797 the given args and kwargs. The first argument to the call will be 798 the DeviceSpecific object itself. If there is no module, or the 799 module does not define the function, return the value of the 800 'default' kwarg (which itself defaults to None).""" 801 if self.module is None or not hasattr(self.module, function_name): 802 return kwargs.get("default", None) 803 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 804 805 def FullOTA_Assertions(self): 806 """Called after emitting the block of assertions at the top of a 807 full OTA package. Implementations can add whatever additional 808 assertions they like.""" 809 return self._DoCall("FullOTA_Assertions") 810 811 def FullOTA_InstallBegin(self): 812 """Called at the start of full OTA installation.""" 813 return self._DoCall("FullOTA_InstallBegin") 814 815 def FullOTA_InstallEnd(self): 816 """Called at the end of full OTA installation; typically this is 817 used to install the image for the device's baseband processor.""" 818 return self._DoCall("FullOTA_InstallEnd") 819 820 def IncrementalOTA_Assertions(self): 821 """Called after emitting the block of assertions at the top of an 822 incremental OTA package. Implementations can add whatever 823 additional assertions they like.""" 824 return self._DoCall("IncrementalOTA_Assertions") 825 826 def IncrementalOTA_VerifyBegin(self): 827 """Called at the start of the verification phase of incremental 828 OTA installation; additional checks can be placed here to abort 829 the script before any changes are made.""" 830 return self._DoCall("IncrementalOTA_VerifyBegin") 831 832 def IncrementalOTA_VerifyEnd(self): 833 """Called at the end of the verification phase of incremental OTA 834 installation; additional checks can be placed here to abort the 835 script before any changes are made.""" 836 return self._DoCall("IncrementalOTA_VerifyEnd") 837 838 def IncrementalOTA_InstallBegin(self): 839 """Called at the start of incremental OTA installation (after 840 verification is complete).""" 841 return self._DoCall("IncrementalOTA_InstallBegin") 842 843 def IncrementalOTA_InstallEnd(self): 844 """Called at the end of incremental OTA installation; typically 845 this is used to install the image for the device's baseband 846 processor.""" 847 return self._DoCall("IncrementalOTA_InstallEnd") 848 849class File(object): 850 def __init__(self, name, data): 851 self.name = name 852 self.data = data 853 self.size = len(data) 854 self.sha1 = sha1(data).hexdigest() 855 856 @classmethod 857 def FromLocalFile(cls, name, diskname): 858 f = open(diskname, "rb") 859 data = f.read() 860 f.close() 861 return File(name, data) 862 863 def WriteToTemp(self): 864 t = tempfile.NamedTemporaryFile() 865 t.write(self.data) 866 t.flush() 867 return t 868 869 def AddToZip(self, z, compression=None): 870 ZipWriteStr(z, self.name, self.data, compression=compression) 871 872DIFF_PROGRAM_BY_EXT = { 873 ".gz" : "imgdiff", 874 ".zip" : ["imgdiff", "-z"], 875 ".jar" : ["imgdiff", "-z"], 876 ".apk" : ["imgdiff", "-z"], 877 ".img" : "imgdiff", 878 } 879 880class Difference(object): 881 def __init__(self, tf, sf, diff_program=None): 882 self.tf = tf 883 self.sf = sf 884 self.patch = None 885 self.diff_program = diff_program 886 887 def ComputePatch(self): 888 """Compute the patch (as a string of data) needed to turn sf into 889 tf. Returns the same tuple as GetPatch().""" 890 891 tf = self.tf 892 sf = self.sf 893 894 if self.diff_program: 895 diff_program = self.diff_program 896 else: 897 ext = os.path.splitext(tf.name)[1] 898 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 899 900 ttemp = tf.WriteToTemp() 901 stemp = sf.WriteToTemp() 902 903 ext = os.path.splitext(tf.name)[1] 904 905 try: 906 ptemp = tempfile.NamedTemporaryFile() 907 if isinstance(diff_program, list): 908 cmd = copy.copy(diff_program) 909 else: 910 cmd = [diff_program] 911 cmd.append(stemp.name) 912 cmd.append(ttemp.name) 913 cmd.append(ptemp.name) 914 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 915 err = [] 916 def run(): 917 _, e = p.communicate() 918 if e: err.append(e) 919 th = threading.Thread(target=run) 920 th.start() 921 th.join(timeout=300) # 5 mins 922 if th.is_alive(): 923 print "WARNING: diff command timed out" 924 p.terminate() 925 th.join(5) 926 if th.is_alive(): 927 p.kill() 928 th.join() 929 930 if err or p.returncode != 0: 931 print "WARNING: failure running %s:\n%s\n" % ( 932 diff_program, "".join(err)) 933 self.patch = None 934 return None, None, None 935 diff = ptemp.read() 936 finally: 937 ptemp.close() 938 stemp.close() 939 ttemp.close() 940 941 self.patch = diff 942 return self.tf, self.sf, self.patch 943 944 945 def GetPatch(self): 946 """Return a tuple (target_file, source_file, patch_data). 947 patch_data may be None if ComputePatch hasn't been called, or if 948 computing the patch failed.""" 949 return self.tf, self.sf, self.patch 950 951 952def ComputeDifferences(diffs): 953 """Call ComputePatch on all the Difference objects in 'diffs'.""" 954 print len(diffs), "diffs to compute" 955 956 # Do the largest files first, to try and reduce the long-pole effect. 957 by_size = [(i.tf.size, i) for i in diffs] 958 by_size.sort(reverse=True) 959 by_size = [i[1] for i in by_size] 960 961 lock = threading.Lock() 962 diff_iter = iter(by_size) # accessed under lock 963 964 def worker(): 965 try: 966 lock.acquire() 967 for d in diff_iter: 968 lock.release() 969 start = time.time() 970 d.ComputePatch() 971 dur = time.time() - start 972 lock.acquire() 973 974 tf, sf, patch = d.GetPatch() 975 if sf.name == tf.name: 976 name = tf.name 977 else: 978 name = "%s (%s)" % (tf.name, sf.name) 979 if patch is None: 980 print "patching failed! %s" % (name,) 981 else: 982 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 983 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 984 lock.release() 985 except Exception, e: 986 print e 987 raise 988 989 # start worker threads; wait for them all to finish. 990 threads = [threading.Thread(target=worker) 991 for i in range(OPTIONS.worker_threads)] 992 for th in threads: 993 th.start() 994 while threads: 995 threads.pop().join() 996 997 998# map recovery.fstab's fs_types to mount/format "partition types" 999PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 1000 "ext4": "EMMC", "emmc": "EMMC", 1001 "f2fs": "EMMC" } 1002 1003def GetTypeAndDevice(mount_point, info): 1004 fstab = info["fstab"] 1005 if fstab: 1006 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 1007 else: 1008 return None 1009 1010 1011def ParseCertificate(data): 1012 """Parse a PEM-format certificate.""" 1013 cert = [] 1014 save = False 1015 for line in data.split("\n"): 1016 if "--END CERTIFICATE--" in line: 1017 break 1018 if save: 1019 cert.append(line) 1020 if "--BEGIN CERTIFICATE--" in line: 1021 save = True 1022 cert = "".join(cert).decode('base64') 1023 return cert 1024 1025def XDelta3(source_path, target_path, output_path): 1026 diff_program = ["xdelta3", "-0", "-B", str(64<<20), "-e", "-f", "-s"] 1027 diff_program.append(source_path) 1028 diff_program.append(target_path) 1029 diff_program.append(output_path) 1030 p = Run(diff_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 1031 p.communicate() 1032 assert p.returncode == 0, "Couldn't produce patch" 1033 1034def XZ(path): 1035 compress_program = ["xz", "-zk", "-9", "--check=crc32"] 1036 compress_program.append(path) 1037 p = Run(compress_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 1038 p.communicate() 1039 assert p.returncode == 0, "Couldn't compress patch" 1040 1041def MakePartitionPatch(source_file, target_file, partition): 1042 with tempfile.NamedTemporaryFile() as output_file: 1043 XDelta3(source_file.name, target_file.name, output_file.name) 1044 XZ(output_file.name) 1045 with open(output_file.name + ".xz") as patch_file: 1046 patch_data = patch_file.read() 1047 os.unlink(patch_file.name) 1048 return File(partition + ".muimg.p", patch_data) 1049 1050def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img, 1051 info_dict=None): 1052 """Generate a binary patch that creates the recovery image starting 1053 with the boot image. (Most of the space in these images is just the 1054 kernel, which is identical for the two, so the resulting patch 1055 should be efficient.) Add it to the output zip, along with a shell 1056 script that is run from init.rc on first boot to actually do the 1057 patching and install the new recovery image. 1058 1059 recovery_img and boot_img should be File objects for the 1060 corresponding images. info should be the dictionary returned by 1061 common.LoadInfoDict() on the input target_files. 1062 """ 1063 1064 if info_dict is None: 1065 info_dict = OPTIONS.info_dict 1066 1067 diff_program = ["imgdiff"] 1068 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat") 1069 if os.path.exists(path): 1070 diff_program.append("-b") 1071 diff_program.append(path) 1072 bonus_args = "-b /system/etc/recovery-resource.dat" 1073 else: 1074 bonus_args = "" 1075 1076 d = Difference(recovery_img, boot_img, diff_program=diff_program) 1077 _, _, patch = d.ComputePatch() 1078 output_sink("recovery-from-boot.p", patch) 1079 1080 td_pair = GetTypeAndDevice("/boot", info_dict) 1081 if not td_pair: 1082 return 1083 boot_type, boot_device = td_pair 1084 td_pair = GetTypeAndDevice("/recovery", info_dict) 1085 if not td_pair: 1086 return 1087 recovery_type, recovery_device = td_pair 1088 1089 sh = """#!/system/bin/sh 1090if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 1091 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1092else 1093 log -t recovery "Recovery image already installed" 1094fi 1095""" % { 'boot_size': boot_img.size, 1096 'boot_sha1': boot_img.sha1, 1097 'recovery_size': recovery_img.size, 1098 'recovery_sha1': recovery_img.sha1, 1099 'boot_type': boot_type, 1100 'boot_device': boot_device, 1101 'recovery_type': recovery_type, 1102 'recovery_device': recovery_device, 1103 'bonus_args': bonus_args, 1104 } 1105 1106 # The install script location moved from /system/etc to /system/bin 1107 # in the L release. Parse the init.rc file to find out where the 1108 # target-files expects it to be, and put it there. 1109 sh_location = "etc/install-recovery.sh" 1110 try: 1111 with open(os.path.join(input_dir, "BOOT", "RAMDISK", "init.rc")) as f: 1112 for line in f: 1113 m = re.match("^service flash_recovery /system/(\S+)\s*$", line) 1114 if m: 1115 sh_location = m.group(1) 1116 print "putting script in", sh_location 1117 break 1118 except (OSError, IOError), e: 1119 print "failed to read init.rc: %s" % (e,) 1120 1121 output_sink(sh_location, sh) 1122