common.py revision c9253822ea31c1d35d3fc2b495b45b476c240a1d
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32try: 33 from hashlib import sha1 as sha1 34except ImportError: 35 from sha import sha as sha1 36 37# missing in Python 2.4 and before 38if not hasattr(os, "SEEK_SET"): 39 os.SEEK_SET = 0 40 41class Options(object): pass 42OPTIONS = Options() 43OPTIONS.search_path = "out/host/linux-x86" 44OPTIONS.signapk_path = "framework/signapk.jar" # Relative to search_path 45OPTIONS.extra_signapk_args = [] 46OPTIONS.java_path = "java" # Use the one on the path by default. 47OPTIONS.public_key_suffix = ".x509.pem" 48OPTIONS.private_key_suffix = ".pk8" 49OPTIONS.verbose = False 50OPTIONS.tempfiles = [] 51OPTIONS.device_specific = None 52OPTIONS.extras = {} 53OPTIONS.info_dict = None 54 55 56# Values for "certificate" in apkcerts that mean special things. 57SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 58 59 60class ExternalError(RuntimeError): pass 61 62 63def Run(args, **kwargs): 64 """Create and return a subprocess.Popen object, printing the command 65 line on the terminal if -v was specified.""" 66 if OPTIONS.verbose: 67 print " running: ", " ".join(args) 68 return subprocess.Popen(args, **kwargs) 69 70 71def CloseInheritedPipes(): 72 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 73 before doing other work.""" 74 if platform.system() != "Darwin": 75 return 76 for d in range(3, 1025): 77 try: 78 stat = os.fstat(d) 79 if stat is not None: 80 pipebit = stat[0] & 0x1000 81 if pipebit != 0: 82 os.close(d) 83 except OSError: 84 pass 85 86 87def LoadInfoDict(input): 88 """Read and parse the META/misc_info.txt key/value pairs from the 89 input target files and return a dict.""" 90 91 def read_helper(fn): 92 if isinstance(input, zipfile.ZipFile): 93 return input.read(fn) 94 else: 95 path = os.path.join(input, *fn.split("/")) 96 try: 97 with open(path) as f: 98 return f.read() 99 except IOError, e: 100 if e.errno == errno.ENOENT: 101 raise KeyError(fn) 102 103 d = {} 104 try: 105 for line in read_helper("META/misc_info.txt").split("\n"): 106 line = line.strip() 107 if not line or line.startswith("#"): continue 108 k, v = line.split("=", 1) 109 d[k] = v 110 except KeyError: 111 # ok if misc_info.txt doesn't exist 112 pass 113 114 # backwards compatibility: These values used to be in their own 115 # files. Look for them, in case we're processing an old 116 # target_files zip. 117 118 if "mkyaffs2_extra_flags" not in d: 119 try: 120 d["mkyaffs2_extra_flags"] = read_helper("META/mkyaffs2-extra-flags.txt").strip() 121 except KeyError: 122 # ok if flags don't exist 123 pass 124 125 if "recovery_api_version" not in d: 126 try: 127 d["recovery_api_version"] = read_helper("META/recovery-api-version.txt").strip() 128 except KeyError: 129 raise ValueError("can't find recovery API version in input target-files") 130 131 if "tool_extensions" not in d: 132 try: 133 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip() 134 except KeyError: 135 # ok if extensions don't exist 136 pass 137 138 if "fstab_version" not in d: 139 d["fstab_version"] = "1" 140 141 try: 142 data = read_helper("META/imagesizes.txt") 143 for line in data.split("\n"): 144 if not line: continue 145 name, value = line.split(" ", 1) 146 if not value: continue 147 if name == "blocksize": 148 d[name] = value 149 else: 150 d[name + "_size"] = value 151 except KeyError: 152 pass 153 154 def makeint(key): 155 if key in d: 156 d[key] = int(d[key], 0) 157 158 makeint("recovery_api_version") 159 makeint("blocksize") 160 makeint("system_size") 161 makeint("userdata_size") 162 makeint("cache_size") 163 makeint("recovery_size") 164 makeint("boot_size") 165 makeint("fstab_version") 166 167 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"]) 168 d["build.prop"] = LoadBuildProp(read_helper) 169 return d 170 171def LoadBuildProp(read_helper): 172 try: 173 data = read_helper("SYSTEM/build.prop") 174 except KeyError: 175 print "Warning: could not find SYSTEM/build.prop in %s" % zip 176 data = "" 177 178 d = {} 179 for line in data.split("\n"): 180 line = line.strip() 181 if not line or line.startswith("#"): continue 182 name, value = line.split("=", 1) 183 d[name] = value 184 return d 185 186def LoadRecoveryFSTab(read_helper, fstab_version): 187 class Partition(object): 188 pass 189 190 try: 191 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab") 192 except KeyError: 193 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab" 194 data = "" 195 196 if fstab_version == 1: 197 d = {} 198 for line in data.split("\n"): 199 line = line.strip() 200 if not line or line.startswith("#"): continue 201 pieces = line.split() 202 if not (3 <= len(pieces) <= 4): 203 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 204 205 p = Partition() 206 p.mount_point = pieces[0] 207 p.fs_type = pieces[1] 208 p.device = pieces[2] 209 p.length = 0 210 options = None 211 if len(pieces) >= 4: 212 if pieces[3].startswith("/"): 213 p.device2 = pieces[3] 214 if len(pieces) >= 5: 215 options = pieces[4] 216 else: 217 p.device2 = None 218 options = pieces[3] 219 else: 220 p.device2 = None 221 222 if options: 223 options = options.split(",") 224 for i in options: 225 if i.startswith("length="): 226 p.length = int(i[7:]) 227 else: 228 print "%s: unknown option \"%s\"" % (p.mount_point, i) 229 230 d[p.mount_point] = p 231 232 elif fstab_version == 2: 233 d = {} 234 for line in data.split("\n"): 235 line = line.strip() 236 if not line or line.startswith("#"): continue 237 pieces = line.split() 238 if len(pieces) != 5: 239 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 240 241 # Ignore entries that are managed by vold 242 options = pieces[4] 243 if "voldmanaged=" in options: continue 244 245 # It's a good line, parse it 246 p = Partition() 247 p.device = pieces[0] 248 p.mount_point = pieces[1] 249 p.fs_type = pieces[2] 250 p.device2 = None 251 p.length = 0 252 253 options = options.split(",") 254 for i in options: 255 if i.startswith("length="): 256 p.length = int(i[7:]) 257 else: 258 # Ignore all unknown options in the unified fstab 259 continue 260 261 d[p.mount_point] = p 262 263 else: 264 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 265 266 return d 267 268 269def DumpInfoDict(d): 270 for k, v in sorted(d.items()): 271 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 272 273def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 274 """Take a kernel, cmdline, and ramdisk directory from the input (in 275 'sourcedir'), and turn them into a boot image. Return the image 276 data, or None if sourcedir does not appear to contains files for 277 building the requested image.""" 278 279 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 280 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 281 return None 282 283 if info_dict is None: 284 info_dict = OPTIONS.info_dict 285 286 ramdisk_img = tempfile.NamedTemporaryFile() 287 img = tempfile.NamedTemporaryFile() 288 289 if os.access(fs_config_file, os.F_OK): 290 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 291 else: 292 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 293 p1 = Run(cmd, stdout=subprocess.PIPE) 294 p2 = Run(["minigzip"], 295 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 296 297 p2.wait() 298 p1.wait() 299 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 300 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 301 302 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 303 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 304 305 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 306 307 fn = os.path.join(sourcedir, "cmdline") 308 if os.access(fn, os.F_OK): 309 cmd.append("--cmdline") 310 cmd.append(open(fn).read().rstrip("\n")) 311 312 fn = os.path.join(sourcedir, "base") 313 if os.access(fn, os.F_OK): 314 cmd.append("--base") 315 cmd.append(open(fn).read().rstrip("\n")) 316 317 fn = os.path.join(sourcedir, "pagesize") 318 if os.access(fn, os.F_OK): 319 cmd.append("--pagesize") 320 cmd.append(open(fn).read().rstrip("\n")) 321 322 args = info_dict.get("mkbootimg_args", None) 323 if args and args.strip(): 324 cmd.extend(shlex.split(args)) 325 326 cmd.extend(["--ramdisk", ramdisk_img.name, 327 "--output", img.name]) 328 329 p = Run(cmd, stdout=subprocess.PIPE) 330 p.communicate() 331 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 332 os.path.basename(sourcedir),) 333 334 img.seek(os.SEEK_SET, 0) 335 data = img.read() 336 337 ramdisk_img.close() 338 img.close() 339 340 return data 341 342 343def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 344 info_dict=None): 345 """Return a File object (with name 'name') with the desired bootable 346 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 347 'prebuilt_name', otherwise construct it from the source files in 348 'unpack_dir'/'tree_subdir'.""" 349 350 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 351 if os.path.exists(prebuilt_path): 352 print "using prebuilt %s..." % (prebuilt_name,) 353 return File.FromLocalFile(name, prebuilt_path) 354 else: 355 print "building image from target_files %s..." % (tree_subdir,) 356 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 357 return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 358 os.path.join(unpack_dir, fs_config), 359 info_dict)) 360 361 362def UnzipTemp(filename, pattern=None): 363 """Unzip the given archive into a temporary directory and return the name. 364 365 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 366 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 367 368 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 369 main file), open for reading. 370 """ 371 372 tmp = tempfile.mkdtemp(prefix="targetfiles-") 373 OPTIONS.tempfiles.append(tmp) 374 375 def unzip_to_dir(filename, dirname): 376 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 377 if pattern is not None: 378 cmd.append(pattern) 379 p = Run(cmd, stdout=subprocess.PIPE) 380 p.communicate() 381 if p.returncode != 0: 382 raise ExternalError("failed to unzip input target-files \"%s\"" % 383 (filename,)) 384 385 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 386 if m: 387 unzip_to_dir(m.group(1), tmp) 388 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 389 filename = m.group(1) 390 else: 391 unzip_to_dir(filename, tmp) 392 393 return tmp, zipfile.ZipFile(filename, "r") 394 395 396def GetKeyPasswords(keylist): 397 """Given a list of keys, prompt the user to enter passwords for 398 those which require them. Return a {key: password} dict. password 399 will be None if the key has no password.""" 400 401 no_passwords = [] 402 need_passwords = [] 403 key_passwords = {} 404 devnull = open("/dev/null", "w+b") 405 for k in sorted(keylist): 406 # We don't need a password for things that aren't really keys. 407 if k in SPECIAL_CERT_STRINGS: 408 no_passwords.append(k) 409 continue 410 411 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 412 "-inform", "DER", "-nocrypt"], 413 stdin=devnull.fileno(), 414 stdout=devnull.fileno(), 415 stderr=subprocess.STDOUT) 416 p.communicate() 417 if p.returncode == 0: 418 # Definitely an unencrypted key. 419 no_passwords.append(k) 420 else: 421 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 422 "-inform", "DER", "-passin", "pass:"], 423 stdin=devnull.fileno(), 424 stdout=devnull.fileno(), 425 stderr=subprocess.PIPE) 426 stdout, stderr = p.communicate() 427 if p.returncode == 0: 428 # Encrypted key with empty string as password. 429 key_passwords[k] = '' 430 elif stderr.startswith('Error decrypting key'): 431 # Definitely encrypted key. 432 # It would have said "Error reading key" if it didn't parse correctly. 433 need_passwords.append(k) 434 else: 435 # Potentially, a type of key that openssl doesn't understand. 436 # We'll let the routines in signapk.jar handle it. 437 no_passwords.append(k) 438 devnull.close() 439 440 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 441 key_passwords.update(dict.fromkeys(no_passwords, None)) 442 return key_passwords 443 444 445def SignFile(input_name, output_name, key, password, align=None, 446 whole_file=False): 447 """Sign the input_name zip/jar/apk, producing output_name. Use the 448 given key and password (the latter may be None if the key does not 449 have a password. 450 451 If align is an integer > 1, zipalign is run to align stored files in 452 the output zip on 'align'-byte boundaries. 453 454 If whole_file is true, use the "-w" option to SignApk to embed a 455 signature that covers the whole file in the archive comment of the 456 zip file. 457 """ 458 459 if align == 0 or align == 1: 460 align = None 461 462 if align: 463 temp = tempfile.NamedTemporaryFile() 464 sign_name = temp.name 465 else: 466 sign_name = output_name 467 468 cmd = [OPTIONS.java_path, "-Xmx2048m", "-jar", 469 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 470 cmd.extend(OPTIONS.extra_signapk_args) 471 if whole_file: 472 cmd.append("-w") 473 cmd.extend([key + OPTIONS.public_key_suffix, 474 key + OPTIONS.private_key_suffix, 475 input_name, sign_name]) 476 477 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 478 if password is not None: 479 password += "\n" 480 p.communicate(password) 481 if p.returncode != 0: 482 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 483 484 if align: 485 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 486 p.communicate() 487 if p.returncode != 0: 488 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 489 temp.close() 490 491 492def CheckSize(data, target, info_dict): 493 """Check the data string passed against the max size limit, if 494 any, for the given target. Raise exception if the data is too big. 495 Print a warning if the data is nearing the maximum size.""" 496 497 if target.endswith(".img"): target = target[:-4] 498 mount_point = "/" + target 499 500 if info_dict["fstab"]: 501 if mount_point == "/userdata": mount_point = "/data" 502 p = info_dict["fstab"][mount_point] 503 fs_type = p.fs_type 504 device = p.device 505 if "/" in device: 506 device = device[device.rfind("/")+1:] 507 limit = info_dict.get(device + "_size", None) 508 if not fs_type or not limit: return 509 510 if fs_type == "yaffs2": 511 # image size should be increased by 1/64th to account for the 512 # spare area (64 bytes per 2k page) 513 limit = limit / 2048 * (2048+64) 514 size = len(data) 515 pct = float(size) * 100.0 / limit 516 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 517 if pct >= 99.0: 518 raise ExternalError(msg) 519 elif pct >= 95.0: 520 print 521 print " WARNING: ", msg 522 print 523 elif OPTIONS.verbose: 524 print " ", msg 525 526 527def ReadApkCerts(tf_zip): 528 """Given a target_files ZipFile, parse the META/apkcerts.txt file 529 and return a {package: cert} dict.""" 530 certmap = {} 531 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 532 line = line.strip() 533 if not line: continue 534 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 535 r'private_key="(.*)"$', line) 536 if m: 537 name, cert, privkey = m.groups() 538 public_key_suffix_len = len(OPTIONS.public_key_suffix) 539 private_key_suffix_len = len(OPTIONS.private_key_suffix) 540 if cert in SPECIAL_CERT_STRINGS and not privkey: 541 certmap[name] = cert 542 elif (cert.endswith(OPTIONS.public_key_suffix) and 543 privkey.endswith(OPTIONS.private_key_suffix) and 544 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 545 certmap[name] = cert[:-public_key_suffix_len] 546 else: 547 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 548 return certmap 549 550 551COMMON_DOCSTRING = """ 552 -p (--path) <dir> 553 Prepend <dir>/bin to the list of places to search for binaries 554 run by this script, and expect to find jars in <dir>/framework. 555 556 -s (--device_specific) <file> 557 Path to the python module containing device-specific 558 releasetools code. 559 560 -x (--extra) <key=value> 561 Add a key/value pair to the 'extras' dict, which device-specific 562 extension code may look at. 563 564 -v (--verbose) 565 Show command lines being executed. 566 567 -h (--help) 568 Display this usage message and exit. 569""" 570 571def Usage(docstring): 572 print docstring.rstrip("\n") 573 print COMMON_DOCSTRING 574 575 576def ParseOptions(argv, 577 docstring, 578 extra_opts="", extra_long_opts=(), 579 extra_option_handler=None): 580 """Parse the options in argv and return any arguments that aren't 581 flags. docstring is the calling module's docstring, to be displayed 582 for errors and -h. extra_opts and extra_long_opts are for flags 583 defined by the caller, which are processed by passing them to 584 extra_option_handler.""" 585 586 try: 587 opts, args = getopt.getopt( 588 argv, "hvp:s:x:" + extra_opts, 589 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 590 "java_path=", "public_key_suffix=", "private_key_suffix=", 591 "device_specific=", "extra="] + 592 list(extra_long_opts)) 593 except getopt.GetoptError, err: 594 Usage(docstring) 595 print "**", str(err), "**" 596 sys.exit(2) 597 598 path_specified = False 599 600 for o, a in opts: 601 if o in ("-h", "--help"): 602 Usage(docstring) 603 sys.exit() 604 elif o in ("-v", "--verbose"): 605 OPTIONS.verbose = True 606 elif o in ("-p", "--path"): 607 OPTIONS.search_path = a 608 elif o in ("--signapk_path",): 609 OPTIONS.signapk_path = a 610 elif o in ("--extra_signapk_args",): 611 OPTIONS.extra_signapk_args = shlex.split(a) 612 elif o in ("--java_path",): 613 OPTIONS.java_path = a 614 elif o in ("--public_key_suffix",): 615 OPTIONS.public_key_suffix = a 616 elif o in ("--private_key_suffix",): 617 OPTIONS.private_key_suffix = a 618 elif o in ("-s", "--device_specific"): 619 OPTIONS.device_specific = a 620 elif o in ("-x", "--extra"): 621 key, value = a.split("=", 1) 622 OPTIONS.extras[key] = value 623 else: 624 if extra_option_handler is None or not extra_option_handler(o, a): 625 assert False, "unknown option \"%s\"" % (o,) 626 627 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 628 os.pathsep + os.environ["PATH"]) 629 630 return args 631 632 633def Cleanup(): 634 for i in OPTIONS.tempfiles: 635 if os.path.isdir(i): 636 shutil.rmtree(i) 637 else: 638 os.remove(i) 639 640 641class PasswordManager(object): 642 def __init__(self): 643 self.editor = os.getenv("EDITOR", None) 644 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 645 646 def GetPasswords(self, items): 647 """Get passwords corresponding to each string in 'items', 648 returning a dict. (The dict may have keys in addition to the 649 values in 'items'.) 650 651 Uses the passwords in $ANDROID_PW_FILE if available, letting the 652 user edit that file to add more needed passwords. If no editor is 653 available, or $ANDROID_PW_FILE isn't define, prompts the user 654 interactively in the ordinary way. 655 """ 656 657 current = self.ReadFile() 658 659 first = True 660 while True: 661 missing = [] 662 for i in items: 663 if i not in current or not current[i]: 664 missing.append(i) 665 # Are all the passwords already in the file? 666 if not missing: return current 667 668 for i in missing: 669 current[i] = "" 670 671 if not first: 672 print "key file %s still missing some passwords." % (self.pwfile,) 673 answer = raw_input("try to edit again? [y]> ").strip() 674 if answer and answer[0] not in 'yY': 675 raise RuntimeError("key passwords unavailable") 676 first = False 677 678 current = self.UpdateAndReadFile(current) 679 680 def PromptResult(self, current): 681 """Prompt the user to enter a value (password) for each key in 682 'current' whose value is fales. Returns a new dict with all the 683 values. 684 """ 685 result = {} 686 for k, v in sorted(current.iteritems()): 687 if v: 688 result[k] = v 689 else: 690 while True: 691 result[k] = getpass.getpass("Enter password for %s key> " 692 % (k,)).strip() 693 if result[k]: break 694 return result 695 696 def UpdateAndReadFile(self, current): 697 if not self.editor or not self.pwfile: 698 return self.PromptResult(current) 699 700 f = open(self.pwfile, "w") 701 os.chmod(self.pwfile, 0600) 702 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 703 f.write("# (Additional spaces are harmless.)\n\n") 704 705 first_line = None 706 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 707 sorted.sort() 708 for i, (_, k, v) in enumerate(sorted): 709 f.write("[[[ %s ]]] %s\n" % (v, k)) 710 if not v and first_line is None: 711 # position cursor on first line with no password. 712 first_line = i + 4 713 f.close() 714 715 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 716 _, _ = p.communicate() 717 718 return self.ReadFile() 719 720 def ReadFile(self): 721 result = {} 722 if self.pwfile is None: return result 723 try: 724 f = open(self.pwfile, "r") 725 for line in f: 726 line = line.strip() 727 if not line or line[0] == '#': continue 728 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 729 if not m: 730 print "failed to parse password file: ", line 731 else: 732 result[m.group(2)] = m.group(1) 733 f.close() 734 except IOError, e: 735 if e.errno != errno.ENOENT: 736 print "error reading password file: ", str(e) 737 return result 738 739 740def ZipWriteStr(zip, filename, data, perms=0644): 741 # use a fixed timestamp so the output is repeatable. 742 zinfo = zipfile.ZipInfo(filename=filename, 743 date_time=(2009, 1, 1, 0, 0, 0)) 744 zinfo.compress_type = zip.compression 745 zinfo.external_attr = perms << 16 746 zip.writestr(zinfo, data) 747 748 749class DeviceSpecificParams(object): 750 module = None 751 def __init__(self, **kwargs): 752 """Keyword arguments to the constructor become attributes of this 753 object, which is passed to all functions in the device-specific 754 module.""" 755 for k, v in kwargs.iteritems(): 756 setattr(self, k, v) 757 self.extras = OPTIONS.extras 758 759 if self.module is None: 760 path = OPTIONS.device_specific 761 if not path: return 762 try: 763 if os.path.isdir(path): 764 info = imp.find_module("releasetools", [path]) 765 else: 766 d, f = os.path.split(path) 767 b, x = os.path.splitext(f) 768 if x == ".py": 769 f = b 770 info = imp.find_module(f, [d]) 771 print "loaded device-specific extensions from", path 772 self.module = imp.load_module("device_specific", *info) 773 except ImportError: 774 print "unable to load device-specific module; assuming none" 775 776 def _DoCall(self, function_name, *args, **kwargs): 777 """Call the named function in the device-specific module, passing 778 the given args and kwargs. The first argument to the call will be 779 the DeviceSpecific object itself. If there is no module, or the 780 module does not define the function, return the value of the 781 'default' kwarg (which itself defaults to None).""" 782 if self.module is None or not hasattr(self.module, function_name): 783 return kwargs.get("default", None) 784 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 785 786 def FullOTA_Assertions(self): 787 """Called after emitting the block of assertions at the top of a 788 full OTA package. Implementations can add whatever additional 789 assertions they like.""" 790 return self._DoCall("FullOTA_Assertions") 791 792 def FullOTA_InstallBegin(self): 793 """Called at the start of full OTA installation.""" 794 return self._DoCall("FullOTA_InstallBegin") 795 796 def FullOTA_InstallEnd(self): 797 """Called at the end of full OTA installation; typically this is 798 used to install the image for the device's baseband processor.""" 799 return self._DoCall("FullOTA_InstallEnd") 800 801 def IncrementalOTA_Assertions(self): 802 """Called after emitting the block of assertions at the top of an 803 incremental OTA package. Implementations can add whatever 804 additional assertions they like.""" 805 return self._DoCall("IncrementalOTA_Assertions") 806 807 def IncrementalOTA_VerifyBegin(self): 808 """Called at the start of the verification phase of incremental 809 OTA installation; additional checks can be placed here to abort 810 the script before any changes are made.""" 811 return self._DoCall("IncrementalOTA_VerifyBegin") 812 813 def IncrementalOTA_VerifyEnd(self): 814 """Called at the end of the verification phase of incremental OTA 815 installation; additional checks can be placed here to abort the 816 script before any changes are made.""" 817 return self._DoCall("IncrementalOTA_VerifyEnd") 818 819 def IncrementalOTA_InstallBegin(self): 820 """Called at the start of incremental OTA installation (after 821 verification is complete).""" 822 return self._DoCall("IncrementalOTA_InstallBegin") 823 824 def IncrementalOTA_InstallEnd(self): 825 """Called at the end of incremental OTA installation; typically 826 this is used to install the image for the device's baseband 827 processor.""" 828 return self._DoCall("IncrementalOTA_InstallEnd") 829 830class File(object): 831 def __init__(self, name, data): 832 self.name = name 833 self.data = data 834 self.size = len(data) 835 self.sha1 = sha1(data).hexdigest() 836 837 @classmethod 838 def FromLocalFile(cls, name, diskname): 839 f = open(diskname, "rb") 840 data = f.read() 841 f.close() 842 return File(name, data) 843 844 def WriteToTemp(self): 845 t = tempfile.NamedTemporaryFile() 846 t.write(self.data) 847 t.flush() 848 return t 849 850 def AddToZip(self, z): 851 ZipWriteStr(z, self.name, self.data) 852 853DIFF_PROGRAM_BY_EXT = { 854 ".gz" : "imgdiff", 855 ".zip" : ["imgdiff", "-z"], 856 ".jar" : ["imgdiff", "-z"], 857 ".apk" : ["imgdiff", "-z"], 858 ".img" : "imgdiff", 859 } 860 861class Difference(object): 862 def __init__(self, tf, sf, diff_program=None): 863 self.tf = tf 864 self.sf = sf 865 self.patch = None 866 self.diff_program = diff_program 867 868 def ComputePatch(self): 869 """Compute the patch (as a string of data) needed to turn sf into 870 tf. Returns the same tuple as GetPatch().""" 871 872 tf = self.tf 873 sf = self.sf 874 875 if self.diff_program: 876 diff_program = self.diff_program 877 else: 878 ext = os.path.splitext(tf.name)[1] 879 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 880 881 ttemp = tf.WriteToTemp() 882 stemp = sf.WriteToTemp() 883 884 ext = os.path.splitext(tf.name)[1] 885 886 try: 887 ptemp = tempfile.NamedTemporaryFile() 888 if isinstance(diff_program, list): 889 cmd = copy.copy(diff_program) 890 else: 891 cmd = [diff_program] 892 cmd.append(stemp.name) 893 cmd.append(ttemp.name) 894 cmd.append(ptemp.name) 895 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 896 _, err = p.communicate() 897 if err or p.returncode != 0: 898 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 899 return None 900 diff = ptemp.read() 901 finally: 902 ptemp.close() 903 stemp.close() 904 ttemp.close() 905 906 self.patch = diff 907 return self.tf, self.sf, self.patch 908 909 910 def GetPatch(self): 911 """Return a tuple (target_file, source_file, patch_data). 912 patch_data may be None if ComputePatch hasn't been called, or if 913 computing the patch failed.""" 914 return self.tf, self.sf, self.patch 915 916 917def ComputeDifferences(diffs): 918 """Call ComputePatch on all the Difference objects in 'diffs'.""" 919 print len(diffs), "diffs to compute" 920 921 # Do the largest files first, to try and reduce the long-pole effect. 922 by_size = [(i.tf.size, i) for i in diffs] 923 by_size.sort(reverse=True) 924 by_size = [i[1] for i in by_size] 925 926 lock = threading.Lock() 927 diff_iter = iter(by_size) # accessed under lock 928 929 def worker(): 930 try: 931 lock.acquire() 932 for d in diff_iter: 933 lock.release() 934 start = time.time() 935 d.ComputePatch() 936 dur = time.time() - start 937 lock.acquire() 938 939 tf, sf, patch = d.GetPatch() 940 if sf.name == tf.name: 941 name = tf.name 942 else: 943 name = "%s (%s)" % (tf.name, sf.name) 944 if patch is None: 945 print "patching failed! %s" % (name,) 946 else: 947 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 948 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 949 lock.release() 950 except Exception, e: 951 print e 952 raise 953 954 # start worker threads; wait for them all to finish. 955 threads = [threading.Thread(target=worker) 956 for i in range(OPTIONS.worker_threads)] 957 for th in threads: 958 th.start() 959 while threads: 960 threads.pop().join() 961 962 963# map recovery.fstab's fs_types to mount/format "partition types" 964PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 965 "ext4": "EMMC", "emmc": "EMMC" } 966 967def GetTypeAndDevice(mount_point, info): 968 fstab = info["fstab"] 969 if fstab: 970 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 971 else: 972 return None 973 974 975def ParseCertificate(data): 976 """Parse a PEM-format certificate.""" 977 cert = [] 978 save = False 979 for line in data.split("\n"): 980 if "--END CERTIFICATE--" in line: 981 break 982 if save: 983 cert.append(line) 984 if "--BEGIN CERTIFICATE--" in line: 985 save = True 986 cert = "".join(cert).decode('base64') 987 return cert 988 989 990def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img): 991 """Generate a binary patch that creates the recovery image starting 992 with the boot image. (Most of the space in these images is just the 993 kernel, which is identical for the two, so the resulting patch 994 should be efficient.) Add it to the output zip, along with a shell 995 script that is run from init.rc on first boot to actually do the 996 patching and install the new recovery image. 997 998 recovery_img and boot_img should be File objects for the 999 corresponding images. info should be the dictionary returned by 1000 common.LoadInfoDict() on the input target_files. 1001 """ 1002 1003 diff_program = ["imgdiff"] 1004 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat") 1005 if os.path.exists(path): 1006 diff_program.append("-b") 1007 diff_program.append(path) 1008 bonus_args = "-b /system/etc/recovery-resource.dat" 1009 else: 1010 bonus_args = "" 1011 1012 d = Difference(recovery_img, boot_img, diff_program=diff_program) 1013 _, _, patch = d.ComputePatch() 1014 output_sink("recovery-from-boot.p", patch) 1015 1016 boot_type, boot_device = GetTypeAndDevice("/boot", OPTIONS.info_dict) 1017 recovery_type, recovery_device = GetTypeAndDevice("/recovery", OPTIONS.info_dict) 1018 1019 sh = """#!/system/bin/sh 1020if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 1021 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1022else 1023 log -t recovery "Recovery image already installed" 1024fi 1025""" % { 'boot_size': boot_img.size, 1026 'boot_sha1': boot_img.sha1, 1027 'recovery_size': recovery_img.size, 1028 'recovery_sha1': recovery_img.sha1, 1029 'boot_type': boot_type, 1030 'boot_device': boot_device, 1031 'recovery_type': recovery_type, 1032 'recovery_device': recovery_device, 1033 'bonus_args': bonus_args, 1034 } 1035 1036 # The install script location moved from /system/etc to /system/bin 1037 # in the L release. Parse the init.rc file to find out where the 1038 # target-files expects it to be, and put it there. 1039 sh_location = "etc/install-recovery.sh" 1040 try: 1041 with open(os.path.join(input_dir, "BOOT", "RAMDISK", "init.rc")) as f: 1042 for line in f: 1043 m = re.match("^service flash_recovery /system/(\S+)\s*$", line) 1044 if m: 1045 sh_location = m.group(1) 1046 print "putting script in", sh_location 1047 break 1048 except (OSError, IOError), e: 1049 print "failed to read init.rc: %s" % (e,) 1050 1051 output_sink(sh_location, sh) 1052