common.py revision 6e836116f764cf5cebf1654df2f17d8222554f6e
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32try: 33 from hashlib import sha1 as sha1 34except ImportError: 35 from sha import sha as sha1 36 37# missing in Python 2.4 and before 38if not hasattr(os, "SEEK_SET"): 39 os.SEEK_SET = 0 40 41class Options(object): pass 42OPTIONS = Options() 43OPTIONS.search_path = "out/host/linux-x86" 44OPTIONS.signapk_path = "framework/signapk.jar" # Relative to search_path 45OPTIONS.extra_signapk_args = [] 46OPTIONS.java_path = "java" # Use the one on the path by default. 47OPTIONS.public_key_suffix = ".x509.pem" 48OPTIONS.private_key_suffix = ".pk8" 49OPTIONS.verbose = False 50OPTIONS.tempfiles = [] 51OPTIONS.device_specific = None 52OPTIONS.extras = {} 53OPTIONS.info_dict = None 54 55 56# Values for "certificate" in apkcerts that mean special things. 57SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 58 59 60class ExternalError(RuntimeError): pass 61 62 63def Run(args, **kwargs): 64 """Create and return a subprocess.Popen object, printing the command 65 line on the terminal if -v was specified.""" 66 if OPTIONS.verbose: 67 print " running: ", " ".join(args) 68 return subprocess.Popen(args, **kwargs) 69 70 71def CloseInheritedPipes(): 72 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 73 before doing other work.""" 74 if platform.system() != "Darwin": 75 return 76 for d in range(3, 1025): 77 try: 78 stat = os.fstat(d) 79 if stat is not None: 80 pipebit = stat[0] & 0x1000 81 if pipebit != 0: 82 os.close(d) 83 except OSError: 84 pass 85 86 87def LoadInfoDict(input): 88 """Read and parse the META/misc_info.txt key/value pairs from the 89 input target files and return a dict.""" 90 91 def read_helper(fn): 92 if isinstance(input, zipfile.ZipFile): 93 return input.read(fn) 94 else: 95 path = os.path.join(input, *fn.split("/")) 96 try: 97 with open(path) as f: 98 return f.read() 99 except IOError, e: 100 if e.errno == errno.ENOENT: 101 raise KeyError(fn) 102 d = {} 103 try: 104 d = LoadDictionaryFromLines(read_helper("META/misc_info.txt").split("\n")) 105 except KeyError: 106 # ok if misc_info.txt doesn't exist 107 pass 108 109 # backwards compatibility: These values used to be in their own 110 # files. Look for them, in case we're processing an old 111 # target_files zip. 112 113 if "mkyaffs2_extra_flags" not in d: 114 try: 115 d["mkyaffs2_extra_flags"] = read_helper("META/mkyaffs2-extra-flags.txt").strip() 116 except KeyError: 117 # ok if flags don't exist 118 pass 119 120 if "recovery_api_version" not in d: 121 try: 122 d["recovery_api_version"] = read_helper("META/recovery-api-version.txt").strip() 123 except KeyError: 124 raise ValueError("can't find recovery API version in input target-files") 125 126 if "tool_extensions" not in d: 127 try: 128 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip() 129 except KeyError: 130 # ok if extensions don't exist 131 pass 132 133 if "fstab_version" not in d: 134 d["fstab_version"] = "1" 135 136 try: 137 data = read_helper("META/imagesizes.txt") 138 for line in data.split("\n"): 139 if not line: continue 140 name, value = line.split(" ", 1) 141 if not value: continue 142 if name == "blocksize": 143 d[name] = value 144 else: 145 d[name + "_size"] = value 146 except KeyError: 147 pass 148 149 def makeint(key): 150 if key in d: 151 d[key] = int(d[key], 0) 152 153 makeint("recovery_api_version") 154 makeint("blocksize") 155 makeint("system_size") 156 makeint("userdata_size") 157 makeint("cache_size") 158 makeint("recovery_size") 159 makeint("boot_size") 160 makeint("fstab_version") 161 162 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"]) 163 d["build.prop"] = LoadBuildProp(read_helper) 164 return d 165 166def LoadBuildProp(read_helper): 167 try: 168 data = read_helper("SYSTEM/build.prop") 169 except KeyError: 170 print "Warning: could not find SYSTEM/build.prop in %s" % zip 171 data = "" 172 return LoadDictionaryFromLines(data.split("\n")) 173 174def LoadDictionaryFromLines(lines): 175 d = {} 176 for line in lines: 177 line = line.strip() 178 if not line or line.startswith("#"): continue 179 if "=" in line: 180 name, value = line.split("=", 1) 181 d[name] = value 182 return d 183 184def LoadRecoveryFSTab(read_helper, fstab_version): 185 class Partition(object): 186 pass 187 188 try: 189 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab") 190 except KeyError: 191 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab" 192 data = "" 193 194 if fstab_version == 1: 195 d = {} 196 for line in data.split("\n"): 197 line = line.strip() 198 if not line or line.startswith("#"): continue 199 pieces = line.split() 200 if not (3 <= len(pieces) <= 4): 201 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 202 203 p = Partition() 204 p.mount_point = pieces[0] 205 p.fs_type = pieces[1] 206 p.device = pieces[2] 207 p.length = 0 208 options = None 209 if len(pieces) >= 4: 210 if pieces[3].startswith("/"): 211 p.device2 = pieces[3] 212 if len(pieces) >= 5: 213 options = pieces[4] 214 else: 215 p.device2 = None 216 options = pieces[3] 217 else: 218 p.device2 = None 219 220 if options: 221 options = options.split(",") 222 for i in options: 223 if i.startswith("length="): 224 p.length = int(i[7:]) 225 else: 226 print "%s: unknown option \"%s\"" % (p.mount_point, i) 227 228 d[p.mount_point] = p 229 230 elif fstab_version == 2: 231 d = {} 232 for line in data.split("\n"): 233 line = line.strip() 234 if not line or line.startswith("#"): continue 235 pieces = line.split() 236 if len(pieces) != 5: 237 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 238 239 # Ignore entries that are managed by vold 240 options = pieces[4] 241 if "voldmanaged=" in options: continue 242 243 # It's a good line, parse it 244 p = Partition() 245 p.device = pieces[0] 246 p.mount_point = pieces[1] 247 p.fs_type = pieces[2] 248 p.device2 = None 249 p.length = 0 250 251 options = options.split(",") 252 for i in options: 253 if i.startswith("length="): 254 p.length = int(i[7:]) 255 else: 256 # Ignore all unknown options in the unified fstab 257 continue 258 259 d[p.mount_point] = p 260 261 else: 262 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 263 264 return d 265 266 267def DumpInfoDict(d): 268 for k, v in sorted(d.items()): 269 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 270 271def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 272 """Take a kernel, cmdline, and ramdisk directory from the input (in 273 'sourcedir'), and turn them into a boot image. Return the image 274 data, or None if sourcedir does not appear to contains files for 275 building the requested image.""" 276 277 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 278 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 279 return None 280 281 if info_dict is None: 282 info_dict = OPTIONS.info_dict 283 284 ramdisk_img = tempfile.NamedTemporaryFile() 285 img = tempfile.NamedTemporaryFile() 286 287 if os.access(fs_config_file, os.F_OK): 288 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 289 else: 290 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 291 p1 = Run(cmd, stdout=subprocess.PIPE) 292 p2 = Run(["minigzip"], 293 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 294 295 p2.wait() 296 p1.wait() 297 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 298 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 299 300 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 301 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 302 303 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 304 305 fn = os.path.join(sourcedir, "cmdline") 306 if os.access(fn, os.F_OK): 307 cmd.append("--cmdline") 308 cmd.append(open(fn).read().rstrip("\n")) 309 310 fn = os.path.join(sourcedir, "base") 311 if os.access(fn, os.F_OK): 312 cmd.append("--base") 313 cmd.append(open(fn).read().rstrip("\n")) 314 315 fn = os.path.join(sourcedir, "pagesize") 316 if os.access(fn, os.F_OK): 317 cmd.append("--pagesize") 318 cmd.append(open(fn).read().rstrip("\n")) 319 320 args = info_dict.get("mkbootimg_args", None) 321 if args and args.strip(): 322 cmd.extend(shlex.split(args)) 323 324 cmd.extend(["--ramdisk", ramdisk_img.name, 325 "--output", img.name]) 326 327 p = Run(cmd, stdout=subprocess.PIPE) 328 p.communicate() 329 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 330 os.path.basename(sourcedir),) 331 332 img.seek(os.SEEK_SET, 0) 333 data = img.read() 334 335 ramdisk_img.close() 336 img.close() 337 338 return data 339 340 341def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 342 info_dict=None): 343 """Return a File object (with name 'name') with the desired bootable 344 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 345 'prebuilt_name', otherwise construct it from the source files in 346 'unpack_dir'/'tree_subdir'.""" 347 348 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 349 if os.path.exists(prebuilt_path): 350 print "using prebuilt %s..." % (prebuilt_name,) 351 return File.FromLocalFile(name, prebuilt_path) 352 else: 353 print "building image from target_files %s..." % (tree_subdir,) 354 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 355 data = BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 356 os.path.join(unpack_dir, fs_config), 357 info_dict) 358 if data: 359 return File(name, data) 360 return None 361 362 363def UnzipTemp(filename, pattern=None): 364 """Unzip the given archive into a temporary directory and return the name. 365 366 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 367 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 368 369 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 370 main file), open for reading. 371 """ 372 373 tmp = tempfile.mkdtemp(prefix="targetfiles-") 374 OPTIONS.tempfiles.append(tmp) 375 376 def unzip_to_dir(filename, dirname): 377 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 378 if pattern is not None: 379 cmd.append(pattern) 380 p = Run(cmd, stdout=subprocess.PIPE) 381 p.communicate() 382 if p.returncode != 0: 383 raise ExternalError("failed to unzip input target-files \"%s\"" % 384 (filename,)) 385 386 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 387 if m: 388 unzip_to_dir(m.group(1), tmp) 389 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 390 filename = m.group(1) 391 else: 392 unzip_to_dir(filename, tmp) 393 394 return tmp, zipfile.ZipFile(filename, "r") 395 396 397def GetKeyPasswords(keylist): 398 """Given a list of keys, prompt the user to enter passwords for 399 those which require them. Return a {key: password} dict. password 400 will be None if the key has no password.""" 401 402 no_passwords = [] 403 need_passwords = [] 404 key_passwords = {} 405 devnull = open("/dev/null", "w+b") 406 for k in sorted(keylist): 407 # We don't need a password for things that aren't really keys. 408 if k in SPECIAL_CERT_STRINGS: 409 no_passwords.append(k) 410 continue 411 412 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 413 "-inform", "DER", "-nocrypt"], 414 stdin=devnull.fileno(), 415 stdout=devnull.fileno(), 416 stderr=subprocess.STDOUT) 417 p.communicate() 418 if p.returncode == 0: 419 # Definitely an unencrypted key. 420 no_passwords.append(k) 421 else: 422 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 423 "-inform", "DER", "-passin", "pass:"], 424 stdin=devnull.fileno(), 425 stdout=devnull.fileno(), 426 stderr=subprocess.PIPE) 427 stdout, stderr = p.communicate() 428 if p.returncode == 0: 429 # Encrypted key with empty string as password. 430 key_passwords[k] = '' 431 elif stderr.startswith('Error decrypting key'): 432 # Definitely encrypted key. 433 # It would have said "Error reading key" if it didn't parse correctly. 434 need_passwords.append(k) 435 else: 436 # Potentially, a type of key that openssl doesn't understand. 437 # We'll let the routines in signapk.jar handle it. 438 no_passwords.append(k) 439 devnull.close() 440 441 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 442 key_passwords.update(dict.fromkeys(no_passwords, None)) 443 return key_passwords 444 445 446def SignFile(input_name, output_name, key, password, align=None, 447 whole_file=False): 448 """Sign the input_name zip/jar/apk, producing output_name. Use the 449 given key and password (the latter may be None if the key does not 450 have a password. 451 452 If align is an integer > 1, zipalign is run to align stored files in 453 the output zip on 'align'-byte boundaries. 454 455 If whole_file is true, use the "-w" option to SignApk to embed a 456 signature that covers the whole file in the archive comment of the 457 zip file. 458 """ 459 460 if align == 0 or align == 1: 461 align = None 462 463 if align: 464 temp = tempfile.NamedTemporaryFile() 465 sign_name = temp.name 466 else: 467 sign_name = output_name 468 469 cmd = [OPTIONS.java_path, "-Xmx2048m", "-jar", 470 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 471 cmd.extend(OPTIONS.extra_signapk_args) 472 if whole_file: 473 cmd.append("-w") 474 cmd.extend([key + OPTIONS.public_key_suffix, 475 key + OPTIONS.private_key_suffix, 476 input_name, sign_name]) 477 478 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 479 if password is not None: 480 password += "\n" 481 p.communicate(password) 482 if p.returncode != 0: 483 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 484 485 if align: 486 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 487 p.communicate() 488 if p.returncode != 0: 489 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 490 temp.close() 491 492 493def CheckSize(data, target, info_dict): 494 """Check the data string passed against the max size limit, if 495 any, for the given target. Raise exception if the data is too big. 496 Print a warning if the data is nearing the maximum size.""" 497 498 if target.endswith(".img"): target = target[:-4] 499 mount_point = "/" + target 500 501 if info_dict["fstab"]: 502 if mount_point == "/userdata": mount_point = "/data" 503 p = info_dict["fstab"][mount_point] 504 fs_type = p.fs_type 505 device = p.device 506 if "/" in device: 507 device = device[device.rfind("/")+1:] 508 limit = info_dict.get(device + "_size", None) 509 if not fs_type or not limit: return 510 511 if fs_type == "yaffs2": 512 # image size should be increased by 1/64th to account for the 513 # spare area (64 bytes per 2k page) 514 limit = limit / 2048 * (2048+64) 515 size = len(data) 516 pct = float(size) * 100.0 / limit 517 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 518 if pct >= 99.0: 519 raise ExternalError(msg) 520 elif pct >= 95.0: 521 print 522 print " WARNING: ", msg 523 print 524 elif OPTIONS.verbose: 525 print " ", msg 526 527 528def ReadApkCerts(tf_zip): 529 """Given a target_files ZipFile, parse the META/apkcerts.txt file 530 and return a {package: cert} dict.""" 531 certmap = {} 532 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 533 line = line.strip() 534 if not line: continue 535 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 536 r'private_key="(.*)"$', line) 537 if m: 538 name, cert, privkey = m.groups() 539 public_key_suffix_len = len(OPTIONS.public_key_suffix) 540 private_key_suffix_len = len(OPTIONS.private_key_suffix) 541 if cert in SPECIAL_CERT_STRINGS and not privkey: 542 certmap[name] = cert 543 elif (cert.endswith(OPTIONS.public_key_suffix) and 544 privkey.endswith(OPTIONS.private_key_suffix) and 545 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 546 certmap[name] = cert[:-public_key_suffix_len] 547 else: 548 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 549 return certmap 550 551 552COMMON_DOCSTRING = """ 553 -p (--path) <dir> 554 Prepend <dir>/bin to the list of places to search for binaries 555 run by this script, and expect to find jars in <dir>/framework. 556 557 -s (--device_specific) <file> 558 Path to the python module containing device-specific 559 releasetools code. 560 561 -x (--extra) <key=value> 562 Add a key/value pair to the 'extras' dict, which device-specific 563 extension code may look at. 564 565 -v (--verbose) 566 Show command lines being executed. 567 568 -h (--help) 569 Display this usage message and exit. 570""" 571 572def Usage(docstring): 573 print docstring.rstrip("\n") 574 print COMMON_DOCSTRING 575 576 577def ParseOptions(argv, 578 docstring, 579 extra_opts="", extra_long_opts=(), 580 extra_option_handler=None): 581 """Parse the options in argv and return any arguments that aren't 582 flags. docstring is the calling module's docstring, to be displayed 583 for errors and -h. extra_opts and extra_long_opts are for flags 584 defined by the caller, which are processed by passing them to 585 extra_option_handler.""" 586 587 try: 588 opts, args = getopt.getopt( 589 argv, "hvp:s:x:" + extra_opts, 590 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 591 "java_path=", "public_key_suffix=", "private_key_suffix=", 592 "device_specific=", "extra="] + 593 list(extra_long_opts)) 594 except getopt.GetoptError, err: 595 Usage(docstring) 596 print "**", str(err), "**" 597 sys.exit(2) 598 599 path_specified = False 600 601 for o, a in opts: 602 if o in ("-h", "--help"): 603 Usage(docstring) 604 sys.exit() 605 elif o in ("-v", "--verbose"): 606 OPTIONS.verbose = True 607 elif o in ("-p", "--path"): 608 OPTIONS.search_path = a 609 elif o in ("--signapk_path",): 610 OPTIONS.signapk_path = a 611 elif o in ("--extra_signapk_args",): 612 OPTIONS.extra_signapk_args = shlex.split(a) 613 elif o in ("--java_path",): 614 OPTIONS.java_path = a 615 elif o in ("--public_key_suffix",): 616 OPTIONS.public_key_suffix = a 617 elif o in ("--private_key_suffix",): 618 OPTIONS.private_key_suffix = a 619 elif o in ("-s", "--device_specific"): 620 OPTIONS.device_specific = a 621 elif o in ("-x", "--extra"): 622 key, value = a.split("=", 1) 623 OPTIONS.extras[key] = value 624 else: 625 if extra_option_handler is None or not extra_option_handler(o, a): 626 assert False, "unknown option \"%s\"" % (o,) 627 628 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 629 os.pathsep + os.environ["PATH"]) 630 631 return args 632 633 634def Cleanup(): 635 for i in OPTIONS.tempfiles: 636 if os.path.isdir(i): 637 shutil.rmtree(i) 638 else: 639 os.remove(i) 640 641 642class PasswordManager(object): 643 def __init__(self): 644 self.editor = os.getenv("EDITOR", None) 645 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 646 647 def GetPasswords(self, items): 648 """Get passwords corresponding to each string in 'items', 649 returning a dict. (The dict may have keys in addition to the 650 values in 'items'.) 651 652 Uses the passwords in $ANDROID_PW_FILE if available, letting the 653 user edit that file to add more needed passwords. If no editor is 654 available, or $ANDROID_PW_FILE isn't define, prompts the user 655 interactively in the ordinary way. 656 """ 657 658 current = self.ReadFile() 659 660 first = True 661 while True: 662 missing = [] 663 for i in items: 664 if i not in current or not current[i]: 665 missing.append(i) 666 # Are all the passwords already in the file? 667 if not missing: return current 668 669 for i in missing: 670 current[i] = "" 671 672 if not first: 673 print "key file %s still missing some passwords." % (self.pwfile,) 674 answer = raw_input("try to edit again? [y]> ").strip() 675 if answer and answer[0] not in 'yY': 676 raise RuntimeError("key passwords unavailable") 677 first = False 678 679 current = self.UpdateAndReadFile(current) 680 681 def PromptResult(self, current): 682 """Prompt the user to enter a value (password) for each key in 683 'current' whose value is fales. Returns a new dict with all the 684 values. 685 """ 686 result = {} 687 for k, v in sorted(current.iteritems()): 688 if v: 689 result[k] = v 690 else: 691 while True: 692 result[k] = getpass.getpass("Enter password for %s key> " 693 % (k,)).strip() 694 if result[k]: break 695 return result 696 697 def UpdateAndReadFile(self, current): 698 if not self.editor or not self.pwfile: 699 return self.PromptResult(current) 700 701 f = open(self.pwfile, "w") 702 os.chmod(self.pwfile, 0600) 703 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 704 f.write("# (Additional spaces are harmless.)\n\n") 705 706 first_line = None 707 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 708 sorted.sort() 709 for i, (_, k, v) in enumerate(sorted): 710 f.write("[[[ %s ]]] %s\n" % (v, k)) 711 if not v and first_line is None: 712 # position cursor on first line with no password. 713 first_line = i + 4 714 f.close() 715 716 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 717 _, _ = p.communicate() 718 719 return self.ReadFile() 720 721 def ReadFile(self): 722 result = {} 723 if self.pwfile is None: return result 724 try: 725 f = open(self.pwfile, "r") 726 for line in f: 727 line = line.strip() 728 if not line or line[0] == '#': continue 729 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 730 if not m: 731 print "failed to parse password file: ", line 732 else: 733 result[m.group(2)] = m.group(1) 734 f.close() 735 except IOError, e: 736 if e.errno != errno.ENOENT: 737 print "error reading password file: ", str(e) 738 return result 739 740 741def ZipWriteStr(zip, filename, data, perms=0644, compression=None): 742 # use a fixed timestamp so the output is repeatable. 743 zinfo = zipfile.ZipInfo(filename=filename, 744 date_time=(2009, 1, 1, 0, 0, 0)) 745 if compression is None: 746 zinfo.compress_type = zip.compression 747 else: 748 zinfo.compress_type = compression 749 zinfo.external_attr = perms << 16 750 zip.writestr(zinfo, data) 751 752 753class DeviceSpecificParams(object): 754 module = None 755 def __init__(self, **kwargs): 756 """Keyword arguments to the constructor become attributes of this 757 object, which is passed to all functions in the device-specific 758 module.""" 759 for k, v in kwargs.iteritems(): 760 setattr(self, k, v) 761 self.extras = OPTIONS.extras 762 763 if self.module is None: 764 path = OPTIONS.device_specific 765 if not path: return 766 try: 767 if os.path.isdir(path): 768 info = imp.find_module("releasetools", [path]) 769 else: 770 d, f = os.path.split(path) 771 b, x = os.path.splitext(f) 772 if x == ".py": 773 f = b 774 info = imp.find_module(f, [d]) 775 print "loaded device-specific extensions from", path 776 self.module = imp.load_module("device_specific", *info) 777 except ImportError: 778 print "unable to load device-specific module; assuming none" 779 780 def _DoCall(self, function_name, *args, **kwargs): 781 """Call the named function in the device-specific module, passing 782 the given args and kwargs. The first argument to the call will be 783 the DeviceSpecific object itself. If there is no module, or the 784 module does not define the function, return the value of the 785 'default' kwarg (which itself defaults to None).""" 786 if self.module is None or not hasattr(self.module, function_name): 787 return kwargs.get("default", None) 788 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 789 790 def FullOTA_Assertions(self): 791 """Called after emitting the block of assertions at the top of a 792 full OTA package. Implementations can add whatever additional 793 assertions they like.""" 794 return self._DoCall("FullOTA_Assertions") 795 796 def FullOTA_InstallBegin(self): 797 """Called at the start of full OTA installation.""" 798 return self._DoCall("FullOTA_InstallBegin") 799 800 def FullOTA_InstallEnd(self): 801 """Called at the end of full OTA installation; typically this is 802 used to install the image for the device's baseband processor.""" 803 return self._DoCall("FullOTA_InstallEnd") 804 805 def IncrementalOTA_Assertions(self): 806 """Called after emitting the block of assertions at the top of an 807 incremental OTA package. Implementations can add whatever 808 additional assertions they like.""" 809 return self._DoCall("IncrementalOTA_Assertions") 810 811 def IncrementalOTA_VerifyBegin(self): 812 """Called at the start of the verification phase of incremental 813 OTA installation; additional checks can be placed here to abort 814 the script before any changes are made.""" 815 return self._DoCall("IncrementalOTA_VerifyBegin") 816 817 def IncrementalOTA_VerifyEnd(self): 818 """Called at the end of the verification phase of incremental OTA 819 installation; additional checks can be placed here to abort the 820 script before any changes are made.""" 821 return self._DoCall("IncrementalOTA_VerifyEnd") 822 823 def IncrementalOTA_InstallBegin(self): 824 """Called at the start of incremental OTA installation (after 825 verification is complete).""" 826 return self._DoCall("IncrementalOTA_InstallBegin") 827 828 def IncrementalOTA_InstallEnd(self): 829 """Called at the end of incremental OTA installation; typically 830 this is used to install the image for the device's baseband 831 processor.""" 832 return self._DoCall("IncrementalOTA_InstallEnd") 833 834class File(object): 835 def __init__(self, name, data): 836 self.name = name 837 self.data = data 838 self.size = len(data) 839 self.sha1 = sha1(data).hexdigest() 840 841 @classmethod 842 def FromLocalFile(cls, name, diskname): 843 f = open(diskname, "rb") 844 data = f.read() 845 f.close() 846 return File(name, data) 847 848 def WriteToTemp(self): 849 t = tempfile.NamedTemporaryFile() 850 t.write(self.data) 851 t.flush() 852 return t 853 854 def AddToZip(self, z, compression=None): 855 ZipWriteStr(z, self.name, self.data, compression=compression) 856 857DIFF_PROGRAM_BY_EXT = { 858 ".gz" : "imgdiff", 859 ".zip" : ["imgdiff", "-z"], 860 ".jar" : ["imgdiff", "-z"], 861 ".apk" : ["imgdiff", "-z"], 862 ".img" : "imgdiff", 863 } 864 865class Difference(object): 866 def __init__(self, tf, sf, diff_program=None): 867 self.tf = tf 868 self.sf = sf 869 self.patch = None 870 self.diff_program = diff_program 871 872 def ComputePatch(self): 873 """Compute the patch (as a string of data) needed to turn sf into 874 tf. Returns the same tuple as GetPatch().""" 875 876 tf = self.tf 877 sf = self.sf 878 879 if self.diff_program: 880 diff_program = self.diff_program 881 else: 882 ext = os.path.splitext(tf.name)[1] 883 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 884 885 ttemp = tf.WriteToTemp() 886 stemp = sf.WriteToTemp() 887 888 ext = os.path.splitext(tf.name)[1] 889 890 try: 891 ptemp = tempfile.NamedTemporaryFile() 892 if isinstance(diff_program, list): 893 cmd = copy.copy(diff_program) 894 else: 895 cmd = [diff_program] 896 cmd.append(stemp.name) 897 cmd.append(ttemp.name) 898 cmd.append(ptemp.name) 899 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 900 _, err = p.communicate() 901 if err or p.returncode != 0: 902 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 903 return None 904 diff = ptemp.read() 905 finally: 906 ptemp.close() 907 stemp.close() 908 ttemp.close() 909 910 self.patch = diff 911 return self.tf, self.sf, self.patch 912 913 914 def GetPatch(self): 915 """Return a tuple (target_file, source_file, patch_data). 916 patch_data may be None if ComputePatch hasn't been called, or if 917 computing the patch failed.""" 918 return self.tf, self.sf, self.patch 919 920 921def ComputeDifferences(diffs): 922 """Call ComputePatch on all the Difference objects in 'diffs'.""" 923 print len(diffs), "diffs to compute" 924 925 # Do the largest files first, to try and reduce the long-pole effect. 926 by_size = [(i.tf.size, i) for i in diffs] 927 by_size.sort(reverse=True) 928 by_size = [i[1] for i in by_size] 929 930 lock = threading.Lock() 931 diff_iter = iter(by_size) # accessed under lock 932 933 def worker(): 934 try: 935 lock.acquire() 936 for d in diff_iter: 937 lock.release() 938 start = time.time() 939 d.ComputePatch() 940 dur = time.time() - start 941 lock.acquire() 942 943 tf, sf, patch = d.GetPatch() 944 if sf.name == tf.name: 945 name = tf.name 946 else: 947 name = "%s (%s)" % (tf.name, sf.name) 948 if patch is None: 949 print "patching failed! %s" % (name,) 950 else: 951 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 952 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 953 lock.release() 954 except Exception, e: 955 print e 956 raise 957 958 # start worker threads; wait for them all to finish. 959 threads = [threading.Thread(target=worker) 960 for i in range(OPTIONS.worker_threads)] 961 for th in threads: 962 th.start() 963 while threads: 964 threads.pop().join() 965 966 967# map recovery.fstab's fs_types to mount/format "partition types" 968PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 969 "ext4": "EMMC", "emmc": "EMMC" } 970 971def GetTypeAndDevice(mount_point, info): 972 fstab = info["fstab"] 973 if fstab: 974 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 975 else: 976 return None 977 978 979def ParseCertificate(data): 980 """Parse a PEM-format certificate.""" 981 cert = [] 982 save = False 983 for line in data.split("\n"): 984 if "--END CERTIFICATE--" in line: 985 break 986 if save: 987 cert.append(line) 988 if "--BEGIN CERTIFICATE--" in line: 989 save = True 990 cert = "".join(cert).decode('base64') 991 return cert 992 993def XDelta3(source_path, target_path, output_path): 994 diff_program = ["xdelta3", "-0", "-B", str(64<<20), "-e", "-f", "-s"] 995 diff_program.append(source_path) 996 diff_program.append(target_path) 997 diff_program.append(output_path) 998 p = Run(diff_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 999 p.communicate() 1000 assert p.returncode == 0, "Couldn't produce patch" 1001 1002def XZ(path): 1003 compress_program = ["xz", "-zk", "-9", "--check=crc32"] 1004 compress_program.append(path) 1005 p = Run(compress_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 1006 p.communicate() 1007 assert p.returncode == 0, "Couldn't compress patch" 1008 1009def MakeSystemPatch(source_file, target_file): 1010 with tempfile.NamedTemporaryFile() as output_file: 1011 XDelta3(source_file.name, target_file.name, output_file.name) 1012 XZ(output_file.name) 1013 with open(output_file.name + ".xz") as patch_file: 1014 patch_data = patch_file.read() 1015 os.unlink(patch_file.name) 1016 return File("system.muimg.p", patch_data) 1017 1018def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img, 1019 info_dict=None): 1020 """Generate a binary patch that creates the recovery image starting 1021 with the boot image. (Most of the space in these images is just the 1022 kernel, which is identical for the two, so the resulting patch 1023 should be efficient.) Add it to the output zip, along with a shell 1024 script that is run from init.rc on first boot to actually do the 1025 patching and install the new recovery image. 1026 1027 recovery_img and boot_img should be File objects for the 1028 corresponding images. info should be the dictionary returned by 1029 common.LoadInfoDict() on the input target_files. 1030 """ 1031 1032 if info_dict is None: 1033 info_dict = OPTIONS.info_dict 1034 1035 diff_program = ["imgdiff"] 1036 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat") 1037 if os.path.exists(path): 1038 diff_program.append("-b") 1039 diff_program.append(path) 1040 bonus_args = "-b /system/etc/recovery-resource.dat" 1041 else: 1042 bonus_args = "" 1043 1044 d = Difference(recovery_img, boot_img, diff_program=diff_program) 1045 _, _, patch = d.ComputePatch() 1046 output_sink("recovery-from-boot.p", patch) 1047 1048 boot_type, boot_device = GetTypeAndDevice("/boot", info_dict) 1049 recovery_type, recovery_device = GetTypeAndDevice("/recovery", info_dict) 1050 1051 sh = """#!/system/bin/sh 1052if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 1053 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1054else 1055 log -t recovery "Recovery image already installed" 1056fi 1057""" % { 'boot_size': boot_img.size, 1058 'boot_sha1': boot_img.sha1, 1059 'recovery_size': recovery_img.size, 1060 'recovery_sha1': recovery_img.sha1, 1061 'boot_type': boot_type, 1062 'boot_device': boot_device, 1063 'recovery_type': recovery_type, 1064 'recovery_device': recovery_device, 1065 'bonus_args': bonus_args, 1066 } 1067 1068 # The install script location moved from /system/etc to /system/bin 1069 # in the L release. Parse the init.rc file to find out where the 1070 # target-files expects it to be, and put it there. 1071 sh_location = "etc/install-recovery.sh" 1072 try: 1073 with open(os.path.join(input_dir, "BOOT", "RAMDISK", "init.rc")) as f: 1074 for line in f: 1075 m = re.match("^service flash_recovery /system/(\S+)\s*$", line) 1076 if m: 1077 sh_location = m.group(1) 1078 print "putting script in", sh_location 1079 break 1080 except (OSError, IOError), e: 1081 print "failed to read init.rc: %s" % (e,) 1082 1083 output_sink(sh_location, sh) 1084