common.py revision 5bfed5a320860de5d44c915c88cf7f72c2cdb574
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32try: 33 from hashlib import sha1 as sha1 34except ImportError: 35 from sha import sha as sha1 36 37# missing in Python 2.4 and before 38if not hasattr(os, "SEEK_SET"): 39 os.SEEK_SET = 0 40 41class Options(object): pass 42OPTIONS = Options() 43OPTIONS.search_path = "out/host/linux-x86" 44OPTIONS.signapk_path = "framework/signapk.jar" # Relative to search_path 45OPTIONS.extra_signapk_args = [] 46OPTIONS.java_path = "java" # Use the one on the path by default. 47OPTIONS.public_key_suffix = ".x509.pem" 48OPTIONS.private_key_suffix = ".pk8" 49OPTIONS.verbose = False 50OPTIONS.tempfiles = [] 51OPTIONS.device_specific = None 52OPTIONS.extras = {} 53OPTIONS.info_dict = None 54 55 56# Values for "certificate" in apkcerts that mean special things. 57SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 58 59 60class ExternalError(RuntimeError): pass 61 62 63def Run(args, **kwargs): 64 """Create and return a subprocess.Popen object, printing the command 65 line on the terminal if -v was specified.""" 66 if OPTIONS.verbose: 67 print " running: ", " ".join(args) 68 return subprocess.Popen(args, **kwargs) 69 70 71def CloseInheritedPipes(): 72 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 73 before doing other work.""" 74 if platform.system() != "Darwin": 75 return 76 for d in range(3, 1025): 77 try: 78 stat = os.fstat(d) 79 if stat is not None: 80 pipebit = stat[0] & 0x1000 81 if pipebit != 0: 82 os.close(d) 83 except OSError: 84 pass 85 86 87def LoadInfoDict(input): 88 """Read and parse the META/misc_info.txt key/value pairs from the 89 input target files and return a dict.""" 90 91 def read_helper(fn): 92 if isinstance(input, zipfile.ZipFile): 93 return input.read(fn) 94 else: 95 path = os.path.join(input, *fn.split("/")) 96 try: 97 with open(path) as f: 98 return f.read() 99 except IOError, e: 100 if e.errno == errno.ENOENT: 101 raise KeyError(fn) 102 d = {} 103 try: 104 d = LoadDictionaryFromLines(read_helper("META/misc_info.txt").split("\n")) 105 except KeyError: 106 # ok if misc_info.txt doesn't exist 107 pass 108 109 # backwards compatibility: These values used to be in their own 110 # files. Look for them, in case we're processing an old 111 # target_files zip. 112 113 if "mkyaffs2_extra_flags" not in d: 114 try: 115 d["mkyaffs2_extra_flags"] = read_helper("META/mkyaffs2-extra-flags.txt").strip() 116 except KeyError: 117 # ok if flags don't exist 118 pass 119 120 if "recovery_api_version" not in d: 121 try: 122 d["recovery_api_version"] = read_helper("META/recovery-api-version.txt").strip() 123 except KeyError: 124 raise ValueError("can't find recovery API version in input target-files") 125 126 if "tool_extensions" not in d: 127 try: 128 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip() 129 except KeyError: 130 # ok if extensions don't exist 131 pass 132 133 if "fstab_version" not in d: 134 d["fstab_version"] = "1" 135 136 try: 137 data = read_helper("META/imagesizes.txt") 138 for line in data.split("\n"): 139 if not line: continue 140 name, value = line.split(" ", 1) 141 if not value: continue 142 if name == "blocksize": 143 d[name] = value 144 else: 145 d[name + "_size"] = value 146 except KeyError: 147 pass 148 149 def makeint(key): 150 if key in d: 151 d[key] = int(d[key], 0) 152 153 makeint("recovery_api_version") 154 makeint("blocksize") 155 makeint("system_size") 156 makeint("userdata_size") 157 makeint("cache_size") 158 makeint("recovery_size") 159 makeint("boot_size") 160 makeint("fstab_version") 161 162 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"]) 163 d["build.prop"] = LoadBuildProp(read_helper) 164 return d 165 166def LoadBuildProp(read_helper): 167 try: 168 data = read_helper("SYSTEM/build.prop") 169 except KeyError: 170 print "Warning: could not find SYSTEM/build.prop in %s" % zip 171 data = "" 172 return LoadDictionaryFromLines(data.split("\n")) 173 174def LoadDictionaryFromLines(lines): 175 d = {} 176 for line in lines: 177 line = line.strip() 178 if not line or line.startswith("#"): continue 179 if "=" in line: 180 name, value = line.split("=", 1) 181 d[name] = value 182 return d 183 184def LoadRecoveryFSTab(read_helper, fstab_version): 185 class Partition(object): 186 pass 187 188 try: 189 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab") 190 except KeyError: 191 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab" 192 data = "" 193 194 if fstab_version == 1: 195 d = {} 196 for line in data.split("\n"): 197 line = line.strip() 198 if not line or line.startswith("#"): continue 199 pieces = line.split() 200 if not (3 <= len(pieces) <= 4): 201 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 202 203 p = Partition() 204 p.mount_point = pieces[0] 205 p.fs_type = pieces[1] 206 p.device = pieces[2] 207 p.length = 0 208 options = None 209 if len(pieces) >= 4: 210 if pieces[3].startswith("/"): 211 p.device2 = pieces[3] 212 if len(pieces) >= 5: 213 options = pieces[4] 214 else: 215 p.device2 = None 216 options = pieces[3] 217 else: 218 p.device2 = None 219 220 if options: 221 options = options.split(",") 222 for i in options: 223 if i.startswith("length="): 224 p.length = int(i[7:]) 225 else: 226 print "%s: unknown option \"%s\"" % (p.mount_point, i) 227 228 d[p.mount_point] = p 229 230 elif fstab_version == 2: 231 d = {} 232 for line in data.split("\n"): 233 line = line.strip() 234 if not line or line.startswith("#"): continue 235 pieces = line.split() 236 if len(pieces) != 5: 237 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 238 239 # Ignore entries that are managed by vold 240 options = pieces[4] 241 if "voldmanaged=" in options: continue 242 243 # It's a good line, parse it 244 p = Partition() 245 p.device = pieces[0] 246 p.mount_point = pieces[1] 247 p.fs_type = pieces[2] 248 p.device2 = None 249 p.length = 0 250 251 options = options.split(",") 252 for i in options: 253 if i.startswith("length="): 254 p.length = int(i[7:]) 255 else: 256 # Ignore all unknown options in the unified fstab 257 continue 258 259 d[p.mount_point] = p 260 261 else: 262 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 263 264 return d 265 266 267def DumpInfoDict(d): 268 for k, v in sorted(d.items()): 269 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 270 271def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 272 """Take a kernel, cmdline, and ramdisk directory from the input (in 273 'sourcedir'), and turn them into a boot image. Return the image 274 data, or None if sourcedir does not appear to contains files for 275 building the requested image.""" 276 277 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 278 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 279 return None 280 281 if info_dict is None: 282 info_dict = OPTIONS.info_dict 283 284 ramdisk_img = tempfile.NamedTemporaryFile() 285 img = tempfile.NamedTemporaryFile() 286 287 if os.access(fs_config_file, os.F_OK): 288 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 289 else: 290 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 291 p1 = Run(cmd, stdout=subprocess.PIPE) 292 p2 = Run(["minigzip"], 293 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 294 295 p2.wait() 296 p1.wait() 297 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 298 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 299 300 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 301 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 302 303 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 304 305 fn = os.path.join(sourcedir, "cmdline") 306 if os.access(fn, os.F_OK): 307 cmd.append("--cmdline") 308 cmd.append(open(fn).read().rstrip("\n")) 309 310 fn = os.path.join(sourcedir, "base") 311 if os.access(fn, os.F_OK): 312 cmd.append("--base") 313 cmd.append(open(fn).read().rstrip("\n")) 314 315 fn = os.path.join(sourcedir, "pagesize") 316 if os.access(fn, os.F_OK): 317 cmd.append("--pagesize") 318 cmd.append(open(fn).read().rstrip("\n")) 319 320 args = info_dict.get("mkbootimg_args", None) 321 if args and args.strip(): 322 cmd.extend(shlex.split(args)) 323 324 cmd.extend(["--ramdisk", ramdisk_img.name, 325 "--output", img.name]) 326 327 p = Run(cmd, stdout=subprocess.PIPE) 328 p.communicate() 329 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 330 os.path.basename(sourcedir),) 331 332 img.seek(os.SEEK_SET, 0) 333 data = img.read() 334 335 ramdisk_img.close() 336 img.close() 337 338 return data 339 340 341def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 342 info_dict=None): 343 """Return a File object (with name 'name') with the desired bootable 344 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 345 'prebuilt_name', otherwise construct it from the source files in 346 'unpack_dir'/'tree_subdir'.""" 347 348 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 349 if os.path.exists(prebuilt_path): 350 print "using prebuilt %s..." % (prebuilt_name,) 351 return File.FromLocalFile(name, prebuilt_path) 352 else: 353 print "building image from target_files %s..." % (tree_subdir,) 354 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 355 data = BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 356 os.path.join(unpack_dir, fs_config), 357 info_dict) 358 if data: 359 return File(name, data) 360 return None 361 362 363def UnzipTemp(filename, pattern=None): 364 """Unzip the given archive into a temporary directory and return the name. 365 366 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 367 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 368 369 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 370 main file), open for reading. 371 """ 372 373 tmp = tempfile.mkdtemp(prefix="targetfiles-") 374 OPTIONS.tempfiles.append(tmp) 375 376 def unzip_to_dir(filename, dirname): 377 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 378 if pattern is not None: 379 cmd.append(pattern) 380 p = Run(cmd, stdout=subprocess.PIPE) 381 p.communicate() 382 if p.returncode != 0: 383 raise ExternalError("failed to unzip input target-files \"%s\"" % 384 (filename,)) 385 386 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 387 if m: 388 unzip_to_dir(m.group(1), tmp) 389 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 390 filename = m.group(1) 391 else: 392 unzip_to_dir(filename, tmp) 393 394 return tmp, zipfile.ZipFile(filename, "r") 395 396 397def GetKeyPasswords(keylist): 398 """Given a list of keys, prompt the user to enter passwords for 399 those which require them. Return a {key: password} dict. password 400 will be None if the key has no password.""" 401 402 no_passwords = [] 403 need_passwords = [] 404 key_passwords = {} 405 devnull = open("/dev/null", "w+b") 406 for k in sorted(keylist): 407 # We don't need a password for things that aren't really keys. 408 if k in SPECIAL_CERT_STRINGS: 409 no_passwords.append(k) 410 continue 411 412 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 413 "-inform", "DER", "-nocrypt"], 414 stdin=devnull.fileno(), 415 stdout=devnull.fileno(), 416 stderr=subprocess.STDOUT) 417 p.communicate() 418 if p.returncode == 0: 419 # Definitely an unencrypted key. 420 no_passwords.append(k) 421 else: 422 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 423 "-inform", "DER", "-passin", "pass:"], 424 stdin=devnull.fileno(), 425 stdout=devnull.fileno(), 426 stderr=subprocess.PIPE) 427 stdout, stderr = p.communicate() 428 if p.returncode == 0: 429 # Encrypted key with empty string as password. 430 key_passwords[k] = '' 431 elif stderr.startswith('Error decrypting key'): 432 # Definitely encrypted key. 433 # It would have said "Error reading key" if it didn't parse correctly. 434 need_passwords.append(k) 435 else: 436 # Potentially, a type of key that openssl doesn't understand. 437 # We'll let the routines in signapk.jar handle it. 438 no_passwords.append(k) 439 devnull.close() 440 441 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 442 key_passwords.update(dict.fromkeys(no_passwords, None)) 443 return key_passwords 444 445 446def SignFile(input_name, output_name, key, password, align=None, 447 whole_file=False): 448 """Sign the input_name zip/jar/apk, producing output_name. Use the 449 given key and password (the latter may be None if the key does not 450 have a password. 451 452 If align is an integer > 1, zipalign is run to align stored files in 453 the output zip on 'align'-byte boundaries. 454 455 If whole_file is true, use the "-w" option to SignApk to embed a 456 signature that covers the whole file in the archive comment of the 457 zip file. 458 """ 459 460 if align == 0 or align == 1: 461 align = None 462 463 if align: 464 temp = tempfile.NamedTemporaryFile() 465 sign_name = temp.name 466 else: 467 sign_name = output_name 468 469 cmd = [OPTIONS.java_path, "-Xmx2048m", "-jar", 470 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 471 cmd.extend(OPTIONS.extra_signapk_args) 472 if whole_file: 473 cmd.append("-w") 474 cmd.extend([key + OPTIONS.public_key_suffix, 475 key + OPTIONS.private_key_suffix, 476 input_name, sign_name]) 477 478 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 479 if password is not None: 480 password += "\n" 481 p.communicate(password) 482 if p.returncode != 0: 483 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 484 485 if align: 486 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 487 p.communicate() 488 if p.returncode != 0: 489 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 490 temp.close() 491 492 493def CheckSize(data, target, info_dict): 494 """Check the data string passed against the max size limit, if 495 any, for the given target. Raise exception if the data is too big. 496 Print a warning if the data is nearing the maximum size.""" 497 498 if target.endswith(".img"): target = target[:-4] 499 mount_point = "/" + target 500 501 fs_type = None 502 limit = None 503 if info_dict["fstab"]: 504 if mount_point == "/userdata": mount_point = "/data" 505 p = info_dict["fstab"][mount_point] 506 fs_type = p.fs_type 507 device = p.device 508 if "/" in device: 509 device = device[device.rfind("/")+1:] 510 limit = info_dict.get(device + "_size", None) 511 if not fs_type or not limit: return 512 513 if fs_type == "yaffs2": 514 # image size should be increased by 1/64th to account for the 515 # spare area (64 bytes per 2k page) 516 limit = limit / 2048 * (2048+64) 517 size = len(data) 518 pct = float(size) * 100.0 / limit 519 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 520 if pct >= 99.0: 521 raise ExternalError(msg) 522 elif pct >= 95.0: 523 print 524 print " WARNING: ", msg 525 print 526 elif OPTIONS.verbose: 527 print " ", msg 528 529 530def ReadApkCerts(tf_zip): 531 """Given a target_files ZipFile, parse the META/apkcerts.txt file 532 and return a {package: cert} dict.""" 533 certmap = {} 534 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 535 line = line.strip() 536 if not line: continue 537 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 538 r'private_key="(.*)"$', line) 539 if m: 540 name, cert, privkey = m.groups() 541 public_key_suffix_len = len(OPTIONS.public_key_suffix) 542 private_key_suffix_len = len(OPTIONS.private_key_suffix) 543 if cert in SPECIAL_CERT_STRINGS and not privkey: 544 certmap[name] = cert 545 elif (cert.endswith(OPTIONS.public_key_suffix) and 546 privkey.endswith(OPTIONS.private_key_suffix) and 547 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 548 certmap[name] = cert[:-public_key_suffix_len] 549 else: 550 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 551 return certmap 552 553 554COMMON_DOCSTRING = """ 555 -p (--path) <dir> 556 Prepend <dir>/bin to the list of places to search for binaries 557 run by this script, and expect to find jars in <dir>/framework. 558 559 -s (--device_specific) <file> 560 Path to the python module containing device-specific 561 releasetools code. 562 563 -x (--extra) <key=value> 564 Add a key/value pair to the 'extras' dict, which device-specific 565 extension code may look at. 566 567 -v (--verbose) 568 Show command lines being executed. 569 570 -h (--help) 571 Display this usage message and exit. 572""" 573 574def Usage(docstring): 575 print docstring.rstrip("\n") 576 print COMMON_DOCSTRING 577 578 579def ParseOptions(argv, 580 docstring, 581 extra_opts="", extra_long_opts=(), 582 extra_option_handler=None): 583 """Parse the options in argv and return any arguments that aren't 584 flags. docstring is the calling module's docstring, to be displayed 585 for errors and -h. extra_opts and extra_long_opts are for flags 586 defined by the caller, which are processed by passing them to 587 extra_option_handler.""" 588 589 try: 590 opts, args = getopt.getopt( 591 argv, "hvp:s:x:" + extra_opts, 592 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 593 "java_path=", "public_key_suffix=", "private_key_suffix=", 594 "device_specific=", "extra="] + 595 list(extra_long_opts)) 596 except getopt.GetoptError, err: 597 Usage(docstring) 598 print "**", str(err), "**" 599 sys.exit(2) 600 601 path_specified = False 602 603 for o, a in opts: 604 if o in ("-h", "--help"): 605 Usage(docstring) 606 sys.exit() 607 elif o in ("-v", "--verbose"): 608 OPTIONS.verbose = True 609 elif o in ("-p", "--path"): 610 OPTIONS.search_path = a 611 elif o in ("--signapk_path",): 612 OPTIONS.signapk_path = a 613 elif o in ("--extra_signapk_args",): 614 OPTIONS.extra_signapk_args = shlex.split(a) 615 elif o in ("--java_path",): 616 OPTIONS.java_path = a 617 elif o in ("--public_key_suffix",): 618 OPTIONS.public_key_suffix = a 619 elif o in ("--private_key_suffix",): 620 OPTIONS.private_key_suffix = a 621 elif o in ("-s", "--device_specific"): 622 OPTIONS.device_specific = a 623 elif o in ("-x", "--extra"): 624 key, value = a.split("=", 1) 625 OPTIONS.extras[key] = value 626 else: 627 if extra_option_handler is None or not extra_option_handler(o, a): 628 assert False, "unknown option \"%s\"" % (o,) 629 630 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 631 os.pathsep + os.environ["PATH"]) 632 633 return args 634 635 636def Cleanup(): 637 for i in OPTIONS.tempfiles: 638 if os.path.isdir(i): 639 shutil.rmtree(i) 640 else: 641 os.remove(i) 642 643 644class PasswordManager(object): 645 def __init__(self): 646 self.editor = os.getenv("EDITOR", None) 647 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 648 649 def GetPasswords(self, items): 650 """Get passwords corresponding to each string in 'items', 651 returning a dict. (The dict may have keys in addition to the 652 values in 'items'.) 653 654 Uses the passwords in $ANDROID_PW_FILE if available, letting the 655 user edit that file to add more needed passwords. If no editor is 656 available, or $ANDROID_PW_FILE isn't define, prompts the user 657 interactively in the ordinary way. 658 """ 659 660 current = self.ReadFile() 661 662 first = True 663 while True: 664 missing = [] 665 for i in items: 666 if i not in current or not current[i]: 667 missing.append(i) 668 # Are all the passwords already in the file? 669 if not missing: return current 670 671 for i in missing: 672 current[i] = "" 673 674 if not first: 675 print "key file %s still missing some passwords." % (self.pwfile,) 676 answer = raw_input("try to edit again? [y]> ").strip() 677 if answer and answer[0] not in 'yY': 678 raise RuntimeError("key passwords unavailable") 679 first = False 680 681 current = self.UpdateAndReadFile(current) 682 683 def PromptResult(self, current): 684 """Prompt the user to enter a value (password) for each key in 685 'current' whose value is fales. Returns a new dict with all the 686 values. 687 """ 688 result = {} 689 for k, v in sorted(current.iteritems()): 690 if v: 691 result[k] = v 692 else: 693 while True: 694 result[k] = getpass.getpass("Enter password for %s key> " 695 % (k,)).strip() 696 if result[k]: break 697 return result 698 699 def UpdateAndReadFile(self, current): 700 if not self.editor or not self.pwfile: 701 return self.PromptResult(current) 702 703 f = open(self.pwfile, "w") 704 os.chmod(self.pwfile, 0600) 705 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 706 f.write("# (Additional spaces are harmless.)\n\n") 707 708 first_line = None 709 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 710 sorted.sort() 711 for i, (_, k, v) in enumerate(sorted): 712 f.write("[[[ %s ]]] %s\n" % (v, k)) 713 if not v and first_line is None: 714 # position cursor on first line with no password. 715 first_line = i + 4 716 f.close() 717 718 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 719 _, _ = p.communicate() 720 721 return self.ReadFile() 722 723 def ReadFile(self): 724 result = {} 725 if self.pwfile is None: return result 726 try: 727 f = open(self.pwfile, "r") 728 for line in f: 729 line = line.strip() 730 if not line or line[0] == '#': continue 731 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 732 if not m: 733 print "failed to parse password file: ", line 734 else: 735 result[m.group(2)] = m.group(1) 736 f.close() 737 except IOError, e: 738 if e.errno != errno.ENOENT: 739 print "error reading password file: ", str(e) 740 return result 741 742 743def ZipWriteStr(zip, filename, data, perms=0644, compression=None): 744 # use a fixed timestamp so the output is repeatable. 745 zinfo = zipfile.ZipInfo(filename=filename, 746 date_time=(2009, 1, 1, 0, 0, 0)) 747 if compression is None: 748 zinfo.compress_type = zip.compression 749 else: 750 zinfo.compress_type = compression 751 zinfo.external_attr = perms << 16 752 zip.writestr(zinfo, data) 753 754 755class DeviceSpecificParams(object): 756 module = None 757 def __init__(self, **kwargs): 758 """Keyword arguments to the constructor become attributes of this 759 object, which is passed to all functions in the device-specific 760 module.""" 761 for k, v in kwargs.iteritems(): 762 setattr(self, k, v) 763 self.extras = OPTIONS.extras 764 765 if self.module is None: 766 path = OPTIONS.device_specific 767 if not path: return 768 try: 769 if os.path.isdir(path): 770 info = imp.find_module("releasetools", [path]) 771 else: 772 d, f = os.path.split(path) 773 b, x = os.path.splitext(f) 774 if x == ".py": 775 f = b 776 info = imp.find_module(f, [d]) 777 print "loaded device-specific extensions from", path 778 self.module = imp.load_module("device_specific", *info) 779 except ImportError: 780 print "unable to load device-specific module; assuming none" 781 782 def _DoCall(self, function_name, *args, **kwargs): 783 """Call the named function in the device-specific module, passing 784 the given args and kwargs. The first argument to the call will be 785 the DeviceSpecific object itself. If there is no module, or the 786 module does not define the function, return the value of the 787 'default' kwarg (which itself defaults to None).""" 788 if self.module is None or not hasattr(self.module, function_name): 789 return kwargs.get("default", None) 790 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 791 792 def FullOTA_Assertions(self): 793 """Called after emitting the block of assertions at the top of a 794 full OTA package. Implementations can add whatever additional 795 assertions they like.""" 796 return self._DoCall("FullOTA_Assertions") 797 798 def FullOTA_InstallBegin(self): 799 """Called at the start of full OTA installation.""" 800 return self._DoCall("FullOTA_InstallBegin") 801 802 def FullOTA_InstallEnd(self): 803 """Called at the end of full OTA installation; typically this is 804 used to install the image for the device's baseband processor.""" 805 return self._DoCall("FullOTA_InstallEnd") 806 807 def IncrementalOTA_Assertions(self): 808 """Called after emitting the block of assertions at the top of an 809 incremental OTA package. Implementations can add whatever 810 additional assertions they like.""" 811 return self._DoCall("IncrementalOTA_Assertions") 812 813 def IncrementalOTA_VerifyBegin(self): 814 """Called at the start of the verification phase of incremental 815 OTA installation; additional checks can be placed here to abort 816 the script before any changes are made.""" 817 return self._DoCall("IncrementalOTA_VerifyBegin") 818 819 def IncrementalOTA_VerifyEnd(self): 820 """Called at the end of the verification phase of incremental OTA 821 installation; additional checks can be placed here to abort the 822 script before any changes are made.""" 823 return self._DoCall("IncrementalOTA_VerifyEnd") 824 825 def IncrementalOTA_InstallBegin(self): 826 """Called at the start of incremental OTA installation (after 827 verification is complete).""" 828 return self._DoCall("IncrementalOTA_InstallBegin") 829 830 def IncrementalOTA_InstallEnd(self): 831 """Called at the end of incremental OTA installation; typically 832 this is used to install the image for the device's baseband 833 processor.""" 834 return self._DoCall("IncrementalOTA_InstallEnd") 835 836class File(object): 837 def __init__(self, name, data): 838 self.name = name 839 self.data = data 840 self.size = len(data) 841 self.sha1 = sha1(data).hexdigest() 842 843 @classmethod 844 def FromLocalFile(cls, name, diskname): 845 f = open(diskname, "rb") 846 data = f.read() 847 f.close() 848 return File(name, data) 849 850 def WriteToTemp(self): 851 t = tempfile.NamedTemporaryFile() 852 t.write(self.data) 853 t.flush() 854 return t 855 856 def AddToZip(self, z, compression=None): 857 ZipWriteStr(z, self.name, self.data, compression=compression) 858 859DIFF_PROGRAM_BY_EXT = { 860 ".gz" : "imgdiff", 861 ".zip" : ["imgdiff", "-z"], 862 ".jar" : ["imgdiff", "-z"], 863 ".apk" : ["imgdiff", "-z"], 864 ".img" : "imgdiff", 865 } 866 867class Difference(object): 868 def __init__(self, tf, sf, diff_program=None): 869 self.tf = tf 870 self.sf = sf 871 self.patch = None 872 self.diff_program = diff_program 873 874 def ComputePatch(self): 875 """Compute the patch (as a string of data) needed to turn sf into 876 tf. Returns the same tuple as GetPatch().""" 877 878 tf = self.tf 879 sf = self.sf 880 881 if self.diff_program: 882 diff_program = self.diff_program 883 else: 884 ext = os.path.splitext(tf.name)[1] 885 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 886 887 ttemp = tf.WriteToTemp() 888 stemp = sf.WriteToTemp() 889 890 ext = os.path.splitext(tf.name)[1] 891 892 try: 893 ptemp = tempfile.NamedTemporaryFile() 894 if isinstance(diff_program, list): 895 cmd = copy.copy(diff_program) 896 else: 897 cmd = [diff_program] 898 cmd.append(stemp.name) 899 cmd.append(ttemp.name) 900 cmd.append(ptemp.name) 901 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 902 _, err = p.communicate() 903 if err or p.returncode != 0: 904 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 905 return None 906 diff = ptemp.read() 907 finally: 908 ptemp.close() 909 stemp.close() 910 ttemp.close() 911 912 self.patch = diff 913 return self.tf, self.sf, self.patch 914 915 916 def GetPatch(self): 917 """Return a tuple (target_file, source_file, patch_data). 918 patch_data may be None if ComputePatch hasn't been called, or if 919 computing the patch failed.""" 920 return self.tf, self.sf, self.patch 921 922 923def ComputeDifferences(diffs): 924 """Call ComputePatch on all the Difference objects in 'diffs'.""" 925 print len(diffs), "diffs to compute" 926 927 # Do the largest files first, to try and reduce the long-pole effect. 928 by_size = [(i.tf.size, i) for i in diffs] 929 by_size.sort(reverse=True) 930 by_size = [i[1] for i in by_size] 931 932 lock = threading.Lock() 933 diff_iter = iter(by_size) # accessed under lock 934 935 def worker(): 936 try: 937 lock.acquire() 938 for d in diff_iter: 939 lock.release() 940 start = time.time() 941 d.ComputePatch() 942 dur = time.time() - start 943 lock.acquire() 944 945 tf, sf, patch = d.GetPatch() 946 if sf.name == tf.name: 947 name = tf.name 948 else: 949 name = "%s (%s)" % (tf.name, sf.name) 950 if patch is None: 951 print "patching failed! %s" % (name,) 952 else: 953 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 954 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 955 lock.release() 956 except Exception, e: 957 print e 958 raise 959 960 # start worker threads; wait for them all to finish. 961 threads = [threading.Thread(target=worker) 962 for i in range(OPTIONS.worker_threads)] 963 for th in threads: 964 th.start() 965 while threads: 966 threads.pop().join() 967 968 969# map recovery.fstab's fs_types to mount/format "partition types" 970PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 971 "ext4": "EMMC", "emmc": "EMMC", 972 "f2fs": "EMMC" } 973 974def GetTypeAndDevice(mount_point, info): 975 fstab = info["fstab"] 976 if fstab: 977 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 978 else: 979 return None 980 981 982def ParseCertificate(data): 983 """Parse a PEM-format certificate.""" 984 cert = [] 985 save = False 986 for line in data.split("\n"): 987 if "--END CERTIFICATE--" in line: 988 break 989 if save: 990 cert.append(line) 991 if "--BEGIN CERTIFICATE--" in line: 992 save = True 993 cert = "".join(cert).decode('base64') 994 return cert 995 996def XDelta3(source_path, target_path, output_path): 997 diff_program = ["xdelta3", "-0", "-B", str(64<<20), "-e", "-f", "-s"] 998 diff_program.append(source_path) 999 diff_program.append(target_path) 1000 diff_program.append(output_path) 1001 p = Run(diff_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 1002 p.communicate() 1003 assert p.returncode == 0, "Couldn't produce patch" 1004 1005def XZ(path): 1006 compress_program = ["xz", "-zk", "-9", "--check=crc32"] 1007 compress_program.append(path) 1008 p = Run(compress_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 1009 p.communicate() 1010 assert p.returncode == 0, "Couldn't compress patch" 1011 1012def MakePartitionPatch(source_file, target_file, partition): 1013 with tempfile.NamedTemporaryFile() as output_file: 1014 XDelta3(source_file.name, target_file.name, output_file.name) 1015 XZ(output_file.name) 1016 with open(output_file.name + ".xz") as patch_file: 1017 patch_data = patch_file.read() 1018 os.unlink(patch_file.name) 1019 return File(partition + ".muimg.p", patch_data) 1020 1021def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img, 1022 info_dict=None): 1023 """Generate a binary patch that creates the recovery image starting 1024 with the boot image. (Most of the space in these images is just the 1025 kernel, which is identical for the two, so the resulting patch 1026 should be efficient.) Add it to the output zip, along with a shell 1027 script that is run from init.rc on first boot to actually do the 1028 patching and install the new recovery image. 1029 1030 recovery_img and boot_img should be File objects for the 1031 corresponding images. info should be the dictionary returned by 1032 common.LoadInfoDict() on the input target_files. 1033 """ 1034 1035 if info_dict is None: 1036 info_dict = OPTIONS.info_dict 1037 1038 diff_program = ["imgdiff"] 1039 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat") 1040 if os.path.exists(path): 1041 diff_program.append("-b") 1042 diff_program.append(path) 1043 bonus_args = "-b /system/etc/recovery-resource.dat" 1044 else: 1045 bonus_args = "" 1046 1047 d = Difference(recovery_img, boot_img, diff_program=diff_program) 1048 _, _, patch = d.ComputePatch() 1049 output_sink("recovery-from-boot.p", patch) 1050 1051 boot_type, boot_device = GetTypeAndDevice("/boot", info_dict) 1052 recovery_type, recovery_device = GetTypeAndDevice("/recovery", info_dict) 1053 1054 sh = """#!/system/bin/sh 1055if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 1056 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1057else 1058 log -t recovery "Recovery image already installed" 1059fi 1060""" % { 'boot_size': boot_img.size, 1061 'boot_sha1': boot_img.sha1, 1062 'recovery_size': recovery_img.size, 1063 'recovery_sha1': recovery_img.sha1, 1064 'boot_type': boot_type, 1065 'boot_device': boot_device, 1066 'recovery_type': recovery_type, 1067 'recovery_device': recovery_device, 1068 'bonus_args': bonus_args, 1069 } 1070 1071 # The install script location moved from /system/etc to /system/bin 1072 # in the L release. Parse the init.rc file to find out where the 1073 # target-files expects it to be, and put it there. 1074 sh_location = "etc/install-recovery.sh" 1075 try: 1076 with open(os.path.join(input_dir, "BOOT", "RAMDISK", "init.rc")) as f: 1077 for line in f: 1078 m = re.match("^service flash_recovery /system/(\S+)\s*$", line) 1079 if m: 1080 sh_location = m.group(1) 1081 print "putting script in", sh_location 1082 break 1083 except (OSError, IOError), e: 1084 print "failed to read init.rc: %s" % (e,) 1085 1086 output_sink(sh_location, sh) 1087