common.py revision a961a09e59dcf130435cf5ff052c4f2d7e3986ea
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32try: 33 from hashlib import sha1 as sha1 34except ImportError: 35 from sha import sha as sha1 36 37# missing in Python 2.4 and before 38if not hasattr(os, "SEEK_SET"): 39 os.SEEK_SET = 0 40 41class Options(object): pass 42OPTIONS = Options() 43OPTIONS.search_path = "out/host/linux-x86" 44OPTIONS.signapk_path = "framework/signapk.jar" # Relative to search_path 45OPTIONS.extra_signapk_args = [] 46OPTIONS.java_path = "java" # Use the one on the path by default. 47OPTIONS.public_key_suffix = ".x509.pem" 48OPTIONS.private_key_suffix = ".pk8" 49OPTIONS.verbose = False 50OPTIONS.tempfiles = [] 51OPTIONS.device_specific = None 52OPTIONS.extras = {} 53OPTIONS.info_dict = None 54 55 56# Values for "certificate" in apkcerts that mean special things. 57SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 58 59 60class ExternalError(RuntimeError): pass 61 62 63def Run(args, **kwargs): 64 """Create and return a subprocess.Popen object, printing the command 65 line on the terminal if -v was specified.""" 66 if OPTIONS.verbose: 67 print " running: ", " ".join(args) 68 return subprocess.Popen(args, **kwargs) 69 70 71def CloseInheritedPipes(): 72 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 73 before doing other work.""" 74 if platform.system() != "Darwin": 75 return 76 for d in range(3, 1025): 77 try: 78 stat = os.fstat(d) 79 if stat is not None: 80 pipebit = stat[0] & 0x1000 81 if pipebit != 0: 82 os.close(d) 83 except OSError: 84 pass 85 86 87def LoadInfoDict(input): 88 """Read and parse the META/misc_info.txt key/value pairs from the 89 input target files and return a dict.""" 90 91 def read_helper(fn): 92 if isinstance(input, zipfile.ZipFile): 93 return input.read(fn) 94 else: 95 path = os.path.join(input, *fn.split("/")) 96 try: 97 with open(path) as f: 98 return f.read() 99 except IOError, e: 100 if e.errno == errno.ENOENT: 101 raise KeyError(fn) 102 d = {} 103 try: 104 d = LoadDictionaryFromLines(read_helper("META/misc_info.txt").split("\n")) 105 except KeyError: 106 # ok if misc_info.txt doesn't exist 107 pass 108 109 # backwards compatibility: These values used to be in their own 110 # files. Look for them, in case we're processing an old 111 # target_files zip. 112 113 if "mkyaffs2_extra_flags" not in d: 114 try: 115 d["mkyaffs2_extra_flags"] = read_helper("META/mkyaffs2-extra-flags.txt").strip() 116 except KeyError: 117 # ok if flags don't exist 118 pass 119 120 if "recovery_api_version" not in d: 121 try: 122 d["recovery_api_version"] = read_helper("META/recovery-api-version.txt").strip() 123 except KeyError: 124 raise ValueError("can't find recovery API version in input target-files") 125 126 if "tool_extensions" not in d: 127 try: 128 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip() 129 except KeyError: 130 # ok if extensions don't exist 131 pass 132 133 if "fstab_version" not in d: 134 d["fstab_version"] = "1" 135 136 try: 137 data = read_helper("META/imagesizes.txt") 138 for line in data.split("\n"): 139 if not line: continue 140 name, value = line.split(" ", 1) 141 if not value: continue 142 if name == "blocksize": 143 d[name] = value 144 else: 145 d[name + "_size"] = value 146 except KeyError: 147 pass 148 149 def makeint(key): 150 if key in d: 151 d[key] = int(d[key], 0) 152 153 makeint("recovery_api_version") 154 makeint("blocksize") 155 makeint("system_size") 156 makeint("vendor_size") 157 makeint("userdata_size") 158 makeint("cache_size") 159 makeint("recovery_size") 160 makeint("boot_size") 161 makeint("fstab_version") 162 163 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"]) 164 d["build.prop"] = LoadBuildProp(read_helper) 165 return d 166 167def LoadBuildProp(read_helper): 168 try: 169 data = read_helper("SYSTEM/build.prop") 170 except KeyError: 171 print "Warning: could not find SYSTEM/build.prop in %s" % zip 172 data = "" 173 return LoadDictionaryFromLines(data.split("\n")) 174 175def LoadDictionaryFromLines(lines): 176 d = {} 177 for line in lines: 178 line = line.strip() 179 if not line or line.startswith("#"): continue 180 if "=" in line: 181 name, value = line.split("=", 1) 182 d[name] = value 183 return d 184 185def LoadRecoveryFSTab(read_helper, fstab_version): 186 class Partition(object): 187 pass 188 189 try: 190 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab") 191 except KeyError: 192 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab" 193 data = "" 194 195 if fstab_version == 1: 196 d = {} 197 for line in data.split("\n"): 198 line = line.strip() 199 if not line or line.startswith("#"): continue 200 pieces = line.split() 201 if not (3 <= len(pieces) <= 4): 202 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 203 204 p = Partition() 205 p.mount_point = pieces[0] 206 p.fs_type = pieces[1] 207 p.device = pieces[2] 208 p.length = 0 209 options = None 210 if len(pieces) >= 4: 211 if pieces[3].startswith("/"): 212 p.device2 = pieces[3] 213 if len(pieces) >= 5: 214 options = pieces[4] 215 else: 216 p.device2 = None 217 options = pieces[3] 218 else: 219 p.device2 = None 220 221 if options: 222 options = options.split(",") 223 for i in options: 224 if i.startswith("length="): 225 p.length = int(i[7:]) 226 else: 227 print "%s: unknown option \"%s\"" % (p.mount_point, i) 228 229 d[p.mount_point] = p 230 231 elif fstab_version == 2: 232 d = {} 233 for line in data.split("\n"): 234 line = line.strip() 235 if not line or line.startswith("#"): continue 236 pieces = line.split() 237 if len(pieces) != 5: 238 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 239 240 # Ignore entries that are managed by vold 241 options = pieces[4] 242 if "voldmanaged=" in options: continue 243 244 # It's a good line, parse it 245 p = Partition() 246 p.device = pieces[0] 247 p.mount_point = pieces[1] 248 p.fs_type = pieces[2] 249 p.device2 = None 250 p.length = 0 251 252 options = options.split(",") 253 for i in options: 254 if i.startswith("length="): 255 p.length = int(i[7:]) 256 else: 257 # Ignore all unknown options in the unified fstab 258 continue 259 260 d[p.mount_point] = p 261 262 else: 263 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 264 265 return d 266 267 268def DumpInfoDict(d): 269 for k, v in sorted(d.items()): 270 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 271 272def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 273 """Take a kernel, cmdline, and ramdisk directory from the input (in 274 'sourcedir'), and turn them into a boot image. Return the image 275 data, or None if sourcedir does not appear to contains files for 276 building the requested image.""" 277 278 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 279 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 280 return None 281 282 if info_dict is None: 283 info_dict = OPTIONS.info_dict 284 285 ramdisk_img = tempfile.NamedTemporaryFile() 286 img = tempfile.NamedTemporaryFile() 287 288 if os.access(fs_config_file, os.F_OK): 289 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 290 else: 291 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 292 p1 = Run(cmd, stdout=subprocess.PIPE) 293 p2 = Run(["minigzip"], 294 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 295 296 p2.wait() 297 p1.wait() 298 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 299 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 300 301 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 302 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 303 304 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 305 306 fn = os.path.join(sourcedir, "cmdline") 307 if os.access(fn, os.F_OK): 308 cmd.append("--cmdline") 309 cmd.append(open(fn).read().rstrip("\n")) 310 311 fn = os.path.join(sourcedir, "base") 312 if os.access(fn, os.F_OK): 313 cmd.append("--base") 314 cmd.append(open(fn).read().rstrip("\n")) 315 316 fn = os.path.join(sourcedir, "pagesize") 317 if os.access(fn, os.F_OK): 318 cmd.append("--pagesize") 319 cmd.append(open(fn).read().rstrip("\n")) 320 321 args = info_dict.get("mkbootimg_args", None) 322 if args and args.strip(): 323 cmd.extend(shlex.split(args)) 324 325 cmd.extend(["--ramdisk", ramdisk_img.name, 326 "--output", img.name]) 327 328 p = Run(cmd, stdout=subprocess.PIPE) 329 p.communicate() 330 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 331 os.path.basename(sourcedir),) 332 333 img.seek(os.SEEK_SET, 0) 334 data = img.read() 335 336 ramdisk_img.close() 337 img.close() 338 339 return data 340 341 342def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 343 info_dict=None): 344 """Return a File object (with name 'name') with the desired bootable 345 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 346 'prebuilt_name', otherwise construct it from the source files in 347 'unpack_dir'/'tree_subdir'.""" 348 349 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 350 if os.path.exists(prebuilt_path): 351 print "using prebuilt %s..." % (prebuilt_name,) 352 return File.FromLocalFile(name, prebuilt_path) 353 else: 354 print "building image from target_files %s..." % (tree_subdir,) 355 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 356 data = BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 357 os.path.join(unpack_dir, fs_config), 358 info_dict) 359 if data: 360 return File(name, data) 361 return None 362 363 364def UnzipTemp(filename, pattern=None): 365 """Unzip the given archive into a temporary directory and return the name. 366 367 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 368 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 369 370 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 371 main file), open for reading. 372 """ 373 374 tmp = tempfile.mkdtemp(prefix="targetfiles-") 375 OPTIONS.tempfiles.append(tmp) 376 377 def unzip_to_dir(filename, dirname): 378 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 379 if pattern is not None: 380 cmd.append(pattern) 381 p = Run(cmd, stdout=subprocess.PIPE) 382 p.communicate() 383 if p.returncode != 0: 384 raise ExternalError("failed to unzip input target-files \"%s\"" % 385 (filename,)) 386 387 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 388 if m: 389 unzip_to_dir(m.group(1), tmp) 390 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 391 filename = m.group(1) 392 else: 393 unzip_to_dir(filename, tmp) 394 395 return tmp, zipfile.ZipFile(filename, "r") 396 397 398def GetKeyPasswords(keylist): 399 """Given a list of keys, prompt the user to enter passwords for 400 those which require them. Return a {key: password} dict. password 401 will be None if the key has no password.""" 402 403 no_passwords = [] 404 need_passwords = [] 405 key_passwords = {} 406 devnull = open("/dev/null", "w+b") 407 for k in sorted(keylist): 408 # We don't need a password for things that aren't really keys. 409 if k in SPECIAL_CERT_STRINGS: 410 no_passwords.append(k) 411 continue 412 413 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 414 "-inform", "DER", "-nocrypt"], 415 stdin=devnull.fileno(), 416 stdout=devnull.fileno(), 417 stderr=subprocess.STDOUT) 418 p.communicate() 419 if p.returncode == 0: 420 # Definitely an unencrypted key. 421 no_passwords.append(k) 422 else: 423 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 424 "-inform", "DER", "-passin", "pass:"], 425 stdin=devnull.fileno(), 426 stdout=devnull.fileno(), 427 stderr=subprocess.PIPE) 428 stdout, stderr = p.communicate() 429 if p.returncode == 0: 430 # Encrypted key with empty string as password. 431 key_passwords[k] = '' 432 elif stderr.startswith('Error decrypting key'): 433 # Definitely encrypted key. 434 # It would have said "Error reading key" if it didn't parse correctly. 435 need_passwords.append(k) 436 else: 437 # Potentially, a type of key that openssl doesn't understand. 438 # We'll let the routines in signapk.jar handle it. 439 no_passwords.append(k) 440 devnull.close() 441 442 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 443 key_passwords.update(dict.fromkeys(no_passwords, None)) 444 return key_passwords 445 446 447def SignFile(input_name, output_name, key, password, align=None, 448 whole_file=False): 449 """Sign the input_name zip/jar/apk, producing output_name. Use the 450 given key and password (the latter may be None if the key does not 451 have a password. 452 453 If align is an integer > 1, zipalign is run to align stored files in 454 the output zip on 'align'-byte boundaries. 455 456 If whole_file is true, use the "-w" option to SignApk to embed a 457 signature that covers the whole file in the archive comment of the 458 zip file. 459 """ 460 461 if align == 0 or align == 1: 462 align = None 463 464 if align: 465 temp = tempfile.NamedTemporaryFile() 466 sign_name = temp.name 467 else: 468 sign_name = output_name 469 470 cmd = [OPTIONS.java_path, "-Xmx2048m", "-jar", 471 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 472 cmd.extend(OPTIONS.extra_signapk_args) 473 if whole_file: 474 cmd.append("-w") 475 cmd.extend([key + OPTIONS.public_key_suffix, 476 key + OPTIONS.private_key_suffix, 477 input_name, sign_name]) 478 479 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 480 if password is not None: 481 password += "\n" 482 p.communicate(password) 483 if p.returncode != 0: 484 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 485 486 if align: 487 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 488 p.communicate() 489 if p.returncode != 0: 490 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 491 temp.close() 492 493 494def CheckSize(data, target, info_dict): 495 """Check the data string passed against the max size limit, if 496 any, for the given target. Raise exception if the data is too big. 497 Print a warning if the data is nearing the maximum size.""" 498 499 if target.endswith(".img"): target = target[:-4] 500 mount_point = "/" + target 501 502 fs_type = None 503 limit = None 504 if info_dict["fstab"]: 505 if mount_point == "/userdata": mount_point = "/data" 506 p = info_dict["fstab"][mount_point] 507 fs_type = p.fs_type 508 device = p.device 509 if "/" in device: 510 device = device[device.rfind("/")+1:] 511 limit = info_dict.get(device + "_size", None) 512 if not fs_type or not limit: return 513 514 if fs_type == "yaffs2": 515 # image size should be increased by 1/64th to account for the 516 # spare area (64 bytes per 2k page) 517 limit = limit / 2048 * (2048+64) 518 size = len(data) 519 pct = float(size) * 100.0 / limit 520 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 521 if pct >= 99.0: 522 raise ExternalError(msg) 523 elif pct >= 95.0: 524 print 525 print " WARNING: ", msg 526 print 527 elif OPTIONS.verbose: 528 print " ", msg 529 530 531def ReadApkCerts(tf_zip): 532 """Given a target_files ZipFile, parse the META/apkcerts.txt file 533 and return a {package: cert} dict.""" 534 certmap = {} 535 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 536 line = line.strip() 537 if not line: continue 538 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 539 r'private_key="(.*)"$', line) 540 if m: 541 name, cert, privkey = m.groups() 542 public_key_suffix_len = len(OPTIONS.public_key_suffix) 543 private_key_suffix_len = len(OPTIONS.private_key_suffix) 544 if cert in SPECIAL_CERT_STRINGS and not privkey: 545 certmap[name] = cert 546 elif (cert.endswith(OPTIONS.public_key_suffix) and 547 privkey.endswith(OPTIONS.private_key_suffix) and 548 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 549 certmap[name] = cert[:-public_key_suffix_len] 550 else: 551 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 552 return certmap 553 554 555COMMON_DOCSTRING = """ 556 -p (--path) <dir> 557 Prepend <dir>/bin to the list of places to search for binaries 558 run by this script, and expect to find jars in <dir>/framework. 559 560 -s (--device_specific) <file> 561 Path to the python module containing device-specific 562 releasetools code. 563 564 -x (--extra) <key=value> 565 Add a key/value pair to the 'extras' dict, which device-specific 566 extension code may look at. 567 568 -v (--verbose) 569 Show command lines being executed. 570 571 -h (--help) 572 Display this usage message and exit. 573""" 574 575def Usage(docstring): 576 print docstring.rstrip("\n") 577 print COMMON_DOCSTRING 578 579 580def ParseOptions(argv, 581 docstring, 582 extra_opts="", extra_long_opts=(), 583 extra_option_handler=None): 584 """Parse the options in argv and return any arguments that aren't 585 flags. docstring is the calling module's docstring, to be displayed 586 for errors and -h. extra_opts and extra_long_opts are for flags 587 defined by the caller, which are processed by passing them to 588 extra_option_handler.""" 589 590 try: 591 opts, args = getopt.getopt( 592 argv, "hvp:s:x:" + extra_opts, 593 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 594 "java_path=", "public_key_suffix=", "private_key_suffix=", 595 "device_specific=", "extra="] + 596 list(extra_long_opts)) 597 except getopt.GetoptError, err: 598 Usage(docstring) 599 print "**", str(err), "**" 600 sys.exit(2) 601 602 path_specified = False 603 604 for o, a in opts: 605 if o in ("-h", "--help"): 606 Usage(docstring) 607 sys.exit() 608 elif o in ("-v", "--verbose"): 609 OPTIONS.verbose = True 610 elif o in ("-p", "--path"): 611 OPTIONS.search_path = a 612 elif o in ("--signapk_path",): 613 OPTIONS.signapk_path = a 614 elif o in ("--extra_signapk_args",): 615 OPTIONS.extra_signapk_args = shlex.split(a) 616 elif o in ("--java_path",): 617 OPTIONS.java_path = a 618 elif o in ("--public_key_suffix",): 619 OPTIONS.public_key_suffix = a 620 elif o in ("--private_key_suffix",): 621 OPTIONS.private_key_suffix = a 622 elif o in ("-s", "--device_specific"): 623 OPTIONS.device_specific = a 624 elif o in ("-x", "--extra"): 625 key, value = a.split("=", 1) 626 OPTIONS.extras[key] = value 627 else: 628 if extra_option_handler is None or not extra_option_handler(o, a): 629 assert False, "unknown option \"%s\"" % (o,) 630 631 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 632 os.pathsep + os.environ["PATH"]) 633 634 return args 635 636 637def Cleanup(): 638 for i in OPTIONS.tempfiles: 639 if os.path.isdir(i): 640 shutil.rmtree(i) 641 else: 642 os.remove(i) 643 644 645class PasswordManager(object): 646 def __init__(self): 647 self.editor = os.getenv("EDITOR", None) 648 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 649 650 def GetPasswords(self, items): 651 """Get passwords corresponding to each string in 'items', 652 returning a dict. (The dict may have keys in addition to the 653 values in 'items'.) 654 655 Uses the passwords in $ANDROID_PW_FILE if available, letting the 656 user edit that file to add more needed passwords. If no editor is 657 available, or $ANDROID_PW_FILE isn't define, prompts the user 658 interactively in the ordinary way. 659 """ 660 661 current = self.ReadFile() 662 663 first = True 664 while True: 665 missing = [] 666 for i in items: 667 if i not in current or not current[i]: 668 missing.append(i) 669 # Are all the passwords already in the file? 670 if not missing: return current 671 672 for i in missing: 673 current[i] = "" 674 675 if not first: 676 print "key file %s still missing some passwords." % (self.pwfile,) 677 answer = raw_input("try to edit again? [y]> ").strip() 678 if answer and answer[0] not in 'yY': 679 raise RuntimeError("key passwords unavailable") 680 first = False 681 682 current = self.UpdateAndReadFile(current) 683 684 def PromptResult(self, current): 685 """Prompt the user to enter a value (password) for each key in 686 'current' whose value is fales. Returns a new dict with all the 687 values. 688 """ 689 result = {} 690 for k, v in sorted(current.iteritems()): 691 if v: 692 result[k] = v 693 else: 694 while True: 695 result[k] = getpass.getpass("Enter password for %s key> " 696 % (k,)).strip() 697 if result[k]: break 698 return result 699 700 def UpdateAndReadFile(self, current): 701 if not self.editor or not self.pwfile: 702 return self.PromptResult(current) 703 704 f = open(self.pwfile, "w") 705 os.chmod(self.pwfile, 0600) 706 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 707 f.write("# (Additional spaces are harmless.)\n\n") 708 709 first_line = None 710 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 711 sorted.sort() 712 for i, (_, k, v) in enumerate(sorted): 713 f.write("[[[ %s ]]] %s\n" % (v, k)) 714 if not v and first_line is None: 715 # position cursor on first line with no password. 716 first_line = i + 4 717 f.close() 718 719 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 720 _, _ = p.communicate() 721 722 return self.ReadFile() 723 724 def ReadFile(self): 725 result = {} 726 if self.pwfile is None: return result 727 try: 728 f = open(self.pwfile, "r") 729 for line in f: 730 line = line.strip() 731 if not line or line[0] == '#': continue 732 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 733 if not m: 734 print "failed to parse password file: ", line 735 else: 736 result[m.group(2)] = m.group(1) 737 f.close() 738 except IOError, e: 739 if e.errno != errno.ENOENT: 740 print "error reading password file: ", str(e) 741 return result 742 743 744def ZipWriteStr(zip, filename, data, perms=0644, compression=None): 745 # use a fixed timestamp so the output is repeatable. 746 zinfo = zipfile.ZipInfo(filename=filename, 747 date_time=(2009, 1, 1, 0, 0, 0)) 748 if compression is None: 749 zinfo.compress_type = zip.compression 750 else: 751 zinfo.compress_type = compression 752 zinfo.external_attr = perms << 16 753 zip.writestr(zinfo, data) 754 755 756class DeviceSpecificParams(object): 757 module = None 758 def __init__(self, **kwargs): 759 """Keyword arguments to the constructor become attributes of this 760 object, which is passed to all functions in the device-specific 761 module.""" 762 for k, v in kwargs.iteritems(): 763 setattr(self, k, v) 764 self.extras = OPTIONS.extras 765 766 if self.module is None: 767 path = OPTIONS.device_specific 768 if not path: return 769 try: 770 if os.path.isdir(path): 771 info = imp.find_module("releasetools", [path]) 772 else: 773 d, f = os.path.split(path) 774 b, x = os.path.splitext(f) 775 if x == ".py": 776 f = b 777 info = imp.find_module(f, [d]) 778 print "loaded device-specific extensions from", path 779 self.module = imp.load_module("device_specific", *info) 780 except ImportError: 781 print "unable to load device-specific module; assuming none" 782 783 def _DoCall(self, function_name, *args, **kwargs): 784 """Call the named function in the device-specific module, passing 785 the given args and kwargs. The first argument to the call will be 786 the DeviceSpecific object itself. If there is no module, or the 787 module does not define the function, return the value of the 788 'default' kwarg (which itself defaults to None).""" 789 if self.module is None or not hasattr(self.module, function_name): 790 return kwargs.get("default", None) 791 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 792 793 def FullOTA_Assertions(self): 794 """Called after emitting the block of assertions at the top of a 795 full OTA package. Implementations can add whatever additional 796 assertions they like.""" 797 return self._DoCall("FullOTA_Assertions") 798 799 def FullOTA_InstallBegin(self): 800 """Called at the start of full OTA installation.""" 801 return self._DoCall("FullOTA_InstallBegin") 802 803 def FullOTA_InstallEnd(self): 804 """Called at the end of full OTA installation; typically this is 805 used to install the image for the device's baseband processor.""" 806 return self._DoCall("FullOTA_InstallEnd") 807 808 def IncrementalOTA_Assertions(self): 809 """Called after emitting the block of assertions at the top of an 810 incremental OTA package. Implementations can add whatever 811 additional assertions they like.""" 812 return self._DoCall("IncrementalOTA_Assertions") 813 814 def IncrementalOTA_VerifyBegin(self): 815 """Called at the start of the verification phase of incremental 816 OTA installation; additional checks can be placed here to abort 817 the script before any changes are made.""" 818 return self._DoCall("IncrementalOTA_VerifyBegin") 819 820 def IncrementalOTA_VerifyEnd(self): 821 """Called at the end of the verification phase of incremental OTA 822 installation; additional checks can be placed here to abort the 823 script before any changes are made.""" 824 return self._DoCall("IncrementalOTA_VerifyEnd") 825 826 def IncrementalOTA_InstallBegin(self): 827 """Called at the start of incremental OTA installation (after 828 verification is complete).""" 829 return self._DoCall("IncrementalOTA_InstallBegin") 830 831 def IncrementalOTA_InstallEnd(self): 832 """Called at the end of incremental OTA installation; typically 833 this is used to install the image for the device's baseband 834 processor.""" 835 return self._DoCall("IncrementalOTA_InstallEnd") 836 837class File(object): 838 def __init__(self, name, data): 839 self.name = name 840 self.data = data 841 self.size = len(data) 842 self.sha1 = sha1(data).hexdigest() 843 844 @classmethod 845 def FromLocalFile(cls, name, diskname): 846 f = open(diskname, "rb") 847 data = f.read() 848 f.close() 849 return File(name, data) 850 851 def WriteToTemp(self): 852 t = tempfile.NamedTemporaryFile() 853 t.write(self.data) 854 t.flush() 855 return t 856 857 def AddToZip(self, z, compression=None): 858 ZipWriteStr(z, self.name, self.data, compression=compression) 859 860DIFF_PROGRAM_BY_EXT = { 861 ".gz" : "imgdiff", 862 ".zip" : ["imgdiff", "-z"], 863 ".jar" : ["imgdiff", "-z"], 864 ".apk" : ["imgdiff", "-z"], 865 ".img" : "imgdiff", 866 } 867 868class Difference(object): 869 def __init__(self, tf, sf, diff_program=None): 870 self.tf = tf 871 self.sf = sf 872 self.patch = None 873 self.diff_program = diff_program 874 875 def ComputePatch(self): 876 """Compute the patch (as a string of data) needed to turn sf into 877 tf. Returns the same tuple as GetPatch().""" 878 879 tf = self.tf 880 sf = self.sf 881 882 if self.diff_program: 883 diff_program = self.diff_program 884 else: 885 ext = os.path.splitext(tf.name)[1] 886 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 887 888 ttemp = tf.WriteToTemp() 889 stemp = sf.WriteToTemp() 890 891 ext = os.path.splitext(tf.name)[1] 892 893 try: 894 ptemp = tempfile.NamedTemporaryFile() 895 if isinstance(diff_program, list): 896 cmd = copy.copy(diff_program) 897 else: 898 cmd = [diff_program] 899 cmd.append(stemp.name) 900 cmd.append(ttemp.name) 901 cmd.append(ptemp.name) 902 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 903 _, err = p.communicate() 904 if err or p.returncode != 0: 905 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 906 return None 907 diff = ptemp.read() 908 finally: 909 ptemp.close() 910 stemp.close() 911 ttemp.close() 912 913 self.patch = diff 914 return self.tf, self.sf, self.patch 915 916 917 def GetPatch(self): 918 """Return a tuple (target_file, source_file, patch_data). 919 patch_data may be None if ComputePatch hasn't been called, or if 920 computing the patch failed.""" 921 return self.tf, self.sf, self.patch 922 923 924def ComputeDifferences(diffs): 925 """Call ComputePatch on all the Difference objects in 'diffs'.""" 926 print len(diffs), "diffs to compute" 927 928 # Do the largest files first, to try and reduce the long-pole effect. 929 by_size = [(i.tf.size, i) for i in diffs] 930 by_size.sort(reverse=True) 931 by_size = [i[1] for i in by_size] 932 933 lock = threading.Lock() 934 diff_iter = iter(by_size) # accessed under lock 935 936 def worker(): 937 try: 938 lock.acquire() 939 for d in diff_iter: 940 lock.release() 941 start = time.time() 942 d.ComputePatch() 943 dur = time.time() - start 944 lock.acquire() 945 946 tf, sf, patch = d.GetPatch() 947 if sf.name == tf.name: 948 name = tf.name 949 else: 950 name = "%s (%s)" % (tf.name, sf.name) 951 if patch is None: 952 print "patching failed! %s" % (name,) 953 else: 954 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 955 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 956 lock.release() 957 except Exception, e: 958 print e 959 raise 960 961 # start worker threads; wait for them all to finish. 962 threads = [threading.Thread(target=worker) 963 for i in range(OPTIONS.worker_threads)] 964 for th in threads: 965 th.start() 966 while threads: 967 threads.pop().join() 968 969 970# map recovery.fstab's fs_types to mount/format "partition types" 971PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 972 "ext4": "EMMC", "emmc": "EMMC", 973 "f2fs": "EMMC" } 974 975def GetTypeAndDevice(mount_point, info): 976 fstab = info["fstab"] 977 if fstab: 978 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 979 else: 980 return None 981 982 983def ParseCertificate(data): 984 """Parse a PEM-format certificate.""" 985 cert = [] 986 save = False 987 for line in data.split("\n"): 988 if "--END CERTIFICATE--" in line: 989 break 990 if save: 991 cert.append(line) 992 if "--BEGIN CERTIFICATE--" in line: 993 save = True 994 cert = "".join(cert).decode('base64') 995 return cert 996 997def XDelta3(source_path, target_path, output_path): 998 diff_program = ["xdelta3", "-0", "-B", str(64<<20), "-e", "-f", "-s"] 999 diff_program.append(source_path) 1000 diff_program.append(target_path) 1001 diff_program.append(output_path) 1002 p = Run(diff_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 1003 p.communicate() 1004 assert p.returncode == 0, "Couldn't produce patch" 1005 1006def XZ(path): 1007 compress_program = ["xz", "-zk", "-9", "--check=crc32"] 1008 compress_program.append(path) 1009 p = Run(compress_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 1010 p.communicate() 1011 assert p.returncode == 0, "Couldn't compress patch" 1012 1013def MakePartitionPatch(source_file, target_file, partition): 1014 with tempfile.NamedTemporaryFile() as output_file: 1015 XDelta3(source_file.name, target_file.name, output_file.name) 1016 XZ(output_file.name) 1017 with open(output_file.name + ".xz") as patch_file: 1018 patch_data = patch_file.read() 1019 os.unlink(patch_file.name) 1020 return File(partition + ".muimg.p", patch_data) 1021 1022def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img, 1023 info_dict=None): 1024 """Generate a binary patch that creates the recovery image starting 1025 with the boot image. (Most of the space in these images is just the 1026 kernel, which is identical for the two, so the resulting patch 1027 should be efficient.) Add it to the output zip, along with a shell 1028 script that is run from init.rc on first boot to actually do the 1029 patching and install the new recovery image. 1030 1031 recovery_img and boot_img should be File objects for the 1032 corresponding images. info should be the dictionary returned by 1033 common.LoadInfoDict() on the input target_files. 1034 """ 1035 1036 if info_dict is None: 1037 info_dict = OPTIONS.info_dict 1038 1039 diff_program = ["imgdiff"] 1040 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat") 1041 if os.path.exists(path): 1042 diff_program.append("-b") 1043 diff_program.append(path) 1044 bonus_args = "-b /system/etc/recovery-resource.dat" 1045 else: 1046 bonus_args = "" 1047 1048 d = Difference(recovery_img, boot_img, diff_program=diff_program) 1049 _, _, patch = d.ComputePatch() 1050 output_sink("recovery-from-boot.p", patch) 1051 1052 td_pair = GetTypeAndDevice("/boot", info_dict) 1053 if not td_pair: 1054 return 1055 boot_type, boot_device = td_pair 1056 td_pair = GetTypeAndDevice("/recovery", info_dict) 1057 if not td_pair: 1058 return 1059 recovery_type, recovery_device = td_pair 1060 1061 sh = """#!/system/bin/sh 1062if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 1063 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1064else 1065 log -t recovery "Recovery image already installed" 1066fi 1067""" % { 'boot_size': boot_img.size, 1068 'boot_sha1': boot_img.sha1, 1069 'recovery_size': recovery_img.size, 1070 'recovery_sha1': recovery_img.sha1, 1071 'boot_type': boot_type, 1072 'boot_device': boot_device, 1073 'recovery_type': recovery_type, 1074 'recovery_device': recovery_device, 1075 'bonus_args': bonus_args, 1076 } 1077 1078 # The install script location moved from /system/etc to /system/bin 1079 # in the L release. Parse the init.rc file to find out where the 1080 # target-files expects it to be, and put it there. 1081 sh_location = "etc/install-recovery.sh" 1082 try: 1083 with open(os.path.join(input_dir, "BOOT", "RAMDISK", "init.rc")) as f: 1084 for line in f: 1085 m = re.match("^service flash_recovery /system/(\S+)\s*$", line) 1086 if m: 1087 sh_location = m.group(1) 1088 print "putting script in", sh_location 1089 break 1090 except (OSError, IOError), e: 1091 print "failed to read init.rc: %s" % (e,) 1092 1093 output_sink(sh_location, sh) 1094