common.py revision 114b46f15dec20ad6b55879fe072220b07dcc099
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32try: 33 from hashlib import sha1 as sha1 34except ImportError: 35 from sha import sha as sha1 36 37# missing in Python 2.4 and before 38if not hasattr(os, "SEEK_SET"): 39 os.SEEK_SET = 0 40 41class Options(object): pass 42OPTIONS = Options() 43OPTIONS.search_path = "out/host/linux-x86" 44OPTIONS.signapk_path = "framework/signapk.jar" # Relative to search_path 45OPTIONS.extra_signapk_args = [] 46OPTIONS.java_path = "java" # Use the one on the path by default. 47OPTIONS.public_key_suffix = ".x509.pem" 48OPTIONS.private_key_suffix = ".pk8" 49OPTIONS.verbose = False 50OPTIONS.tempfiles = [] 51OPTIONS.device_specific = None 52OPTIONS.extras = {} 53OPTIONS.info_dict = None 54 55 56# Values for "certificate" in apkcerts that mean special things. 57SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 58 59 60class ExternalError(RuntimeError): pass 61 62 63def Run(args, **kwargs): 64 """Create and return a subprocess.Popen object, printing the command 65 line on the terminal if -v was specified.""" 66 if OPTIONS.verbose: 67 print " running: ", " ".join(args) 68 return subprocess.Popen(args, **kwargs) 69 70 71def CloseInheritedPipes(): 72 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 73 before doing other work.""" 74 if platform.system() != "Darwin": 75 return 76 for d in range(3, 1025): 77 try: 78 stat = os.fstat(d) 79 if stat is not None: 80 pipebit = stat[0] & 0x1000 81 if pipebit != 0: 82 os.close(d) 83 except OSError: 84 pass 85 86 87def LoadInfoDict(input): 88 """Read and parse the META/misc_info.txt key/value pairs from the 89 input target files and return a dict.""" 90 91 def read_helper(fn): 92 if isinstance(input, zipfile.ZipFile): 93 return input.read(fn) 94 else: 95 path = os.path.join(input, *fn.split("/")) 96 try: 97 with open(path) as f: 98 return f.read() 99 except IOError, e: 100 if e.errno == errno.ENOENT: 101 raise KeyError(fn) 102 103 d = {} 104 try: 105 for line in read_helper("META/misc_info.txt").split("\n"): 106 line = line.strip() 107 if not line or line.startswith("#"): continue 108 k, v = line.split("=", 1) 109 d[k] = v 110 except KeyError: 111 # ok if misc_info.txt doesn't exist 112 pass 113 114 # backwards compatibility: These values used to be in their own 115 # files. Look for them, in case we're processing an old 116 # target_files zip. 117 118 if "mkyaffs2_extra_flags" not in d: 119 try: 120 d["mkyaffs2_extra_flags"] = read_helper("META/mkyaffs2-extra-flags.txt").strip() 121 except KeyError: 122 # ok if flags don't exist 123 pass 124 125 if "recovery_api_version" not in d: 126 try: 127 d["recovery_api_version"] = read_helper("META/recovery-api-version.txt").strip() 128 except KeyError: 129 raise ValueError("can't find recovery API version in input target-files") 130 131 if "tool_extensions" not in d: 132 try: 133 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip() 134 except KeyError: 135 # ok if extensions don't exist 136 pass 137 138 if "fstab_version" not in d: 139 d["fstab_version"] = "1" 140 141 try: 142 data = read_helper("META/imagesizes.txt") 143 for line in data.split("\n"): 144 if not line: continue 145 name, value = line.split(" ", 1) 146 if not value: continue 147 if name == "blocksize": 148 d[name] = value 149 else: 150 d[name + "_size"] = value 151 except KeyError: 152 pass 153 154 def makeint(key): 155 if key in d: 156 d[key] = int(d[key], 0) 157 158 makeint("recovery_api_version") 159 makeint("blocksize") 160 makeint("system_size") 161 makeint("userdata_size") 162 makeint("cache_size") 163 makeint("recovery_size") 164 makeint("boot_size") 165 makeint("fstab_version") 166 167 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"]) 168 d["build.prop"] = LoadBuildProp(read_helper) 169 return d 170 171def LoadBuildProp(read_helper): 172 try: 173 data = read_helper("SYSTEM/build.prop") 174 except KeyError: 175 print "Warning: could not find SYSTEM/build.prop in %s" % zip 176 data = "" 177 178 d = {} 179 for line in data.split("\n"): 180 line = line.strip() 181 if not line or line.startswith("#"): continue 182 if "=" in line: 183 name, value = line.split("=", 1) 184 d[name] = value 185 return d 186 187def LoadRecoveryFSTab(read_helper, fstab_version): 188 class Partition(object): 189 pass 190 191 try: 192 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab") 193 except KeyError: 194 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab" 195 data = "" 196 197 if fstab_version == 1: 198 d = {} 199 for line in data.split("\n"): 200 line = line.strip() 201 if not line or line.startswith("#"): continue 202 pieces = line.split() 203 if not (3 <= len(pieces) <= 4): 204 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 205 206 p = Partition() 207 p.mount_point = pieces[0] 208 p.fs_type = pieces[1] 209 p.device = pieces[2] 210 p.length = 0 211 options = None 212 if len(pieces) >= 4: 213 if pieces[3].startswith("/"): 214 p.device2 = pieces[3] 215 if len(pieces) >= 5: 216 options = pieces[4] 217 else: 218 p.device2 = None 219 options = pieces[3] 220 else: 221 p.device2 = None 222 223 if options: 224 options = options.split(",") 225 for i in options: 226 if i.startswith("length="): 227 p.length = int(i[7:]) 228 else: 229 print "%s: unknown option \"%s\"" % (p.mount_point, i) 230 231 d[p.mount_point] = p 232 233 elif fstab_version == 2: 234 d = {} 235 for line in data.split("\n"): 236 line = line.strip() 237 if not line or line.startswith("#"): continue 238 pieces = line.split() 239 if len(pieces) != 5: 240 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 241 242 # Ignore entries that are managed by vold 243 options = pieces[4] 244 if "voldmanaged=" in options: continue 245 246 # It's a good line, parse it 247 p = Partition() 248 p.device = pieces[0] 249 p.mount_point = pieces[1] 250 p.fs_type = pieces[2] 251 p.device2 = None 252 p.length = 0 253 254 options = options.split(",") 255 for i in options: 256 if i.startswith("length="): 257 p.length = int(i[7:]) 258 else: 259 # Ignore all unknown options in the unified fstab 260 continue 261 262 d[p.mount_point] = p 263 264 else: 265 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 266 267 return d 268 269 270def DumpInfoDict(d): 271 for k, v in sorted(d.items()): 272 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 273 274def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 275 """Take a kernel, cmdline, and ramdisk directory from the input (in 276 'sourcedir'), and turn them into a boot image. Return the image 277 data, or None if sourcedir does not appear to contains files for 278 building the requested image.""" 279 280 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 281 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 282 return None 283 284 if info_dict is None: 285 info_dict = OPTIONS.info_dict 286 287 ramdisk_img = tempfile.NamedTemporaryFile() 288 img = tempfile.NamedTemporaryFile() 289 290 if os.access(fs_config_file, os.F_OK): 291 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 292 else: 293 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 294 p1 = Run(cmd, stdout=subprocess.PIPE) 295 p2 = Run(["minigzip"], 296 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 297 298 p2.wait() 299 p1.wait() 300 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 301 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 302 303 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 304 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 305 306 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 307 308 fn = os.path.join(sourcedir, "cmdline") 309 if os.access(fn, os.F_OK): 310 cmd.append("--cmdline") 311 cmd.append(open(fn).read().rstrip("\n")) 312 313 fn = os.path.join(sourcedir, "base") 314 if os.access(fn, os.F_OK): 315 cmd.append("--base") 316 cmd.append(open(fn).read().rstrip("\n")) 317 318 fn = os.path.join(sourcedir, "pagesize") 319 if os.access(fn, os.F_OK): 320 cmd.append("--pagesize") 321 cmd.append(open(fn).read().rstrip("\n")) 322 323 args = info_dict.get("mkbootimg_args", None) 324 if args and args.strip(): 325 cmd.extend(shlex.split(args)) 326 327 cmd.extend(["--ramdisk", ramdisk_img.name, 328 "--output", img.name]) 329 330 p = Run(cmd, stdout=subprocess.PIPE) 331 p.communicate() 332 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 333 os.path.basename(sourcedir),) 334 335 img.seek(os.SEEK_SET, 0) 336 data = img.read() 337 338 ramdisk_img.close() 339 img.close() 340 341 return data 342 343 344def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 345 info_dict=None): 346 """Return a File object (with name 'name') with the desired bootable 347 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 348 'prebuilt_name', otherwise construct it from the source files in 349 'unpack_dir'/'tree_subdir'.""" 350 351 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 352 if os.path.exists(prebuilt_path): 353 print "using prebuilt %s..." % (prebuilt_name,) 354 return File.FromLocalFile(name, prebuilt_path) 355 else: 356 print "building image from target_files %s..." % (tree_subdir,) 357 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 358 data = BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 359 os.path.join(unpack_dir, fs_config), 360 info_dict) 361 if data: 362 return File(name, data) 363 return None 364 365 366def UnzipTemp(filename, pattern=None): 367 """Unzip the given archive into a temporary directory and return the name. 368 369 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 370 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 371 372 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 373 main file), open for reading. 374 """ 375 376 tmp = tempfile.mkdtemp(prefix="targetfiles-") 377 OPTIONS.tempfiles.append(tmp) 378 379 def unzip_to_dir(filename, dirname): 380 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 381 if pattern is not None: 382 cmd.append(pattern) 383 p = Run(cmd, stdout=subprocess.PIPE) 384 p.communicate() 385 if p.returncode != 0: 386 raise ExternalError("failed to unzip input target-files \"%s\"" % 387 (filename,)) 388 389 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 390 if m: 391 unzip_to_dir(m.group(1), tmp) 392 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 393 filename = m.group(1) 394 else: 395 unzip_to_dir(filename, tmp) 396 397 return tmp, zipfile.ZipFile(filename, "r") 398 399 400def GetKeyPasswords(keylist): 401 """Given a list of keys, prompt the user to enter passwords for 402 those which require them. Return a {key: password} dict. password 403 will be None if the key has no password.""" 404 405 no_passwords = [] 406 need_passwords = [] 407 key_passwords = {} 408 devnull = open("/dev/null", "w+b") 409 for k in sorted(keylist): 410 # We don't need a password for things that aren't really keys. 411 if k in SPECIAL_CERT_STRINGS: 412 no_passwords.append(k) 413 continue 414 415 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 416 "-inform", "DER", "-nocrypt"], 417 stdin=devnull.fileno(), 418 stdout=devnull.fileno(), 419 stderr=subprocess.STDOUT) 420 p.communicate() 421 if p.returncode == 0: 422 # Definitely an unencrypted key. 423 no_passwords.append(k) 424 else: 425 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 426 "-inform", "DER", "-passin", "pass:"], 427 stdin=devnull.fileno(), 428 stdout=devnull.fileno(), 429 stderr=subprocess.PIPE) 430 stdout, stderr = p.communicate() 431 if p.returncode == 0: 432 # Encrypted key with empty string as password. 433 key_passwords[k] = '' 434 elif stderr.startswith('Error decrypting key'): 435 # Definitely encrypted key. 436 # It would have said "Error reading key" if it didn't parse correctly. 437 need_passwords.append(k) 438 else: 439 # Potentially, a type of key that openssl doesn't understand. 440 # We'll let the routines in signapk.jar handle it. 441 no_passwords.append(k) 442 devnull.close() 443 444 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 445 key_passwords.update(dict.fromkeys(no_passwords, None)) 446 return key_passwords 447 448 449def SignFile(input_name, output_name, key, password, align=None, 450 whole_file=False): 451 """Sign the input_name zip/jar/apk, producing output_name. Use the 452 given key and password (the latter may be None if the key does not 453 have a password. 454 455 If align is an integer > 1, zipalign is run to align stored files in 456 the output zip on 'align'-byte boundaries. 457 458 If whole_file is true, use the "-w" option to SignApk to embed a 459 signature that covers the whole file in the archive comment of the 460 zip file. 461 """ 462 463 if align == 0 or align == 1: 464 align = None 465 466 if align: 467 temp = tempfile.NamedTemporaryFile() 468 sign_name = temp.name 469 else: 470 sign_name = output_name 471 472 cmd = [OPTIONS.java_path, "-Xmx2048m", "-jar", 473 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 474 cmd.extend(OPTIONS.extra_signapk_args) 475 if whole_file: 476 cmd.append("-w") 477 cmd.extend([key + OPTIONS.public_key_suffix, 478 key + OPTIONS.private_key_suffix, 479 input_name, sign_name]) 480 481 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 482 if password is not None: 483 password += "\n" 484 p.communicate(password) 485 if p.returncode != 0: 486 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 487 488 if align: 489 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 490 p.communicate() 491 if p.returncode != 0: 492 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 493 temp.close() 494 495 496def CheckSize(data, target, info_dict): 497 """Check the data string passed against the max size limit, if 498 any, for the given target. Raise exception if the data is too big. 499 Print a warning if the data is nearing the maximum size.""" 500 501 if target.endswith(".img"): target = target[:-4] 502 mount_point = "/" + target 503 504 if info_dict["fstab"]: 505 if mount_point == "/userdata": mount_point = "/data" 506 p = info_dict["fstab"][mount_point] 507 fs_type = p.fs_type 508 device = p.device 509 if "/" in device: 510 device = device[device.rfind("/")+1:] 511 limit = info_dict.get(device + "_size", None) 512 if not fs_type or not limit: return 513 514 if fs_type == "yaffs2": 515 # image size should be increased by 1/64th to account for the 516 # spare area (64 bytes per 2k page) 517 limit = limit / 2048 * (2048+64) 518 size = len(data) 519 pct = float(size) * 100.0 / limit 520 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 521 if pct >= 99.0: 522 raise ExternalError(msg) 523 elif pct >= 95.0: 524 print 525 print " WARNING: ", msg 526 print 527 elif OPTIONS.verbose: 528 print " ", msg 529 530 531def ReadApkCerts(tf_zip): 532 """Given a target_files ZipFile, parse the META/apkcerts.txt file 533 and return a {package: cert} dict.""" 534 certmap = {} 535 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 536 line = line.strip() 537 if not line: continue 538 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 539 r'private_key="(.*)"$', line) 540 if m: 541 name, cert, privkey = m.groups() 542 public_key_suffix_len = len(OPTIONS.public_key_suffix) 543 private_key_suffix_len = len(OPTIONS.private_key_suffix) 544 if cert in SPECIAL_CERT_STRINGS and not privkey: 545 certmap[name] = cert 546 elif (cert.endswith(OPTIONS.public_key_suffix) and 547 privkey.endswith(OPTIONS.private_key_suffix) and 548 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 549 certmap[name] = cert[:-public_key_suffix_len] 550 else: 551 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 552 return certmap 553 554 555COMMON_DOCSTRING = """ 556 -p (--path) <dir> 557 Prepend <dir>/bin to the list of places to search for binaries 558 run by this script, and expect to find jars in <dir>/framework. 559 560 -s (--device_specific) <file> 561 Path to the python module containing device-specific 562 releasetools code. 563 564 -x (--extra) <key=value> 565 Add a key/value pair to the 'extras' dict, which device-specific 566 extension code may look at. 567 568 -v (--verbose) 569 Show command lines being executed. 570 571 -h (--help) 572 Display this usage message and exit. 573""" 574 575def Usage(docstring): 576 print docstring.rstrip("\n") 577 print COMMON_DOCSTRING 578 579 580def ParseOptions(argv, 581 docstring, 582 extra_opts="", extra_long_opts=(), 583 extra_option_handler=None): 584 """Parse the options in argv and return any arguments that aren't 585 flags. docstring is the calling module's docstring, to be displayed 586 for errors and -h. extra_opts and extra_long_opts are for flags 587 defined by the caller, which are processed by passing them to 588 extra_option_handler.""" 589 590 try: 591 opts, args = getopt.getopt( 592 argv, "hvp:s:x:" + extra_opts, 593 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 594 "java_path=", "public_key_suffix=", "private_key_suffix=", 595 "device_specific=", "extra="] + 596 list(extra_long_opts)) 597 except getopt.GetoptError, err: 598 Usage(docstring) 599 print "**", str(err), "**" 600 sys.exit(2) 601 602 path_specified = False 603 604 for o, a in opts: 605 if o in ("-h", "--help"): 606 Usage(docstring) 607 sys.exit() 608 elif o in ("-v", "--verbose"): 609 OPTIONS.verbose = True 610 elif o in ("-p", "--path"): 611 OPTIONS.search_path = a 612 elif o in ("--signapk_path",): 613 OPTIONS.signapk_path = a 614 elif o in ("--extra_signapk_args",): 615 OPTIONS.extra_signapk_args = shlex.split(a) 616 elif o in ("--java_path",): 617 OPTIONS.java_path = a 618 elif o in ("--public_key_suffix",): 619 OPTIONS.public_key_suffix = a 620 elif o in ("--private_key_suffix",): 621 OPTIONS.private_key_suffix = a 622 elif o in ("-s", "--device_specific"): 623 OPTIONS.device_specific = a 624 elif o in ("-x", "--extra"): 625 key, value = a.split("=", 1) 626 OPTIONS.extras[key] = value 627 else: 628 if extra_option_handler is None or not extra_option_handler(o, a): 629 assert False, "unknown option \"%s\"" % (o,) 630 631 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 632 os.pathsep + os.environ["PATH"]) 633 634 return args 635 636 637def Cleanup(): 638 for i in OPTIONS.tempfiles: 639 if os.path.isdir(i): 640 shutil.rmtree(i) 641 else: 642 os.remove(i) 643 644 645class PasswordManager(object): 646 def __init__(self): 647 self.editor = os.getenv("EDITOR", None) 648 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 649 650 def GetPasswords(self, items): 651 """Get passwords corresponding to each string in 'items', 652 returning a dict. (The dict may have keys in addition to the 653 values in 'items'.) 654 655 Uses the passwords in $ANDROID_PW_FILE if available, letting the 656 user edit that file to add more needed passwords. If no editor is 657 available, or $ANDROID_PW_FILE isn't define, prompts the user 658 interactively in the ordinary way. 659 """ 660 661 current = self.ReadFile() 662 663 first = True 664 while True: 665 missing = [] 666 for i in items: 667 if i not in current or not current[i]: 668 missing.append(i) 669 # Are all the passwords already in the file? 670 if not missing: return current 671 672 for i in missing: 673 current[i] = "" 674 675 if not first: 676 print "key file %s still missing some passwords." % (self.pwfile,) 677 answer = raw_input("try to edit again? [y]> ").strip() 678 if answer and answer[0] not in 'yY': 679 raise RuntimeError("key passwords unavailable") 680 first = False 681 682 current = self.UpdateAndReadFile(current) 683 684 def PromptResult(self, current): 685 """Prompt the user to enter a value (password) for each key in 686 'current' whose value is fales. Returns a new dict with all the 687 values. 688 """ 689 result = {} 690 for k, v in sorted(current.iteritems()): 691 if v: 692 result[k] = v 693 else: 694 while True: 695 result[k] = getpass.getpass("Enter password for %s key> " 696 % (k,)).strip() 697 if result[k]: break 698 return result 699 700 def UpdateAndReadFile(self, current): 701 if not self.editor or not self.pwfile: 702 return self.PromptResult(current) 703 704 f = open(self.pwfile, "w") 705 os.chmod(self.pwfile, 0600) 706 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 707 f.write("# (Additional spaces are harmless.)\n\n") 708 709 first_line = None 710 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 711 sorted.sort() 712 for i, (_, k, v) in enumerate(sorted): 713 f.write("[[[ %s ]]] %s\n" % (v, k)) 714 if not v and first_line is None: 715 # position cursor on first line with no password. 716 first_line = i + 4 717 f.close() 718 719 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 720 _, _ = p.communicate() 721 722 return self.ReadFile() 723 724 def ReadFile(self): 725 result = {} 726 if self.pwfile is None: return result 727 try: 728 f = open(self.pwfile, "r") 729 for line in f: 730 line = line.strip() 731 if not line or line[0] == '#': continue 732 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 733 if not m: 734 print "failed to parse password file: ", line 735 else: 736 result[m.group(2)] = m.group(1) 737 f.close() 738 except IOError, e: 739 if e.errno != errno.ENOENT: 740 print "error reading password file: ", str(e) 741 return result 742 743 744def ZipWriteStr(zip, filename, data, perms=0644, compression=None): 745 # use a fixed timestamp so the output is repeatable. 746 zinfo = zipfile.ZipInfo(filename=filename, 747 date_time=(2009, 1, 1, 0, 0, 0)) 748 if compression is None: 749 zinfo.compress_type = zip.compression 750 else: 751 zinfo.compress_type = compression 752 zinfo.external_attr = perms << 16 753 zip.writestr(zinfo, data) 754 755 756class DeviceSpecificParams(object): 757 module = None 758 def __init__(self, **kwargs): 759 """Keyword arguments to the constructor become attributes of this 760 object, which is passed to all functions in the device-specific 761 module.""" 762 for k, v in kwargs.iteritems(): 763 setattr(self, k, v) 764 self.extras = OPTIONS.extras 765 766 if self.module is None: 767 path = OPTIONS.device_specific 768 if not path: return 769 try: 770 if os.path.isdir(path): 771 info = imp.find_module("releasetools", [path]) 772 else: 773 d, f = os.path.split(path) 774 b, x = os.path.splitext(f) 775 if x == ".py": 776 f = b 777 info = imp.find_module(f, [d]) 778 print "loaded device-specific extensions from", path 779 self.module = imp.load_module("device_specific", *info) 780 except ImportError: 781 print "unable to load device-specific module; assuming none" 782 783 def _DoCall(self, function_name, *args, **kwargs): 784 """Call the named function in the device-specific module, passing 785 the given args and kwargs. The first argument to the call will be 786 the DeviceSpecific object itself. If there is no module, or the 787 module does not define the function, return the value of the 788 'default' kwarg (which itself defaults to None).""" 789 if self.module is None or not hasattr(self.module, function_name): 790 return kwargs.get("default", None) 791 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 792 793 def FullOTA_Assertions(self): 794 """Called after emitting the block of assertions at the top of a 795 full OTA package. Implementations can add whatever additional 796 assertions they like.""" 797 return self._DoCall("FullOTA_Assertions") 798 799 def FullOTA_InstallBegin(self): 800 """Called at the start of full OTA installation.""" 801 return self._DoCall("FullOTA_InstallBegin") 802 803 def FullOTA_InstallEnd(self): 804 """Called at the end of full OTA installation; typically this is 805 used to install the image for the device's baseband processor.""" 806 return self._DoCall("FullOTA_InstallEnd") 807 808 def IncrementalOTA_Assertions(self): 809 """Called after emitting the block of assertions at the top of an 810 incremental OTA package. Implementations can add whatever 811 additional assertions they like.""" 812 return self._DoCall("IncrementalOTA_Assertions") 813 814 def IncrementalOTA_VerifyBegin(self): 815 """Called at the start of the verification phase of incremental 816 OTA installation; additional checks can be placed here to abort 817 the script before any changes are made.""" 818 return self._DoCall("IncrementalOTA_VerifyBegin") 819 820 def IncrementalOTA_VerifyEnd(self): 821 """Called at the end of the verification phase of incremental OTA 822 installation; additional checks can be placed here to abort the 823 script before any changes are made.""" 824 return self._DoCall("IncrementalOTA_VerifyEnd") 825 826 def IncrementalOTA_InstallBegin(self): 827 """Called at the start of incremental OTA installation (after 828 verification is complete).""" 829 return self._DoCall("IncrementalOTA_InstallBegin") 830 831 def IncrementalOTA_InstallEnd(self): 832 """Called at the end of incremental OTA installation; typically 833 this is used to install the image for the device's baseband 834 processor.""" 835 return self._DoCall("IncrementalOTA_InstallEnd") 836 837class File(object): 838 def __init__(self, name, data): 839 self.name = name 840 self.data = data 841 self.size = len(data) 842 self.sha1 = sha1(data).hexdigest() 843 844 @classmethod 845 def FromLocalFile(cls, name, diskname): 846 f = open(diskname, "rb") 847 data = f.read() 848 f.close() 849 return File(name, data) 850 851 def WriteToTemp(self): 852 t = tempfile.NamedTemporaryFile() 853 t.write(self.data) 854 t.flush() 855 return t 856 857 def AddToZip(self, z, compression=None): 858 ZipWriteStr(z, self.name, self.data, compression=compression) 859 860DIFF_PROGRAM_BY_EXT = { 861 ".gz" : "imgdiff", 862 ".zip" : ["imgdiff", "-z"], 863 ".jar" : ["imgdiff", "-z"], 864 ".apk" : ["imgdiff", "-z"], 865 ".img" : "imgdiff", 866 } 867 868class Difference(object): 869 def __init__(self, tf, sf, diff_program=None): 870 self.tf = tf 871 self.sf = sf 872 self.patch = None 873 self.diff_program = diff_program 874 875 def ComputePatch(self): 876 """Compute the patch (as a string of data) needed to turn sf into 877 tf. Returns the same tuple as GetPatch().""" 878 879 tf = self.tf 880 sf = self.sf 881 882 if self.diff_program: 883 diff_program = self.diff_program 884 else: 885 ext = os.path.splitext(tf.name)[1] 886 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 887 888 ttemp = tf.WriteToTemp() 889 stemp = sf.WriteToTemp() 890 891 ext = os.path.splitext(tf.name)[1] 892 893 try: 894 ptemp = tempfile.NamedTemporaryFile() 895 if isinstance(diff_program, list): 896 cmd = copy.copy(diff_program) 897 else: 898 cmd = [diff_program] 899 cmd.append(stemp.name) 900 cmd.append(ttemp.name) 901 cmd.append(ptemp.name) 902 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 903 _, err = p.communicate() 904 if err or p.returncode != 0: 905 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 906 return None 907 diff = ptemp.read() 908 finally: 909 ptemp.close() 910 stemp.close() 911 ttemp.close() 912 913 self.patch = diff 914 return self.tf, self.sf, self.patch 915 916 917 def GetPatch(self): 918 """Return a tuple (target_file, source_file, patch_data). 919 patch_data may be None if ComputePatch hasn't been called, or if 920 computing the patch failed.""" 921 return self.tf, self.sf, self.patch 922 923 924def ComputeDifferences(diffs): 925 """Call ComputePatch on all the Difference objects in 'diffs'.""" 926 print len(diffs), "diffs to compute" 927 928 # Do the largest files first, to try and reduce the long-pole effect. 929 by_size = [(i.tf.size, i) for i in diffs] 930 by_size.sort(reverse=True) 931 by_size = [i[1] for i in by_size] 932 933 lock = threading.Lock() 934 diff_iter = iter(by_size) # accessed under lock 935 936 def worker(): 937 try: 938 lock.acquire() 939 for d in diff_iter: 940 lock.release() 941 start = time.time() 942 d.ComputePatch() 943 dur = time.time() - start 944 lock.acquire() 945 946 tf, sf, patch = d.GetPatch() 947 if sf.name == tf.name: 948 name = tf.name 949 else: 950 name = "%s (%s)" % (tf.name, sf.name) 951 if patch is None: 952 print "patching failed! %s" % (name,) 953 else: 954 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 955 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 956 lock.release() 957 except Exception, e: 958 print e 959 raise 960 961 # start worker threads; wait for them all to finish. 962 threads = [threading.Thread(target=worker) 963 for i in range(OPTIONS.worker_threads)] 964 for th in threads: 965 th.start() 966 while threads: 967 threads.pop().join() 968 969 970# map recovery.fstab's fs_types to mount/format "partition types" 971PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 972 "ext4": "EMMC", "emmc": "EMMC" } 973 974def GetTypeAndDevice(mount_point, info): 975 fstab = info["fstab"] 976 if fstab: 977 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 978 else: 979 return None 980 981 982def ParseCertificate(data): 983 """Parse a PEM-format certificate.""" 984 cert = [] 985 save = False 986 for line in data.split("\n"): 987 if "--END CERTIFICATE--" in line: 988 break 989 if save: 990 cert.append(line) 991 if "--BEGIN CERTIFICATE--" in line: 992 save = True 993 cert = "".join(cert).decode('base64') 994 return cert 995 996def XDelta3(source_path, target_path, output_path): 997 diff_program = ["xdelta3", "-0", "-B", str(64<<20), "-e", "-f", "-s"] 998 diff_program.append(source_path) 999 diff_program.append(target_path) 1000 diff_program.append(output_path) 1001 p = Run(diff_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 1002 p.communicate() 1003 assert p.returncode == 0, "Couldn't produce patch" 1004 1005def XZ(path): 1006 compress_program = ["xz", "-zk", "-9", "--check=crc32"] 1007 compress_program.append(path) 1008 p = Run(compress_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 1009 p.communicate() 1010 assert p.returncode == 0, "Couldn't compress patch" 1011 1012def MakeSystemPatch(source_file, target_file): 1013 with tempfile.NamedTemporaryFile() as output_file: 1014 XDelta3(source_file.name, target_file.name, output_file.name) 1015 XZ(output_file.name) 1016 with open(output_file.name + ".xz") as patch_file: 1017 patch_data = patch_file.read() 1018 os.unlink(patch_file.name) 1019 return File("system.muimg.p", patch_data) 1020 1021def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img, 1022 info_dict=None): 1023 """Generate a binary patch that creates the recovery image starting 1024 with the boot image. (Most of the space in these images is just the 1025 kernel, which is identical for the two, so the resulting patch 1026 should be efficient.) Add it to the output zip, along with a shell 1027 script that is run from init.rc on first boot to actually do the 1028 patching and install the new recovery image. 1029 1030 recovery_img and boot_img should be File objects for the 1031 corresponding images. info should be the dictionary returned by 1032 common.LoadInfoDict() on the input target_files. 1033 """ 1034 1035 if info_dict is None: 1036 info_dict = OPTIONS.info_dict 1037 1038 diff_program = ["imgdiff"] 1039 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat") 1040 if os.path.exists(path): 1041 diff_program.append("-b") 1042 diff_program.append(path) 1043 bonus_args = "-b /system/etc/recovery-resource.dat" 1044 else: 1045 bonus_args = "" 1046 1047 d = Difference(recovery_img, boot_img, diff_program=diff_program) 1048 _, _, patch = d.ComputePatch() 1049 output_sink("recovery-from-boot.p", patch) 1050 1051 boot_type, boot_device = GetTypeAndDevice("/boot", info_dict) 1052 recovery_type, recovery_device = GetTypeAndDevice("/recovery", info_dict) 1053 1054 sh = """#!/system/bin/sh 1055if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 1056 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1057else 1058 log -t recovery "Recovery image already installed" 1059fi 1060""" % { 'boot_size': boot_img.size, 1061 'boot_sha1': boot_img.sha1, 1062 'recovery_size': recovery_img.size, 1063 'recovery_sha1': recovery_img.sha1, 1064 'boot_type': boot_type, 1065 'boot_device': boot_device, 1066 'recovery_type': recovery_type, 1067 'recovery_device': recovery_device, 1068 'bonus_args': bonus_args, 1069 } 1070 1071 # The install script location moved from /system/etc to /system/bin 1072 # in the L release. Parse the init.rc file to find out where the 1073 # target-files expects it to be, and put it there. 1074 sh_location = "etc/install-recovery.sh" 1075 try: 1076 with open(os.path.join(input_dir, "BOOT", "RAMDISK", "init.rc")) as f: 1077 for line in f: 1078 m = re.match("^service flash_recovery /system/(\S+)\s*$", line) 1079 if m: 1080 sh_location = m.group(1) 1081 print "putting script in", sh_location 1082 break 1083 except (OSError, IOError), e: 1084 print "failed to read init.rc: %s" % (e,) 1085 1086 output_sink(sh_location, sh) 1087