common.py revision 486de126e398938b4f38d3e8065ad520244fe5e3
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import re 22import sha 23import shutil 24import subprocess 25import sys 26import tempfile 27import threading 28import time 29import zipfile 30 31# missing in Python 2.4 and before 32if not hasattr(os, "SEEK_SET"): 33 os.SEEK_SET = 0 34 35class Options(object): pass 36OPTIONS = Options() 37OPTIONS.search_path = "out/host/linux-x86" 38OPTIONS.max_image_size = {} 39OPTIONS.verbose = False 40OPTIONS.tempfiles = [] 41OPTIONS.device_specific = None 42OPTIONS.extras = {} 43OPTIONS.mkyaffs2_extra_flags = None 44OPTIONS.info_dict = None 45 46 47# Values for "certificate" in apkcerts that mean special things. 48SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 49 50 51class ExternalError(RuntimeError): pass 52 53 54def Run(args, **kwargs): 55 """Create and return a subprocess.Popen object, printing the command 56 line on the terminal if -v was specified.""" 57 if OPTIONS.verbose: 58 print " running: ", " ".join(args) 59 return subprocess.Popen(args, **kwargs) 60 61 62def LoadInfoDict(): 63 """Read and parse the META/misc_info.txt key/value pairs from the 64 input target files and return a dict.""" 65 66 d = {} 67 try: 68 for line in open(os.path.join(OPTIONS.input_tmp, "META", "misc_info.txt")): 69 line = line.strip() 70 if not line or line.startswith("#"): continue 71 k, v = line.split("=", 1) 72 d[k] = v 73 except IOError, e: 74 if e.errno == errno.ENOENT: 75 # ok if misc_info.txt file doesn't exist 76 pass 77 else: 78 raise 79 80 if "fs_type" not in d: info["fs_type"] = "yaffs2" 81 if "partition_type" not in d: info["partition_type"] = "MTD" 82 83 return d 84 85 86def LoadMaxSizes(info): 87 """Load the maximum allowable images sizes from the input 88 target_files. Uses the imagesizes.txt file if it's available 89 (pre-gingerbread target_files), or the more general info dict (which 90 must be passed in) if not.""" 91 OPTIONS.max_image_size = {} 92 try: 93 for line in open(os.path.join(OPTIONS.input_tmp, "META", "imagesizes.txt")): 94 pieces = line.split() 95 if len(pieces) != 2: continue 96 image = pieces[0] 97 size = int(pieces[1], 0) 98 OPTIONS.max_image_size[image + ".img"] = size 99 except IOError, e: 100 if e.errno == errno.ENOENT: 101 def copy(x, y): 102 if x+y in info: OPTIONS.max_image_size[x+".img"] = int(info[x+y], 0) 103 copy("blocksize", "") 104 copy("boot", "_size") 105 copy("recovery", "_size") 106 copy("system", "_size") 107 copy("userdata", "_size") 108 else: 109 raise 110 111 112def LoadMkyaffs2ExtraFlags(): 113 """Load mkyaffs2 extra flags.""" 114 try: 115 fn = os.path.join(OPTIONS.input_tmp, "META", "mkyaffs2-extra-flags.txt"); 116 if os.access(fn, os.F_OK): 117 OPTIONS.mkyaffs2_extra_flags = open(fn).read().rstrip("\n") 118 except IOError, e: 119 if e.errno == errno.ENOENT: 120 pass 121 122 123def BuildAndAddBootableImage(sourcedir, targetname, output_zip): 124 """Take a kernel, cmdline, and ramdisk directory from the input (in 125 'sourcedir'), and turn them into a boot image. Put the boot image 126 into the output zip file under the name 'targetname'. Returns 127 targetname on success or None on failure (if sourcedir does not 128 appear to contain files for the requested image).""" 129 130 print "creating %s..." % (targetname,) 131 132 img = BuildBootableImage(sourcedir) 133 if img is None: 134 return None 135 136 CheckSize(img, targetname) 137 ZipWriteStr(output_zip, targetname, img) 138 return targetname 139 140def BuildBootableImage(sourcedir): 141 """Take a kernel, cmdline, and ramdisk directory from the input (in 142 'sourcedir'), and turn them into a boot image. Return the image 143 data, or None if sourcedir does not appear to contains files for 144 building the requested image.""" 145 146 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 147 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 148 return None 149 150 ramdisk_img = tempfile.NamedTemporaryFile() 151 img = tempfile.NamedTemporaryFile() 152 153 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")], 154 stdout=subprocess.PIPE) 155 p2 = Run(["minigzip"], 156 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 157 158 p2.wait() 159 p1.wait() 160 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 161 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 162 163 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")] 164 165 fn = os.path.join(sourcedir, "cmdline") 166 if os.access(fn, os.F_OK): 167 cmd.append("--cmdline") 168 cmd.append(open(fn).read().rstrip("\n")) 169 170 fn = os.path.join(sourcedir, "base") 171 if os.access(fn, os.F_OK): 172 cmd.append("--base") 173 cmd.append(open(fn).read().rstrip("\n")) 174 175 fn = os.path.join(sourcedir, "pagesize") 176 if os.access(fn, os.F_OK): 177 cmd.append("--pagesize") 178 cmd.append(open(fn).read().rstrip("\n")) 179 180 cmd.extend(["--ramdisk", ramdisk_img.name, 181 "--output", img.name]) 182 183 p = Run(cmd, stdout=subprocess.PIPE) 184 p.communicate() 185 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 186 os.path.basename(sourcedir),) 187 188 img.seek(os.SEEK_SET, 0) 189 data = img.read() 190 191 ramdisk_img.close() 192 img.close() 193 194 return data 195 196 197def AddRecovery(output_zip): 198 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "RECOVERY"), 199 "recovery.img", output_zip) 200 201def AddBoot(output_zip): 202 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "BOOT"), 203 "boot.img", output_zip) 204 205def UnzipTemp(filename, pattern=None): 206 """Unzip the given archive into a temporary directory and return the name.""" 207 208 tmp = tempfile.mkdtemp(prefix="targetfiles-") 209 OPTIONS.tempfiles.append(tmp) 210 cmd = ["unzip", "-o", "-q", filename, "-d", tmp] 211 if pattern is not None: 212 cmd.append(pattern) 213 p = Run(cmd, stdout=subprocess.PIPE) 214 p.communicate() 215 if p.returncode != 0: 216 raise ExternalError("failed to unzip input target-files \"%s\"" % 217 (filename,)) 218 return tmp 219 220 221def GetKeyPasswords(keylist): 222 """Given a list of keys, prompt the user to enter passwords for 223 those which require them. Return a {key: password} dict. password 224 will be None if the key has no password.""" 225 226 no_passwords = [] 227 need_passwords = [] 228 devnull = open("/dev/null", "w+b") 229 for k in sorted(keylist): 230 # We don't need a password for things that aren't really keys. 231 if k in SPECIAL_CERT_STRINGS: 232 no_passwords.append(k) 233 continue 234 235 p = Run(["openssl", "pkcs8", "-in", k+".pk8", 236 "-inform", "DER", "-nocrypt"], 237 stdin=devnull.fileno(), 238 stdout=devnull.fileno(), 239 stderr=subprocess.STDOUT) 240 p.communicate() 241 if p.returncode == 0: 242 no_passwords.append(k) 243 else: 244 need_passwords.append(k) 245 devnull.close() 246 247 key_passwords = PasswordManager().GetPasswords(need_passwords) 248 key_passwords.update(dict.fromkeys(no_passwords, None)) 249 return key_passwords 250 251 252def SignFile(input_name, output_name, key, password, align=None, 253 whole_file=False): 254 """Sign the input_name zip/jar/apk, producing output_name. Use the 255 given key and password (the latter may be None if the key does not 256 have a password. 257 258 If align is an integer > 1, zipalign is run to align stored files in 259 the output zip on 'align'-byte boundaries. 260 261 If whole_file is true, use the "-w" option to SignApk to embed a 262 signature that covers the whole file in the archive comment of the 263 zip file. 264 """ 265 266 if align == 0 or align == 1: 267 align = None 268 269 if align: 270 temp = tempfile.NamedTemporaryFile() 271 sign_name = temp.name 272 else: 273 sign_name = output_name 274 275 cmd = ["java", "-Xmx512m", "-jar", 276 os.path.join(OPTIONS.search_path, "framework", "signapk.jar")] 277 if whole_file: 278 cmd.append("-w") 279 cmd.extend([key + ".x509.pem", key + ".pk8", 280 input_name, sign_name]) 281 282 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 283 if password is not None: 284 password += "\n" 285 p.communicate(password) 286 if p.returncode != 0: 287 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 288 289 if align: 290 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 291 p.communicate() 292 if p.returncode != 0: 293 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 294 temp.close() 295 296 297def CheckSize(data, target): 298 """Check the data string passed against the max size limit, if 299 any, for the given target. Raise exception if the data is too big. 300 Print a warning if the data is nearing the maximum size.""" 301 302 fs_type = OPTIONS.info_dict.get("fs_type", None) 303 if not fs_type: return 304 305 limit = OPTIONS.max_image_size.get(target, None) 306 if limit is None: return 307 308 if fs_type == "yaffs2": 309 # image size should be increased by 1/64th to account for the 310 # spare area (64 bytes per 2k page) 311 limit = limit / 2048 * (2048+64) 312 size = len(data) 313 pct = float(size) * 100.0 / limit 314 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 315 if pct >= 99.0: 316 raise ExternalError(msg) 317 elif pct >= 95.0: 318 print 319 print " WARNING: ", msg 320 print 321 elif OPTIONS.verbose: 322 print " ", msg 323 324 325def ReadApkCerts(tf_zip): 326 """Given a target_files ZipFile, parse the META/apkcerts.txt file 327 and return a {package: cert} dict.""" 328 certmap = {} 329 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 330 line = line.strip() 331 if not line: continue 332 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 333 r'private_key="(.*)"$', line) 334 if m: 335 name, cert, privkey = m.groups() 336 if cert in SPECIAL_CERT_STRINGS and not privkey: 337 certmap[name] = cert 338 elif (cert.endswith(".x509.pem") and 339 privkey.endswith(".pk8") and 340 cert[:-9] == privkey[:-4]): 341 certmap[name] = cert[:-9] 342 else: 343 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 344 return certmap 345 346 347COMMON_DOCSTRING = """ 348 -p (--path) <dir> 349 Prepend <dir>/bin to the list of places to search for binaries 350 run by this script, and expect to find jars in <dir>/framework. 351 352 -s (--device_specific) <file> 353 Path to the python module containing device-specific 354 releasetools code. 355 356 -x (--extra) <key=value> 357 Add a key/value pair to the 'extras' dict, which device-specific 358 extension code may look at. 359 360 -v (--verbose) 361 Show command lines being executed. 362 363 -h (--help) 364 Display this usage message and exit. 365""" 366 367def Usage(docstring): 368 print docstring.rstrip("\n") 369 print COMMON_DOCSTRING 370 371 372def ParseOptions(argv, 373 docstring, 374 extra_opts="", extra_long_opts=(), 375 extra_option_handler=None): 376 """Parse the options in argv and return any arguments that aren't 377 flags. docstring is the calling module's docstring, to be displayed 378 for errors and -h. extra_opts and extra_long_opts are for flags 379 defined by the caller, which are processed by passing them to 380 extra_option_handler.""" 381 382 try: 383 opts, args = getopt.getopt( 384 argv, "hvp:s:x:" + extra_opts, 385 ["help", "verbose", "path=", "device_specific=", "extra="] + 386 list(extra_long_opts)) 387 except getopt.GetoptError, err: 388 Usage(docstring) 389 print "**", str(err), "**" 390 sys.exit(2) 391 392 path_specified = False 393 394 for o, a in opts: 395 if o in ("-h", "--help"): 396 Usage(docstring) 397 sys.exit() 398 elif o in ("-v", "--verbose"): 399 OPTIONS.verbose = True 400 elif o in ("-p", "--path"): 401 OPTIONS.search_path = a 402 elif o in ("-s", "--device_specific"): 403 OPTIONS.device_specific = a 404 elif o in ("-x", "--extra"): 405 key, value = a.split("=", 1) 406 OPTIONS.extras[key] = value 407 else: 408 if extra_option_handler is None or not extra_option_handler(o, a): 409 assert False, "unknown option \"%s\"" % (o,) 410 411 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 412 os.pathsep + os.environ["PATH"]) 413 414 return args 415 416 417def Cleanup(): 418 for i in OPTIONS.tempfiles: 419 if os.path.isdir(i): 420 shutil.rmtree(i) 421 else: 422 os.remove(i) 423 424 425class PasswordManager(object): 426 def __init__(self): 427 self.editor = os.getenv("EDITOR", None) 428 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 429 430 def GetPasswords(self, items): 431 """Get passwords corresponding to each string in 'items', 432 returning a dict. (The dict may have keys in addition to the 433 values in 'items'.) 434 435 Uses the passwords in $ANDROID_PW_FILE if available, letting the 436 user edit that file to add more needed passwords. If no editor is 437 available, or $ANDROID_PW_FILE isn't define, prompts the user 438 interactively in the ordinary way. 439 """ 440 441 current = self.ReadFile() 442 443 first = True 444 while True: 445 missing = [] 446 for i in items: 447 if i not in current or not current[i]: 448 missing.append(i) 449 # Are all the passwords already in the file? 450 if not missing: return current 451 452 for i in missing: 453 current[i] = "" 454 455 if not first: 456 print "key file %s still missing some passwords." % (self.pwfile,) 457 answer = raw_input("try to edit again? [y]> ").strip() 458 if answer and answer[0] not in 'yY': 459 raise RuntimeError("key passwords unavailable") 460 first = False 461 462 current = self.UpdateAndReadFile(current) 463 464 def PromptResult(self, current): 465 """Prompt the user to enter a value (password) for each key in 466 'current' whose value is fales. Returns a new dict with all the 467 values. 468 """ 469 result = {} 470 for k, v in sorted(current.iteritems()): 471 if v: 472 result[k] = v 473 else: 474 while True: 475 result[k] = getpass.getpass("Enter password for %s key> " 476 % (k,)).strip() 477 if result[k]: break 478 return result 479 480 def UpdateAndReadFile(self, current): 481 if not self.editor or not self.pwfile: 482 return self.PromptResult(current) 483 484 f = open(self.pwfile, "w") 485 os.chmod(self.pwfile, 0600) 486 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 487 f.write("# (Additional spaces are harmless.)\n\n") 488 489 first_line = None 490 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 491 sorted.sort() 492 for i, (_, k, v) in enumerate(sorted): 493 f.write("[[[ %s ]]] %s\n" % (v, k)) 494 if not v and first_line is None: 495 # position cursor on first line with no password. 496 first_line = i + 4 497 f.close() 498 499 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 500 _, _ = p.communicate() 501 502 return self.ReadFile() 503 504 def ReadFile(self): 505 result = {} 506 if self.pwfile is None: return result 507 try: 508 f = open(self.pwfile, "r") 509 for line in f: 510 line = line.strip() 511 if not line or line[0] == '#': continue 512 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 513 if not m: 514 print "failed to parse password file: ", line 515 else: 516 result[m.group(2)] = m.group(1) 517 f.close() 518 except IOError, e: 519 if e.errno != errno.ENOENT: 520 print "error reading password file: ", str(e) 521 return result 522 523 524def ZipWriteStr(zip, filename, data, perms=0644): 525 # use a fixed timestamp so the output is repeatable. 526 zinfo = zipfile.ZipInfo(filename=filename, 527 date_time=(2009, 1, 1, 0, 0, 0)) 528 zinfo.compress_type = zip.compression 529 zinfo.external_attr = perms << 16 530 zip.writestr(zinfo, data) 531 532 533class DeviceSpecificParams(object): 534 module = None 535 def __init__(self, **kwargs): 536 """Keyword arguments to the constructor become attributes of this 537 object, which is passed to all functions in the device-specific 538 module.""" 539 for k, v in kwargs.iteritems(): 540 setattr(self, k, v) 541 self.extras = OPTIONS.extras 542 543 if self.module is None: 544 path = OPTIONS.device_specific 545 if not path: return 546 try: 547 if os.path.isdir(path): 548 info = imp.find_module("releasetools", [path]) 549 else: 550 d, f = os.path.split(path) 551 b, x = os.path.splitext(f) 552 if x == ".py": 553 f = b 554 info = imp.find_module(f, [d]) 555 self.module = imp.load_module("device_specific", *info) 556 except ImportError: 557 print "unable to load device-specific module; assuming none" 558 559 def _DoCall(self, function_name, *args, **kwargs): 560 """Call the named function in the device-specific module, passing 561 the given args and kwargs. The first argument to the call will be 562 the DeviceSpecific object itself. If there is no module, or the 563 module does not define the function, return the value of the 564 'default' kwarg (which itself defaults to None).""" 565 if self.module is None or not hasattr(self.module, function_name): 566 return kwargs.get("default", None) 567 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 568 569 def FullOTA_Assertions(self): 570 """Called after emitting the block of assertions at the top of a 571 full OTA package. Implementations can add whatever additional 572 assertions they like.""" 573 return self._DoCall("FullOTA_Assertions") 574 575 def FullOTA_InstallEnd(self): 576 """Called at the end of full OTA installation; typically this is 577 used to install the image for the device's baseband processor.""" 578 return self._DoCall("FullOTA_InstallEnd") 579 580 def IncrementalOTA_Assertions(self): 581 """Called after emitting the block of assertions at the top of an 582 incremental OTA package. Implementations can add whatever 583 additional assertions they like.""" 584 return self._DoCall("IncrementalOTA_Assertions") 585 586 def IncrementalOTA_VerifyEnd(self): 587 """Called at the end of the verification phase of incremental OTA 588 installation; additional checks can be placed here to abort the 589 script before any changes are made.""" 590 return self._DoCall("IncrementalOTA_VerifyEnd") 591 592 def IncrementalOTA_InstallEnd(self): 593 """Called at the end of incremental OTA installation; typically 594 this is used to install the image for the device's baseband 595 processor.""" 596 return self._DoCall("IncrementalOTA_InstallEnd") 597 598class File(object): 599 def __init__(self, name, data): 600 self.name = name 601 self.data = data 602 self.size = len(data) 603 self.sha1 = sha.sha(data).hexdigest() 604 605 def WriteToTemp(self): 606 t = tempfile.NamedTemporaryFile() 607 t.write(self.data) 608 t.flush() 609 return t 610 611 def AddToZip(self, z): 612 ZipWriteStr(z, self.name, self.data) 613 614DIFF_PROGRAM_BY_EXT = { 615 ".gz" : "imgdiff", 616 ".zip" : ["imgdiff", "-z"], 617 ".jar" : ["imgdiff", "-z"], 618 ".apk" : ["imgdiff", "-z"], 619 ".img" : "imgdiff", 620 } 621 622class Difference(object): 623 def __init__(self, tf, sf): 624 self.tf = tf 625 self.sf = sf 626 self.patch = None 627 628 def ComputePatch(self): 629 """Compute the patch (as a string of data) needed to turn sf into 630 tf. Returns the same tuple as GetPatch().""" 631 632 tf = self.tf 633 sf = self.sf 634 635 ext = os.path.splitext(tf.name)[1] 636 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 637 638 ttemp = tf.WriteToTemp() 639 stemp = sf.WriteToTemp() 640 641 ext = os.path.splitext(tf.name)[1] 642 643 try: 644 ptemp = tempfile.NamedTemporaryFile() 645 if isinstance(diff_program, list): 646 cmd = copy.copy(diff_program) 647 else: 648 cmd = [diff_program] 649 cmd.append(stemp.name) 650 cmd.append(ttemp.name) 651 cmd.append(ptemp.name) 652 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 653 _, err = p.communicate() 654 if err or p.returncode != 0: 655 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 656 return None 657 diff = ptemp.read() 658 finally: 659 ptemp.close() 660 stemp.close() 661 ttemp.close() 662 663 self.patch = diff 664 return self.tf, self.sf, self.patch 665 666 667 def GetPatch(self): 668 """Return a tuple (target_file, source_file, patch_data). 669 patch_data may be None if ComputePatch hasn't been called, or if 670 computing the patch failed.""" 671 return self.tf, self.sf, self.patch 672 673 674def ComputeDifferences(diffs): 675 """Call ComputePatch on all the Difference objects in 'diffs'.""" 676 print len(diffs), "diffs to compute" 677 678 # Do the largest files first, to try and reduce the long-pole effect. 679 by_size = [(i.tf.size, i) for i in diffs] 680 by_size.sort(reverse=True) 681 by_size = [i[1] for i in by_size] 682 683 lock = threading.Lock() 684 diff_iter = iter(by_size) # accessed under lock 685 686 def worker(): 687 try: 688 lock.acquire() 689 for d in diff_iter: 690 lock.release() 691 start = time.time() 692 d.ComputePatch() 693 dur = time.time() - start 694 lock.acquire() 695 696 tf, sf, patch = d.GetPatch() 697 if sf.name == tf.name: 698 name = tf.name 699 else: 700 name = "%s (%s)" % (tf.name, sf.name) 701 if patch is None: 702 print "patching failed! %s" % (name,) 703 else: 704 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 705 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 706 lock.release() 707 except Exception, e: 708 print e 709 raise 710 711 # start worker threads; wait for them all to finish. 712 threads = [threading.Thread(target=worker) 713 for i in range(OPTIONS.worker_threads)] 714 for th in threads: 715 th.start() 716 while threads: 717 threads.pop().join() 718