common.py revision ea5d7a9de7660bef5b9c68f6372a92d4b2f2f1f6
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import re 22import sha 23import shutil 24import subprocess 25import sys 26import tempfile 27import threading 28import time 29import zipfile 30 31# missing in Python 2.4 and before 32if not hasattr(os, "SEEK_SET"): 33 os.SEEK_SET = 0 34 35class Options(object): pass 36OPTIONS = Options() 37OPTIONS.search_path = "out/host/linux-x86" 38OPTIONS.max_image_size = {} 39OPTIONS.verbose = False 40OPTIONS.tempfiles = [] 41OPTIONS.device_specific = None 42OPTIONS.extras = {} 43OPTIONS.mkyaffs2_extra_flags = None 44 45 46# Values for "certificate" in apkcerts that mean special things. 47SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 48 49 50class ExternalError(RuntimeError): pass 51 52 53def Run(args, **kwargs): 54 """Create and return a subprocess.Popen object, printing the command 55 line on the terminal if -v was specified.""" 56 if OPTIONS.verbose: 57 print " running: ", " ".join(args) 58 return subprocess.Popen(args, **kwargs) 59 60 61def LoadMaxSizes(): 62 """Load the maximum allowable images sizes from the input 63 target_files size.""" 64 OPTIONS.max_image_size = {} 65 try: 66 for line in open(os.path.join(OPTIONS.input_tmp, "META", "imagesizes.txt")): 67 pieces = line.split() 68 if len(pieces) != 2: continue 69 image = pieces[0] 70 size = int(pieces[1]) 71 OPTIONS.max_image_size[image + ".img"] = size 72 except IOError, e: 73 if e.errno == errno.ENOENT: 74 pass 75 76 77def LoadMkyaffs2ExtraFlags(): 78 """Load mkyaffs2 extra flags.""" 79 try: 80 fn = os.path.join(OPTIONS.input_tmp, "META", "mkyaffs2-extra-flags.txt"); 81 if os.access(fn, os.F_OK): 82 OPTIONS.mkyaffs2_extra_flags = open(fn).read().rstrip("\n") 83 except IOError, e: 84 if e.errno == errno.ENOENT: 85 pass 86 87 88def BuildAndAddBootableImage(sourcedir, targetname, output_zip): 89 """Take a kernel, cmdline, and ramdisk directory from the input (in 90 'sourcedir'), and turn them into a boot image. Put the boot image 91 into the output zip file under the name 'targetname'. Returns 92 targetname on success or None on failure (if sourcedir does not 93 appear to contain files for the requested image).""" 94 95 print "creating %s..." % (targetname,) 96 97 img = BuildBootableImage(sourcedir) 98 if img is None: 99 return None 100 101 CheckSize(img, targetname) 102 ZipWriteStr(output_zip, targetname, img) 103 return targetname 104 105def BuildBootableImage(sourcedir): 106 """Take a kernel, cmdline, and ramdisk directory from the input (in 107 'sourcedir'), and turn them into a boot image. Return the image 108 data, or None if sourcedir does not appear to contains files for 109 building the requested image.""" 110 111 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 112 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 113 return None 114 115 ramdisk_img = tempfile.NamedTemporaryFile() 116 img = tempfile.NamedTemporaryFile() 117 118 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")], 119 stdout=subprocess.PIPE) 120 p2 = Run(["minigzip"], 121 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 122 123 p2.wait() 124 p1.wait() 125 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 126 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 127 128 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")] 129 130 fn = os.path.join(sourcedir, "cmdline") 131 if os.access(fn, os.F_OK): 132 cmd.append("--cmdline") 133 cmd.append(open(fn).read().rstrip("\n")) 134 135 fn = os.path.join(sourcedir, "base") 136 if os.access(fn, os.F_OK): 137 cmd.append("--base") 138 cmd.append(open(fn).read().rstrip("\n")) 139 140 fn = os.path.join(sourcedir, "pagesize") 141 if os.access(fn, os.F_OK): 142 cmd.append("--pagesize") 143 cmd.append(open(fn).read().rstrip("\n")) 144 145 cmd.extend(["--ramdisk", ramdisk_img.name, 146 "--output", img.name]) 147 148 p = Run(cmd, stdout=subprocess.PIPE) 149 p.communicate() 150 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 151 os.path.basename(sourcedir),) 152 153 img.seek(os.SEEK_SET, 0) 154 data = img.read() 155 156 ramdisk_img.close() 157 img.close() 158 159 return data 160 161 162def AddRecovery(output_zip): 163 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "RECOVERY"), 164 "recovery.img", output_zip) 165 166def AddBoot(output_zip): 167 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "BOOT"), 168 "boot.img", output_zip) 169 170def UnzipTemp(filename, pattern=None): 171 """Unzip the given archive into a temporary directory and return the name.""" 172 173 tmp = tempfile.mkdtemp(prefix="targetfiles-") 174 OPTIONS.tempfiles.append(tmp) 175 cmd = ["unzip", "-o", "-q", filename, "-d", tmp] 176 if pattern is not None: 177 cmd.append(pattern) 178 p = Run(cmd, stdout=subprocess.PIPE) 179 p.communicate() 180 if p.returncode != 0: 181 raise ExternalError("failed to unzip input target-files \"%s\"" % 182 (filename,)) 183 return tmp 184 185 186def GetKeyPasswords(keylist): 187 """Given a list of keys, prompt the user to enter passwords for 188 those which require them. Return a {key: password} dict. password 189 will be None if the key has no password.""" 190 191 no_passwords = [] 192 need_passwords = [] 193 devnull = open("/dev/null", "w+b") 194 for k in sorted(keylist): 195 # We don't need a password for things that aren't really keys. 196 if k in SPECIAL_CERT_STRINGS: 197 no_passwords.append(k) 198 continue 199 200 p = Run(["openssl", "pkcs8", "-in", k+".pk8", 201 "-inform", "DER", "-nocrypt"], 202 stdin=devnull.fileno(), 203 stdout=devnull.fileno(), 204 stderr=subprocess.STDOUT) 205 p.communicate() 206 if p.returncode == 0: 207 no_passwords.append(k) 208 else: 209 need_passwords.append(k) 210 devnull.close() 211 212 key_passwords = PasswordManager().GetPasswords(need_passwords) 213 key_passwords.update(dict.fromkeys(no_passwords, None)) 214 return key_passwords 215 216 217def SignFile(input_name, output_name, key, password, align=None, 218 whole_file=False): 219 """Sign the input_name zip/jar/apk, producing output_name. Use the 220 given key and password (the latter may be None if the key does not 221 have a password. 222 223 If align is an integer > 1, zipalign is run to align stored files in 224 the output zip on 'align'-byte boundaries. 225 226 If whole_file is true, use the "-w" option to SignApk to embed a 227 signature that covers the whole file in the archive comment of the 228 zip file. 229 """ 230 231 if align == 0 or align == 1: 232 align = None 233 234 if align: 235 temp = tempfile.NamedTemporaryFile() 236 sign_name = temp.name 237 else: 238 sign_name = output_name 239 240 cmd = ["java", "-Xmx512m", "-jar", 241 os.path.join(OPTIONS.search_path, "framework", "signapk.jar")] 242 if whole_file: 243 cmd.append("-w") 244 cmd.extend([key + ".x509.pem", key + ".pk8", 245 input_name, sign_name]) 246 247 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 248 if password is not None: 249 password += "\n" 250 p.communicate(password) 251 if p.returncode != 0: 252 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 253 254 if align: 255 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 256 p.communicate() 257 if p.returncode != 0: 258 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 259 temp.close() 260 261 262def CheckSize(data, target): 263 """Check the data string passed against the max size limit, if 264 any, for the given target. Raise exception if the data is too big. 265 Print a warning if the data is nearing the maximum size.""" 266 limit = OPTIONS.max_image_size.get(target, None) 267 if limit is None: return 268 269 size = len(data) 270 pct = float(size) * 100.0 / limit 271 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 272 if pct >= 99.0: 273 raise ExternalError(msg) 274 elif pct >= 95.0: 275 print 276 print " WARNING: ", msg 277 print 278 elif OPTIONS.verbose: 279 print " ", msg 280 281 282def ReadApkCerts(tf_zip): 283 """Given a target_files ZipFile, parse the META/apkcerts.txt file 284 and return a {package: cert} dict.""" 285 certmap = {} 286 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 287 line = line.strip() 288 if not line: continue 289 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 290 r'private_key="(.*)"$', line) 291 if m: 292 name, cert, privkey = m.groups() 293 if cert in SPECIAL_CERT_STRINGS and not privkey: 294 certmap[name] = cert 295 elif (cert.endswith(".x509.pem") and 296 privkey.endswith(".pk8") and 297 cert[:-9] == privkey[:-4]): 298 certmap[name] = cert[:-9] 299 else: 300 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 301 return certmap 302 303 304COMMON_DOCSTRING = """ 305 -p (--path) <dir> 306 Prepend <dir>/bin to the list of places to search for binaries 307 run by this script, and expect to find jars in <dir>/framework. 308 309 -s (--device_specific) <file> 310 Path to the python module containing device-specific 311 releasetools code. 312 313 -x (--extra) <key=value> 314 Add a key/value pair to the 'extras' dict, which device-specific 315 extension code may look at. 316 317 -v (--verbose) 318 Show command lines being executed. 319 320 -h (--help) 321 Display this usage message and exit. 322""" 323 324def Usage(docstring): 325 print docstring.rstrip("\n") 326 print COMMON_DOCSTRING 327 328 329def ParseOptions(argv, 330 docstring, 331 extra_opts="", extra_long_opts=(), 332 extra_option_handler=None): 333 """Parse the options in argv and return any arguments that aren't 334 flags. docstring is the calling module's docstring, to be displayed 335 for errors and -h. extra_opts and extra_long_opts are for flags 336 defined by the caller, which are processed by passing them to 337 extra_option_handler.""" 338 339 try: 340 opts, args = getopt.getopt( 341 argv, "hvp:s:x:" + extra_opts, 342 ["help", "verbose", "path=", "device_specific=", "extra="] + 343 list(extra_long_opts)) 344 except getopt.GetoptError, err: 345 Usage(docstring) 346 print "**", str(err), "**" 347 sys.exit(2) 348 349 path_specified = False 350 351 for o, a in opts: 352 if o in ("-h", "--help"): 353 Usage(docstring) 354 sys.exit() 355 elif o in ("-v", "--verbose"): 356 OPTIONS.verbose = True 357 elif o in ("-p", "--path"): 358 OPTIONS.search_path = a 359 elif o in ("-s", "--device_specific"): 360 OPTIONS.device_specific = a 361 elif o in ("-x", "--extra"): 362 key, value = a.split("=", 1) 363 OPTIONS.extras[key] = value 364 else: 365 if extra_option_handler is None or not extra_option_handler(o, a): 366 assert False, "unknown option \"%s\"" % (o,) 367 368 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 369 os.pathsep + os.environ["PATH"]) 370 371 return args 372 373 374def Cleanup(): 375 for i in OPTIONS.tempfiles: 376 if os.path.isdir(i): 377 shutil.rmtree(i) 378 else: 379 os.remove(i) 380 381 382class PasswordManager(object): 383 def __init__(self): 384 self.editor = os.getenv("EDITOR", None) 385 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 386 387 def GetPasswords(self, items): 388 """Get passwords corresponding to each string in 'items', 389 returning a dict. (The dict may have keys in addition to the 390 values in 'items'.) 391 392 Uses the passwords in $ANDROID_PW_FILE if available, letting the 393 user edit that file to add more needed passwords. If no editor is 394 available, or $ANDROID_PW_FILE isn't define, prompts the user 395 interactively in the ordinary way. 396 """ 397 398 current = self.ReadFile() 399 400 first = True 401 while True: 402 missing = [] 403 for i in items: 404 if i not in current or not current[i]: 405 missing.append(i) 406 # Are all the passwords already in the file? 407 if not missing: return current 408 409 for i in missing: 410 current[i] = "" 411 412 if not first: 413 print "key file %s still missing some passwords." % (self.pwfile,) 414 answer = raw_input("try to edit again? [y]> ").strip() 415 if answer and answer[0] not in 'yY': 416 raise RuntimeError("key passwords unavailable") 417 first = False 418 419 current = self.UpdateAndReadFile(current) 420 421 def PromptResult(self, current): 422 """Prompt the user to enter a value (password) for each key in 423 'current' whose value is fales. Returns a new dict with all the 424 values. 425 """ 426 result = {} 427 for k, v in sorted(current.iteritems()): 428 if v: 429 result[k] = v 430 else: 431 while True: 432 result[k] = getpass.getpass("Enter password for %s key> " 433 % (k,)).strip() 434 if result[k]: break 435 return result 436 437 def UpdateAndReadFile(self, current): 438 if not self.editor or not self.pwfile: 439 return self.PromptResult(current) 440 441 f = open(self.pwfile, "w") 442 os.chmod(self.pwfile, 0600) 443 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 444 f.write("# (Additional spaces are harmless.)\n\n") 445 446 first_line = None 447 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 448 sorted.sort() 449 for i, (_, k, v) in enumerate(sorted): 450 f.write("[[[ %s ]]] %s\n" % (v, k)) 451 if not v and first_line is None: 452 # position cursor on first line with no password. 453 first_line = i + 4 454 f.close() 455 456 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 457 _, _ = p.communicate() 458 459 return self.ReadFile() 460 461 def ReadFile(self): 462 result = {} 463 if self.pwfile is None: return result 464 try: 465 f = open(self.pwfile, "r") 466 for line in f: 467 line = line.strip() 468 if not line or line[0] == '#': continue 469 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 470 if not m: 471 print "failed to parse password file: ", line 472 else: 473 result[m.group(2)] = m.group(1) 474 f.close() 475 except IOError, e: 476 if e.errno != errno.ENOENT: 477 print "error reading password file: ", str(e) 478 return result 479 480 481def ZipWriteStr(zip, filename, data, perms=0644): 482 # use a fixed timestamp so the output is repeatable. 483 zinfo = zipfile.ZipInfo(filename=filename, 484 date_time=(2009, 1, 1, 0, 0, 0)) 485 zinfo.compress_type = zip.compression 486 zinfo.external_attr = perms << 16 487 zip.writestr(zinfo, data) 488 489 490class DeviceSpecificParams(object): 491 module = None 492 def __init__(self, **kwargs): 493 """Keyword arguments to the constructor become attributes of this 494 object, which is passed to all functions in the device-specific 495 module.""" 496 for k, v in kwargs.iteritems(): 497 setattr(self, k, v) 498 self.extras = OPTIONS.extras 499 500 if self.module is None: 501 path = OPTIONS.device_specific 502 if not path: return 503 try: 504 if os.path.isdir(path): 505 info = imp.find_module("releasetools", [path]) 506 else: 507 d, f = os.path.split(path) 508 b, x = os.path.splitext(f) 509 if x == ".py": 510 f = b 511 info = imp.find_module(f, [d]) 512 self.module = imp.load_module("device_specific", *info) 513 except ImportError: 514 print "unable to load device-specific module; assuming none" 515 516 def _DoCall(self, function_name, *args, **kwargs): 517 """Call the named function in the device-specific module, passing 518 the given args and kwargs. The first argument to the call will be 519 the DeviceSpecific object itself. If there is no module, or the 520 module does not define the function, return the value of the 521 'default' kwarg (which itself defaults to None).""" 522 if self.module is None or not hasattr(self.module, function_name): 523 return kwargs.get("default", None) 524 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 525 526 def FullOTA_Assertions(self): 527 """Called after emitting the block of assertions at the top of a 528 full OTA package. Implementations can add whatever additional 529 assertions they like.""" 530 return self._DoCall("FullOTA_Assertions") 531 532 def FullOTA_InstallEnd(self): 533 """Called at the end of full OTA installation; typically this is 534 used to install the image for the device's baseband processor.""" 535 return self._DoCall("FullOTA_InstallEnd") 536 537 def IncrementalOTA_Assertions(self): 538 """Called after emitting the block of assertions at the top of an 539 incremental OTA package. Implementations can add whatever 540 additional assertions they like.""" 541 return self._DoCall("IncrementalOTA_Assertions") 542 543 def IncrementalOTA_VerifyEnd(self): 544 """Called at the end of the verification phase of incremental OTA 545 installation; additional checks can be placed here to abort the 546 script before any changes are made.""" 547 return self._DoCall("IncrementalOTA_VerifyEnd") 548 549 def IncrementalOTA_InstallEnd(self): 550 """Called at the end of incremental OTA installation; typically 551 this is used to install the image for the device's baseband 552 processor.""" 553 return self._DoCall("IncrementalOTA_InstallEnd") 554 555class File(object): 556 def __init__(self, name, data): 557 self.name = name 558 self.data = data 559 self.size = len(data) 560 self.sha1 = sha.sha(data).hexdigest() 561 562 def WriteToTemp(self): 563 t = tempfile.NamedTemporaryFile() 564 t.write(self.data) 565 t.flush() 566 return t 567 568 def AddToZip(self, z): 569 ZipWriteStr(z, self.name, self.data) 570 571DIFF_PROGRAM_BY_EXT = { 572 ".gz" : "imgdiff", 573 ".zip" : ["imgdiff", "-z"], 574 ".jar" : ["imgdiff", "-z"], 575 ".apk" : ["imgdiff", "-z"], 576 ".img" : "imgdiff", 577 } 578 579class Difference(object): 580 def __init__(self, tf, sf): 581 self.tf = tf 582 self.sf = sf 583 self.patch = None 584 585 def ComputePatch(self): 586 """Compute the patch (as a string of data) needed to turn sf into 587 tf. Returns the same tuple as GetPatch().""" 588 589 tf = self.tf 590 sf = self.sf 591 592 ext = os.path.splitext(tf.name)[1] 593 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 594 595 ttemp = tf.WriteToTemp() 596 stemp = sf.WriteToTemp() 597 598 ext = os.path.splitext(tf.name)[1] 599 600 try: 601 ptemp = tempfile.NamedTemporaryFile() 602 if isinstance(diff_program, list): 603 cmd = copy.copy(diff_program) 604 else: 605 cmd = [diff_program] 606 cmd.append(stemp.name) 607 cmd.append(ttemp.name) 608 cmd.append(ptemp.name) 609 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 610 _, err = p.communicate() 611 if err or p.returncode != 0: 612 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 613 return None 614 diff = ptemp.read() 615 finally: 616 ptemp.close() 617 stemp.close() 618 ttemp.close() 619 620 self.patch = diff 621 return self.tf, self.sf, self.patch 622 623 624 def GetPatch(self): 625 """Return a tuple (target_file, source_file, patch_data). 626 patch_data may be None if ComputePatch hasn't been called, or if 627 computing the patch failed.""" 628 return self.tf, self.sf, self.patch 629 630 631def ComputeDifferences(diffs): 632 """Call ComputePatch on all the Difference objects in 'diffs'.""" 633 print len(diffs), "diffs to compute" 634 635 # Do the largest files first, to try and reduce the long-pole effect. 636 by_size = [(i.tf.size, i) for i in diffs] 637 by_size.sort(reverse=True) 638 by_size = [i[1] for i in by_size] 639 640 lock = threading.Lock() 641 diff_iter = iter(by_size) # accessed under lock 642 643 def worker(): 644 try: 645 lock.acquire() 646 for d in diff_iter: 647 lock.release() 648 start = time.time() 649 d.ComputePatch() 650 dur = time.time() - start 651 lock.acquire() 652 653 tf, sf, patch = d.GetPatch() 654 if sf.name == tf.name: 655 name = tf.name 656 else: 657 name = "%s (%s)" % (tf.name, sf.name) 658 if patch is None: 659 print "patching failed! %s" % (name,) 660 else: 661 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 662 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 663 lock.release() 664 except Exception, e: 665 print e 666 raise 667 668 # start worker threads; wait for them all to finish. 669 threads = [threading.Thread(target=worker) 670 for i in range(OPTIONS.worker_threads)] 671 for th in threads: 672 th.start() 673 while threads: 674 threads.pop().join() 675