common.py revision b4c7d32cbac0ef36062c4e2f348b47e027610eef
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import errno 16import getopt 17import getpass 18import imp 19import os 20import re 21import shutil 22import subprocess 23import sys 24import tempfile 25import zipfile 26 27# missing in Python 2.4 and before 28if not hasattr(os, "SEEK_SET"): 29 os.SEEK_SET = 0 30 31class Options(object): pass 32OPTIONS = Options() 33OPTIONS.search_path = "out/host/linux-x86" 34OPTIONS.max_image_size = {} 35OPTIONS.verbose = False 36OPTIONS.tempfiles = [] 37OPTIONS.device_specific = None 38OPTIONS.extras = {} 39 40 41# Values for "certificate" in apkcerts that mean special things. 42SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 43 44 45class ExternalError(RuntimeError): pass 46 47 48def Run(args, **kwargs): 49 """Create and return a subprocess.Popen object, printing the command 50 line on the terminal if -v was specified.""" 51 if OPTIONS.verbose: 52 print " running: ", " ".join(args) 53 return subprocess.Popen(args, **kwargs) 54 55 56def LoadInfoDict(): 57 """Read and parse the META/misc_info.txt key/value pairs from the 58 input target files and return a dict.""" 59 60 d = {} 61 try: 62 for line in open(os.path.join(OPTIONS.input_tmp, "META", "misc_info.txt")): 63 line = line.strip() 64 if not line or line.startswith("#"): continue 65 k, v = line.split("=", 1) 66 d[k] = v 67 except IOError, e: 68 if e.errno == errno.ENOENT: 69 # ok if misc_info.txt file doesn't exist 70 pass 71 else: 72 raise 73 74 if "fs_type" not in d: info["fs_type"] = "yaffs2" 75 if "partition_type" not in d: info["partition_type"] = "MTD" 76 77 return d 78 79 80def LoadMaxSizes(info): 81 """Load the maximum allowable images sizes from the input 82 target_files. Uses the imagesizes.txt file if it's available 83 (pre-honeycomb target_files), or the more general info dict (which 84 must be passed in) if not.""" 85 OPTIONS.max_image_size = {} 86 try: 87 for line in open(os.path.join(OPTIONS.input_tmp, "META", "imagesizes.txt")): 88 pieces = line.split() 89 if len(pieces) != 2: continue 90 image = pieces[0] 91 size = int(pieces[1]) 92 OPTIONS.max_image_size[image + ".img"] = size 93 except IOError, e: 94 if e.errno == errno.ENOENT: 95 def copy(x, y): 96 if x in info: OPTIONS.max_image_size[x+".img"] = int(info[x+y]) 97 copy("blocksize", "") 98 copy("boot", "_size") 99 copy("recovery", "_size") 100 copy("system", "_size") 101 copy("userdata", "_size") 102 else: 103 raise 104 105 106def BuildAndAddBootableImage(sourcedir, targetname, output_zip): 107 """Take a kernel, cmdline, and ramdisk directory from the input (in 108 'sourcedir'), and turn them into a boot image. Put the boot image 109 into the output zip file under the name 'targetname'. Returns 110 targetname on success or None on failure (if sourcedir does not 111 appear to contain files for the requested image).""" 112 113 print "creating %s..." % (targetname,) 114 115 img = BuildBootableImage(sourcedir) 116 if img is None: 117 return None 118 119 CheckSize(img, targetname) 120 ZipWriteStr(output_zip, targetname, img) 121 return targetname 122 123def BuildBootableImage(sourcedir): 124 """Take a kernel, cmdline, and ramdisk directory from the input (in 125 'sourcedir'), and turn them into a boot image. Return the image 126 data, or None if sourcedir does not appear to contains files for 127 building the requested image.""" 128 129 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 130 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 131 return None 132 133 ramdisk_img = tempfile.NamedTemporaryFile() 134 img = tempfile.NamedTemporaryFile() 135 136 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")], 137 stdout=subprocess.PIPE) 138 p2 = Run(["minigzip"], 139 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 140 141 p2.wait() 142 p1.wait() 143 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 144 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 145 146 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")] 147 148 fn = os.path.join(sourcedir, "cmdline") 149 if os.access(fn, os.F_OK): 150 cmd.append("--cmdline") 151 cmd.append(open(fn).read().rstrip("\n")) 152 153 fn = os.path.join(sourcedir, "base") 154 if os.access(fn, os.F_OK): 155 cmd.append("--base") 156 cmd.append(open(fn).read().rstrip("\n")) 157 158 cmd.extend(["--ramdisk", ramdisk_img.name, 159 "--output", img.name]) 160 161 p = Run(cmd, stdout=subprocess.PIPE) 162 p.communicate() 163 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 164 os.path.basename(sourcedir),) 165 166 img.seek(os.SEEK_SET, 0) 167 data = img.read() 168 169 ramdisk_img.close() 170 img.close() 171 172 return data 173 174 175def AddRecovery(output_zip): 176 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "RECOVERY"), 177 "recovery.img", output_zip) 178 179def AddBoot(output_zip): 180 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "BOOT"), 181 "boot.img", output_zip) 182 183def UnzipTemp(filename, pattern=None): 184 """Unzip the given archive into a temporary directory and return the name.""" 185 186 tmp = tempfile.mkdtemp(prefix="targetfiles-") 187 OPTIONS.tempfiles.append(tmp) 188 cmd = ["unzip", "-o", "-q", filename, "-d", tmp] 189 if pattern is not None: 190 cmd.append(pattern) 191 p = Run(cmd, stdout=subprocess.PIPE) 192 p.communicate() 193 if p.returncode != 0: 194 raise ExternalError("failed to unzip input target-files \"%s\"" % 195 (filename,)) 196 return tmp 197 198 199def GetKeyPasswords(keylist): 200 """Given a list of keys, prompt the user to enter passwords for 201 those which require them. Return a {key: password} dict. password 202 will be None if the key has no password.""" 203 204 no_passwords = [] 205 need_passwords = [] 206 devnull = open("/dev/null", "w+b") 207 for k in sorted(keylist): 208 # We don't need a password for things that aren't really keys. 209 if k in SPECIAL_CERT_STRINGS: 210 no_passwords.append(k) 211 continue 212 213 p = Run(["openssl", "pkcs8", "-in", k+".pk8", 214 "-inform", "DER", "-nocrypt"], 215 stdin=devnull.fileno(), 216 stdout=devnull.fileno(), 217 stderr=subprocess.STDOUT) 218 p.communicate() 219 if p.returncode == 0: 220 no_passwords.append(k) 221 else: 222 need_passwords.append(k) 223 devnull.close() 224 225 key_passwords = PasswordManager().GetPasswords(need_passwords) 226 key_passwords.update(dict.fromkeys(no_passwords, None)) 227 return key_passwords 228 229 230def SignFile(input_name, output_name, key, password, align=None, 231 whole_file=False): 232 """Sign the input_name zip/jar/apk, producing output_name. Use the 233 given key and password (the latter may be None if the key does not 234 have a password. 235 236 If align is an integer > 1, zipalign is run to align stored files in 237 the output zip on 'align'-byte boundaries. 238 239 If whole_file is true, use the "-w" option to SignApk to embed a 240 signature that covers the whole file in the archive comment of the 241 zip file. 242 """ 243 244 if align == 0 or align == 1: 245 align = None 246 247 if align: 248 temp = tempfile.NamedTemporaryFile() 249 sign_name = temp.name 250 else: 251 sign_name = output_name 252 253 cmd = ["java", "-Xmx512m", "-jar", 254 os.path.join(OPTIONS.search_path, "framework", "signapk.jar")] 255 if whole_file: 256 cmd.append("-w") 257 cmd.extend([key + ".x509.pem", key + ".pk8", 258 input_name, sign_name]) 259 260 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 261 if password is not None: 262 password += "\n" 263 p.communicate(password) 264 if p.returncode != 0: 265 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 266 267 if align: 268 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 269 p.communicate() 270 if p.returncode != 0: 271 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 272 temp.close() 273 274 275def CheckSize(data, target): 276 """Check the data string passed against the max size limit, if 277 any, for the given target. Raise exception if the data is too big. 278 Print a warning if the data is nearing the maximum size.""" 279 limit = OPTIONS.max_image_size.get(target, None) 280 if limit is None: return 281 282 size = len(data) 283 pct = float(size) * 100.0 / limit 284 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 285 if pct >= 99.0: 286 raise ExternalError(msg) 287 elif pct >= 95.0: 288 print 289 print " WARNING: ", msg 290 print 291 elif OPTIONS.verbose: 292 print " ", msg 293 294 295def ReadApkCerts(tf_zip): 296 """Given a target_files ZipFile, parse the META/apkcerts.txt file 297 and return a {package: cert} dict.""" 298 certmap = {} 299 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 300 line = line.strip() 301 if not line: continue 302 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 303 r'private_key="(.*)"$', line) 304 if m: 305 name, cert, privkey = m.groups() 306 if cert in SPECIAL_CERT_STRINGS and not privkey: 307 certmap[name] = cert 308 elif (cert.endswith(".x509.pem") and 309 privkey.endswith(".pk8") and 310 cert[:-9] == privkey[:-4]): 311 certmap[name] = cert[:-9] 312 else: 313 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 314 return certmap 315 316 317COMMON_DOCSTRING = """ 318 -p (--path) <dir> 319 Prepend <dir>/bin to the list of places to search for binaries 320 run by this script, and expect to find jars in <dir>/framework. 321 322 -s (--device_specific) <file> 323 Path to the python module containing device-specific 324 releasetools code. 325 326 -x (--extra) <key=value> 327 Add a key/value pair to the 'extras' dict, which device-specific 328 extension code may look at. 329 330 -v (--verbose) 331 Show command lines being executed. 332 333 -h (--help) 334 Display this usage message and exit. 335""" 336 337def Usage(docstring): 338 print docstring.rstrip("\n") 339 print COMMON_DOCSTRING 340 341 342def ParseOptions(argv, 343 docstring, 344 extra_opts="", extra_long_opts=(), 345 extra_option_handler=None): 346 """Parse the options in argv and return any arguments that aren't 347 flags. docstring is the calling module's docstring, to be displayed 348 for errors and -h. extra_opts and extra_long_opts are for flags 349 defined by the caller, which are processed by passing them to 350 extra_option_handler.""" 351 352 try: 353 opts, args = getopt.getopt( 354 argv, "hvp:s:x:" + extra_opts, 355 ["help", "verbose", "path=", "device_specific=", "extra="] + 356 list(extra_long_opts)) 357 except getopt.GetoptError, err: 358 Usage(docstring) 359 print "**", str(err), "**" 360 sys.exit(2) 361 362 path_specified = False 363 364 for o, a in opts: 365 if o in ("-h", "--help"): 366 Usage(docstring) 367 sys.exit() 368 elif o in ("-v", "--verbose"): 369 OPTIONS.verbose = True 370 elif o in ("-p", "--path"): 371 OPTIONS.search_path = a 372 elif o in ("-s", "--device_specific"): 373 OPTIONS.device_specific = a 374 elif o in ("-x", "--extra"): 375 key, value = a.split("=", 1) 376 OPTIONS.extras[key] = value 377 else: 378 if extra_option_handler is None or not extra_option_handler(o, a): 379 assert False, "unknown option \"%s\"" % (o,) 380 381 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 382 os.pathsep + os.environ["PATH"]) 383 384 return args 385 386 387def Cleanup(): 388 for i in OPTIONS.tempfiles: 389 if os.path.isdir(i): 390 shutil.rmtree(i) 391 else: 392 os.remove(i) 393 394 395class PasswordManager(object): 396 def __init__(self): 397 self.editor = os.getenv("EDITOR", None) 398 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 399 400 def GetPasswords(self, items): 401 """Get passwords corresponding to each string in 'items', 402 returning a dict. (The dict may have keys in addition to the 403 values in 'items'.) 404 405 Uses the passwords in $ANDROID_PW_FILE if available, letting the 406 user edit that file to add more needed passwords. If no editor is 407 available, or $ANDROID_PW_FILE isn't define, prompts the user 408 interactively in the ordinary way. 409 """ 410 411 current = self.ReadFile() 412 413 first = True 414 while True: 415 missing = [] 416 for i in items: 417 if i not in current or not current[i]: 418 missing.append(i) 419 # Are all the passwords already in the file? 420 if not missing: return current 421 422 for i in missing: 423 current[i] = "" 424 425 if not first: 426 print "key file %s still missing some passwords." % (self.pwfile,) 427 answer = raw_input("try to edit again? [y]> ").strip() 428 if answer and answer[0] not in 'yY': 429 raise RuntimeError("key passwords unavailable") 430 first = False 431 432 current = self.UpdateAndReadFile(current) 433 434 def PromptResult(self, current): 435 """Prompt the user to enter a value (password) for each key in 436 'current' whose value is fales. Returns a new dict with all the 437 values. 438 """ 439 result = {} 440 for k, v in sorted(current.iteritems()): 441 if v: 442 result[k] = v 443 else: 444 while True: 445 result[k] = getpass.getpass("Enter password for %s key> " 446 % (k,)).strip() 447 if result[k]: break 448 return result 449 450 def UpdateAndReadFile(self, current): 451 if not self.editor or not self.pwfile: 452 return self.PromptResult(current) 453 454 f = open(self.pwfile, "w") 455 os.chmod(self.pwfile, 0600) 456 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 457 f.write("# (Additional spaces are harmless.)\n\n") 458 459 first_line = None 460 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 461 sorted.sort() 462 for i, (_, k, v) in enumerate(sorted): 463 f.write("[[[ %s ]]] %s\n" % (v, k)) 464 if not v and first_line is None: 465 # position cursor on first line with no password. 466 first_line = i + 4 467 f.close() 468 469 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 470 _, _ = p.communicate() 471 472 return self.ReadFile() 473 474 def ReadFile(self): 475 result = {} 476 if self.pwfile is None: return result 477 try: 478 f = open(self.pwfile, "r") 479 for line in f: 480 line = line.strip() 481 if not line or line[0] == '#': continue 482 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 483 if not m: 484 print "failed to parse password file: ", line 485 else: 486 result[m.group(2)] = m.group(1) 487 f.close() 488 except IOError, e: 489 if e.errno != errno.ENOENT: 490 print "error reading password file: ", str(e) 491 return result 492 493 494def ZipWriteStr(zip, filename, data, perms=0644): 495 # use a fixed timestamp so the output is repeatable. 496 zinfo = zipfile.ZipInfo(filename=filename, 497 date_time=(2009, 1, 1, 0, 0, 0)) 498 zinfo.compress_type = zip.compression 499 zinfo.external_attr = perms << 16 500 zip.writestr(zinfo, data) 501 502 503class DeviceSpecificParams(object): 504 module = None 505 def __init__(self, **kwargs): 506 """Keyword arguments to the constructor become attributes of this 507 object, which is passed to all functions in the device-specific 508 module.""" 509 for k, v in kwargs.iteritems(): 510 setattr(self, k, v) 511 self.extras = OPTIONS.extras 512 513 if self.module is None: 514 path = OPTIONS.device_specific 515 if not path: return 516 try: 517 if os.path.isdir(path): 518 info = imp.find_module("releasetools", [path]) 519 else: 520 d, f = os.path.split(path) 521 b, x = os.path.splitext(f) 522 if x == ".py": 523 f = b 524 info = imp.find_module(f, [d]) 525 self.module = imp.load_module("device_specific", *info) 526 except ImportError: 527 print "unable to load device-specific module; assuming none" 528 529 def _DoCall(self, function_name, *args, **kwargs): 530 """Call the named function in the device-specific module, passing 531 the given args and kwargs. The first argument to the call will be 532 the DeviceSpecific object itself. If there is no module, or the 533 module does not define the function, return the value of the 534 'default' kwarg (which itself defaults to None).""" 535 if self.module is None or not hasattr(self.module, function_name): 536 return kwargs.get("default", None) 537 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 538 539 def FullOTA_Assertions(self): 540 """Called after emitting the block of assertions at the top of a 541 full OTA package. Implementations can add whatever additional 542 assertions they like.""" 543 return self._DoCall("FullOTA_Assertions") 544 545 def FullOTA_InstallEnd(self): 546 """Called at the end of full OTA installation; typically this is 547 used to install the image for the device's baseband processor.""" 548 return self._DoCall("FullOTA_InstallEnd") 549 550 def IncrementalOTA_Assertions(self): 551 """Called after emitting the block of assertions at the top of an 552 incremental OTA package. Implementations can add whatever 553 additional assertions they like.""" 554 return self._DoCall("IncrementalOTA_Assertions") 555 556 def IncrementalOTA_VerifyEnd(self): 557 """Called at the end of the verification phase of incremental OTA 558 installation; additional checks can be placed here to abort the 559 script before any changes are made.""" 560 return self._DoCall("IncrementalOTA_VerifyEnd") 561 562 def IncrementalOTA_InstallEnd(self): 563 """Called at the end of incremental OTA installation; typically 564 this is used to install the image for the device's baseband 565 processor.""" 566 return self._DoCall("IncrementalOTA_InstallEnd") 567