common.py revision d421f57d22009e4d9ce71298c2f93636ff9e7d9d
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import errno 16import getopt 17import getpass 18import imp 19import os 20import re 21import shutil 22import subprocess 23import sys 24import tempfile 25import zipfile 26 27# missing in Python 2.4 and before 28if not hasattr(os, "SEEK_SET"): 29 os.SEEK_SET = 0 30 31class Options(object): pass 32OPTIONS = Options() 33OPTIONS.search_path = "out/host/linux-x86" 34OPTIONS.max_image_size = {} 35OPTIONS.verbose = False 36OPTIONS.tempfiles = [] 37OPTIONS.device_specific = None 38OPTIONS.extras = {} 39OPTIONS.mkyaffs2_extra_flags = None 40 41 42# Values for "certificate" in apkcerts that mean special things. 43SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 44 45 46class ExternalError(RuntimeError): pass 47 48 49def Run(args, **kwargs): 50 """Create and return a subprocess.Popen object, printing the command 51 line on the terminal if -v was specified.""" 52 if OPTIONS.verbose: 53 print " running: ", " ".join(args) 54 return subprocess.Popen(args, **kwargs) 55 56 57def LoadMaxSizes(): 58 """Load the maximum allowable images sizes from the input 59 target_files size.""" 60 OPTIONS.max_image_size = {} 61 try: 62 for line in open(os.path.join(OPTIONS.input_tmp, "META", "imagesizes.txt")): 63 pieces = line.split() 64 if len(pieces) != 2: continue 65 image = pieces[0] 66 size = int(pieces[1]) 67 OPTIONS.max_image_size[image + ".img"] = size 68 except IOError, e: 69 if e.errno == errno.ENOENT: 70 pass 71 72 73def LoadMkyaffs2ExtraFlags(): 74 """Load mkyaffs2 extra flags.""" 75 try: 76 fn = os.path.join(OPTIONS.input_tmp, "META", "mkyaffs2-extra-flags.txt"); 77 if os.access(fn, os.F_OK): 78 OPTIONS.mkyaffs2_extra_flags = open(fn).read().rstrip("\n") 79 except IOError, e: 80 if e.errno == errno.ENOENT: 81 pass 82 83 84def BuildAndAddBootableImage(sourcedir, targetname, output_zip): 85 """Take a kernel, cmdline, and ramdisk directory from the input (in 86 'sourcedir'), and turn them into a boot image. Put the boot image 87 into the output zip file under the name 'targetname'. Returns 88 targetname on success or None on failure (if sourcedir does not 89 appear to contain files for the requested image).""" 90 91 print "creating %s..." % (targetname,) 92 93 img = BuildBootableImage(sourcedir) 94 if img is None: 95 return None 96 97 CheckSize(img, targetname) 98 ZipWriteStr(output_zip, targetname, img) 99 return targetname 100 101def BuildBootableImage(sourcedir): 102 """Take a kernel, cmdline, and ramdisk directory from the input (in 103 'sourcedir'), and turn them into a boot image. Return the image 104 data, or None if sourcedir does not appear to contains files for 105 building the requested image.""" 106 107 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 108 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 109 return None 110 111 ramdisk_img = tempfile.NamedTemporaryFile() 112 img = tempfile.NamedTemporaryFile() 113 114 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")], 115 stdout=subprocess.PIPE) 116 p2 = Run(["minigzip"], 117 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 118 119 p2.wait() 120 p1.wait() 121 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 122 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 123 124 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")] 125 126 fn = os.path.join(sourcedir, "cmdline") 127 if os.access(fn, os.F_OK): 128 cmd.append("--cmdline") 129 cmd.append(open(fn).read().rstrip("\n")) 130 131 fn = os.path.join(sourcedir, "base") 132 if os.access(fn, os.F_OK): 133 cmd.append("--base") 134 cmd.append(open(fn).read().rstrip("\n")) 135 136 fn = os.path.join(sourcedir, "pagesize") 137 if os.access(fn, os.F_OK): 138 cmd.append("--pagesize") 139 cmd.append(open(fn).read().rstrip("\n")) 140 141 cmd.extend(["--ramdisk", ramdisk_img.name, 142 "--output", img.name]) 143 144 p = Run(cmd, stdout=subprocess.PIPE) 145 p.communicate() 146 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 147 os.path.basename(sourcedir),) 148 149 img.seek(os.SEEK_SET, 0) 150 data = img.read() 151 152 ramdisk_img.close() 153 img.close() 154 155 return data 156 157 158def AddRecovery(output_zip): 159 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "RECOVERY"), 160 "recovery.img", output_zip) 161 162def AddBoot(output_zip): 163 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "BOOT"), 164 "boot.img", output_zip) 165 166def UnzipTemp(filename, pattern=None): 167 """Unzip the given archive into a temporary directory and return the name.""" 168 169 tmp = tempfile.mkdtemp(prefix="targetfiles-") 170 OPTIONS.tempfiles.append(tmp) 171 cmd = ["unzip", "-o", "-q", filename, "-d", tmp] 172 if pattern is not None: 173 cmd.append(pattern) 174 p = Run(cmd, stdout=subprocess.PIPE) 175 p.communicate() 176 if p.returncode != 0: 177 raise ExternalError("failed to unzip input target-files \"%s\"" % 178 (filename,)) 179 return tmp 180 181 182def GetKeyPasswords(keylist): 183 """Given a list of keys, prompt the user to enter passwords for 184 those which require them. Return a {key: password} dict. password 185 will be None if the key has no password.""" 186 187 no_passwords = [] 188 need_passwords = [] 189 devnull = open("/dev/null", "w+b") 190 for k in sorted(keylist): 191 # We don't need a password for things that aren't really keys. 192 if k in SPECIAL_CERT_STRINGS: 193 no_passwords.append(k) 194 continue 195 196 p = Run(["openssl", "pkcs8", "-in", k+".pk8", 197 "-inform", "DER", "-nocrypt"], 198 stdin=devnull.fileno(), 199 stdout=devnull.fileno(), 200 stderr=subprocess.STDOUT) 201 p.communicate() 202 if p.returncode == 0: 203 no_passwords.append(k) 204 else: 205 need_passwords.append(k) 206 devnull.close() 207 208 key_passwords = PasswordManager().GetPasswords(need_passwords) 209 key_passwords.update(dict.fromkeys(no_passwords, None)) 210 return key_passwords 211 212 213def SignFile(input_name, output_name, key, password, align=None, 214 whole_file=False): 215 """Sign the input_name zip/jar/apk, producing output_name. Use the 216 given key and password (the latter may be None if the key does not 217 have a password. 218 219 If align is an integer > 1, zipalign is run to align stored files in 220 the output zip on 'align'-byte boundaries. 221 222 If whole_file is true, use the "-w" option to SignApk to embed a 223 signature that covers the whole file in the archive comment of the 224 zip file. 225 """ 226 227 if align == 0 or align == 1: 228 align = None 229 230 if align: 231 temp = tempfile.NamedTemporaryFile() 232 sign_name = temp.name 233 else: 234 sign_name = output_name 235 236 cmd = ["java", "-Xmx512m", "-jar", 237 os.path.join(OPTIONS.search_path, "framework", "signapk.jar")] 238 if whole_file: 239 cmd.append("-w") 240 cmd.extend([key + ".x509.pem", key + ".pk8", 241 input_name, sign_name]) 242 243 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 244 if password is not None: 245 password += "\n" 246 p.communicate(password) 247 if p.returncode != 0: 248 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 249 250 if align: 251 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 252 p.communicate() 253 if p.returncode != 0: 254 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 255 temp.close() 256 257 258def CheckSize(data, target): 259 """Check the data string passed against the max size limit, if 260 any, for the given target. Raise exception if the data is too big. 261 Print a warning if the data is nearing the maximum size.""" 262 limit = OPTIONS.max_image_size.get(target, None) 263 if limit is None: return 264 265 size = len(data) 266 pct = float(size) * 100.0 / limit 267 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 268 if pct >= 99.0: 269 raise ExternalError(msg) 270 elif pct >= 95.0: 271 print 272 print " WARNING: ", msg 273 print 274 elif OPTIONS.verbose: 275 print " ", msg 276 277 278def ReadApkCerts(tf_zip): 279 """Given a target_files ZipFile, parse the META/apkcerts.txt file 280 and return a {package: cert} dict.""" 281 certmap = {} 282 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 283 line = line.strip() 284 if not line: continue 285 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 286 r'private_key="(.*)"$', line) 287 if m: 288 name, cert, privkey = m.groups() 289 if cert in SPECIAL_CERT_STRINGS and not privkey: 290 certmap[name] = cert 291 elif (cert.endswith(".x509.pem") and 292 privkey.endswith(".pk8") and 293 cert[:-9] == privkey[:-4]): 294 certmap[name] = cert[:-9] 295 else: 296 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 297 return certmap 298 299 300COMMON_DOCSTRING = """ 301 -p (--path) <dir> 302 Prepend <dir>/bin to the list of places to search for binaries 303 run by this script, and expect to find jars in <dir>/framework. 304 305 -s (--device_specific) <file> 306 Path to the python module containing device-specific 307 releasetools code. 308 309 -x (--extra) <key=value> 310 Add a key/value pair to the 'extras' dict, which device-specific 311 extension code may look at. 312 313 -v (--verbose) 314 Show command lines being executed. 315 316 -h (--help) 317 Display this usage message and exit. 318""" 319 320def Usage(docstring): 321 print docstring.rstrip("\n") 322 print COMMON_DOCSTRING 323 324 325def ParseOptions(argv, 326 docstring, 327 extra_opts="", extra_long_opts=(), 328 extra_option_handler=None): 329 """Parse the options in argv and return any arguments that aren't 330 flags. docstring is the calling module's docstring, to be displayed 331 for errors and -h. extra_opts and extra_long_opts are for flags 332 defined by the caller, which are processed by passing them to 333 extra_option_handler.""" 334 335 try: 336 opts, args = getopt.getopt( 337 argv, "hvp:s:x:" + extra_opts, 338 ["help", "verbose", "path=", "device_specific=", "extra="] + 339 list(extra_long_opts)) 340 except getopt.GetoptError, err: 341 Usage(docstring) 342 print "**", str(err), "**" 343 sys.exit(2) 344 345 path_specified = False 346 347 for o, a in opts: 348 if o in ("-h", "--help"): 349 Usage(docstring) 350 sys.exit() 351 elif o in ("-v", "--verbose"): 352 OPTIONS.verbose = True 353 elif o in ("-p", "--path"): 354 OPTIONS.search_path = a 355 elif o in ("-s", "--device_specific"): 356 OPTIONS.device_specific = a 357 elif o in ("-x", "--extra"): 358 key, value = a.split("=", 1) 359 OPTIONS.extras[key] = value 360 else: 361 if extra_option_handler is None or not extra_option_handler(o, a): 362 assert False, "unknown option \"%s\"" % (o,) 363 364 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 365 os.pathsep + os.environ["PATH"]) 366 367 return args 368 369 370def Cleanup(): 371 for i in OPTIONS.tempfiles: 372 if os.path.isdir(i): 373 shutil.rmtree(i) 374 else: 375 os.remove(i) 376 377 378class PasswordManager(object): 379 def __init__(self): 380 self.editor = os.getenv("EDITOR", None) 381 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 382 383 def GetPasswords(self, items): 384 """Get passwords corresponding to each string in 'items', 385 returning a dict. (The dict may have keys in addition to the 386 values in 'items'.) 387 388 Uses the passwords in $ANDROID_PW_FILE if available, letting the 389 user edit that file to add more needed passwords. If no editor is 390 available, or $ANDROID_PW_FILE isn't define, prompts the user 391 interactively in the ordinary way. 392 """ 393 394 current = self.ReadFile() 395 396 first = True 397 while True: 398 missing = [] 399 for i in items: 400 if i not in current or not current[i]: 401 missing.append(i) 402 # Are all the passwords already in the file? 403 if not missing: return current 404 405 for i in missing: 406 current[i] = "" 407 408 if not first: 409 print "key file %s still missing some passwords." % (self.pwfile,) 410 answer = raw_input("try to edit again? [y]> ").strip() 411 if answer and answer[0] not in 'yY': 412 raise RuntimeError("key passwords unavailable") 413 first = False 414 415 current = self.UpdateAndReadFile(current) 416 417 def PromptResult(self, current): 418 """Prompt the user to enter a value (password) for each key in 419 'current' whose value is fales. Returns a new dict with all the 420 values. 421 """ 422 result = {} 423 for k, v in sorted(current.iteritems()): 424 if v: 425 result[k] = v 426 else: 427 while True: 428 result[k] = getpass.getpass("Enter password for %s key> " 429 % (k,)).strip() 430 if result[k]: break 431 return result 432 433 def UpdateAndReadFile(self, current): 434 if not self.editor or not self.pwfile: 435 return self.PromptResult(current) 436 437 f = open(self.pwfile, "w") 438 os.chmod(self.pwfile, 0600) 439 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 440 f.write("# (Additional spaces are harmless.)\n\n") 441 442 first_line = None 443 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 444 sorted.sort() 445 for i, (_, k, v) in enumerate(sorted): 446 f.write("[[[ %s ]]] %s\n" % (v, k)) 447 if not v and first_line is None: 448 # position cursor on first line with no password. 449 first_line = i + 4 450 f.close() 451 452 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 453 _, _ = p.communicate() 454 455 return self.ReadFile() 456 457 def ReadFile(self): 458 result = {} 459 if self.pwfile is None: return result 460 try: 461 f = open(self.pwfile, "r") 462 for line in f: 463 line = line.strip() 464 if not line or line[0] == '#': continue 465 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 466 if not m: 467 print "failed to parse password file: ", line 468 else: 469 result[m.group(2)] = m.group(1) 470 f.close() 471 except IOError, e: 472 if e.errno != errno.ENOENT: 473 print "error reading password file: ", str(e) 474 return result 475 476 477def ZipWriteStr(zip, filename, data, perms=0644): 478 # use a fixed timestamp so the output is repeatable. 479 zinfo = zipfile.ZipInfo(filename=filename, 480 date_time=(2009, 1, 1, 0, 0, 0)) 481 zinfo.compress_type = zip.compression 482 zinfo.external_attr = perms << 16 483 zip.writestr(zinfo, data) 484 485 486class DeviceSpecificParams(object): 487 module = None 488 def __init__(self, **kwargs): 489 """Keyword arguments to the constructor become attributes of this 490 object, which is passed to all functions in the device-specific 491 module.""" 492 for k, v in kwargs.iteritems(): 493 setattr(self, k, v) 494 self.extras = OPTIONS.extras 495 496 if self.module is None: 497 path = OPTIONS.device_specific 498 if not path: return 499 try: 500 if os.path.isdir(path): 501 info = imp.find_module("releasetools", [path]) 502 else: 503 d, f = os.path.split(path) 504 b, x = os.path.splitext(f) 505 if x == ".py": 506 f = b 507 info = imp.find_module(f, [d]) 508 self.module = imp.load_module("device_specific", *info) 509 except ImportError: 510 print "unable to load device-specific module; assuming none" 511 512 def _DoCall(self, function_name, *args, **kwargs): 513 """Call the named function in the device-specific module, passing 514 the given args and kwargs. The first argument to the call will be 515 the DeviceSpecific object itself. If there is no module, or the 516 module does not define the function, return the value of the 517 'default' kwarg (which itself defaults to None).""" 518 if self.module is None or not hasattr(self.module, function_name): 519 return kwargs.get("default", None) 520 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 521 522 def FullOTA_Assertions(self): 523 """Called after emitting the block of assertions at the top of a 524 full OTA package. Implementations can add whatever additional 525 assertions they like.""" 526 return self._DoCall("FullOTA_Assertions") 527 528 def FullOTA_InstallEnd(self): 529 """Called at the end of full OTA installation; typically this is 530 used to install the image for the device's baseband processor.""" 531 return self._DoCall("FullOTA_InstallEnd") 532 533 def IncrementalOTA_Assertions(self): 534 """Called after emitting the block of assertions at the top of an 535 incremental OTA package. Implementations can add whatever 536 additional assertions they like.""" 537 return self._DoCall("IncrementalOTA_Assertions") 538 539 def IncrementalOTA_VerifyEnd(self): 540 """Called at the end of the verification phase of incremental OTA 541 installation; additional checks can be placed here to abort the 542 script before any changes are made.""" 543 return self._DoCall("IncrementalOTA_VerifyEnd") 544 545 def IncrementalOTA_InstallEnd(self): 546 """Called at the end of incremental OTA installation; typically 547 this is used to install the image for the device's baseband 548 processor.""" 549 return self._DoCall("IncrementalOTA_InstallEnd") 550