common.py revision 75f1736469d6ebfb51f7d2abfa74039d9be54d8d
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import errno 16import getopt 17import getpass 18import imp 19import os 20import re 21import shutil 22import subprocess 23import sys 24import tempfile 25import zipfile 26 27# missing in Python 2.4 and before 28if not hasattr(os, "SEEK_SET"): 29 os.SEEK_SET = 0 30 31class Options(object): pass 32OPTIONS = Options() 33OPTIONS.search_path = "out/host/linux-x86" 34OPTIONS.max_image_size = {} 35OPTIONS.verbose = False 36OPTIONS.tempfiles = [] 37OPTIONS.device_specific = None 38OPTIONS.extras = {} 39 40class ExternalError(RuntimeError): pass 41 42 43def Run(args, **kwargs): 44 """Create and return a subprocess.Popen object, printing the command 45 line on the terminal if -v was specified.""" 46 if OPTIONS.verbose: 47 print " running: ", " ".join(args) 48 return subprocess.Popen(args, **kwargs) 49 50 51def LoadMaxSizes(): 52 """Load the maximum allowable images sizes from the input 53 target_files size.""" 54 OPTIONS.max_image_size = {} 55 try: 56 for line in open(os.path.join(OPTIONS.input_tmp, "META", "imagesizes.txt")): 57 pieces = line.split() 58 if len(pieces) != 2: continue 59 image = pieces[0] 60 size = int(pieces[1]) 61 OPTIONS.max_image_size[image + ".img"] = size 62 except IOError, e: 63 if e.errno == errno.ENOENT: 64 pass 65 66 67def BuildAndAddBootableImage(sourcedir, targetname, output_zip): 68 """Take a kernel, cmdline, and ramdisk directory from the input (in 69 'sourcedir'), and turn them into a boot image. Put the boot image 70 into the output zip file under the name 'targetname'. Returns 71 targetname on success or None on failure (if sourcedir does not 72 appear to contain files for the requested image).""" 73 74 print "creating %s..." % (targetname,) 75 76 img = BuildBootableImage(sourcedir) 77 if img is None: 78 return None 79 80 CheckSize(img, targetname) 81 ZipWriteStr(output_zip, targetname, img) 82 return targetname 83 84def BuildBootableImage(sourcedir): 85 """Take a kernel, cmdline, and ramdisk directory from the input (in 86 'sourcedir'), and turn them into a boot image. Return the image 87 data, or None if sourcedir does not appear to contains files for 88 building the requested image.""" 89 90 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 91 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 92 return None 93 94 ramdisk_img = tempfile.NamedTemporaryFile() 95 img = tempfile.NamedTemporaryFile() 96 97 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")], 98 stdout=subprocess.PIPE) 99 p2 = Run(["minigzip"], 100 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 101 102 p2.wait() 103 p1.wait() 104 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 105 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 106 107 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")] 108 109 fn = os.path.join(sourcedir, "cmdline") 110 if os.access(fn, os.F_OK): 111 cmd.append("--cmdline") 112 cmd.append(open(fn).read().rstrip("\n")) 113 114 fn = os.path.join(sourcedir, "base") 115 if os.access(fn, os.F_OK): 116 cmd.append("--base") 117 cmd.append(open(fn).read().rstrip("\n")) 118 119 cmd.extend(["--ramdisk", ramdisk_img.name, 120 "--output", img.name]) 121 122 p = Run(cmd, stdout=subprocess.PIPE) 123 p.communicate() 124 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 125 os.path.basename(sourcedir),) 126 127 img.seek(os.SEEK_SET, 0) 128 data = img.read() 129 130 ramdisk_img.close() 131 img.close() 132 133 return data 134 135 136def AddRecovery(output_zip): 137 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "RECOVERY"), 138 "recovery.img", output_zip) 139 140def AddBoot(output_zip): 141 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "BOOT"), 142 "boot.img", output_zip) 143 144def UnzipTemp(filename, pattern=None): 145 """Unzip the given archive into a temporary directory and return the name.""" 146 147 tmp = tempfile.mkdtemp(prefix="targetfiles-") 148 OPTIONS.tempfiles.append(tmp) 149 cmd = ["unzip", "-o", "-q", filename, "-d", tmp] 150 if pattern is not None: 151 cmd.append(pattern) 152 p = Run(cmd, stdout=subprocess.PIPE) 153 p.communicate() 154 if p.returncode != 0: 155 raise ExternalError("failed to unzip input target-files \"%s\"" % 156 (filename,)) 157 return tmp 158 159 160def GetKeyPasswords(keylist): 161 """Given a list of keys, prompt the user to enter passwords for 162 those which require them. Return a {key: password} dict. password 163 will be None if the key has no password.""" 164 165 no_passwords = [] 166 need_passwords = [] 167 devnull = open("/dev/null", "w+b") 168 for k in sorted(keylist): 169 # An empty-string key is used to mean don't re-sign this package. 170 # Obviously we don't need a password for this non-key. 171 if not k: 172 no_passwords.append(k) 173 continue 174 175 p = Run(["openssl", "pkcs8", "-in", k+".pk8", 176 "-inform", "DER", "-nocrypt"], 177 stdin=devnull.fileno(), 178 stdout=devnull.fileno(), 179 stderr=subprocess.STDOUT) 180 p.communicate() 181 if p.returncode == 0: 182 no_passwords.append(k) 183 else: 184 need_passwords.append(k) 185 devnull.close() 186 187 key_passwords = PasswordManager().GetPasswords(need_passwords) 188 key_passwords.update(dict.fromkeys(no_passwords, None)) 189 return key_passwords 190 191 192def SignFile(input_name, output_name, key, password, align=None, 193 whole_file=False): 194 """Sign the input_name zip/jar/apk, producing output_name. Use the 195 given key and password (the latter may be None if the key does not 196 have a password. 197 198 If align is an integer > 1, zipalign is run to align stored files in 199 the output zip on 'align'-byte boundaries. 200 201 If whole_file is true, use the "-w" option to SignApk to embed a 202 signature that covers the whole file in the archive comment of the 203 zip file. 204 """ 205 206 if align == 0 or align == 1: 207 align = None 208 209 if align: 210 temp = tempfile.NamedTemporaryFile() 211 sign_name = temp.name 212 else: 213 sign_name = output_name 214 215 cmd = ["java", "-Xmx512m", "-jar", 216 os.path.join(OPTIONS.search_path, "framework", "signapk.jar")] 217 if whole_file: 218 cmd.append("-w") 219 cmd.extend([key + ".x509.pem", key + ".pk8", 220 input_name, sign_name]) 221 222 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 223 if password is not None: 224 password += "\n" 225 p.communicate(password) 226 if p.returncode != 0: 227 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 228 229 if align: 230 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 231 p.communicate() 232 if p.returncode != 0: 233 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 234 temp.close() 235 236 237def CheckSize(data, target): 238 """Check the data string passed against the max size limit, if 239 any, for the given target. Raise exception if the data is too big. 240 Print a warning if the data is nearing the maximum size.""" 241 limit = OPTIONS.max_image_size.get(target, None) 242 if limit is None: return 243 244 size = len(data) 245 pct = float(size) * 100.0 / limit 246 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 247 if pct >= 99.0: 248 raise ExternalError(msg) 249 elif pct >= 95.0: 250 print 251 print " WARNING: ", msg 252 print 253 elif OPTIONS.verbose: 254 print " ", msg 255 256 257COMMON_DOCSTRING = """ 258 -p (--path) <dir> 259 Prepend <dir>/bin to the list of places to search for binaries 260 run by this script, and expect to find jars in <dir>/framework. 261 262 -s (--device_specific) <file> 263 Path to the python module containing device-specific 264 releasetools code. 265 266 -x (--extra) <key=value> 267 Add a key/value pair to the 'extras' dict, which device-specific 268 extension code may look at. 269 270 -v (--verbose) 271 Show command lines being executed. 272 273 -h (--help) 274 Display this usage message and exit. 275""" 276 277def Usage(docstring): 278 print docstring.rstrip("\n") 279 print COMMON_DOCSTRING 280 281 282def ParseOptions(argv, 283 docstring, 284 extra_opts="", extra_long_opts=(), 285 extra_option_handler=None): 286 """Parse the options in argv and return any arguments that aren't 287 flags. docstring is the calling module's docstring, to be displayed 288 for errors and -h. extra_opts and extra_long_opts are for flags 289 defined by the caller, which are processed by passing them to 290 extra_option_handler.""" 291 292 try: 293 opts, args = getopt.getopt( 294 argv, "hvp:s:x:" + extra_opts, 295 ["help", "verbose", "path=", "device_specific=", "extra="] + 296 list(extra_long_opts)) 297 except getopt.GetoptError, err: 298 Usage(docstring) 299 print "**", str(err), "**" 300 sys.exit(2) 301 302 path_specified = False 303 304 for o, a in opts: 305 if o in ("-h", "--help"): 306 Usage(docstring) 307 sys.exit() 308 elif o in ("-v", "--verbose"): 309 OPTIONS.verbose = True 310 elif o in ("-p", "--path"): 311 OPTIONS.search_path = a 312 elif o in ("-s", "--device_specific"): 313 OPTIONS.device_specific = a 314 elif o in ("-x", "--extra"): 315 key, value = a.split("=", 1) 316 OPTIONS.extras[key] = value 317 else: 318 if extra_option_handler is None or not extra_option_handler(o, a): 319 assert False, "unknown option \"%s\"" % (o,) 320 321 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 322 os.pathsep + os.environ["PATH"]) 323 324 return args 325 326 327def Cleanup(): 328 for i in OPTIONS.tempfiles: 329 if os.path.isdir(i): 330 shutil.rmtree(i) 331 else: 332 os.remove(i) 333 334 335class PasswordManager(object): 336 def __init__(self): 337 self.editor = os.getenv("EDITOR", None) 338 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 339 340 def GetPasswords(self, items): 341 """Get passwords corresponding to each string in 'items', 342 returning a dict. (The dict may have keys in addition to the 343 values in 'items'.) 344 345 Uses the passwords in $ANDROID_PW_FILE if available, letting the 346 user edit that file to add more needed passwords. If no editor is 347 available, or $ANDROID_PW_FILE isn't define, prompts the user 348 interactively in the ordinary way. 349 """ 350 351 current = self.ReadFile() 352 353 first = True 354 while True: 355 missing = [] 356 for i in items: 357 if i not in current or not current[i]: 358 missing.append(i) 359 # Are all the passwords already in the file? 360 if not missing: return current 361 362 for i in missing: 363 current[i] = "" 364 365 if not first: 366 print "key file %s still missing some passwords." % (self.pwfile,) 367 answer = raw_input("try to edit again? [y]> ").strip() 368 if answer and answer[0] not in 'yY': 369 raise RuntimeError("key passwords unavailable") 370 first = False 371 372 current = self.UpdateAndReadFile(current) 373 374 def PromptResult(self, current): 375 """Prompt the user to enter a value (password) for each key in 376 'current' whose value is fales. Returns a new dict with all the 377 values. 378 """ 379 result = {} 380 for k, v in sorted(current.iteritems()): 381 if v: 382 result[k] = v 383 else: 384 while True: 385 result[k] = getpass.getpass("Enter password for %s key> " 386 % (k,)).strip() 387 if result[k]: break 388 return result 389 390 def UpdateAndReadFile(self, current): 391 if not self.editor or not self.pwfile: 392 return self.PromptResult(current) 393 394 f = open(self.pwfile, "w") 395 os.chmod(self.pwfile, 0600) 396 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 397 f.write("# (Additional spaces are harmless.)\n\n") 398 399 first_line = None 400 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 401 sorted.sort() 402 for i, (_, k, v) in enumerate(sorted): 403 f.write("[[[ %s ]]] %s\n" % (v, k)) 404 if not v and first_line is None: 405 # position cursor on first line with no password. 406 first_line = i + 4 407 f.close() 408 409 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 410 _, _ = p.communicate() 411 412 return self.ReadFile() 413 414 def ReadFile(self): 415 result = {} 416 if self.pwfile is None: return result 417 try: 418 f = open(self.pwfile, "r") 419 for line in f: 420 line = line.strip() 421 if not line or line[0] == '#': continue 422 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 423 if not m: 424 print "failed to parse password file: ", line 425 else: 426 result[m.group(2)] = m.group(1) 427 f.close() 428 except IOError, e: 429 if e.errno != errno.ENOENT: 430 print "error reading password file: ", str(e) 431 return result 432 433 434def ZipWriteStr(zip, filename, data, perms=0644): 435 # use a fixed timestamp so the output is repeatable. 436 zinfo = zipfile.ZipInfo(filename=filename, 437 date_time=(2009, 1, 1, 0, 0, 0)) 438 zinfo.compress_type = zip.compression 439 zinfo.external_attr = perms << 16 440 zip.writestr(zinfo, data) 441 442 443class DeviceSpecificParams(object): 444 module = None 445 def __init__(self, **kwargs): 446 """Keyword arguments to the constructor become attributes of this 447 object, which is passed to all functions in the device-specific 448 module.""" 449 for k, v in kwargs.iteritems(): 450 setattr(self, k, v) 451 self.extras = OPTIONS.extras 452 453 if self.module is None: 454 path = OPTIONS.device_specific 455 if not path: return 456 try: 457 if os.path.isdir(path): 458 info = imp.find_module("releasetools", [path]) 459 else: 460 d, f = os.path.split(path) 461 b, x = os.path.splitext(f) 462 if x == ".py": 463 f = b 464 info = imp.find_module(f, [d]) 465 self.module = imp.load_module("device_specific", *info) 466 except ImportError: 467 print "unable to load device-specific module; assuming none" 468 469 def _DoCall(self, function_name, *args, **kwargs): 470 """Call the named function in the device-specific module, passing 471 the given args and kwargs. The first argument to the call will be 472 the DeviceSpecific object itself. If there is no module, or the 473 module does not define the function, return the value of the 474 'default' kwarg (which itself defaults to None).""" 475 if self.module is None or not hasattr(self.module, function_name): 476 return kwargs.get("default", None) 477 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 478 479 def FullOTA_Assertions(self): 480 """Called after emitting the block of assertions at the top of a 481 full OTA package. Implementations can add whatever additional 482 assertions they like.""" 483 return self._DoCall("FullOTA_Assertions") 484 485 def FullOTA_InstallEnd(self): 486 """Called at the end of full OTA installation; typically this is 487 used to install the image for the device's baseband processor.""" 488 return self._DoCall("FullOTA_InstallEnd") 489 490 def IncrementalOTA_Assertions(self): 491 """Called after emitting the block of assertions at the top of an 492 incremental OTA package. Implementations can add whatever 493 additional assertions they like.""" 494 return self._DoCall("IncrementalOTA_Assertions") 495 496 def IncrementalOTA_VerifyEnd(self): 497 """Called at the end of the verification phase of incremental OTA 498 installation; additional checks can be placed here to abort the 499 script before any changes are made.""" 500 return self._DoCall("IncrementalOTA_VerifyEnd") 501 502 def IncrementalOTA_InstallEnd(self): 503 """Called at the end of incremental OTA installation; typically 504 this is used to install the image for the device's baseband 505 processor.""" 506 return self._DoCall("IncrementalOTA_InstallEnd") 507