common.py revision 8e2f2b9b207ba4615cb0fb49d29ff65203aad279
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import errno 16import getopt 17import getpass 18import imp 19import os 20import re 21import shutil 22import subprocess 23import sys 24import tempfile 25import zipfile 26 27# missing in Python 2.4 and before 28if not hasattr(os, "SEEK_SET"): 29 os.SEEK_SET = 0 30 31class Options(object): pass 32OPTIONS = Options() 33OPTIONS.search_path = "out/host/linux-x86" 34OPTIONS.max_image_size = {} 35OPTIONS.verbose = False 36OPTIONS.tempfiles = [] 37OPTIONS.device_specific = None 38 39class ExternalError(RuntimeError): pass 40 41 42def Run(args, **kwargs): 43 """Create and return a subprocess.Popen object, printing the command 44 line on the terminal if -v was specified.""" 45 if OPTIONS.verbose: 46 print " running: ", " ".join(args) 47 return subprocess.Popen(args, **kwargs) 48 49 50def LoadBoardConfig(fn): 51 """Parse a board_config.mk file looking for lines that specify the 52 maximum size of various images, and parse them into the 53 OPTIONS.max_image_size dict.""" 54 OPTIONS.max_image_size = {} 55 for line in open(fn): 56 line = line.strip() 57 m = re.match(r"BOARD_(BOOT|RECOVERY|SYSTEM|USERDATA)IMAGE_MAX_SIZE" 58 r"\s*:=\s*(\d+)", line) 59 if not m: continue 60 61 OPTIONS.max_image_size[m.group(1).lower() + ".img"] = int(m.group(2)) 62 63 64def BuildAndAddBootableImage(sourcedir, targetname, output_zip): 65 """Take a kernel, cmdline, and ramdisk directory from the input (in 66 'sourcedir'), and turn them into a boot image. Put the boot image 67 into the output zip file under the name 'targetname'. Returns 68 targetname on success or None on failure (if sourcedir does not 69 appear to contain files for the requested image).""" 70 71 print "creating %s..." % (targetname,) 72 73 img = BuildBootableImage(sourcedir) 74 if img is None: 75 return None 76 77 CheckSize(img, targetname) 78 ZipWriteStr(output_zip, targetname, img) 79 return targetname 80 81def BuildBootableImage(sourcedir): 82 """Take a kernel, cmdline, and ramdisk directory from the input (in 83 'sourcedir'), and turn them into a boot image. Return the image 84 data, or None if sourcedir does not appear to contains files for 85 building the requested image.""" 86 87 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 88 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 89 return None 90 91 ramdisk_img = tempfile.NamedTemporaryFile() 92 img = tempfile.NamedTemporaryFile() 93 94 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")], 95 stdout=subprocess.PIPE) 96 p2 = Run(["minigzip"], 97 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 98 99 p2.wait() 100 p1.wait() 101 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 102 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 103 104 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")] 105 106 fn = os.path.join(sourcedir, "cmdline") 107 if os.access(fn, os.F_OK): 108 cmd.append("--cmdline") 109 cmd.append(open(fn).read().rstrip("\n")) 110 111 fn = os.path.join(sourcedir, "base") 112 if os.access(fn, os.F_OK): 113 cmd.append("--base") 114 cmd.append(open(fn).read().rstrip("\n")) 115 116 cmd.extend(["--ramdisk", ramdisk_img.name, 117 "--output", img.name]) 118 119 p = Run(cmd, stdout=subprocess.PIPE) 120 p.communicate() 121 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 122 os.path.basename(sourcedir),) 123 124 img.seek(os.SEEK_SET, 0) 125 data = img.read() 126 127 ramdisk_img.close() 128 img.close() 129 130 return data 131 132 133def AddRecovery(output_zip): 134 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "RECOVERY"), 135 "recovery.img", output_zip) 136 137def AddBoot(output_zip): 138 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "BOOT"), 139 "boot.img", output_zip) 140 141def UnzipTemp(filename): 142 """Unzip the given archive into a temporary directory and return the name.""" 143 144 tmp = tempfile.mkdtemp(prefix="targetfiles-") 145 OPTIONS.tempfiles.append(tmp) 146 p = Run(["unzip", "-q", filename, "-d", tmp], stdout=subprocess.PIPE) 147 p.communicate() 148 if p.returncode != 0: 149 raise ExternalError("failed to unzip input target-files \"%s\"" % 150 (filename,)) 151 return tmp 152 153 154def GetKeyPasswords(keylist): 155 """Given a list of keys, prompt the user to enter passwords for 156 those which require them. Return a {key: password} dict. password 157 will be None if the key has no password.""" 158 159 no_passwords = [] 160 need_passwords = [] 161 devnull = open("/dev/null", "w+b") 162 for k in sorted(keylist): 163 # An empty-string key is used to mean don't re-sign this package. 164 # Obviously we don't need a password for this non-key. 165 if not k: 166 no_passwords.append(k) 167 continue 168 169 p = Run(["openssl", "pkcs8", "-in", k+".pk8", 170 "-inform", "DER", "-nocrypt"], 171 stdin=devnull.fileno(), 172 stdout=devnull.fileno(), 173 stderr=subprocess.STDOUT) 174 p.communicate() 175 if p.returncode == 0: 176 no_passwords.append(k) 177 else: 178 need_passwords.append(k) 179 devnull.close() 180 181 key_passwords = PasswordManager().GetPasswords(need_passwords) 182 key_passwords.update(dict.fromkeys(no_passwords, None)) 183 return key_passwords 184 185 186def SignFile(input_name, output_name, key, password, align=None): 187 """Sign the input_name zip/jar/apk, producing output_name. Use the 188 given key and password (the latter may be None if the key does not 189 have a password. 190 191 If align is an integer > 1, zipalign is run to align stored files in 192 the output zip on 'align'-byte boundaries. 193 """ 194 if align == 0 or align == 1: 195 align = None 196 197 if align: 198 temp = tempfile.NamedTemporaryFile() 199 sign_name = temp.name 200 else: 201 sign_name = output_name 202 203 p = Run(["java", "-jar", 204 os.path.join(OPTIONS.search_path, "framework", "signapk.jar"), 205 key + ".x509.pem", 206 key + ".pk8", 207 input_name, sign_name], 208 stdin=subprocess.PIPE, 209 stdout=subprocess.PIPE) 210 if password is not None: 211 password += "\n" 212 p.communicate(password) 213 if p.returncode != 0: 214 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 215 216 if align: 217 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 218 p.communicate() 219 if p.returncode != 0: 220 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 221 temp.close() 222 223 224def CheckSize(data, target): 225 """Check the data string passed against the max size limit, if 226 any, for the given target. Raise exception if the data is too big. 227 Print a warning if the data is nearing the maximum size.""" 228 limit = OPTIONS.max_image_size.get(target, None) 229 if limit is None: return 230 231 size = len(data) 232 pct = float(size) * 100.0 / limit 233 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 234 if pct >= 99.0: 235 raise ExternalError(msg) 236 elif pct >= 95.0: 237 print 238 print " WARNING: ", msg 239 print 240 elif OPTIONS.verbose: 241 print " ", msg 242 243 244COMMON_DOCSTRING = """ 245 -p (--path) <dir> 246 Prepend <dir>/bin to the list of places to search for binaries 247 run by this script, and expect to find jars in <dir>/framework. 248 249 -s (--device_specific) <file> 250 Path to the python module containing device-specific 251 releasetools code. 252 253 -v (--verbose) 254 Show command lines being executed. 255 256 -h (--help) 257 Display this usage message and exit. 258""" 259 260def Usage(docstring): 261 print docstring.rstrip("\n") 262 print COMMON_DOCSTRING 263 264 265def ParseOptions(argv, 266 docstring, 267 extra_opts="", extra_long_opts=(), 268 extra_option_handler=None): 269 """Parse the options in argv and return any arguments that aren't 270 flags. docstring is the calling module's docstring, to be displayed 271 for errors and -h. extra_opts and extra_long_opts are for flags 272 defined by the caller, which are processed by passing them to 273 extra_option_handler.""" 274 275 try: 276 opts, args = getopt.getopt( 277 argv, "hvp:s:" + extra_opts, 278 ["help", "verbose", "path=", "device_specific="] + 279 list(extra_long_opts)) 280 except getopt.GetoptError, err: 281 Usage(docstring) 282 print "**", str(err), "**" 283 sys.exit(2) 284 285 path_specified = False 286 287 for o, a in opts: 288 if o in ("-h", "--help"): 289 Usage(docstring) 290 sys.exit() 291 elif o in ("-v", "--verbose"): 292 OPTIONS.verbose = True 293 elif o in ("-p", "--path"): 294 OPTIONS.search_path = a 295 elif o in ("-s", "--device_specific"): 296 OPTIONS.device_specific = a 297 else: 298 if extra_option_handler is None or not extra_option_handler(o, a): 299 assert False, "unknown option \"%s\"" % (o,) 300 301 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 302 os.pathsep + os.environ["PATH"]) 303 304 return args 305 306 307def Cleanup(): 308 for i in OPTIONS.tempfiles: 309 if os.path.isdir(i): 310 shutil.rmtree(i) 311 else: 312 os.remove(i) 313 314 315class PasswordManager(object): 316 def __init__(self): 317 self.editor = os.getenv("EDITOR", None) 318 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 319 320 def GetPasswords(self, items): 321 """Get passwords corresponding to each string in 'items', 322 returning a dict. (The dict may have keys in addition to the 323 values in 'items'.) 324 325 Uses the passwords in $ANDROID_PW_FILE if available, letting the 326 user edit that file to add more needed passwords. If no editor is 327 available, or $ANDROID_PW_FILE isn't define, prompts the user 328 interactively in the ordinary way. 329 """ 330 331 current = self.ReadFile() 332 333 first = True 334 while True: 335 missing = [] 336 for i in items: 337 if i not in current or not current[i]: 338 missing.append(i) 339 # Are all the passwords already in the file? 340 if not missing: return current 341 342 for i in missing: 343 current[i] = "" 344 345 if not first: 346 print "key file %s still missing some passwords." % (self.pwfile,) 347 answer = raw_input("try to edit again? [y]> ").strip() 348 if answer and answer[0] not in 'yY': 349 raise RuntimeError("key passwords unavailable") 350 first = False 351 352 current = self.UpdateAndReadFile(current) 353 354 def PromptResult(self, current): 355 """Prompt the user to enter a value (password) for each key in 356 'current' whose value is fales. Returns a new dict with all the 357 values. 358 """ 359 result = {} 360 for k, v in sorted(current.iteritems()): 361 if v: 362 result[k] = v 363 else: 364 while True: 365 result[k] = getpass.getpass("Enter password for %s key> " 366 % (k,)).strip() 367 if result[k]: break 368 return result 369 370 def UpdateAndReadFile(self, current): 371 if not self.editor or not self.pwfile: 372 return self.PromptResult(current) 373 374 f = open(self.pwfile, "w") 375 os.chmod(self.pwfile, 0600) 376 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 377 f.write("# (Additional spaces are harmless.)\n\n") 378 379 first_line = None 380 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 381 sorted.sort() 382 for i, (_, k, v) in enumerate(sorted): 383 f.write("[[[ %s ]]] %s\n" % (v, k)) 384 if not v and first_line is None: 385 # position cursor on first line with no password. 386 first_line = i + 4 387 f.close() 388 389 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 390 _, _ = p.communicate() 391 392 return self.ReadFile() 393 394 def ReadFile(self): 395 result = {} 396 if self.pwfile is None: return result 397 try: 398 f = open(self.pwfile, "r") 399 for line in f: 400 line = line.strip() 401 if not line or line[0] == '#': continue 402 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 403 if not m: 404 print "failed to parse password file: ", line 405 else: 406 result[m.group(2)] = m.group(1) 407 f.close() 408 except IOError, e: 409 if e.errno != errno.ENOENT: 410 print "error reading password file: ", str(e) 411 return result 412 413 414def ZipWriteStr(zip, filename, data, perms=0644): 415 # use a fixed timestamp so the output is repeatable. 416 zinfo = zipfile.ZipInfo(filename=filename, 417 date_time=(2009, 1, 1, 0, 0, 0)) 418 zinfo.compress_type = zip.compression 419 zinfo.external_attr = perms << 16 420 zip.writestr(zinfo, data) 421 422 423class DeviceSpecificParams(object): 424 module = None 425 def __init__(self, **kwargs): 426 """Keyword arguments to the constructor become attributes of this 427 object, which is passed to all functions in the device-specific 428 module.""" 429 for k, v in kwargs.iteritems(): 430 setattr(self, k, v) 431 432 if self.module is None: 433 path = OPTIONS.device_specific 434 if path is None: return 435 try: 436 if os.path.isdir(path): 437 info = imp.find_module("releasetools", [path]) 438 else: 439 d, f = os.path.split(path) 440 b, x = os.path.splitext(f) 441 if x == ".py": 442 f = b 443 info = imp.find_module(f, [d]) 444 self.module = imp.load_module("device_specific", *info) 445 except ImportError: 446 print "unable to load device-specific module; assuming none" 447 448 def _DoCall(self, function_name, *args, **kwargs): 449 """Call the named function in the device-specific module, passing 450 the given args and kwargs. The first argument to the call will be 451 the DeviceSpecific object itself. If there is no module, or the 452 module does not define the function, return the value of the 453 'default' kwarg (which itself defaults to None).""" 454 if self.module is None or not hasattr(self.module, function_name): 455 return kwargs.get("default", None) 456 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 457 458 def FullOTA_Assertions(self): 459 """Called after emitting the block of assertions at the top of a 460 full OTA package. Implementations can add whatever additional 461 assertions they like.""" 462 return self._DoCall("FullOTA_Assertions") 463 464 def FullOTA_InstallEnd(self): 465 """Called at the end of full OTA installation; typically this is 466 used to install the image for the device's baseband processor.""" 467 return self._DoCall("FullOTA_InstallEnd") 468 469 def IncrementalOTA_Assertions(self): 470 """Called after emitting the block of assertions at the top of an 471 incremental OTA package. Implementations can add whatever 472 additional assertions they like.""" 473 return self._DoCall("IncrementalOTA_Assertions") 474 475 def IncrementalOTA_VerifyEnd(self): 476 """Called at the end of the verification phase of incremental OTA 477 installation; additional checks can be placed here to abort the 478 script before any changes are made.""" 479 return self._DoCall("IncrementalOTA_VerifyEnd") 480 481 def IncrementalOTA_InstallEnd(self): 482 """Called at the end of incremental OTA installation; typically 483 this is used to install the image for the device's baseband 484 processor.""" 485 return self._DoCall("IncrementalOTA_InstallEnd") 486