common.py revision 05d3dea519688b61d86e30c2d4b99ff494aeca73
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import errno 16import getopt 17import getpass 18import imp 19import os 20import re 21import shutil 22import subprocess 23import sys 24import tempfile 25import zipfile 26 27# missing in Python 2.4 and before 28if not hasattr(os, "SEEK_SET"): 29 os.SEEK_SET = 0 30 31class Options(object): pass 32OPTIONS = Options() 33OPTIONS.search_path = "out/host/linux-x86" 34OPTIONS.max_image_size = {} 35OPTIONS.verbose = False 36OPTIONS.tempfiles = [] 37OPTIONS.device_specific = None 38 39class ExternalError(RuntimeError): pass 40 41 42def Run(args, **kwargs): 43 """Create and return a subprocess.Popen object, printing the command 44 line on the terminal if -v was specified.""" 45 if OPTIONS.verbose: 46 print " running: ", " ".join(args) 47 return subprocess.Popen(args, **kwargs) 48 49 50def LoadBoardConfig(fn): 51 """Parse a board_config.mk file looking for lines that specify the 52 maximum size of various images, and parse them into the 53 OPTIONS.max_image_size dict.""" 54 OPTIONS.max_image_size = {} 55 for line in open(fn): 56 line = line.strip() 57 m = re.match(r"BOARD_(BOOT|RECOVERY|SYSTEM|USERDATA)IMAGE_MAX_SIZE" 58 r"\s*:=\s*(\d+)", line) 59 if not m: continue 60 61 OPTIONS.max_image_size[m.group(1).lower() + ".img"] = int(m.group(2)) 62 63 64def BuildAndAddBootableImage(sourcedir, targetname, output_zip): 65 """Take a kernel, cmdline, and ramdisk directory from the input (in 66 'sourcedir'), and turn them into a boot image. Put the boot image 67 into the output zip file under the name 'targetname'.""" 68 69 print "creating %s..." % (targetname,) 70 71 img = BuildBootableImage(sourcedir) 72 73 CheckSize(img, targetname) 74 ZipWriteStr(output_zip, targetname, img) 75 76def BuildBootableImage(sourcedir): 77 """Take a kernel, cmdline, and ramdisk directory from the input (in 78 'sourcedir'), and turn them into a boot image. Return the image data.""" 79 80 ramdisk_img = tempfile.NamedTemporaryFile() 81 img = tempfile.NamedTemporaryFile() 82 83 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")], 84 stdout=subprocess.PIPE) 85 p2 = Run(["minigzip"], 86 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 87 88 p2.wait() 89 p1.wait() 90 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 91 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 92 93 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")] 94 95 fn = os.path.join(sourcedir, "cmdline") 96 if os.access(fn, os.F_OK): 97 cmd.append("--cmdline") 98 cmd.append(open(fn).read().rstrip("\n")) 99 100 fn = os.path.join(sourcedir, "base") 101 if os.access(fn, os.F_OK): 102 cmd.append("--base") 103 cmd.append(open(fn).read().rstrip("\n")) 104 105 cmd.extend(["--ramdisk", ramdisk_img.name, 106 "--output", img.name]) 107 108 p = Run(cmd, stdout=subprocess.PIPE) 109 p.communicate() 110 assert p.returncode == 0, "mkbootimg of %s image failed" % (targetname,) 111 112 img.seek(os.SEEK_SET, 0) 113 data = img.read() 114 115 ramdisk_img.close() 116 img.close() 117 118 return data 119 120 121def AddRecovery(output_zip): 122 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "RECOVERY"), 123 "recovery.img", output_zip) 124 125def AddBoot(output_zip): 126 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "BOOT"), 127 "boot.img", output_zip) 128 129def UnzipTemp(filename): 130 """Unzip the given archive into a temporary directory and return the name.""" 131 132 tmp = tempfile.mkdtemp(prefix="targetfiles-") 133 OPTIONS.tempfiles.append(tmp) 134 p = Run(["unzip", "-q", filename, "-d", tmp], stdout=subprocess.PIPE) 135 p.communicate() 136 if p.returncode != 0: 137 raise ExternalError("failed to unzip input target-files \"%s\"" % 138 (filename,)) 139 return tmp 140 141 142def GetKeyPasswords(keylist): 143 """Given a list of keys, prompt the user to enter passwords for 144 those which require them. Return a {key: password} dict. password 145 will be None if the key has no password.""" 146 147 no_passwords = [] 148 need_passwords = [] 149 devnull = open("/dev/null", "w+b") 150 for k in sorted(keylist): 151 # An empty-string key is used to mean don't re-sign this package. 152 # Obviously we don't need a password for this non-key. 153 if not k: 154 no_passwords.append(k) 155 continue 156 157 p = Run(["openssl", "pkcs8", "-in", k+".pk8", 158 "-inform", "DER", "-nocrypt"], 159 stdin=devnull.fileno(), 160 stdout=devnull.fileno(), 161 stderr=subprocess.STDOUT) 162 p.communicate() 163 if p.returncode == 0: 164 no_passwords.append(k) 165 else: 166 need_passwords.append(k) 167 devnull.close() 168 169 key_passwords = PasswordManager().GetPasswords(need_passwords) 170 key_passwords.update(dict.fromkeys(no_passwords, None)) 171 return key_passwords 172 173 174def SignFile(input_name, output_name, key, password, align=None): 175 """Sign the input_name zip/jar/apk, producing output_name. Use the 176 given key and password (the latter may be None if the key does not 177 have a password. 178 179 If align is an integer > 1, zipalign is run to align stored files in 180 the output zip on 'align'-byte boundaries. 181 """ 182 if align == 0 or align == 1: 183 align = None 184 185 if align: 186 temp = tempfile.NamedTemporaryFile() 187 sign_name = temp.name 188 else: 189 sign_name = output_name 190 191 p = Run(["java", "-jar", 192 os.path.join(OPTIONS.search_path, "framework", "signapk.jar"), 193 key + ".x509.pem", 194 key + ".pk8", 195 input_name, sign_name], 196 stdin=subprocess.PIPE, 197 stdout=subprocess.PIPE) 198 if password is not None: 199 password += "\n" 200 p.communicate(password) 201 if p.returncode != 0: 202 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 203 204 if align: 205 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 206 p.communicate() 207 if p.returncode != 0: 208 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 209 temp.close() 210 211 212def CheckSize(data, target): 213 """Check the data string passed against the max size limit, if 214 any, for the given target. Raise exception if the data is too big. 215 Print a warning if the data is nearing the maximum size.""" 216 limit = OPTIONS.max_image_size.get(target, None) 217 if limit is None: return 218 219 size = len(data) 220 pct = float(size) * 100.0 / limit 221 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 222 if pct >= 99.0: 223 raise ExternalError(msg) 224 elif pct >= 95.0: 225 print 226 print " WARNING: ", msg 227 print 228 elif OPTIONS.verbose: 229 print " ", msg 230 231 232COMMON_DOCSTRING = """ 233 -p (--path) <dir> 234 Prepend <dir>/bin to the list of places to search for binaries 235 run by this script, and expect to find jars in <dir>/framework. 236 237 -s (--device_specific) <file> 238 Path to the python module containing device-specific 239 releasetools code. 240 241 -v (--verbose) 242 Show command lines being executed. 243 244 -h (--help) 245 Display this usage message and exit. 246""" 247 248def Usage(docstring): 249 print docstring.rstrip("\n") 250 print COMMON_DOCSTRING 251 252 253def ParseOptions(argv, 254 docstring, 255 extra_opts="", extra_long_opts=(), 256 extra_option_handler=None): 257 """Parse the options in argv and return any arguments that aren't 258 flags. docstring is the calling module's docstring, to be displayed 259 for errors and -h. extra_opts and extra_long_opts are for flags 260 defined by the caller, which are processed by passing them to 261 extra_option_handler.""" 262 263 try: 264 opts, args = getopt.getopt( 265 argv, "hvp:s:" + extra_opts, 266 ["help", "verbose", "path=", "device_specific="] + 267 list(extra_long_opts)) 268 except getopt.GetoptError, err: 269 Usage(docstring) 270 print "**", str(err), "**" 271 sys.exit(2) 272 273 path_specified = False 274 275 for o, a in opts: 276 if o in ("-h", "--help"): 277 Usage(docstring) 278 sys.exit() 279 elif o in ("-v", "--verbose"): 280 OPTIONS.verbose = True 281 elif o in ("-p", "--path"): 282 OPTIONS.search_path = a 283 elif o in ("-s", "--device_specific"): 284 OPTIONS.device_specific = a 285 else: 286 if extra_option_handler is None or not extra_option_handler(o, a): 287 assert False, "unknown option \"%s\"" % (o,) 288 289 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 290 os.pathsep + os.environ["PATH"]) 291 292 return args 293 294 295def Cleanup(): 296 for i in OPTIONS.tempfiles: 297 if os.path.isdir(i): 298 shutil.rmtree(i) 299 else: 300 os.remove(i) 301 302 303class PasswordManager(object): 304 def __init__(self): 305 self.editor = os.getenv("EDITOR", None) 306 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 307 308 def GetPasswords(self, items): 309 """Get passwords corresponding to each string in 'items', 310 returning a dict. (The dict may have keys in addition to the 311 values in 'items'.) 312 313 Uses the passwords in $ANDROID_PW_FILE if available, letting the 314 user edit that file to add more needed passwords. If no editor is 315 available, or $ANDROID_PW_FILE isn't define, prompts the user 316 interactively in the ordinary way. 317 """ 318 319 current = self.ReadFile() 320 321 first = True 322 while True: 323 missing = [] 324 for i in items: 325 if i not in current or not current[i]: 326 missing.append(i) 327 # Are all the passwords already in the file? 328 if not missing: return current 329 330 for i in missing: 331 current[i] = "" 332 333 if not first: 334 print "key file %s still missing some passwords." % (self.pwfile,) 335 answer = raw_input("try to edit again? [y]> ").strip() 336 if answer and answer[0] not in 'yY': 337 raise RuntimeError("key passwords unavailable") 338 first = False 339 340 current = self.UpdateAndReadFile(current) 341 342 def PromptResult(self, current): 343 """Prompt the user to enter a value (password) for each key in 344 'current' whose value is fales. Returns a new dict with all the 345 values. 346 """ 347 result = {} 348 for k, v in sorted(current.iteritems()): 349 if v: 350 result[k] = v 351 else: 352 while True: 353 result[k] = getpass.getpass("Enter password for %s key> " 354 % (k,)).strip() 355 if result[k]: break 356 return result 357 358 def UpdateAndReadFile(self, current): 359 if not self.editor or not self.pwfile: 360 return self.PromptResult(current) 361 362 f = open(self.pwfile, "w") 363 os.chmod(self.pwfile, 0600) 364 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 365 f.write("# (Additional spaces are harmless.)\n\n") 366 367 first_line = None 368 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 369 sorted.sort() 370 for i, (_, k, v) in enumerate(sorted): 371 f.write("[[[ %s ]]] %s\n" % (v, k)) 372 if not v and first_line is None: 373 # position cursor on first line with no password. 374 first_line = i + 4 375 f.close() 376 377 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 378 _, _ = p.communicate() 379 380 return self.ReadFile() 381 382 def ReadFile(self): 383 result = {} 384 if self.pwfile is None: return result 385 try: 386 f = open(self.pwfile, "r") 387 for line in f: 388 line = line.strip() 389 if not line or line[0] == '#': continue 390 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 391 if not m: 392 print "failed to parse password file: ", line 393 else: 394 result[m.group(2)] = m.group(1) 395 f.close() 396 except IOError, e: 397 if e.errno != errno.ENOENT: 398 print "error reading password file: ", str(e) 399 return result 400 401 402def ZipWriteStr(zip, filename, data, perms=0644): 403 # use a fixed timestamp so the output is repeatable. 404 zinfo = zipfile.ZipInfo(filename=filename, 405 date_time=(2009, 1, 1, 0, 0, 0)) 406 zinfo.compress_type = zip.compression 407 zinfo.external_attr = perms << 16 408 zip.writestr(zinfo, data) 409 410 411class DeviceSpecificParams(object): 412 module = None 413 def __init__(self, **kwargs): 414 """Keyword arguments to the constructor become attributes of this 415 object, which is passed to all functions in the device-specific 416 module.""" 417 for k, v in kwargs.iteritems(): 418 setattr(self, k, v) 419 420 if self.module is None: 421 path = OPTIONS.device_specific 422 if path is None: return 423 if os.path.isdir(path): 424 info = imp.find_module("releasetools", [path]) 425 else: 426 d, f = os.path.split(path) 427 b, x = os.path.splitext(f) 428 if x == ".py": 429 f = b 430 info = imp.find_module(f, [d]) 431 if not info or info[0] is None: 432 raise ValueError("unable to find device-specific module") 433 self.module = imp.load_module("device_specific", *info) 434 435 def _DoCall(self, function_name, *args, **kwargs): 436 """Call the named function in the device-specific module, passing 437 the given args and kwargs. The first argument to the call will be 438 the DeviceSpecific object itself. If there is no module, or the 439 module does not define the function, return the value of the 440 'default' kwarg (which itself defaults to None).""" 441 if self.module is None or not hasattr(self.module, function_name): 442 return kwargs.get("default", None) 443 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 444 445 def FullOTA_Assertions(self): 446 """Called after emitting the block of assertions at the top of a 447 full OTA package. Implementations can add whatever additional 448 assertions they like.""" 449 return self._DoCall("FullOTA_Assertions") 450 451 def FullOTA_InstallEnd(self): 452 """Called at the end of full OTA installation; typically this is 453 used to install the image for the device's baseband processor.""" 454 return self._DoCall("FullOTA_InstallEnd") 455 456 def IncrementalOTA_Assertions(self): 457 """Called after emitting the block of assertions at the top of an 458 incremental OTA package. Implementations can add whatever 459 additional assertions they like.""" 460 return self._DoCall("IncrementalOTA_Assertions") 461 462 def IncrementalOTA_VerifyEnd(self): 463 """Called at the end of the verification phase of incremental OTA 464 installation; additional checks can be placed here to abort the 465 script before any changes are made.""" 466 return self._DoCall("IncrementalOTA_VerifyEnd") 467 468 def IncrementalOTA_InstallEnd(self): 469 """Called at the end of incremental OTA installation; typically 470 this is used to install the image for the device's baseband 471 processor.""" 472 return self._DoCall("IncrementalOTA_InstallEnd") 473