common.py revision 4de6b5bfc89de95b18514e75329a2ec7491bebbd 
1# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import errno
16import getopt
17import getpass
18import imp
19import os
20import re
21import shutil
22import subprocess
23import sys
24import tempfile
25import zipfile
26
27# missing in Python 2.4 and before
28if not hasattr(os, "SEEK_SET"):
29  os.SEEK_SET = 0
30
31class Options(object): pass
32OPTIONS = Options()
33OPTIONS.search_path = "out/host/linux-x86"
34OPTIONS.max_image_size = {}
35OPTIONS.verbose = False
36OPTIONS.tempfiles = []
37OPTIONS.device_specific = None
38OPTIONS.extras = {}
39
40
41# Values for "certificate" in apkcerts that mean special things.
42SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL")
43
44
45class ExternalError(RuntimeError): pass
46
47
48def Run(args, **kwargs):
49  """Create and return a subprocess.Popen object, printing the command
50  line on the terminal if -v was specified."""
51  if OPTIONS.verbose:
52    print "  running: ", " ".join(args)
53  return subprocess.Popen(args, **kwargs)
54
55
56def LoadMaxSizes():
57  """Load the maximum allowable images sizes from the input
58  target_files size."""
59  OPTIONS.max_image_size = {}
60  try:
61    for line in open(os.path.join(OPTIONS.input_tmp, "META", "imagesizes.txt")):
62      pieces = line.split()
63      if len(pieces) != 2: continue
64      image = pieces[0]
65      size = int(pieces[1])
66      OPTIONS.max_image_size[image + ".img"] = size
67  except IOError, e:
68    if e.errno == errno.ENOENT:
69      pass
70
71
72def BuildAndAddBootableImage(sourcedir, targetname, output_zip):
73  """Take a kernel, cmdline, and ramdisk directory from the input (in
74  'sourcedir'), and turn them into a boot image.  Put the boot image
75  into the output zip file under the name 'targetname'.  Returns
76  targetname on success or None on failure (if sourcedir does not
77  appear to contain files for the requested image)."""
78
79  print "creating %s..." % (targetname,)
80
81  img = BuildBootableImage(sourcedir)
82  if img is None:
83    return None
84
85  CheckSize(img, targetname)
86  ZipWriteStr(output_zip, targetname, img)
87  return targetname
88
89def BuildBootableImage(sourcedir):
90  """Take a kernel, cmdline, and ramdisk directory from the input (in
91  'sourcedir'), and turn them into a boot image.  Return the image
92  data, or None if sourcedir does not appear to contains files for
93  building the requested image."""
94
95  if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or
96      not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)):
97    return None
98
99  ramdisk_img = tempfile.NamedTemporaryFile()
100  img = tempfile.NamedTemporaryFile()
101
102  p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")],
103           stdout=subprocess.PIPE)
104  p2 = Run(["minigzip"],
105           stdin=p1.stdout, stdout=ramdisk_img.file.fileno())
106
107  p2.wait()
108  p1.wait()
109  assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,)
110  assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,)
111
112  cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")]
113
114  fn = os.path.join(sourcedir, "cmdline")
115  if os.access(fn, os.F_OK):
116    cmd.append("--cmdline")
117    cmd.append(open(fn).read().rstrip("\n"))
118
119  fn = os.path.join(sourcedir, "base")
120  if os.access(fn, os.F_OK):
121    cmd.append("--base")
122    cmd.append(open(fn).read().rstrip("\n"))
123
124  fn = os.path.join(sourcedir, "pagesize")
125  if os.access(fn, os.F_OK):
126    cmd.append("--pagesize")
127    cmd.append(open(fn).read().rstrip("\n"))
128
129  cmd.extend(["--ramdisk", ramdisk_img.name,
130              "--output", img.name])
131
132  p = Run(cmd, stdout=subprocess.PIPE)
133  p.communicate()
134  assert p.returncode == 0, "mkbootimg of %s image failed" % (
135      os.path.basename(sourcedir),)
136
137  img.seek(os.SEEK_SET, 0)
138  data = img.read()
139
140  ramdisk_img.close()
141  img.close()
142
143  return data
144
145
146def AddRecovery(output_zip):
147  BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "RECOVERY"),
148                           "recovery.img", output_zip)
149
150def AddBoot(output_zip):
151  BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "BOOT"),
152                           "boot.img", output_zip)
153
154def UnzipTemp(filename, pattern=None):
155  """Unzip the given archive into a temporary directory and return the name."""
156
157  tmp = tempfile.mkdtemp(prefix="targetfiles-")
158  OPTIONS.tempfiles.append(tmp)
159  cmd = ["unzip", "-o", "-q", filename, "-d", tmp]
160  if pattern is not None:
161    cmd.append(pattern)
162  p = Run(cmd, stdout=subprocess.PIPE)
163  p.communicate()
164  if p.returncode != 0:
165    raise ExternalError("failed to unzip input target-files \"%s\"" %
166                        (filename,))
167  return tmp
168
169
170def GetKeyPasswords(keylist):
171  """Given a list of keys, prompt the user to enter passwords for
172  those which require them.  Return a {key: password} dict.  password
173  will be None if the key has no password."""
174
175  no_passwords = []
176  need_passwords = []
177  devnull = open("/dev/null", "w+b")
178  for k in sorted(keylist):
179    # We don't need a password for things that aren't really keys.
180    if k in SPECIAL_CERT_STRINGS:
181      no_passwords.append(k)
182      continue
183
184    p = Run(["openssl", "pkcs8", "-in", k+".pk8",
185             "-inform", "DER", "-nocrypt"],
186            stdin=devnull.fileno(),
187            stdout=devnull.fileno(),
188            stderr=subprocess.STDOUT)
189    p.communicate()
190    if p.returncode == 0:
191      no_passwords.append(k)
192    else:
193      need_passwords.append(k)
194  devnull.close()
195
196  key_passwords = PasswordManager().GetPasswords(need_passwords)
197  key_passwords.update(dict.fromkeys(no_passwords, None))
198  return key_passwords
199
200
201def SignFile(input_name, output_name, key, password, align=None,
202             whole_file=False):
203  """Sign the input_name zip/jar/apk, producing output_name.  Use the
204  given key and password (the latter may be None if the key does not
205  have a password.
206
207  If align is an integer > 1, zipalign is run to align stored files in
208  the output zip on 'align'-byte boundaries.
209
210  If whole_file is true, use the "-w" option to SignApk to embed a
211  signature that covers the whole file in the archive comment of the
212  zip file.
213  """
214
215  if align == 0 or align == 1:
216    align = None
217
218  if align:
219    temp = tempfile.NamedTemporaryFile()
220    sign_name = temp.name
221  else:
222    sign_name = output_name
223
224  cmd = ["java", "-Xmx512m", "-jar",
225           os.path.join(OPTIONS.search_path, "framework", "signapk.jar")]
226  if whole_file:
227    cmd.append("-w")
228  cmd.extend([key + ".x509.pem", key + ".pk8",
229              input_name, sign_name])
230
231  p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
232  if password is not None:
233    password += "\n"
234  p.communicate(password)
235  if p.returncode != 0:
236    raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,))
237
238  if align:
239    p = Run(["zipalign", "-f", str(align), sign_name, output_name])
240    p.communicate()
241    if p.returncode != 0:
242      raise ExternalError("zipalign failed: return code %s" % (p.returncode,))
243    temp.close()
244
245
246def CheckSize(data, target):
247  """Check the data string passed against the max size limit, if
248  any, for the given target.  Raise exception if the data is too big.
249  Print a warning if the data is nearing the maximum size."""
250  limit = OPTIONS.max_image_size.get(target, None)
251  if limit is None: return
252
253  size = len(data)
254  pct = float(size) * 100.0 / limit
255  msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit)
256  if pct >= 99.0:
257    raise ExternalError(msg)
258  elif pct >= 95.0:
259    print
260    print "  WARNING: ", msg
261    print
262  elif OPTIONS.verbose:
263    print "  ", msg
264
265
266def ReadApkCerts(tf_zip):
267  """Given a target_files ZipFile, parse the META/apkcerts.txt file
268  and return a {package: cert} dict."""
269  certmap = {}
270  for line in tf_zip.read("META/apkcerts.txt").split("\n"):
271    line = line.strip()
272    if not line: continue
273    m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+'
274                 r'private_key="(.*)"$', line)
275    if m:
276      name, cert, privkey = m.groups()
277      if cert in SPECIAL_CERT_STRINGS and not privkey:
278        certmap[name] = cert
279      elif (cert.endswith(".x509.pem") and
280            privkey.endswith(".pk8") and
281            cert[:-9] == privkey[:-4]):
282        certmap[name] = cert[:-9]
283      else:
284        raise ValueError("failed to parse line from apkcerts.txt:\n" + line)
285  return certmap
286
287
288COMMON_DOCSTRING = """
289  -p  (--path)  <dir>
290      Prepend <dir>/bin to the list of places to search for binaries
291      run by this script, and expect to find jars in <dir>/framework.
292
293  -s  (--device_specific) <file>
294      Path to the python module containing device-specific
295      releasetools code.
296
297  -x  (--extra)  <key=value>
298      Add a key/value pair to the 'extras' dict, which device-specific
299      extension code may look at.
300
301  -v  (--verbose)
302      Show command lines being executed.
303
304  -h  (--help)
305      Display this usage message and exit.
306"""
307
308def Usage(docstring):
309  print docstring.rstrip("\n")
310  print COMMON_DOCSTRING
311
312
313def ParseOptions(argv,
314                 docstring,
315                 extra_opts="", extra_long_opts=(),
316                 extra_option_handler=None):
317  """Parse the options in argv and return any arguments that aren't
318  flags.  docstring is the calling module's docstring, to be displayed
319  for errors and -h.  extra_opts and extra_long_opts are for flags
320  defined by the caller, which are processed by passing them to
321  extra_option_handler."""
322
323  try:
324    opts, args = getopt.getopt(
325        argv, "hvp:s:x:" + extra_opts,
326        ["help", "verbose", "path=", "device_specific=", "extra="] +
327          list(extra_long_opts))
328  except getopt.GetoptError, err:
329    Usage(docstring)
330    print "**", str(err), "**"
331    sys.exit(2)
332
333  path_specified = False
334
335  for o, a in opts:
336    if o in ("-h", "--help"):
337      Usage(docstring)
338      sys.exit()
339    elif o in ("-v", "--verbose"):
340      OPTIONS.verbose = True
341    elif o in ("-p", "--path"):
342      OPTIONS.search_path = a
343    elif o in ("-s", "--device_specific"):
344      OPTIONS.device_specific = a
345    elif o in ("-x", "--extra"):
346      key, value = a.split("=", 1)
347      OPTIONS.extras[key] = value
348    else:
349      if extra_option_handler is None or not extra_option_handler(o, a):
350        assert False, "unknown option \"%s\"" % (o,)
351
352  os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") +
353                        os.pathsep + os.environ["PATH"])
354
355  return args
356
357
358def Cleanup():
359  for i in OPTIONS.tempfiles:
360    if os.path.isdir(i):
361      shutil.rmtree(i)
362    else:
363      os.remove(i)
364
365
366class PasswordManager(object):
367  def __init__(self):
368    self.editor = os.getenv("EDITOR", None)
369    self.pwfile = os.getenv("ANDROID_PW_FILE", None)
370
371  def GetPasswords(self, items):
372    """Get passwords corresponding to each string in 'items',
373    returning a dict.  (The dict may have keys in addition to the
374    values in 'items'.)
375
376    Uses the passwords in $ANDROID_PW_FILE if available, letting the
377    user edit that file to add more needed passwords.  If no editor is
378    available, or $ANDROID_PW_FILE isn't define, prompts the user
379    interactively in the ordinary way.
380    """
381
382    current = self.ReadFile()
383
384    first = True
385    while True:
386      missing = []
387      for i in items:
388        if i not in current or not current[i]:
389          missing.append(i)
390      # Are all the passwords already in the file?
391      if not missing: return current
392
393      for i in missing:
394        current[i] = ""
395
396      if not first:
397        print "key file %s still missing some passwords." % (self.pwfile,)
398        answer = raw_input("try to edit again? [y]> ").strip()
399        if answer and answer[0] not in 'yY':
400          raise RuntimeError("key passwords unavailable")
401      first = False
402
403      current = self.UpdateAndReadFile(current)
404
405  def PromptResult(self, current):
406    """Prompt the user to enter a value (password) for each key in
407    'current' whose value is fales.  Returns a new dict with all the
408    values.
409    """
410    result = {}
411    for k, v in sorted(current.iteritems()):
412      if v:
413        result[k] = v
414      else:
415        while True:
416          result[k] = getpass.getpass("Enter password for %s key> "
417                                      % (k,)).strip()
418          if result[k]: break
419    return result
420
421  def UpdateAndReadFile(self, current):
422    if not self.editor or not self.pwfile:
423      return self.PromptResult(current)
424
425    f = open(self.pwfile, "w")
426    os.chmod(self.pwfile, 0600)
427    f.write("# Enter key passwords between the [[[ ]]] brackets.\n")
428    f.write("# (Additional spaces are harmless.)\n\n")
429
430    first_line = None
431    sorted = [(not v, k, v) for (k, v) in current.iteritems()]
432    sorted.sort()
433    for i, (_, k, v) in enumerate(sorted):
434      f.write("[[[  %s  ]]] %s\n" % (v, k))
435      if not v and first_line is None:
436        # position cursor on first line with no password.
437        first_line = i + 4
438    f.close()
439
440    p = Run([self.editor, "+%d" % (first_line,), self.pwfile])
441    _, _ = p.communicate()
442
443    return self.ReadFile()
444
445  def ReadFile(self):
446    result = {}
447    if self.pwfile is None: return result
448    try:
449      f = open(self.pwfile, "r")
450      for line in f:
451        line = line.strip()
452        if not line or line[0] == '#': continue
453        m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line)
454        if not m:
455          print "failed to parse password file: ", line
456        else:
457          result[m.group(2)] = m.group(1)
458      f.close()
459    except IOError, e:
460      if e.errno != errno.ENOENT:
461        print "error reading password file: ", str(e)
462    return result
463
464
465def ZipWriteStr(zip, filename, data, perms=0644):
466  # use a fixed timestamp so the output is repeatable.
467  zinfo = zipfile.ZipInfo(filename=filename,
468                          date_time=(2009, 1, 1, 0, 0, 0))
469  zinfo.compress_type = zip.compression
470  zinfo.external_attr = perms << 16
471  zip.writestr(zinfo, data)
472
473
474class DeviceSpecificParams(object):
475  module = None
476  def __init__(self, **kwargs):
477    """Keyword arguments to the constructor become attributes of this
478    object, which is passed to all functions in the device-specific
479    module."""
480    for k, v in kwargs.iteritems():
481      setattr(self, k, v)
482    self.extras = OPTIONS.extras
483
484    if self.module is None:
485      path = OPTIONS.device_specific
486      if not path: return
487      try:
488        if os.path.isdir(path):
489          info = imp.find_module("releasetools", [path])
490        else:
491          d, f = os.path.split(path)
492          b, x = os.path.splitext(f)
493          if x == ".py":
494            f = b
495          info = imp.find_module(f, [d])
496        self.module = imp.load_module("device_specific", *info)
497      except ImportError:
498        print "unable to load device-specific module; assuming none"
499
500  def _DoCall(self, function_name, *args, **kwargs):
501    """Call the named function in the device-specific module, passing
502    the given args and kwargs.  The first argument to the call will be
503    the DeviceSpecific object itself.  If there is no module, or the
504    module does not define the function, return the value of the
505    'default' kwarg (which itself defaults to None)."""
506    if self.module is None or not hasattr(self.module, function_name):
507      return kwargs.get("default", None)
508    return getattr(self.module, function_name)(*((self,) + args), **kwargs)
509
510  def FullOTA_Assertions(self):
511    """Called after emitting the block of assertions at the top of a
512    full OTA package.  Implementations can add whatever additional
513    assertions they like."""
514    return self._DoCall("FullOTA_Assertions")
515
516  def FullOTA_InstallEnd(self):
517    """Called at the end of full OTA installation; typically this is
518    used to install the image for the device's baseband processor."""
519    return self._DoCall("FullOTA_InstallEnd")
520
521  def IncrementalOTA_Assertions(self):
522    """Called after emitting the block of assertions at the top of an
523    incremental OTA package.  Implementations can add whatever
524    additional assertions they like."""
525    return self._DoCall("IncrementalOTA_Assertions")
526
527  def IncrementalOTA_VerifyEnd(self):
528    """Called at the end of the verification phase of incremental OTA
529    installation; additional checks can be placed here to abort the
530    script before any changes are made."""
531    return self._DoCall("IncrementalOTA_VerifyEnd")
532
533  def IncrementalOTA_InstallEnd(self):
534    """Called at the end of incremental OTA installation; typically
535    this is used to install the image for the device's baseband
536    processor."""
537    return self._DoCall("IncrementalOTA_InstallEnd")
538