common.py revision 5ecba70ce5302c34996bc70a0f8dc9ee6cbb2dea 
1# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import errno
16import getopt
17import getpass
18import imp
19import os
20import re
21import shutil
22import subprocess
23import sys
24import tempfile
25import zipfile
26
27# missing in Python 2.4 and before
28if not hasattr(os, "SEEK_SET"):
29  os.SEEK_SET = 0
30
31class Options(object): pass
32OPTIONS = Options()
33OPTIONS.search_path = "out/host/linux-x86"
34OPTIONS.max_image_size = {}
35OPTIONS.verbose = False
36OPTIONS.tempfiles = []
37OPTIONS.device_specific = None
38OPTIONS.extras = {}
39
40class ExternalError(RuntimeError): pass
41
42
43def Run(args, **kwargs):
44  """Create and return a subprocess.Popen object, printing the command
45  line on the terminal if -v was specified."""
46  if OPTIONS.verbose:
47    print "  running: ", " ".join(args)
48  return subprocess.Popen(args, **kwargs)
49
50
51def LoadMaxSizes():
52  """Load the maximum allowable images sizes from the input
53  target_files size."""
54  OPTIONS.max_image_size = {}
55  try:
56    for line in open(os.path.join(OPTIONS.input_tmp, "META", "imagesizes.txt")):
57      pieces = line.split()
58      if len(pieces) != 2: continue
59      image = pieces[0]
60      size = int(pieces[1])
61      OPTIONS.max_image_size[image + ".img"] = size
62  except IOError, e:
63    if e.errno == errno.ENOENT:
64      pass
65
66
67def BuildAndAddBootableImage(sourcedir, targetname, output_zip):
68  """Take a kernel, cmdline, and ramdisk directory from the input (in
69  'sourcedir'), and turn them into a boot image.  Put the boot image
70  into the output zip file under the name 'targetname'.  Returns
71  targetname on success or None on failure (if sourcedir does not
72  appear to contain files for the requested image)."""
73
74  print "creating %s..." % (targetname,)
75
76  img = BuildBootableImage(sourcedir)
77  if img is None:
78    return None
79
80  CheckSize(img, targetname)
81  ZipWriteStr(output_zip, targetname, img)
82  return targetname
83
84def BuildBootableImage(sourcedir):
85  """Take a kernel, cmdline, and ramdisk directory from the input (in
86  'sourcedir'), and turn them into a boot image.  Return the image
87  data, or None if sourcedir does not appear to contains files for
88  building the requested image."""
89
90  if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or
91      not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)):
92    return None
93
94  ramdisk_img = tempfile.NamedTemporaryFile()
95  img = tempfile.NamedTemporaryFile()
96
97  p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")],
98           stdout=subprocess.PIPE)
99  p2 = Run(["minigzip"],
100           stdin=p1.stdout, stdout=ramdisk_img.file.fileno())
101
102  p2.wait()
103  p1.wait()
104  assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,)
105  assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,)
106
107  cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")]
108
109  fn = os.path.join(sourcedir, "cmdline")
110  if os.access(fn, os.F_OK):
111    cmd.append("--cmdline")
112    cmd.append(open(fn).read().rstrip("\n"))
113
114  fn = os.path.join(sourcedir, "base")
115  if os.access(fn, os.F_OK):
116    cmd.append("--base")
117    cmd.append(open(fn).read().rstrip("\n"))
118
119  cmd.extend(["--ramdisk", ramdisk_img.name,
120              "--output", img.name])
121
122  p = Run(cmd, stdout=subprocess.PIPE)
123  p.communicate()
124  assert p.returncode == 0, "mkbootimg of %s image failed" % (
125      os.path.basename(sourcedir),)
126
127  img.seek(os.SEEK_SET, 0)
128  data = img.read()
129
130  ramdisk_img.close()
131  img.close()
132
133  return data
134
135
136def AddRecovery(output_zip):
137  BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "RECOVERY"),
138                           "recovery.img", output_zip)
139
140def AddBoot(output_zip):
141  BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "BOOT"),
142                           "boot.img", output_zip)
143
144def UnzipTemp(filename):
145  """Unzip the given archive into a temporary directory and return the name."""
146
147  tmp = tempfile.mkdtemp(prefix="targetfiles-")
148  OPTIONS.tempfiles.append(tmp)
149  p = Run(["unzip", "-o", "-q", filename, "-d", tmp], stdout=subprocess.PIPE)
150  p.communicate()
151  if p.returncode != 0:
152    raise ExternalError("failed to unzip input target-files \"%s\"" %
153                        (filename,))
154  return tmp
155
156
157def GetKeyPasswords(keylist):
158  """Given a list of keys, prompt the user to enter passwords for
159  those which require them.  Return a {key: password} dict.  password
160  will be None if the key has no password."""
161
162  no_passwords = []
163  need_passwords = []
164  devnull = open("/dev/null", "w+b")
165  for k in sorted(keylist):
166    # An empty-string key is used to mean don't re-sign this package.
167    # Obviously we don't need a password for this non-key.
168    if not k:
169      no_passwords.append(k)
170      continue
171
172    p = Run(["openssl", "pkcs8", "-in", k+".pk8",
173             "-inform", "DER", "-nocrypt"],
174            stdin=devnull.fileno(),
175            stdout=devnull.fileno(),
176            stderr=subprocess.STDOUT)
177    p.communicate()
178    if p.returncode == 0:
179      no_passwords.append(k)
180    else:
181      need_passwords.append(k)
182  devnull.close()
183
184  key_passwords = PasswordManager().GetPasswords(need_passwords)
185  key_passwords.update(dict.fromkeys(no_passwords, None))
186  return key_passwords
187
188
189def SignFile(input_name, output_name, key, password, align=None,
190             whole_file=False):
191  """Sign the input_name zip/jar/apk, producing output_name.  Use the
192  given key and password (the latter may be None if the key does not
193  have a password.
194
195  If align is an integer > 1, zipalign is run to align stored files in
196  the output zip on 'align'-byte boundaries.
197
198  If whole_file is true, use the "-w" option to SignApk to embed a
199  signature that covers the whole file in the archive comment of the
200  zip file.
201  """
202
203  if align == 0 or align == 1:
204    align = None
205
206  if align:
207    temp = tempfile.NamedTemporaryFile()
208    sign_name = temp.name
209  else:
210    sign_name = output_name
211
212  cmd = ["java", "-Xmx512m", "-jar",
213           os.path.join(OPTIONS.search_path, "framework", "signapk.jar")]
214  if whole_file:
215    cmd.append("-w")
216  cmd.extend([key + ".x509.pem", key + ".pk8",
217              input_name, sign_name])
218
219  p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
220  if password is not None:
221    password += "\n"
222  p.communicate(password)
223  if p.returncode != 0:
224    raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,))
225
226  if align:
227    p = Run(["zipalign", "-f", str(align), sign_name, output_name])
228    p.communicate()
229    if p.returncode != 0:
230      raise ExternalError("zipalign failed: return code %s" % (p.returncode,))
231    temp.close()
232
233
234def CheckSize(data, target):
235  """Check the data string passed against the max size limit, if
236  any, for the given target.  Raise exception if the data is too big.
237  Print a warning if the data is nearing the maximum size."""
238  limit = OPTIONS.max_image_size.get(target, None)
239  if limit is None: return
240
241  size = len(data)
242  pct = float(size) * 100.0 / limit
243  msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit)
244  if pct >= 99.0:
245    raise ExternalError(msg)
246  elif pct >= 95.0:
247    print
248    print "  WARNING: ", msg
249    print
250  elif OPTIONS.verbose:
251    print "  ", msg
252
253
254COMMON_DOCSTRING = """
255  -p  (--path)  <dir>
256      Prepend <dir>/bin to the list of places to search for binaries
257      run by this script, and expect to find jars in <dir>/framework.
258
259  -s  (--device_specific) <file>
260      Path to the python module containing device-specific
261      releasetools code.
262
263  -x  (--extra)  <key=value>
264      Add a key/value pair to the 'extras' dict, which device-specific
265      extension code may look at.
266
267  -v  (--verbose)
268      Show command lines being executed.
269
270  -h  (--help)
271      Display this usage message and exit.
272"""
273
274def Usage(docstring):
275  print docstring.rstrip("\n")
276  print COMMON_DOCSTRING
277
278
279def ParseOptions(argv,
280                 docstring,
281                 extra_opts="", extra_long_opts=(),
282                 extra_option_handler=None):
283  """Parse the options in argv and return any arguments that aren't
284  flags.  docstring is the calling module's docstring, to be displayed
285  for errors and -h.  extra_opts and extra_long_opts are for flags
286  defined by the caller, which are processed by passing them to
287  extra_option_handler."""
288
289  try:
290    opts, args = getopt.getopt(
291        argv, "hvp:s:x:" + extra_opts,
292        ["help", "verbose", "path=", "device_specific=", "extra="] +
293          list(extra_long_opts))
294  except getopt.GetoptError, err:
295    Usage(docstring)
296    print "**", str(err), "**"
297    sys.exit(2)
298
299  path_specified = False
300
301  for o, a in opts:
302    if o in ("-h", "--help"):
303      Usage(docstring)
304      sys.exit()
305    elif o in ("-v", "--verbose"):
306      OPTIONS.verbose = True
307    elif o in ("-p", "--path"):
308      OPTIONS.search_path = a
309    elif o in ("-s", "--device_specific"):
310      OPTIONS.device_specific = a
311    elif o in ("-x", "--extra"):
312      key, value = a.split("=", 1)
313      OPTIONS.extras[key] = value
314    else:
315      if extra_option_handler is None or not extra_option_handler(o, a):
316        assert False, "unknown option \"%s\"" % (o,)
317
318  os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") +
319                        os.pathsep + os.environ["PATH"])
320
321  return args
322
323
324def Cleanup():
325  for i in OPTIONS.tempfiles:
326    if os.path.isdir(i):
327      shutil.rmtree(i)
328    else:
329      os.remove(i)
330
331
332class PasswordManager(object):
333  def __init__(self):
334    self.editor = os.getenv("EDITOR", None)
335    self.pwfile = os.getenv("ANDROID_PW_FILE", None)
336
337  def GetPasswords(self, items):
338    """Get passwords corresponding to each string in 'items',
339    returning a dict.  (The dict may have keys in addition to the
340    values in 'items'.)
341
342    Uses the passwords in $ANDROID_PW_FILE if available, letting the
343    user edit that file to add more needed passwords.  If no editor is
344    available, or $ANDROID_PW_FILE isn't define, prompts the user
345    interactively in the ordinary way.
346    """
347
348    current = self.ReadFile()
349
350    first = True
351    while True:
352      missing = []
353      for i in items:
354        if i not in current or not current[i]:
355          missing.append(i)
356      # Are all the passwords already in the file?
357      if not missing: return current
358
359      for i in missing:
360        current[i] = ""
361
362      if not first:
363        print "key file %s still missing some passwords." % (self.pwfile,)
364        answer = raw_input("try to edit again? [y]> ").strip()
365        if answer and answer[0] not in 'yY':
366          raise RuntimeError("key passwords unavailable")
367      first = False
368
369      current = self.UpdateAndReadFile(current)
370
371  def PromptResult(self, current):
372    """Prompt the user to enter a value (password) for each key in
373    'current' whose value is fales.  Returns a new dict with all the
374    values.
375    """
376    result = {}
377    for k, v in sorted(current.iteritems()):
378      if v:
379        result[k] = v
380      else:
381        while True:
382          result[k] = getpass.getpass("Enter password for %s key> "
383                                      % (k,)).strip()
384          if result[k]: break
385    return result
386
387  def UpdateAndReadFile(self, current):
388    if not self.editor or not self.pwfile:
389      return self.PromptResult(current)
390
391    f = open(self.pwfile, "w")
392    os.chmod(self.pwfile, 0600)
393    f.write("# Enter key passwords between the [[[ ]]] brackets.\n")
394    f.write("# (Additional spaces are harmless.)\n\n")
395
396    first_line = None
397    sorted = [(not v, k, v) for (k, v) in current.iteritems()]
398    sorted.sort()
399    for i, (_, k, v) in enumerate(sorted):
400      f.write("[[[  %s  ]]] %s\n" % (v, k))
401      if not v and first_line is None:
402        # position cursor on first line with no password.
403        first_line = i + 4
404    f.close()
405
406    p = Run([self.editor, "+%d" % (first_line,), self.pwfile])
407    _, _ = p.communicate()
408
409    return self.ReadFile()
410
411  def ReadFile(self):
412    result = {}
413    if self.pwfile is None: return result
414    try:
415      f = open(self.pwfile, "r")
416      for line in f:
417        line = line.strip()
418        if not line or line[0] == '#': continue
419        m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line)
420        if not m:
421          print "failed to parse password file: ", line
422        else:
423          result[m.group(2)] = m.group(1)
424      f.close()
425    except IOError, e:
426      if e.errno != errno.ENOENT:
427        print "error reading password file: ", str(e)
428    return result
429
430
431def ZipWriteStr(zip, filename, data, perms=0644):
432  # use a fixed timestamp so the output is repeatable.
433  zinfo = zipfile.ZipInfo(filename=filename,
434                          date_time=(2009, 1, 1, 0, 0, 0))
435  zinfo.compress_type = zip.compression
436  zinfo.external_attr = perms << 16
437  zip.writestr(zinfo, data)
438
439
440class DeviceSpecificParams(object):
441  module = None
442  def __init__(self, **kwargs):
443    """Keyword arguments to the constructor become attributes of this
444    object, which is passed to all functions in the device-specific
445    module."""
446    for k, v in kwargs.iteritems():
447      setattr(self, k, v)
448    self.extras = OPTIONS.extras
449
450    if self.module is None:
451      path = OPTIONS.device_specific
452      if not path: return
453      try:
454        if os.path.isdir(path):
455          info = imp.find_module("releasetools", [path])
456        else:
457          d, f = os.path.split(path)
458          b, x = os.path.splitext(f)
459          if x == ".py":
460            f = b
461          info = imp.find_module(f, [d])
462        self.module = imp.load_module("device_specific", *info)
463      except ImportError:
464        print "unable to load device-specific module; assuming none"
465
466  def _DoCall(self, function_name, *args, **kwargs):
467    """Call the named function in the device-specific module, passing
468    the given args and kwargs.  The first argument to the call will be
469    the DeviceSpecific object itself.  If there is no module, or the
470    module does not define the function, return the value of the
471    'default' kwarg (which itself defaults to None)."""
472    if self.module is None or not hasattr(self.module, function_name):
473      return kwargs.get("default", None)
474    return getattr(self.module, function_name)(*((self,) + args), **kwargs)
475
476  def FullOTA_Assertions(self):
477    """Called after emitting the block of assertions at the top of a
478    full OTA package.  Implementations can add whatever additional
479    assertions they like."""
480    return self._DoCall("FullOTA_Assertions")
481
482  def FullOTA_InstallEnd(self):
483    """Called at the end of full OTA installation; typically this is
484    used to install the image for the device's baseband processor."""
485    return self._DoCall("FullOTA_InstallEnd")
486
487  def IncrementalOTA_Assertions(self):
488    """Called after emitting the block of assertions at the top of an
489    incremental OTA package.  Implementations can add whatever
490    additional assertions they like."""
491    return self._DoCall("IncrementalOTA_Assertions")
492
493  def IncrementalOTA_VerifyEnd(self):
494    """Called at the end of the verification phase of incremental OTA
495    installation; additional checks can be placed here to abort the
496    script before any changes are made."""
497    return self._DoCall("IncrementalOTA_VerifyEnd")
498
499  def IncrementalOTA_InstallEnd(self):
500    """Called at the end of incremental OTA installation; typically
501    this is used to install the image for the device's baseband
502    processor."""
503    return self._DoCall("IncrementalOTA_InstallEnd")
504