build_image.py revision 4e66343b2b28ac249214c9f5cb5de936f0f209ad 
1#!/usr/bin/env python
2#
3# Copyright (C) 2011 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Build image output_image_file from input_directory and properties_file.
19
20Usage:  build_image input_directory properties_file output_image_file
21
22"""
23import os
24import os.path
25import re
26import subprocess
27import sys
28import commands
29import shutil
30import tempfile
31
32FIXED_SALT = "aee087a5be3b982978c923f566a94613496b417f2af592639bc80d141e34dfe7"
33
34def RunCommand(cmd):
35  """Echo and run the given command.
36
37  Args:
38    cmd: the command represented as a list of strings.
39  Returns:
40    A tuple of the output and the exit code.
41  """
42  print "Running: ", " ".join(cmd)
43  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
44  output, _ = p.communicate()
45  print "%s" % (output.rstrip(),)
46  return (output, p.returncode)
47
48def GetVerityTreeSize(partition_size):
49  cmd = "build_verity_tree -s %d"
50  cmd %= partition_size
51  status, output = commands.getstatusoutput(cmd)
52  if status:
53    print output
54    return False, 0
55  return True, int(output)
56
57def GetVerityMetadataSize(partition_size):
58  cmd = "system/extras/verity/build_verity_metadata.py -s %d"
59  cmd %= partition_size
60  status, output = commands.getstatusoutput(cmd)
61  if status:
62    print output
63    return False, 0
64  return True, int(output)
65
66def AdjustPartitionSizeForVerity(partition_size):
67  """Modifies the provided partition size to account for the verity metadata.
68
69  This information is used to size the created image appropriately.
70  Args:
71    partition_size: the size of the partition to be verified.
72  Returns:
73    The size of the partition adjusted for verity metadata.
74  """
75  success, verity_tree_size = GetVerityTreeSize(partition_size)
76  if not success:
77    return 0
78  success, verity_metadata_size = GetVerityMetadataSize(partition_size)
79  if not success:
80    return 0
81  return partition_size - verity_tree_size - verity_metadata_size
82
83def BuildVerityTree(sparse_image_path, verity_image_path, prop_dict):
84  cmd = "build_verity_tree -A %s %s %s" % (
85      FIXED_SALT, sparse_image_path, verity_image_path)
86  print cmd
87  status, output = commands.getstatusoutput(cmd)
88  if status:
89    print "Could not build verity tree! Error: %s" % output
90    return False
91  root, salt = output.split()
92  prop_dict["verity_root_hash"] = root
93  prop_dict["verity_salt"] = salt
94  return True
95
96def BuildVerityMetadata(image_size, verity_metadata_path, root_hash, salt,
97                        block_device, signer_path, key):
98  cmd_template = (
99      "system/extras/verity/build_verity_metadata.py %s %s %s %s %s %s %s")
100  cmd = cmd_template % (image_size, verity_metadata_path, root_hash, salt,
101                        block_device, signer_path, key)
102  print cmd
103  status, output = commands.getstatusoutput(cmd)
104  if status:
105    print "Could not build verity metadata! Error: %s" % output
106    return False
107  return True
108
109def Append2Simg(sparse_image_path, unsparse_image_path, error_message):
110  """Appends the unsparse image to the given sparse image.
111
112  Args:
113    sparse_image_path: the path to the (sparse) image
114    unsparse_image_path: the path to the (unsparse) image
115  Returns:
116    True on success, False on failure.
117  """
118  cmd = "append2simg %s %s"
119  cmd %= (sparse_image_path, unsparse_image_path)
120  print cmd
121  status, output = commands.getstatusoutput(cmd)
122  if status:
123    print "%s: %s" % (error_message, output)
124    return False
125  return True
126
127def BuildVerifiedImage(data_image_path, verity_image_path,
128                       verity_metadata_path):
129  if not Append2Simg(data_image_path, verity_metadata_path,
130                     "Could not append verity metadata!"):
131    return False
132  if not Append2Simg(data_image_path, verity_image_path,
133                     "Could not append verity tree!"):
134    return False
135  return True
136
137def UnsparseImage(sparse_image_path, replace=True):
138  img_dir = os.path.dirname(sparse_image_path)
139  unsparse_image_path = "unsparse_" + os.path.basename(sparse_image_path)
140  unsparse_image_path = os.path.join(img_dir, unsparse_image_path)
141  if os.path.exists(unsparse_image_path):
142    if replace:
143      os.unlink(unsparse_image_path)
144    else:
145      return True, unsparse_image_path
146  inflate_command = ["simg2img", sparse_image_path, unsparse_image_path]
147  (_, exit_code) = RunCommand(inflate_command)
148  if exit_code != 0:
149    os.remove(unsparse_image_path)
150    return False, None
151  return True, unsparse_image_path
152
153def MakeVerityEnabledImage(out_file, prop_dict):
154  """Creates an image that is verifiable using dm-verity.
155
156  Args:
157    out_file: the location to write the verifiable image at
158    prop_dict: a dictionary of properties required for image creation and
159               verification
160  Returns:
161    True on success, False otherwise.
162  """
163  # get properties
164  image_size = prop_dict["partition_size"]
165  block_dev = prop_dict["verity_block_device"]
166  signer_key = prop_dict["verity_key"] + ".pk8"
167  signer_path = prop_dict["verity_signer_cmd"]
168
169  # make a tempdir
170  tempdir_name = tempfile.mkdtemp(suffix="_verity_images")
171
172  # get partial image paths
173  verity_image_path = os.path.join(tempdir_name, "verity.img")
174  verity_metadata_path = os.path.join(tempdir_name, "verity_metadata.img")
175
176  # build the verity tree and get the root hash and salt
177  if not BuildVerityTree(out_file, verity_image_path, prop_dict):
178    shutil.rmtree(tempdir_name, ignore_errors=True)
179    return False
180
181  # build the metadata blocks
182  root_hash = prop_dict["verity_root_hash"]
183  salt = prop_dict["verity_salt"]
184  if not BuildVerityMetadata(image_size, verity_metadata_path, root_hash, salt,
185                             block_dev, signer_path, signer_key):
186    shutil.rmtree(tempdir_name, ignore_errors=True)
187    return False
188
189  # build the full verified image
190  if not BuildVerifiedImage(out_file,
191                            verity_image_path,
192                            verity_metadata_path):
193    shutil.rmtree(tempdir_name, ignore_errors=True)
194    return False
195
196  shutil.rmtree(tempdir_name, ignore_errors=True)
197  return True
198
199def BuildImage(in_dir, prop_dict, out_file):
200  """Build an image to out_file from in_dir with property prop_dict.
201
202  Args:
203    in_dir: path of input directory.
204    prop_dict: property dictionary.
205    out_file: path of the output image file.
206
207  Returns:
208    True iff the image is built successfully.
209  """
210  # system_root_image=true: build a system.img that combines the contents of
211  # /system and the ramdisk, and can be mounted at the root of the file system.
212  origin_in = in_dir
213  fs_config = prop_dict.get("fs_config")
214  if (prop_dict.get("system_root_image") == "true"
215      and prop_dict["mount_point"] == "system"):
216    in_dir = tempfile.mkdtemp()
217    # Change the mount point to "/"
218    prop_dict["mount_point"] = "/"
219    if fs_config:
220      # We need to merge the fs_config files of system and ramdisk.
221      fd, merged_fs_config = tempfile.mkstemp(prefix="root_fs_config",
222                                              suffix=".txt")
223      os.close(fd)
224      with open(merged_fs_config, "w") as fw:
225        if "ramdisk_fs_config" in prop_dict:
226          with open(prop_dict["ramdisk_fs_config"]) as fr:
227            fw.writelines(fr.readlines())
228        with open(fs_config) as fr:
229          fw.writelines(fr.readlines())
230      fs_config = merged_fs_config
231
232  build_command = []
233  fs_type = prop_dict.get("fs_type", "")
234  run_fsck = False
235
236  fs_spans_partition = True
237  if fs_type.startswith("squash"):
238    fs_spans_partition = False
239
240  is_verity_partition = "verity_block_device" in prop_dict
241  verity_supported = prop_dict.get("verity") == "true"
242  # Adjust the partition size to make room for the hashes if this is to be
243  # verified.
244  if verity_supported and is_verity_partition and fs_spans_partition:
245    partition_size = int(prop_dict.get("partition_size"))
246    adjusted_size = AdjustPartitionSizeForVerity(partition_size)
247    if not adjusted_size:
248      return False
249    prop_dict["partition_size"] = str(adjusted_size)
250    prop_dict["original_partition_size"] = str(partition_size)
251
252  if fs_type.startswith("ext"):
253    build_command = ["mkuserimg.sh"]
254    if "extfs_sparse_flag" in prop_dict:
255      build_command.append(prop_dict["extfs_sparse_flag"])
256      run_fsck = True
257    build_command.extend([in_dir, out_file, fs_type,
258                          prop_dict["mount_point"]])
259    build_command.append(prop_dict["partition_size"])
260    if "journal_size" in prop_dict:
261      build_command.extend(["-j", prop_dict["journal_size"]])
262    if "timestamp" in prop_dict:
263      build_command.extend(["-T", str(prop_dict["timestamp"])])
264    if fs_config:
265      build_command.extend(["-C", fs_config])
266    if "block_list" in prop_dict:
267      build_command.extend(["-B", prop_dict["block_list"]])
268    build_command.extend(["-L", prop_dict["mount_point"]])
269    if "selinux_fc" in prop_dict:
270      build_command.append(prop_dict["selinux_fc"])
271  elif fs_type.startswith("squash"):
272    build_command = ["mksquashfsimage.sh"]
273    build_command.extend([in_dir, out_file])
274    build_command.extend(["-s"])
275    build_command.extend(["-m", prop_dict["mount_point"]])
276    if "selinux_fc" in prop_dict:
277      build_command.extend(["-c", prop_dict["selinux_fc"]])
278  elif fs_type.startswith("f2fs"):
279    build_command = ["mkf2fsuserimg.sh"]
280    build_command.extend([out_file, prop_dict["partition_size"]])
281  else:
282    build_command = ["mkyaffs2image", "-f"]
283    if prop_dict.get("mkyaffs2_extra_flags", None):
284      build_command.extend(prop_dict["mkyaffs2_extra_flags"].split())
285    build_command.append(in_dir)
286    build_command.append(out_file)
287    if "selinux_fc" in prop_dict:
288      build_command.append(prop_dict["selinux_fc"])
289      build_command.append(prop_dict["mount_point"])
290
291  if in_dir != origin_in:
292    # Construct a staging directory of the root file system.
293    ramdisk_dir = prop_dict.get("ramdisk_dir")
294    if ramdisk_dir:
295      shutil.rmtree(in_dir)
296      shutil.copytree(ramdisk_dir, in_dir, symlinks=True)
297    staging_system = os.path.join(in_dir, "system")
298    shutil.rmtree(staging_system, ignore_errors=True)
299    shutil.copytree(origin_in, staging_system, symlinks=True)
300
301  reserved_blocks = prop_dict.get("has_ext4_reserved_blocks") == "true"
302  ext4fs_output = None
303
304  try:
305    if reserved_blocks and fs_type.startswith("ext4"):
306      (ext4fs_output, exit_code) = RunCommand(build_command)
307    else:
308      (_, exit_code) = RunCommand(build_command)
309  finally:
310    if in_dir != origin_in:
311      # Clean up temporary directories and files.
312      shutil.rmtree(in_dir, ignore_errors=True)
313      if fs_config:
314        os.remove(fs_config)
315  if exit_code != 0:
316    return False
317
318  # Bug: 21522719, 22023465
319  # There are some reserved blocks on ext4 FS (lesser of 4096 blocks and 2%).
320  # We need to deduct those blocks from the available space, since they are
321  # not writable even with root privilege. It only affects devices using
322  # file-based OTA and a kernel version of 3.10 or greater (currently just
323  # sprout).
324  if reserved_blocks and fs_type.startswith("ext4"):
325    assert ext4fs_output is not None
326    ext4fs_stats = re.compile(
327        r'Created filesystem with .* (?P<used_blocks>[0-9]+)/'
328        r'(?P<total_blocks>[0-9]+) blocks')
329    m = ext4fs_stats.match(ext4fs_output.strip().split('\n')[-1])
330    used_blocks = int(m.groupdict().get('used_blocks'))
331    total_blocks = int(m.groupdict().get('total_blocks'))
332    reserved_blocks = min(4096, int(total_blocks * 0.02))
333    adjusted_blocks = total_blocks - reserved_blocks
334    if used_blocks > adjusted_blocks:
335      mount_point = prop_dict.get("mount_point")
336      print("Error: Not enough room on %s (total: %d blocks, used: %d blocks, "
337            "reserved: %d blocks, available: %d blocks)" % (
338                mount_point, total_blocks, used_blocks, reserved_blocks,
339                adjusted_blocks))
340      return False
341
342  if not fs_spans_partition:
343    mount_point = prop_dict.get("mount_point")
344    partition_size = int(prop_dict.get("partition_size"))
345    image_size = os.stat(out_file).st_size
346    if image_size > partition_size:
347      print("Error: %s image size of %d is larger than partition size of "
348            "%d" % (mount_point, image_size, partition_size))
349      return False
350    if verity_supported and is_verity_partition:
351      if 2 * image_size - AdjustPartitionSizeForVerity(image_size) > partition_size:
352        print "Error: No more room on %s to fit verity data" % mount_point
353        return False
354    prop_dict["original_partition_size"] = prop_dict["partition_size"]
355    prop_dict["partition_size"] = str(image_size)
356
357  # create the verified image if this is to be verified
358  if verity_supported and is_verity_partition:
359    if not MakeVerityEnabledImage(out_file, prop_dict):
360      return False
361
362  if run_fsck and prop_dict.get("skip_fsck") != "true":
363    success, unsparse_image = UnsparseImage(out_file, replace=False)
364    if not success:
365      return False
366
367    # Run e2fsck on the inflated image file
368    e2fsck_command = ["e2fsck", "-f", "-n", unsparse_image]
369    (_, exit_code) = RunCommand(e2fsck_command)
370
371    os.remove(unsparse_image)
372
373  return exit_code == 0
374
375
376def ImagePropFromGlobalDict(glob_dict, mount_point):
377  """Build an image property dictionary from the global dictionary.
378
379  Args:
380    glob_dict: the global dictionary from the build system.
381    mount_point: such as "system", "data" etc.
382  """
383  d = {}
384  if "build.prop" in glob_dict:
385    bp = glob_dict["build.prop"]
386    if "ro.build.date.utc" in bp:
387      d["timestamp"] = bp["ro.build.date.utc"]
388
389  def copy_prop(src_p, dest_p):
390    if src_p in glob_dict:
391      d[dest_p] = str(glob_dict[src_p])
392
393  common_props = (
394      "extfs_sparse_flag",
395      "mkyaffs2_extra_flags",
396      "selinux_fc",
397      "skip_fsck",
398      "verity",
399      "verity_key",
400      "verity_signer_cmd"
401      )
402  for p in common_props:
403    copy_prop(p, p)
404
405  d["mount_point"] = mount_point
406  if mount_point == "system":
407    copy_prop("fs_type", "fs_type")
408    # Copy the generic sysetem fs type first, override with specific one if
409    # available.
410    copy_prop("system_fs_type", "fs_type")
411    copy_prop("system_size", "partition_size")
412    copy_prop("system_journal_size", "journal_size")
413    copy_prop("system_verity_block_device", "verity_block_device")
414    copy_prop("system_root_image", "system_root_image")
415    copy_prop("ramdisk_dir", "ramdisk_dir")
416    copy_prop("has_ext4_reserved_blocks", "has_ext4_reserved_blocks")
417  elif mount_point == "data":
418    # Copy the generic fs type first, override with specific one if available.
419    copy_prop("fs_type", "fs_type")
420    copy_prop("userdata_fs_type", "fs_type")
421    copy_prop("userdata_size", "partition_size")
422  elif mount_point == "cache":
423    copy_prop("cache_fs_type", "fs_type")
424    copy_prop("cache_size", "partition_size")
425  elif mount_point == "vendor":
426    copy_prop("vendor_fs_type", "fs_type")
427    copy_prop("vendor_size", "partition_size")
428    copy_prop("vendor_journal_size", "journal_size")
429    copy_prop("vendor_verity_block_device", "verity_block_device")
430    copy_prop("has_ext4_reserved_blocks", "has_ext4_reserved_blocks")
431  elif mount_point == "oem":
432    copy_prop("fs_type", "fs_type")
433    copy_prop("oem_size", "partition_size")
434    copy_prop("oem_journal_size", "journal_size")
435    copy_prop("has_ext4_reserved_blocks", "has_ext4_reserved_blocks")
436
437  return d
438
439
440def LoadGlobalDict(filename):
441  """Load "name=value" pairs from filename"""
442  d = {}
443  f = open(filename)
444  for line in f:
445    line = line.strip()
446    if not line or line.startswith("#"):
447      continue
448    k, v = line.split("=", 1)
449    d[k] = v
450  f.close()
451  return d
452
453
454def main(argv):
455  if len(argv) != 3:
456    print __doc__
457    sys.exit(1)
458
459  in_dir = argv[0]
460  glob_dict_file = argv[1]
461  out_file = argv[2]
462
463  glob_dict = LoadGlobalDict(glob_dict_file)
464  if "mount_point" in glob_dict:
465    # The caller knows the mount point and provides a dictionay needed by
466    # BuildImage().
467    image_properties = glob_dict
468  else:
469    image_filename = os.path.basename(out_file)
470    mount_point = ""
471    if image_filename == "system.img":
472      mount_point = "system"
473    elif image_filename == "userdata.img":
474      mount_point = "data"
475    elif image_filename == "cache.img":
476      mount_point = "cache"
477    elif image_filename == "vendor.img":
478      mount_point = "vendor"
479    elif image_filename == "oem.img":
480      mount_point = "oem"
481    else:
482      print >> sys.stderr, "error: unknown image file name ", image_filename
483      exit(1)
484
485    image_properties = ImagePropFromGlobalDict(glob_dict, mount_point)
486
487  if not BuildImage(in_dir, image_properties, out_file):
488    print >> sys.stderr, "error: failed to build %s from %s" % (out_file,
489                                                                in_dir)
490    exit(1)
491
492
493if __name__ == '__main__':
494  main(sys.argv[1:])
495