add_img_to_target_files.py revision ae3fdd2436854a35951de7c4bd98a2361e1f2123
1#!/usr/bin/env python 2# 3# Copyright (C) 2014 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile that does not contain images (ie, does 19not have an IMAGES/ top-level subdirectory), produce the images and 20add them to the zipfile. 21 22Usage: add_img_to_target_files target_files 23""" 24 25import sys 26 27if sys.hexversion < 0x02070000: 28 print >> sys.stderr, "Python 2.7 or newer is required." 29 sys.exit(1) 30 31import datetime 32import errno 33import os 34import shlex 35import shutil 36import subprocess 37import tempfile 38import zipfile 39 40import build_image 41import common 42import sparse_img 43 44OPTIONS = common.OPTIONS 45 46OPTIONS.add_missing = False 47OPTIONS.rebuild_recovery = False 48OPTIONS.replace_verity_public_key = False 49OPTIONS.replace_verity_private_key = False 50OPTIONS.verity_signer_path = None 51 52def GetCareMap(which, imgname): 53 """Generate care_map of system (or vendor) partition""" 54 55 assert which in ("system", "vendor") 56 _, blk_device = common.GetTypeAndDevice("/" + which, OPTIONS.info_dict) 57 58 simg = sparse_img.SparseImage(imgname) 59 care_map_list = [] 60 care_map_list.append(blk_device) 61 care_map_list.append(simg.care_map.to_string_raw()) 62 return care_map_list 63 64 65def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None): 66 """Turn the contents of SYSTEM into a system image and store it in 67 output_zip. Returns the name of the system image file.""" 68 69 prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system.img") 70 if os.path.exists(prebuilt_path): 71 print "system.img already exists in %s, no need to rebuild..." % (prefix,) 72 return prebuilt_path 73 74 def output_sink(fn, data): 75 ofile = open(os.path.join(OPTIONS.input_tmp, "SYSTEM", fn), "w") 76 ofile.write(data) 77 ofile.close() 78 79 if OPTIONS.rebuild_recovery: 80 print "Building new recovery patch" 81 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img, 82 boot_img, info_dict=OPTIONS.info_dict) 83 84 block_list = common.MakeTempFile(prefix="system-blocklist-", suffix=".map") 85 imgname = BuildSystem(OPTIONS.input_tmp, OPTIONS.info_dict, 86 block_list=block_list) 87 88 # If requested, calculate and add dm-verity integrity hashes and 89 # metadata to system.img. 90 if OPTIONS.info_dict.get("board_bvb_enable", None) == "true": 91 bvbtool = os.getenv('BVBTOOL') or "bvbtool" 92 cmd = [bvbtool, "add_image_hashes", "--image", imgname] 93 args = OPTIONS.info_dict.get("board_bvb_add_image_hashes_args", None) 94 if args and args.strip(): 95 cmd.extend(shlex.split(args)) 96 p = common.Run(cmd, stdout=subprocess.PIPE) 97 p.communicate() 98 assert p.returncode == 0, "bvbtool add_image_hashes of %s image failed" % ( 99 os.path.basename(OPTIONS.input_tmp),) 100 101 common.ZipWrite(output_zip, imgname, prefix + "system.img") 102 common.ZipWrite(output_zip, block_list, prefix + "system.map") 103 return imgname 104 105 106def BuildSystem(input_dir, info_dict, block_list=None): 107 """Build the (sparse) system image and return the name of a temp 108 file containing it.""" 109 return CreateImage(input_dir, info_dict, "system", block_list=block_list) 110 111 112def AddSystemOther(output_zip, prefix="IMAGES/"): 113 """Turn the contents of SYSTEM_OTHER into a system_other image 114 and store it in output_zip.""" 115 116 prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system_other.img") 117 if os.path.exists(prebuilt_path): 118 print "system_other.img already exists in %s, no need to rebuild..." % (prefix,) 119 return 120 121 imgname = BuildSystemOther(OPTIONS.input_tmp, OPTIONS.info_dict) 122 common.ZipWrite(output_zip, imgname, prefix + "system_other.img") 123 124def BuildSystemOther(input_dir, info_dict): 125 """Build the (sparse) system_other image and return the name of a temp 126 file containing it.""" 127 return CreateImage(input_dir, info_dict, "system_other", block_list=None) 128 129 130def AddVendor(output_zip, prefix="IMAGES/"): 131 """Turn the contents of VENDOR into a vendor image and store in it 132 output_zip.""" 133 134 prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "vendor.img") 135 if os.path.exists(prebuilt_path): 136 print "vendor.img already exists in %s, no need to rebuild..." % (prefix,) 137 return prebuilt_path 138 139 block_list = common.MakeTempFile(prefix="vendor-blocklist-", suffix=".map") 140 imgname = BuildVendor(OPTIONS.input_tmp, OPTIONS.info_dict, 141 block_list=block_list) 142 common.ZipWrite(output_zip, imgname, prefix + "vendor.img") 143 common.ZipWrite(output_zip, block_list, prefix + "vendor.map") 144 return imgname 145 146 147def BuildVendor(input_dir, info_dict, block_list=None): 148 """Build the (sparse) vendor image and return the name of a temp 149 file containing it.""" 150 return CreateImage(input_dir, info_dict, "vendor", block_list=block_list) 151 152 153def CreateImage(input_dir, info_dict, what, block_list=None): 154 print "creating " + what + ".img..." 155 156 img = common.MakeTempFile(prefix=what + "-", suffix=".img") 157 158 # The name of the directory it is making an image out of matters to 159 # mkyaffs2image. It wants "system" but we have a directory named 160 # "SYSTEM", so create a symlink. 161 try: 162 os.symlink(os.path.join(input_dir, what.upper()), 163 os.path.join(input_dir, what)) 164 except OSError as e: 165 # bogus error on my mac version? 166 # File "./build/tools/releasetools/img_from_target_files" 167 # os.path.join(OPTIONS.input_tmp, "system")) 168 # OSError: [Errno 17] File exists 169 if e.errno == errno.EEXIST: 170 pass 171 172 image_props = build_image.ImagePropFromGlobalDict(info_dict, what) 173 fstab = info_dict["fstab"] 174 mount_point = "/" + what 175 if fstab and mount_point in fstab: 176 image_props["fs_type"] = fstab[mount_point].fs_type 177 178 # Use a fixed timestamp (01/01/2009) when packaging the image. 179 # Bug: 24377993 180 epoch = datetime.datetime.fromtimestamp(0) 181 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 182 image_props["timestamp"] = int(timestamp) 183 184 if what == "system": 185 fs_config_prefix = "" 186 else: 187 fs_config_prefix = what + "_" 188 189 fs_config = os.path.join( 190 input_dir, "META/" + fs_config_prefix + "filesystem_config.txt") 191 if not os.path.exists(fs_config): 192 fs_config = None 193 194 # Override values loaded from info_dict. 195 if fs_config: 196 image_props["fs_config"] = fs_config 197 if block_list: 198 image_props["block_list"] = block_list 199 200 succ = build_image.BuildImage(os.path.join(input_dir, what), 201 image_props, img) 202 assert succ, "build " + what + ".img image failed" 203 204 return img 205 206 207def AddUserdata(output_zip, prefix="IMAGES/"): 208 """Create a userdata image and store it in output_zip. 209 210 In most case we just create and store an empty userdata.img; 211 But the invoker can also request to create userdata.img with real 212 data from the target files, by setting "userdata_img_with_data=true" 213 in OPTIONS.info_dict. 214 """ 215 216 prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "userdata.img") 217 if os.path.exists(prebuilt_path): 218 print "userdata.img already exists in %s, no need to rebuild..." % (prefix,) 219 return 220 221 # Skip userdata.img if no size. 222 image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict, "data") 223 if not image_props.get("partition_size"): 224 return 225 226 print "creating userdata.img..." 227 228 # Use a fixed timestamp (01/01/2009) when packaging the image. 229 # Bug: 24377993 230 epoch = datetime.datetime.fromtimestamp(0) 231 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 232 image_props["timestamp"] = int(timestamp) 233 234 # The name of the directory it is making an image out of matters to 235 # mkyaffs2image. So we create a temp dir, and within it we create an 236 # empty dir named "data", or a symlink to the DATA dir, 237 # and build the image from that. 238 temp_dir = tempfile.mkdtemp() 239 user_dir = os.path.join(temp_dir, "data") 240 empty = (OPTIONS.info_dict.get("userdata_img_with_data") != "true") 241 if empty: 242 # Create an empty dir. 243 os.mkdir(user_dir) 244 else: 245 # Symlink to the DATA dir. 246 os.symlink(os.path.join(OPTIONS.input_tmp, "DATA"), 247 user_dir) 248 249 img = tempfile.NamedTemporaryFile() 250 251 fstab = OPTIONS.info_dict["fstab"] 252 if fstab: 253 image_props["fs_type"] = fstab["/data"].fs_type 254 succ = build_image.BuildImage(user_dir, image_props, img.name) 255 assert succ, "build userdata.img image failed" 256 257 common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict) 258 common.ZipWrite(output_zip, img.name, prefix + "userdata.img") 259 img.close() 260 shutil.rmtree(temp_dir) 261 262 263def AddPartitionTable(output_zip, prefix="IMAGES/"): 264 """Create a partition table image and store it in output_zip.""" 265 266 _, img_file_name = tempfile.mkstemp() 267 _, bpt_file_name = tempfile.mkstemp() 268 269 # use BPTTOOL from environ, or "bpttool" if empty or not set. 270 bpttool = os.getenv("BPTTOOL") or "bpttool" 271 cmd = [bpttool, "make_table", "--output_json", bpt_file_name, 272 "--output_gpt", img_file_name] 273 input_files_str = OPTIONS.info_dict["board_bpt_input_files"] 274 input_files = input_files_str.split(" ") 275 for i in input_files: 276 cmd.extend(["--input", i]) 277 disk_size = OPTIONS.info_dict.get("board_bpt_disk_size") 278 if disk_size: 279 cmd.extend(["--disk_size", disk_size]) 280 args = OPTIONS.info_dict.get("board_bpt_make_table_args") 281 if args: 282 cmd.extend(shlex.split(args)) 283 284 p = common.Run(cmd, stdout=subprocess.PIPE) 285 p.communicate() 286 assert p.returncode == 0, "bpttool make_table failed" 287 288 common.ZipWrite(output_zip, img_file_name, prefix + "partition-table.img") 289 common.ZipWrite(output_zip, bpt_file_name, prefix + "partition-table.bpt") 290 291 292def AddCache(output_zip, prefix="IMAGES/"): 293 """Create an empty cache image and store it in output_zip.""" 294 295 prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "cache.img") 296 if os.path.exists(prebuilt_path): 297 print "cache.img already exists in %s, no need to rebuild..." % (prefix,) 298 return 299 300 image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict, "cache") 301 # The build system has to explicitly request for cache.img. 302 if "fs_type" not in image_props: 303 return 304 305 print "creating cache.img..." 306 307 # Use a fixed timestamp (01/01/2009) when packaging the image. 308 # Bug: 24377993 309 epoch = datetime.datetime.fromtimestamp(0) 310 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 311 image_props["timestamp"] = int(timestamp) 312 313 # The name of the directory it is making an image out of matters to 314 # mkyaffs2image. So we create a temp dir, and within it we create an 315 # empty dir named "cache", and build the image from that. 316 temp_dir = tempfile.mkdtemp() 317 user_dir = os.path.join(temp_dir, "cache") 318 os.mkdir(user_dir) 319 img = tempfile.NamedTemporaryFile() 320 321 fstab = OPTIONS.info_dict["fstab"] 322 if fstab: 323 image_props["fs_type"] = fstab["/cache"].fs_type 324 succ = build_image.BuildImage(user_dir, image_props, img.name) 325 assert succ, "build cache.img image failed" 326 327 common.CheckSize(img.name, "cache.img", OPTIONS.info_dict) 328 common.ZipWrite(output_zip, img.name, prefix + "cache.img") 329 img.close() 330 os.rmdir(user_dir) 331 os.rmdir(temp_dir) 332 333 334def AddImagesToTargetFiles(filename): 335 OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename) 336 337 if not OPTIONS.add_missing: 338 for n in input_zip.namelist(): 339 if n.startswith("IMAGES/"): 340 print "target_files appears to already contain images." 341 sys.exit(1) 342 343 try: 344 input_zip.getinfo("VENDOR/") 345 has_vendor = True 346 except KeyError: 347 has_vendor = False 348 349 has_system_other = "SYSTEM_OTHER/" in input_zip.namelist() 350 351 OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp) 352 353 common.ZipClose(input_zip) 354 output_zip = zipfile.ZipFile(filename, "a", 355 compression=zipfile.ZIP_DEFLATED, 356 allowZip64=True) 357 358 has_recovery = (OPTIONS.info_dict.get("no_recovery") != "true") 359 system_root_image = (OPTIONS.info_dict.get("system_root_image", None) == "true") 360 board_bvb_enable = (OPTIONS.info_dict.get("board_bvb_enable", None) == "true") 361 362 # Brillo Verified Boot is incompatible with certain 363 # configurations. Explicitly check for these. 364 if board_bvb_enable: 365 assert not has_recovery, "has_recovery incompatible with bvb" 366 assert not system_root_image, "system_root_image incompatible with bvb" 367 assert not OPTIONS.rebuild_recovery, "rebuild_recovery incompatible with bvb" 368 assert not has_vendor, "VENDOR images currently incompatible with bvb" 369 370 def banner(s): 371 print "\n\n++++ " + s + " ++++\n\n" 372 373 prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", "boot.img") 374 boot_image = None 375 if os.path.exists(prebuilt_path): 376 banner("boot") 377 print "boot.img already exists in IMAGES/, no need to rebuild..." 378 if OPTIONS.rebuild_recovery: 379 boot_image = common.GetBootableImage( 380 "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT") 381 else: 382 if board_bvb_enable: 383 # With Brillo Verified Boot, we need to build system.img before 384 # boot.img since the latter includes the dm-verity root hash and 385 # salt for the former. 386 pass 387 else: 388 banner("boot") 389 boot_image = common.GetBootableImage( 390 "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT") 391 if boot_image: 392 boot_image.AddToZip(output_zip) 393 394 recovery_image = None 395 if has_recovery: 396 banner("recovery") 397 prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", "recovery.img") 398 if os.path.exists(prebuilt_path): 399 print "recovery.img already exists in IMAGES/, no need to rebuild..." 400 if OPTIONS.rebuild_recovery: 401 recovery_image = common.GetBootableImage( 402 "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, 403 "RECOVERY") 404 else: 405 recovery_image = common.GetBootableImage( 406 "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY") 407 if recovery_image: 408 recovery_image.AddToZip(output_zip) 409 410 banner("system") 411 system_img_path = AddSystem( 412 output_zip, recovery_img=recovery_image, boot_img=boot_image) 413 if OPTIONS.info_dict.get("board_bvb_enable", None) == "true": 414 # If we're using Brillo Verified Boot, we can now build boot.img 415 # given that we have system.img. 416 banner("boot") 417 boot_image = common.GetBootableImage( 418 "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT", 419 system_img_path=system_img_path) 420 if boot_image: 421 boot_image.AddToZip(output_zip) 422 vendor_img_path = None 423 if has_vendor: 424 banner("vendor") 425 vendor_img_path = AddVendor(output_zip) 426 if has_system_other: 427 banner("system_other") 428 AddSystemOther(output_zip) 429 banner("userdata") 430 AddUserdata(output_zip) 431 banner("cache") 432 AddCache(output_zip) 433 if OPTIONS.info_dict.get("board_bpt_enable", None) == "true": 434 banner("partition-table") 435 AddPartitionTable(output_zip) 436 437 # For devices using A/B update, copy over images from RADIO/ and/or 438 # VENDOR_IMAGES/ to IMAGES/ and make sure we have all the needed 439 # images ready under IMAGES/. All images should have '.img' as extension. 440 banner("radio") 441 ab_partitions = os.path.join(OPTIONS.input_tmp, "META", "ab_partitions.txt") 442 if os.path.exists(ab_partitions): 443 with open(ab_partitions, 'r') as f: 444 lines = f.readlines() 445 # For devices using A/B update, generate care_map for system and vendor 446 # partitions (if present), then write this file to target_files package. 447 care_map_list = [] 448 for line in lines: 449 if line.strip() == "system" and OPTIONS.info_dict.get( 450 "system_verity_block_device", None) is not None: 451 assert os.path.exists(system_img_path) 452 care_map_list += GetCareMap("system", system_img_path) 453 if line.strip() == "vendor" and OPTIONS.info_dict.get( 454 "vendor_verity_block_device", None) is not None: 455 assert os.path.exists(vendor_img_path) 456 care_map_list += GetCareMap("vendor", vendor_img_path) 457 458 img_name = line.strip() + ".img" 459 prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name) 460 if os.path.exists(prebuilt_path): 461 print "%s already exists, no need to overwrite..." % (img_name,) 462 continue 463 464 img_radio_path = os.path.join(OPTIONS.input_tmp, "RADIO", img_name) 465 img_vendor_dir = os.path.join( 466 OPTIONS.input_tmp, "VENDOR_IMAGES") 467 if os.path.exists(img_radio_path): 468 common.ZipWrite(output_zip, img_radio_path, 469 os.path.join("IMAGES", img_name)) 470 else: 471 for root, _, files in os.walk(img_vendor_dir): 472 if img_name in files: 473 common.ZipWrite(output_zip, os.path.join(root, img_name), 474 os.path.join("IMAGES", img_name)) 475 break 476 477 # Zip spec says: All slashes MUST be forward slashes. 478 img_path = 'IMAGES/' + img_name 479 assert img_path in output_zip.namelist(), "cannot find " + img_name 480 481 if care_map_list: 482 file_path = "META/care_map.txt" 483 common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list)) 484 485 common.ZipClose(output_zip) 486 487def main(argv): 488 def option_handler(o, a): 489 if o in ("-a", "--add_missing"): 490 OPTIONS.add_missing = True 491 elif o in ("-r", "--rebuild_recovery",): 492 OPTIONS.rebuild_recovery = True 493 elif o == "--replace_verity_private_key": 494 OPTIONS.replace_verity_private_key = (True, a) 495 elif o == "--replace_verity_public_key": 496 OPTIONS.replace_verity_public_key = (True, a) 497 elif o == "--verity_signer_path": 498 OPTIONS.verity_signer_path = a 499 else: 500 return False 501 return True 502 503 args = common.ParseOptions( 504 argv, __doc__, extra_opts="ar", 505 extra_long_opts=["add_missing", "rebuild_recovery", 506 "replace_verity_public_key=", 507 "replace_verity_private_key=", 508 "verity_signer_path="], 509 extra_option_handler=option_handler) 510 511 512 if len(args) != 1: 513 common.Usage(__doc__) 514 sys.exit(1) 515 516 AddImagesToTargetFiles(args[0]) 517 print "done." 518 519if __name__ == '__main__': 520 try: 521 common.CloseInheritedPipes() 522 main(sys.argv[1:]) 523 except common.ExternalError as e: 524 print 525 print " ERROR: %s" % (e,) 526 print 527 sys.exit(1) 528 finally: 529 common.Cleanup() 530