add_img_to_target_files.py revision 955629b4aa650fbdc681346d1f07906fabfb3226
1#!/usr/bin/env python 2# 3# Copyright (C) 2014 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile that does not contain images (ie, does 19not have an IMAGES/ top-level subdirectory), produce the images and 20add them to the zipfile. 21 22Usage: add_img_to_target_files [flag] target_files 23 24 -a (--add_missing) 25 Build and add missing images to "IMAGES/". If this option is 26 not specified, this script will simply exit when "IMAGES/" 27 directory exists in the target file. 28 29 -r (--rebuild_recovery) 30 Rebuild the recovery patch and write it to the system image. Only 31 meaningful when system image needs to be rebuilt. 32 33 --replace_verity_private_key 34 Replace the private key used for verity signing. (same as the option 35 in sign_target_files_apks) 36 37 --replace_verity_public_key 38 Replace the certificate (public key) used for verity verification. (same 39 as the option in sign_target_files_apks) 40 41 --is_signing 42 Skip building & adding the images for "userdata" and "cache" if we 43 are signing the target files. 44""" 45 46from __future__ import print_function 47 48import sys 49 50if sys.hexversion < 0x02070000: 51 print("Python 2.7 or newer is required.", file=sys.stderr) 52 sys.exit(1) 53 54import datetime 55import errno 56import os 57import shlex 58import shutil 59import subprocess 60import tempfile 61import zipfile 62 63import build_image 64import common 65import rangelib 66import sparse_img 67 68OPTIONS = common.OPTIONS 69 70OPTIONS.add_missing = False 71OPTIONS.rebuild_recovery = False 72OPTIONS.replace_verity_public_key = False 73OPTIONS.replace_verity_private_key = False 74OPTIONS.is_signing = False 75 76def GetCareMap(which, imgname): 77 """Generate care_map of system (or vendor) partition""" 78 79 assert which in ("system", "vendor") 80 81 simg = sparse_img.SparseImage(imgname) 82 care_map_list = [] 83 care_map_list.append(which) 84 85 care_map_ranges = simg.care_map 86 key = which + "_adjusted_partition_size" 87 adjusted_blocks = OPTIONS.info_dict.get(key) 88 if adjusted_blocks: 89 assert adjusted_blocks > 0, "blocks should be positive for " + which 90 care_map_ranges = care_map_ranges.intersect(rangelib.RangeSet( 91 "0-%d" % (adjusted_blocks,))) 92 93 care_map_list.append(care_map_ranges.to_string_raw()) 94 return care_map_list 95 96 97def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None): 98 """Turn the contents of SYSTEM into a system image and store it in 99 output_zip. Returns the name of the system image file.""" 100 101 prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system.img") 102 if os.path.exists(prebuilt_path): 103 print("system.img already exists in %s, no need to rebuild..." % (prefix,)) 104 return prebuilt_path 105 106 def output_sink(fn, data): 107 ofile = open(os.path.join(OPTIONS.input_tmp, "SYSTEM", fn), "w") 108 ofile.write(data) 109 ofile.close() 110 111 if OPTIONS.rebuild_recovery: 112 print("Building new recovery patch") 113 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img, 114 boot_img, info_dict=OPTIONS.info_dict) 115 116 block_list = common.MakeTempFile(prefix="system-blocklist-", suffix=".map") 117 imgname = BuildSystem(OPTIONS.input_tmp, OPTIONS.info_dict, 118 block_list=block_list) 119 120 common.ZipWrite(output_zip, imgname, prefix + "system.img") 121 common.ZipWrite(output_zip, block_list, prefix + "system.map") 122 return imgname 123 124 125def BuildSystem(input_dir, info_dict, block_list=None): 126 """Build the (sparse) system image and return the name of a temp 127 file containing it.""" 128 return CreateImage(input_dir, info_dict, "system", block_list=block_list) 129 130 131def AddSystemOther(output_zip, prefix="IMAGES/"): 132 """Turn the contents of SYSTEM_OTHER into a system_other image 133 and store it in output_zip.""" 134 135 prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system_other.img") 136 if os.path.exists(prebuilt_path): 137 print("system_other.img already exists in %s, no need to rebuild..." % ( 138 prefix,)) 139 return 140 141 imgname = BuildSystemOther(OPTIONS.input_tmp, OPTIONS.info_dict) 142 common.ZipWrite(output_zip, imgname, prefix + "system_other.img") 143 144def BuildSystemOther(input_dir, info_dict): 145 """Build the (sparse) system_other image and return the name of a temp 146 file containing it.""" 147 return CreateImage(input_dir, info_dict, "system_other", block_list=None) 148 149 150def AddVendor(output_zip, prefix="IMAGES/"): 151 """Turn the contents of VENDOR into a vendor image and store in it 152 output_zip.""" 153 154 prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "vendor.img") 155 if os.path.exists(prebuilt_path): 156 print("vendor.img already exists in %s, no need to rebuild..." % (prefix,)) 157 return prebuilt_path 158 159 block_list = common.MakeTempFile(prefix="vendor-blocklist-", suffix=".map") 160 imgname = BuildVendor(OPTIONS.input_tmp, OPTIONS.info_dict, 161 block_list=block_list) 162 common.ZipWrite(output_zip, imgname, prefix + "vendor.img") 163 common.ZipWrite(output_zip, block_list, prefix + "vendor.map") 164 return imgname 165 166 167def BuildVendor(input_dir, info_dict, block_list=None): 168 """Build the (sparse) vendor image and return the name of a temp 169 file containing it.""" 170 return CreateImage(input_dir, info_dict, "vendor", block_list=block_list) 171 172 173def CreateImage(input_dir, info_dict, what, block_list=None): 174 print("creating " + what + ".img...") 175 176 img = common.MakeTempFile(prefix=what + "-", suffix=".img") 177 178 # The name of the directory it is making an image out of matters to 179 # mkyaffs2image. It wants "system" but we have a directory named 180 # "SYSTEM", so create a symlink. 181 try: 182 os.symlink(os.path.join(input_dir, what.upper()), 183 os.path.join(input_dir, what)) 184 except OSError as e: 185 # bogus error on my mac version? 186 # File "./build/tools/releasetools/img_from_target_files" 187 # os.path.join(OPTIONS.input_tmp, "system")) 188 # OSError: [Errno 17] File exists 189 if e.errno == errno.EEXIST: 190 pass 191 192 image_props = build_image.ImagePropFromGlobalDict(info_dict, what) 193 fstab = info_dict["fstab"] 194 mount_point = "/" + what 195 if fstab and mount_point in fstab: 196 image_props["fs_type"] = fstab[mount_point].fs_type 197 198 # Use a fixed timestamp (01/01/2009) when packaging the image. 199 # Bug: 24377993 200 epoch = datetime.datetime.fromtimestamp(0) 201 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 202 image_props["timestamp"] = int(timestamp) 203 204 if what == "system": 205 fs_config_prefix = "" 206 else: 207 fs_config_prefix = what + "_" 208 209 fs_config = os.path.join( 210 input_dir, "META/" + fs_config_prefix + "filesystem_config.txt") 211 if not os.path.exists(fs_config): 212 fs_config = None 213 214 # Override values loaded from info_dict. 215 if fs_config: 216 image_props["fs_config"] = fs_config 217 if block_list: 218 image_props["block_list"] = block_list 219 220 succ = build_image.BuildImage(os.path.join(input_dir, what), 221 image_props, img) 222 assert succ, "build " + what + ".img image failed" 223 224 is_verity_partition = "verity_block_device" in image_props 225 verity_supported = image_props.get("verity") == "true" 226 if is_verity_partition and verity_supported: 227 adjusted_blocks_value = image_props.get("partition_size") 228 if adjusted_blocks_value: 229 adjusted_blocks_key = what + "_adjusted_partition_size" 230 info_dict[adjusted_blocks_key] = int(adjusted_blocks_value)/4096 - 1 231 232 return img 233 234 235def AddUserdata(output_zip, prefix="IMAGES/"): 236 """Create a userdata image and store it in output_zip. 237 238 In most case we just create and store an empty userdata.img; 239 But the invoker can also request to create userdata.img with real 240 data from the target files, by setting "userdata_img_with_data=true" 241 in OPTIONS.info_dict. 242 """ 243 244 prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "userdata.img") 245 if os.path.exists(prebuilt_path): 246 print("userdata.img already exists in %s, no need to rebuild..." % ( 247 prefix,)) 248 return 249 250 # Skip userdata.img if no size. 251 image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict, "data") 252 if not image_props.get("partition_size"): 253 return 254 255 print("creating userdata.img...") 256 257 # Use a fixed timestamp (01/01/2009) when packaging the image. 258 # Bug: 24377993 259 epoch = datetime.datetime.fromtimestamp(0) 260 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 261 image_props["timestamp"] = int(timestamp) 262 263 # The name of the directory it is making an image out of matters to 264 # mkyaffs2image. So we create a temp dir, and within it we create an 265 # empty dir named "data", or a symlink to the DATA dir, 266 # and build the image from that. 267 temp_dir = tempfile.mkdtemp() 268 user_dir = os.path.join(temp_dir, "data") 269 empty = (OPTIONS.info_dict.get("userdata_img_with_data") != "true") 270 if empty: 271 # Create an empty dir. 272 os.mkdir(user_dir) 273 else: 274 # Symlink to the DATA dir. 275 os.symlink(os.path.join(OPTIONS.input_tmp, "DATA"), 276 user_dir) 277 278 img = tempfile.NamedTemporaryFile() 279 280 fstab = OPTIONS.info_dict["fstab"] 281 if fstab: 282 image_props["fs_type"] = fstab["/data"].fs_type 283 succ = build_image.BuildImage(user_dir, image_props, img.name) 284 assert succ, "build userdata.img image failed" 285 286 common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict) 287 common.ZipWrite(output_zip, img.name, prefix + "userdata.img") 288 img.close() 289 shutil.rmtree(temp_dir) 290 291 292def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"): 293 """Create a VBMeta image and store it in output_zip.""" 294 _, img_file_name = tempfile.mkstemp() 295 avbtool = os.getenv('AVBTOOL') or "avbtool" 296 cmd = [avbtool, "make_vbmeta_image", 297 "--output", img_file_name, 298 "--include_descriptors_from_image", boot_img_path, 299 "--include_descriptors_from_image", system_img_path, 300 "--generate_dm_verity_cmdline_from_hashtree", system_img_path] 301 common.AppendAVBSigningArgs(cmd) 302 args = OPTIONS.info_dict.get("board_avb_make_vbmeta_image_args", None) 303 if args and args.strip(): 304 cmd.extend(shlex.split(args)) 305 p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 306 p.communicate() 307 assert p.returncode == 0, "avbtool make_vbmeta_image failed" 308 common.ZipWrite(output_zip, img_file_name, prefix + "vbmeta.img") 309 310 311def AddPartitionTable(output_zip, prefix="IMAGES/"): 312 """Create a partition table image and store it in output_zip.""" 313 314 _, img_file_name = tempfile.mkstemp() 315 _, bpt_file_name = tempfile.mkstemp() 316 317 # use BPTTOOL from environ, or "bpttool" if empty or not set. 318 bpttool = os.getenv("BPTTOOL") or "bpttool" 319 cmd = [bpttool, "make_table", "--output_json", bpt_file_name, 320 "--output_gpt", img_file_name] 321 input_files_str = OPTIONS.info_dict["board_bpt_input_files"] 322 input_files = input_files_str.split(" ") 323 for i in input_files: 324 cmd.extend(["--input", i]) 325 disk_size = OPTIONS.info_dict.get("board_bpt_disk_size") 326 if disk_size: 327 cmd.extend(["--disk_size", disk_size]) 328 args = OPTIONS.info_dict.get("board_bpt_make_table_args") 329 if args: 330 cmd.extend(shlex.split(args)) 331 332 p = common.Run(cmd, stdout=subprocess.PIPE) 333 p.communicate() 334 assert p.returncode == 0, "bpttool make_table failed" 335 336 common.ZipWrite(output_zip, img_file_name, prefix + "partition-table.img") 337 common.ZipWrite(output_zip, bpt_file_name, prefix + "partition-table.bpt") 338 339 340def AddCache(output_zip, prefix="IMAGES/"): 341 """Create an empty cache image and store it in output_zip.""" 342 343 prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "cache.img") 344 if os.path.exists(prebuilt_path): 345 print("cache.img already exists in %s, no need to rebuild..." % (prefix,)) 346 return 347 348 image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict, "cache") 349 # The build system has to explicitly request for cache.img. 350 if "fs_type" not in image_props: 351 return 352 353 print("creating cache.img...") 354 355 # Use a fixed timestamp (01/01/2009) when packaging the image. 356 # Bug: 24377993 357 epoch = datetime.datetime.fromtimestamp(0) 358 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 359 image_props["timestamp"] = int(timestamp) 360 361 # The name of the directory it is making an image out of matters to 362 # mkyaffs2image. So we create a temp dir, and within it we create an 363 # empty dir named "cache", and build the image from that. 364 temp_dir = tempfile.mkdtemp() 365 user_dir = os.path.join(temp_dir, "cache") 366 os.mkdir(user_dir) 367 img = tempfile.NamedTemporaryFile() 368 369 fstab = OPTIONS.info_dict["fstab"] 370 if fstab: 371 image_props["fs_type"] = fstab["/cache"].fs_type 372 succ = build_image.BuildImage(user_dir, image_props, img.name) 373 assert succ, "build cache.img image failed" 374 375 common.CheckSize(img.name, "cache.img", OPTIONS.info_dict) 376 common.ZipWrite(output_zip, img.name, prefix + "cache.img") 377 img.close() 378 os.rmdir(user_dir) 379 os.rmdir(temp_dir) 380 381 382def AddImagesToTargetFiles(filename): 383 OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename) 384 385 if not OPTIONS.add_missing: 386 for n in input_zip.namelist(): 387 if n.startswith("IMAGES/"): 388 print("target_files appears to already contain images.") 389 sys.exit(1) 390 391 try: 392 input_zip.getinfo("VENDOR/") 393 has_vendor = True 394 except KeyError: 395 has_vendor = False 396 397 has_system_other = "SYSTEM_OTHER/" in input_zip.namelist() 398 399 OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp) 400 401 common.ZipClose(input_zip) 402 output_zip = zipfile.ZipFile(filename, "a", 403 compression=zipfile.ZIP_DEFLATED, 404 allowZip64=True) 405 406 has_recovery = (OPTIONS.info_dict.get("no_recovery") != "true") 407 system_root_image = (OPTIONS.info_dict.get("system_root_image", None) == "true") 408 409 def banner(s): 410 print("\n\n++++ " + s + " ++++\n\n") 411 412 prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", "boot.img") 413 boot_image = None 414 if os.path.exists(prebuilt_path): 415 banner("boot") 416 print("boot.img already exists in IMAGES/, no need to rebuild...") 417 if OPTIONS.rebuild_recovery: 418 boot_image = common.GetBootableImage( 419 "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT") 420 else: 421 banner("boot") 422 boot_image = common.GetBootableImage( 423 "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT") 424 if boot_image: 425 boot_image.AddToZip(output_zip) 426 427 recovery_image = None 428 if has_recovery: 429 banner("recovery") 430 prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", "recovery.img") 431 if os.path.exists(prebuilt_path): 432 print("recovery.img already exists in IMAGES/, no need to rebuild...") 433 if OPTIONS.rebuild_recovery: 434 recovery_image = common.GetBootableImage( 435 "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, 436 "RECOVERY") 437 else: 438 recovery_image = common.GetBootableImage( 439 "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY") 440 if recovery_image: 441 recovery_image.AddToZip(output_zip) 442 443 banner("recovery (two-step image)") 444 # The special recovery.img for two-step package use. 445 recovery_two_step_image = common.GetBootableImage( 446 "IMAGES/recovery-two-step.img", "recovery-two-step.img", 447 OPTIONS.input_tmp, "RECOVERY", two_step_image=True) 448 if recovery_two_step_image: 449 recovery_two_step_image.AddToZip(output_zip) 450 451 banner("system") 452 system_img_path = AddSystem( 453 output_zip, recovery_img=recovery_image, boot_img=boot_image) 454 vendor_img_path = None 455 if has_vendor: 456 banner("vendor") 457 vendor_img_path = AddVendor(output_zip) 458 if has_system_other: 459 banner("system_other") 460 AddSystemOther(output_zip) 461 if not OPTIONS.is_signing: 462 banner("userdata") 463 AddUserdata(output_zip) 464 banner("cache") 465 AddCache(output_zip) 466 if OPTIONS.info_dict.get("board_bpt_enable", None) == "true": 467 banner("partition-table") 468 AddPartitionTable(output_zip) 469 if OPTIONS.info_dict.get("board_avb_enable", None) == "true": 470 banner("vbmeta") 471 boot_contents = boot_image.WriteToTemp() 472 AddVBMeta(output_zip, boot_contents.name, system_img_path) 473 474 # For devices using A/B update, copy over images from RADIO/ and/or 475 # VENDOR_IMAGES/ to IMAGES/ and make sure we have all the needed 476 # images ready under IMAGES/. All images should have '.img' as extension. 477 banner("radio") 478 ab_partitions = os.path.join(OPTIONS.input_tmp, "META", "ab_partitions.txt") 479 if os.path.exists(ab_partitions): 480 with open(ab_partitions, 'r') as f: 481 lines = f.readlines() 482 # For devices using A/B update, generate care_map for system and vendor 483 # partitions (if present), then write this file to target_files package. 484 care_map_list = [] 485 for line in lines: 486 if line.strip() == "system" and OPTIONS.info_dict.get( 487 "system_verity_block_device", None) is not None: 488 assert os.path.exists(system_img_path) 489 care_map_list += GetCareMap("system", system_img_path) 490 if line.strip() == "vendor" and OPTIONS.info_dict.get( 491 "vendor_verity_block_device", None) is not None: 492 assert os.path.exists(vendor_img_path) 493 care_map_list += GetCareMap("vendor", vendor_img_path) 494 495 img_name = line.strip() + ".img" 496 prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name) 497 if os.path.exists(prebuilt_path): 498 print("%s already exists, no need to overwrite..." % (img_name,)) 499 continue 500 501 img_radio_path = os.path.join(OPTIONS.input_tmp, "RADIO", img_name) 502 img_vendor_dir = os.path.join( 503 OPTIONS.input_tmp, "VENDOR_IMAGES") 504 if os.path.exists(img_radio_path): 505 common.ZipWrite(output_zip, img_radio_path, 506 os.path.join("IMAGES", img_name)) 507 else: 508 for root, _, files in os.walk(img_vendor_dir): 509 if img_name in files: 510 common.ZipWrite(output_zip, os.path.join(root, img_name), 511 os.path.join("IMAGES", img_name)) 512 break 513 514 # Zip spec says: All slashes MUST be forward slashes. 515 img_path = 'IMAGES/' + img_name 516 assert img_path in output_zip.namelist(), "cannot find " + img_name 517 518 if care_map_list: 519 file_path = "META/care_map.txt" 520 common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list)) 521 522 common.ZipClose(output_zip) 523 524def main(argv): 525 def option_handler(o, a): 526 if o in ("-a", "--add_missing"): 527 OPTIONS.add_missing = True 528 elif o in ("-r", "--rebuild_recovery",): 529 OPTIONS.rebuild_recovery = True 530 elif o == "--replace_verity_private_key": 531 OPTIONS.replace_verity_private_key = (True, a) 532 elif o == "--replace_verity_public_key": 533 OPTIONS.replace_verity_public_key = (True, a) 534 elif o == "--is_signing": 535 OPTIONS.is_signing = True 536 else: 537 return False 538 return True 539 540 args = common.ParseOptions( 541 argv, __doc__, extra_opts="ar", 542 extra_long_opts=["add_missing", "rebuild_recovery", 543 "replace_verity_public_key=", 544 "replace_verity_private_key=", 545 "is_signing"], 546 extra_option_handler=option_handler) 547 548 549 if len(args) != 1: 550 common.Usage(__doc__) 551 sys.exit(1) 552 553 AddImagesToTargetFiles(args[0]) 554 print("done.") 555 556if __name__ == '__main__': 557 try: 558 common.CloseInheritedPipes() 559 main(sys.argv[1:]) 560 except common.ExternalError as e: 561 print("\n ERROR: %s\n" % (e,)) 562 sys.exit(1) 563 finally: 564 common.Cleanup() 565