output_file.c revision 14e28d39f7f094225c1ddae8fa43bd792c621a8f
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define _FILE_OFFSET_BITS 64 18#define _LARGEFILE64_SOURCE 1 19 20#include <fcntl.h> 21#include <limits.h> 22#include <stdbool.h> 23#include <stddef.h> 24#include <stdlib.h> 25#include <string.h> 26#include <sys/stat.h> 27#include <sys/types.h> 28#include <unistd.h> 29#include <zlib.h> 30 31#include "output_file.h" 32#include "sparse_format.h" 33#include "sparse_crc32.h" 34 35#ifndef USE_MINGW 36#include <sys/mman.h> 37#define O_BINARY 0 38#else 39#define ftruncate64 ftruncate 40#endif 41 42#if defined(__APPLE__) && defined(__MACH__) 43#define lseek64 lseek 44#define ftruncate64 ftruncate 45#define mmap64 mmap 46#define off64_t off_t 47#endif 48 49#define min(a, b) \ 50 ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; }) 51 52#define SPARSE_HEADER_MAJOR_VER 1 53#define SPARSE_HEADER_MINOR_VER 0 54#define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) 55#define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) 56 57#define container_of(inner, outer_t, elem) \ 58 ((outer_t *)((char *)inner - offsetof(outer_t, elem))) 59 60struct output_file_ops { 61 int (*open)(struct output_file *, int fd); 62 int (*skip)(struct output_file *, int64_t); 63 int (*pad)(struct output_file *, int64_t); 64 int (*write)(struct output_file *, void *, int); 65 void (*close)(struct output_file *); 66}; 67 68struct sparse_file_ops { 69 int (*write_data_chunk)(struct output_file *out, unsigned int len, 70 void *data); 71 int (*write_fill_chunk)(struct output_file *out, unsigned int len, 72 uint32_t fill_val); 73 int (*write_skip_chunk)(struct output_file *out, int64_t len); 74 int (*write_end_chunk)(struct output_file *out); 75}; 76 77struct output_file { 78 int64_t cur_out_ptr; 79 unsigned int chunk_cnt; 80 uint32_t crc32; 81 struct output_file_ops *ops; 82 struct sparse_file_ops *sparse_ops; 83 int use_crc; 84 unsigned int block_size; 85 int64_t len; 86 char *zero_buf; 87 uint32_t *fill_buf; 88 char *buf; 89}; 90 91struct output_file_gz { 92 struct output_file out; 93 gzFile gz_fd; 94}; 95 96#define to_output_file_gz(_o) \ 97 container_of((_o), struct output_file_gz, out) 98 99struct output_file_normal { 100 struct output_file out; 101 int fd; 102}; 103 104#define to_output_file_normal(_o) \ 105 container_of((_o), struct output_file_normal, out) 106 107struct output_file_callback { 108 struct output_file out; 109 void *priv; 110 int (*write)(void *priv, const void *buf, int len); 111}; 112 113#define to_output_file_callback(_o) \ 114 container_of((_o), struct output_file_callback, out) 115 116static int file_open(struct output_file *out, int fd) 117{ 118 struct output_file_normal *outn = to_output_file_normal(out); 119 120 outn->fd = fd; 121 return 0; 122} 123 124static int file_skip(struct output_file *out, int64_t cnt) 125{ 126 off64_t ret; 127 struct output_file_normal *outn = to_output_file_normal(out); 128 129 ret = lseek64(outn->fd, cnt, SEEK_CUR); 130 if (ret < 0) { 131 error_errno("lseek64"); 132 return -1; 133 } 134 return 0; 135} 136 137static int file_pad(struct output_file *out, int64_t len) 138{ 139 int ret; 140 struct output_file_normal *outn = to_output_file_normal(out); 141 142 ret = ftruncate64(outn->fd, len); 143 if (ret < 0) { 144 return -errno; 145 } 146 147 return 0; 148} 149 150static int file_write(struct output_file *out, void *data, int len) 151{ 152 int ret; 153 struct output_file_normal *outn = to_output_file_normal(out); 154 155 ret = write(outn->fd, data, len); 156 if (ret < 0) { 157 error_errno("write"); 158 return -1; 159 } else if (ret < len) { 160 error("incomplete write"); 161 return -1; 162 } 163 164 return 0; 165} 166 167static void file_close(struct output_file *out) 168{ 169 struct output_file_normal *outn = to_output_file_normal(out); 170 171 free(outn); 172} 173 174static struct output_file_ops file_ops = { 175 .open = file_open, 176 .skip = file_skip, 177 .pad = file_pad, 178 .write = file_write, 179 .close = file_close, 180}; 181 182static int gz_file_open(struct output_file *out, int fd) 183{ 184 struct output_file_gz *outgz = to_output_file_gz(out); 185 186 outgz->gz_fd = gzdopen(fd, "wb9"); 187 if (!outgz->gz_fd) { 188 error_errno("gzopen"); 189 return -errno; 190 } 191 192 return 0; 193} 194 195 196static int gz_file_skip(struct output_file *out, int64_t cnt) 197{ 198 off64_t ret; 199 struct output_file_gz *outgz = to_output_file_gz(out); 200 201 ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR); 202 if (ret < 0) { 203 error_errno("gzseek"); 204 return -1; 205 } 206 return 0; 207} 208 209static int gz_file_pad(struct output_file *out, int64_t len) 210{ 211 off64_t ret; 212 struct output_file_gz *outgz = to_output_file_gz(out); 213 214 ret = gztell(outgz->gz_fd); 215 if (ret < 0) { 216 return -1; 217 } 218 219 if (ret >= len) { 220 return 0; 221 } 222 223 ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET); 224 if (ret < 0) { 225 return -1; 226 } 227 228 gzwrite(outgz->gz_fd, "", 1); 229 230 return 0; 231} 232 233static int gz_file_write(struct output_file *out, void *data, int len) 234{ 235 int ret; 236 struct output_file_gz *outgz = to_output_file_gz(out); 237 238 ret = gzwrite(outgz->gz_fd, data, len); 239 if (ret < 0) { 240 error_errno("gzwrite"); 241 return -1; 242 } else if (ret < len) { 243 error("incomplete gzwrite"); 244 return -1; 245 } 246 247 return 0; 248} 249 250static void gz_file_close(struct output_file *out) 251{ 252 struct output_file_gz *outgz = to_output_file_gz(out); 253 254 gzclose(outgz->gz_fd); 255 free(outgz); 256} 257 258static struct output_file_ops gz_file_ops = { 259 .open = gz_file_open, 260 .skip = gz_file_skip, 261 .pad = gz_file_pad, 262 .write = gz_file_write, 263 .close = gz_file_close, 264}; 265 266static int callback_file_open(struct output_file *out, int fd) 267{ 268 return 0; 269} 270 271static int callback_file_skip(struct output_file *out, int64_t off) 272{ 273 struct output_file_callback *outc = to_output_file_callback(out); 274 int to_write; 275 int ret; 276 277 while (off > 0) { 278 to_write = min(off, (int64_t)INT_MAX); 279 ret = outc->write(outc->priv, NULL, to_write); 280 if (ret < 0) { 281 return ret; 282 } 283 off -= to_write; 284 } 285 286 return 0; 287} 288 289static int callback_file_pad(struct output_file *out, int64_t len) 290{ 291 return -1; 292} 293 294static int callback_file_write(struct output_file *out, void *data, int len) 295{ 296 int ret; 297 struct output_file_callback *outc = to_output_file_callback(out); 298 299 return outc->write(outc->priv, data, len); 300} 301 302static void callback_file_close(struct output_file *out) 303{ 304 struct output_file_callback *outc = to_output_file_callback(out); 305 306 free(outc); 307} 308 309static struct output_file_ops callback_file_ops = { 310 .open = callback_file_open, 311 .skip = callback_file_skip, 312 .pad = callback_file_pad, 313 .write = callback_file_write, 314 .close = callback_file_close, 315}; 316 317int read_all(int fd, void *buf, size_t len) 318{ 319 size_t total = 0; 320 int ret; 321 char *ptr = buf; 322 323 while (total < len) { 324 ret = read(fd, ptr, len - total); 325 326 if (ret < 0) 327 return -errno; 328 329 if (ret == 0) 330 return -EINVAL; 331 332 ptr += ret; 333 total += ret; 334 } 335 336 return 0; 337} 338 339static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len) 340{ 341 chunk_header_t chunk_header; 342 int ret, chunk; 343 344 if (skip_len % out->block_size) { 345 error("don't care size %llu is not a multiple of the block size %u", 346 skip_len, out->block_size); 347 return -1; 348 } 349 350 /* We are skipping data, so emit a don't care chunk. */ 351 chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE; 352 chunk_header.reserved1 = 0; 353 chunk_header.chunk_sz = skip_len / out->block_size; 354 chunk_header.total_sz = CHUNK_HEADER_LEN; 355 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 356 if (ret < 0) 357 return -1; 358 359 out->cur_out_ptr += skip_len; 360 out->chunk_cnt++; 361 362 return 0; 363} 364 365static int write_sparse_fill_chunk(struct output_file *out, unsigned int len, 366 uint32_t fill_val) 367{ 368 chunk_header_t chunk_header; 369 int rnd_up_len, zero_len, count; 370 int ret; 371 unsigned int i; 372 373 /* Round up the fill length to a multiple of the block size */ 374 rnd_up_len = ALIGN(len, out->block_size); 375 376 /* Finally we can safely emit a chunk of data */ 377 chunk_header.chunk_type = CHUNK_TYPE_FILL; 378 chunk_header.reserved1 = 0; 379 chunk_header.chunk_sz = rnd_up_len / out->block_size; 380 chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val); 381 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 382 383 if (ret < 0) 384 return -1; 385 ret = out->ops->write(out, &fill_val, sizeof(fill_val)); 386 if (ret < 0) 387 return -1; 388 389 if (out->use_crc) { 390 count = out->block_size / sizeof(uint32_t); 391 while (count--) 392 out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t)); 393 } 394 395 out->cur_out_ptr += rnd_up_len; 396 out->chunk_cnt++; 397 398 return 0; 399} 400 401static int write_sparse_data_chunk(struct output_file *out, unsigned int len, 402 void *data) 403{ 404 chunk_header_t chunk_header; 405 int rnd_up_len, zero_len; 406 int ret; 407 408 /* Round up the data length to a multiple of the block size */ 409 rnd_up_len = ALIGN(len, out->block_size); 410 zero_len = rnd_up_len - len; 411 412 /* Finally we can safely emit a chunk of data */ 413 chunk_header.chunk_type = CHUNK_TYPE_RAW; 414 chunk_header.reserved1 = 0; 415 chunk_header.chunk_sz = rnd_up_len / out->block_size; 416 chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len; 417 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 418 419 if (ret < 0) 420 return -1; 421 ret = out->ops->write(out, data, len); 422 if (ret < 0) 423 return -1; 424 if (zero_len) { 425 ret = out->ops->write(out, out->zero_buf, zero_len); 426 if (ret < 0) 427 return -1; 428 } 429 430 if (out->use_crc) { 431 out->crc32 = sparse_crc32(out->crc32, data, len); 432 if (zero_len) 433 out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len); 434 } 435 436 out->cur_out_ptr += rnd_up_len; 437 out->chunk_cnt++; 438 439 return 0; 440} 441 442int write_sparse_end_chunk(struct output_file *out) 443{ 444 chunk_header_t chunk_header; 445 int ret; 446 447 if (out->use_crc) { 448 chunk_header.chunk_type = CHUNK_TYPE_CRC32; 449 chunk_header.reserved1 = 0; 450 chunk_header.chunk_sz = 0; 451 chunk_header.total_sz = CHUNK_HEADER_LEN + 4; 452 453 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 454 if (ret < 0) { 455 return ret; 456 } 457 out->ops->write(out, &out->crc32, 4); 458 if (ret < 0) { 459 return ret; 460 } 461 462 out->chunk_cnt++; 463 } 464 465 return 0; 466} 467 468static struct sparse_file_ops sparse_file_ops = { 469 .write_data_chunk = write_sparse_data_chunk, 470 .write_fill_chunk = write_sparse_fill_chunk, 471 .write_skip_chunk = write_sparse_skip_chunk, 472 .write_end_chunk = write_sparse_end_chunk, 473}; 474 475static int write_normal_data_chunk(struct output_file *out, unsigned int len, 476 void *data) 477{ 478 int ret; 479 unsigned int rnd_up_len = ALIGN(len, out->block_size); 480 481 ret = out->ops->write(out, data, len); 482 if (ret < 0) { 483 return ret; 484 } 485 486 if (rnd_up_len > len) { 487 ret = out->ops->skip(out, rnd_up_len - len); 488 } 489 490 return ret; 491} 492 493static int write_normal_fill_chunk(struct output_file *out, unsigned int len, 494 uint32_t fill_val) 495{ 496 int ret; 497 unsigned int i; 498 unsigned int write_len; 499 500 /* Initialize fill_buf with the fill_val */ 501 for (i = 0; i < out->block_size / sizeof(uint32_t); i++) { 502 out->fill_buf[i] = fill_val; 503 } 504 505 while (len) { 506 write_len = min(len, out->block_size); 507 ret = out->ops->write(out, out->fill_buf, write_len); 508 if (ret < 0) { 509 return ret; 510 } 511 512 len -= write_len; 513 } 514 515 return 0; 516} 517 518static int write_normal_skip_chunk(struct output_file *out, int64_t len) 519{ 520 return out->ops->skip(out, len); 521} 522 523int write_normal_end_chunk(struct output_file *out) 524{ 525 return out->ops->pad(out, out->len); 526} 527 528static struct sparse_file_ops normal_file_ops = { 529 .write_data_chunk = write_normal_data_chunk, 530 .write_fill_chunk = write_normal_fill_chunk, 531 .write_skip_chunk = write_normal_skip_chunk, 532 .write_end_chunk = write_normal_end_chunk, 533}; 534 535void output_file_close(struct output_file *out) 536{ 537 int ret; 538 539 out->sparse_ops->write_end_chunk(out); 540 out->ops->close(out); 541} 542 543static int output_file_init(struct output_file *out, int block_size, 544 int64_t len, bool sparse, int chunks, bool crc) 545{ 546 int ret; 547 548 out->len = len; 549 out->block_size = block_size; 550 out->cur_out_ptr = 0ll; 551 out->chunk_cnt = 0; 552 out->crc32 = 0; 553 out->use_crc = crc; 554 555 out->zero_buf = calloc(block_size, 1); 556 if (!out->zero_buf) { 557 error_errno("malloc zero_buf"); 558 return -ENOMEM; 559 } 560 561 out->fill_buf = calloc(block_size, 1); 562 if (!out->fill_buf) { 563 error_errno("malloc fill_buf"); 564 ret = -ENOMEM; 565 goto err_fill_buf; 566 } 567 568 if (sparse) { 569 out->sparse_ops = &sparse_file_ops; 570 } else { 571 out->sparse_ops = &normal_file_ops; 572 } 573 574 if (sparse) { 575 sparse_header_t sparse_header = { 576 .magic = SPARSE_HEADER_MAGIC, 577 .major_version = SPARSE_HEADER_MAJOR_VER, 578 .minor_version = SPARSE_HEADER_MINOR_VER, 579 .file_hdr_sz = SPARSE_HEADER_LEN, 580 .chunk_hdr_sz = CHUNK_HEADER_LEN, 581 .blk_sz = out->block_size, 582 .total_blks = out->len / out->block_size, 583 .total_chunks = chunks, 584 .image_checksum = 0 585 }; 586 587 if (out->use_crc) { 588 sparse_header.total_chunks++; 589 } 590 591 ret = out->ops->write(out, &sparse_header, sizeof(sparse_header)); 592 if (ret < 0) { 593 goto err_write; 594 } 595 } 596 597 return 0; 598 599err_write: 600 free(out->fill_buf); 601err_fill_buf: 602 free(out->zero_buf); 603 return ret; 604} 605 606static struct output_file *output_file_new_gz(void) 607{ 608 struct output_file_gz *outgz = calloc(1, sizeof(struct output_file_gz)); 609 if (!outgz) { 610 error_errno("malloc struct outgz"); 611 return NULL; 612 } 613 614 outgz->out.ops = &gz_file_ops; 615 616 return &outgz->out; 617} 618 619static struct output_file *output_file_new_normal(void) 620{ 621 struct output_file_normal *outn = calloc(1, sizeof(struct output_file_normal)); 622 if (!outn) { 623 error_errno("malloc struct outn"); 624 return NULL; 625 } 626 627 outn->out.ops = &file_ops; 628 629 return &outn->out; 630} 631 632struct output_file *output_file_open_callback(int (*write)(void *, const void *, int), 633 void *priv, unsigned int block_size, int64_t len, int gz, int sparse, 634 int chunks, int crc) 635{ 636 int ret; 637 struct output_file_callback *outc; 638 639 outc = calloc(1, sizeof(struct output_file_callback)); 640 if (!outc) { 641 error_errno("malloc struct outc"); 642 return NULL; 643 } 644 645 outc->out.ops = &callback_file_ops; 646 outc->priv = priv; 647 outc->write = write; 648 649 ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc); 650 if (ret < 0) { 651 free(outc); 652 return NULL; 653 } 654 655 return &outc->out; 656} 657 658struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len, 659 int gz, int sparse, int chunks, int crc) 660{ 661 int ret; 662 struct output_file *out; 663 664 if (gz) { 665 out = output_file_new_gz(); 666 } else { 667 out = output_file_new_normal(); 668 } 669 if (!out) { 670 return NULL; 671 } 672 673 out->ops->open(out, fd); 674 675 ret = output_file_init(out, block_size, len, sparse, chunks, crc); 676 if (ret < 0) { 677 free(out); 678 return NULL; 679 } 680 681 return out; 682} 683 684/* Write a contiguous region of data blocks from a memory buffer */ 685int write_data_chunk(struct output_file *out, unsigned int len, void *data) 686{ 687 return out->sparse_ops->write_data_chunk(out, len, data); 688} 689 690/* Write a contiguous region of data blocks with a fill value */ 691int write_fill_chunk(struct output_file *out, unsigned int len, 692 uint32_t fill_val) 693{ 694 return out->sparse_ops->write_fill_chunk(out, len, fill_val); 695} 696 697int write_fd_chunk(struct output_file *out, unsigned int len, 698 int fd, int64_t offset) 699{ 700 int ret; 701 int64_t aligned_offset; 702 int aligned_diff; 703 int buffer_size; 704 char *ptr; 705 706 aligned_offset = offset & ~(4096 - 1); 707 aligned_diff = offset - aligned_offset; 708 buffer_size = len + aligned_diff; 709 710#ifndef USE_MINGW 711 char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd, 712 aligned_offset); 713 if (data == MAP_FAILED) { 714 return -errno; 715 } 716 ptr = data + aligned_diff; 717#else 718 off64_t pos; 719 char *data = malloc(len); 720 if (!data) { 721 return -errno; 722 } 723 pos = lseek64(fd, offset, SEEK_SET); 724 if (pos < 0) { 725 free(data); 726 return -errno; 727 } 728 ret = read_all(fd, data, len); 729 if (ret < 0) { 730 free(data); 731 return ret; 732 } 733 ptr = data; 734#endif 735 736 ret = out->sparse_ops->write_data_chunk(out, len, ptr); 737 738#ifndef USE_MINGW 739 munmap(data, buffer_size); 740#else 741 free(data); 742#endif 743 744 return ret; 745} 746 747/* Write a contiguous region of data blocks from a file */ 748int write_file_chunk(struct output_file *out, unsigned int len, 749 const char *file, int64_t offset) 750{ 751 int ret; 752 753 int file_fd = open(file, O_RDONLY | O_BINARY); 754 if (file_fd < 0) { 755 return -errno; 756 } 757 758 ret = write_fd_chunk(out, len, file_fd, offset); 759 760 close(file_fd); 761 762 return ret; 763} 764 765int write_skip_chunk(struct output_file *out, int64_t len) 766{ 767 return out->sparse_ops->write_skip_chunk(out, len); 768} 769