output_file.c revision b43828b247fd4f1e0373584de0504004c69eeac9
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define _FILE_OFFSET_BITS 64 18#define _LARGEFILE64_SOURCE 1 19 20#include <fcntl.h> 21#include <limits.h> 22#include <stdbool.h> 23#include <stddef.h> 24#include <stdlib.h> 25#include <string.h> 26#include <sys/stat.h> 27#include <sys/types.h> 28#include <unistd.h> 29#include <zlib.h> 30 31#include "output_file.h" 32#include "sparse_format.h" 33#include "sparse_crc32.h" 34 35#ifndef USE_MINGW 36#include <sys/mman.h> 37#define O_BINARY 0 38#else 39#define ftruncate64 ftruncate 40#endif 41 42#if defined(__APPLE__) && defined(__MACH__) 43#define lseek64 lseek 44#define ftruncate64 ftruncate 45#define mmap64 mmap 46#define off64_t off_t 47#endif 48 49#ifdef __BIONIC__ 50extern void* __mmap2(void *, size_t, int, int, int, off_t); 51static inline void *mmap64(void *addr, size_t length, int prot, int flags, 52 int fd, off64_t offset) 53{ 54 return __mmap2(addr, length, prot, flags, fd, offset >> 12); 55} 56#endif 57 58#define min(a, b) \ 59 ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; }) 60 61#define SPARSE_HEADER_MAJOR_VER 1 62#define SPARSE_HEADER_MINOR_VER 0 63#define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) 64#define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) 65 66#define container_of(inner, outer_t, elem) \ 67 ((outer_t *)((char *)inner - offsetof(outer_t, elem))) 68 69struct output_file_ops { 70 int (*open)(struct output_file *, int fd); 71 int (*skip)(struct output_file *, int64_t); 72 int (*pad)(struct output_file *, int64_t); 73 int (*write)(struct output_file *, void *, int); 74 void (*close)(struct output_file *); 75}; 76 77struct sparse_file_ops { 78 int (*write_data_chunk)(struct output_file *out, unsigned int len, 79 void *data); 80 int (*write_fill_chunk)(struct output_file *out, unsigned int len, 81 uint32_t fill_val); 82 int (*write_skip_chunk)(struct output_file *out, int64_t len); 83 int (*write_end_chunk)(struct output_file *out); 84}; 85 86struct output_file { 87 int64_t cur_out_ptr; 88 unsigned int chunk_cnt; 89 uint32_t crc32; 90 struct output_file_ops *ops; 91 struct sparse_file_ops *sparse_ops; 92 int use_crc; 93 unsigned int block_size; 94 int64_t len; 95 char *zero_buf; 96 uint32_t *fill_buf; 97 char *buf; 98}; 99 100struct output_file_gz { 101 struct output_file out; 102 gzFile gz_fd; 103}; 104 105#define to_output_file_gz(_o) \ 106 container_of((_o), struct output_file_gz, out) 107 108struct output_file_normal { 109 struct output_file out; 110 int fd; 111}; 112 113#define to_output_file_normal(_o) \ 114 container_of((_o), struct output_file_normal, out) 115 116struct output_file_callback { 117 struct output_file out; 118 void *priv; 119 int (*write)(void *priv, const void *buf, int len); 120}; 121 122#define to_output_file_callback(_o) \ 123 container_of((_o), struct output_file_callback, out) 124 125static int file_open(struct output_file *out, int fd) 126{ 127 struct output_file_normal *outn = to_output_file_normal(out); 128 129 outn->fd = fd; 130 return 0; 131} 132 133static int file_skip(struct output_file *out, int64_t cnt) 134{ 135 off64_t ret; 136 struct output_file_normal *outn = to_output_file_normal(out); 137 138 ret = lseek64(outn->fd, cnt, SEEK_CUR); 139 if (ret < 0) { 140 error_errno("lseek64"); 141 return -1; 142 } 143 return 0; 144} 145 146static int file_pad(struct output_file *out, int64_t len) 147{ 148 int ret; 149 struct output_file_normal *outn = to_output_file_normal(out); 150 151 ret = ftruncate64(outn->fd, len); 152 if (ret < 0) { 153 return -errno; 154 } 155 156 return 0; 157} 158 159static int file_write(struct output_file *out, void *data, int len) 160{ 161 int ret; 162 struct output_file_normal *outn = to_output_file_normal(out); 163 164 ret = write(outn->fd, data, len); 165 if (ret < 0) { 166 error_errno("write"); 167 return -1; 168 } else if (ret < len) { 169 error("incomplete write"); 170 return -1; 171 } 172 173 return 0; 174} 175 176static void file_close(struct output_file *out) 177{ 178 struct output_file_normal *outn = to_output_file_normal(out); 179 180 free(outn); 181} 182 183static struct output_file_ops file_ops = { 184 .open = file_open, 185 .skip = file_skip, 186 .pad = file_pad, 187 .write = file_write, 188 .close = file_close, 189}; 190 191static int gz_file_open(struct output_file *out, int fd) 192{ 193 struct output_file_gz *outgz = to_output_file_gz(out); 194 195 outgz->gz_fd = gzdopen(fd, "wb9"); 196 if (!outgz->gz_fd) { 197 error_errno("gzopen"); 198 return -errno; 199 } 200 201 return 0; 202} 203 204 205static int gz_file_skip(struct output_file *out, int64_t cnt) 206{ 207 off64_t ret; 208 struct output_file_gz *outgz = to_output_file_gz(out); 209 210 ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR); 211 if (ret < 0) { 212 error_errno("gzseek"); 213 return -1; 214 } 215 return 0; 216} 217 218static int gz_file_pad(struct output_file *out, int64_t len) 219{ 220 off64_t ret; 221 struct output_file_gz *outgz = to_output_file_gz(out); 222 223 ret = gztell(outgz->gz_fd); 224 if (ret < 0) { 225 return -1; 226 } 227 228 if (ret >= len) { 229 return 0; 230 } 231 232 ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET); 233 if (ret < 0) { 234 return -1; 235 } 236 237 gzwrite(outgz->gz_fd, "", 1); 238 239 return 0; 240} 241 242static int gz_file_write(struct output_file *out, void *data, int len) 243{ 244 int ret; 245 struct output_file_gz *outgz = to_output_file_gz(out); 246 247 ret = gzwrite(outgz->gz_fd, data, len); 248 if (ret < 0) { 249 error_errno("gzwrite"); 250 return -1; 251 } else if (ret < len) { 252 error("incomplete gzwrite"); 253 return -1; 254 } 255 256 return 0; 257} 258 259static void gz_file_close(struct output_file *out) 260{ 261 struct output_file_gz *outgz = to_output_file_gz(out); 262 263 gzclose(outgz->gz_fd); 264 free(outgz); 265} 266 267static struct output_file_ops gz_file_ops = { 268 .open = gz_file_open, 269 .skip = gz_file_skip, 270 .pad = gz_file_pad, 271 .write = gz_file_write, 272 .close = gz_file_close, 273}; 274 275static int callback_file_open(struct output_file *out, int fd) 276{ 277 return 0; 278} 279 280static int callback_file_skip(struct output_file *out, int64_t off) 281{ 282 struct output_file_callback *outc = to_output_file_callback(out); 283 int to_write; 284 int ret; 285 286 while (off > 0) { 287 to_write = min(off, (int64_t)INT_MAX); 288 ret = outc->write(outc->priv, NULL, to_write); 289 if (ret < 0) { 290 return ret; 291 } 292 off -= to_write; 293 } 294 295 return 0; 296} 297 298static int callback_file_pad(struct output_file *out, int64_t len) 299{ 300 return -1; 301} 302 303static int callback_file_write(struct output_file *out, void *data, int len) 304{ 305 int ret; 306 struct output_file_callback *outc = to_output_file_callback(out); 307 308 return outc->write(outc->priv, data, len); 309} 310 311static void callback_file_close(struct output_file *out) 312{ 313 struct output_file_callback *outc = to_output_file_callback(out); 314 315 free(outc); 316} 317 318static struct output_file_ops callback_file_ops = { 319 .open = callback_file_open, 320 .skip = callback_file_skip, 321 .pad = callback_file_pad, 322 .write = callback_file_write, 323 .close = callback_file_close, 324}; 325 326int read_all(int fd, void *buf, size_t len) 327{ 328 size_t total = 0; 329 int ret; 330 char *ptr = buf; 331 332 while (total < len) { 333 ret = read(fd, ptr, len - total); 334 335 if (ret < 0) 336 return -errno; 337 338 if (ret == 0) 339 return -EINVAL; 340 341 ptr += ret; 342 total += ret; 343 } 344 345 return 0; 346} 347 348static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len) 349{ 350 chunk_header_t chunk_header; 351 int ret, chunk; 352 353 if (skip_len % out->block_size) { 354 error("don't care size %llu is not a multiple of the block size %u", 355 skip_len, out->block_size); 356 return -1; 357 } 358 359 /* We are skipping data, so emit a don't care chunk. */ 360 chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE; 361 chunk_header.reserved1 = 0; 362 chunk_header.chunk_sz = skip_len / out->block_size; 363 chunk_header.total_sz = CHUNK_HEADER_LEN; 364 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 365 if (ret < 0) 366 return -1; 367 368 out->cur_out_ptr += skip_len; 369 out->chunk_cnt++; 370 371 return 0; 372} 373 374static int write_sparse_fill_chunk(struct output_file *out, unsigned int len, 375 uint32_t fill_val) 376{ 377 chunk_header_t chunk_header; 378 int rnd_up_len, zero_len, count; 379 int ret; 380 unsigned int i; 381 382 /* Round up the fill length to a multiple of the block size */ 383 rnd_up_len = ALIGN(len, out->block_size); 384 385 /* Finally we can safely emit a chunk of data */ 386 chunk_header.chunk_type = CHUNK_TYPE_FILL; 387 chunk_header.reserved1 = 0; 388 chunk_header.chunk_sz = rnd_up_len / out->block_size; 389 chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val); 390 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 391 392 if (ret < 0) 393 return -1; 394 ret = out->ops->write(out, &fill_val, sizeof(fill_val)); 395 if (ret < 0) 396 return -1; 397 398 if (out->use_crc) { 399 count = out->block_size / sizeof(uint32_t); 400 while (count--) 401 out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t)); 402 } 403 404 out->cur_out_ptr += rnd_up_len; 405 out->chunk_cnt++; 406 407 return 0; 408} 409 410static int write_sparse_data_chunk(struct output_file *out, unsigned int len, 411 void *data) 412{ 413 chunk_header_t chunk_header; 414 int rnd_up_len, zero_len; 415 int ret; 416 417 /* Round up the data length to a multiple of the block size */ 418 rnd_up_len = ALIGN(len, out->block_size); 419 zero_len = rnd_up_len - len; 420 421 /* Finally we can safely emit a chunk of data */ 422 chunk_header.chunk_type = CHUNK_TYPE_RAW; 423 chunk_header.reserved1 = 0; 424 chunk_header.chunk_sz = rnd_up_len / out->block_size; 425 chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len; 426 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 427 428 if (ret < 0) 429 return -1; 430 ret = out->ops->write(out, data, len); 431 if (ret < 0) 432 return -1; 433 if (zero_len) { 434 ret = out->ops->write(out, out->zero_buf, zero_len); 435 if (ret < 0) 436 return -1; 437 } 438 439 if (out->use_crc) { 440 out->crc32 = sparse_crc32(out->crc32, data, len); 441 if (zero_len) 442 out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len); 443 } 444 445 out->cur_out_ptr += rnd_up_len; 446 out->chunk_cnt++; 447 448 return 0; 449} 450 451int write_sparse_end_chunk(struct output_file *out) 452{ 453 chunk_header_t chunk_header; 454 int ret; 455 456 if (out->use_crc) { 457 chunk_header.chunk_type = CHUNK_TYPE_CRC32; 458 chunk_header.reserved1 = 0; 459 chunk_header.chunk_sz = 0; 460 chunk_header.total_sz = CHUNK_HEADER_LEN + 4; 461 462 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 463 if (ret < 0) { 464 return ret; 465 } 466 out->ops->write(out, &out->crc32, 4); 467 if (ret < 0) { 468 return ret; 469 } 470 471 out->chunk_cnt++; 472 } 473 474 return 0; 475} 476 477static struct sparse_file_ops sparse_file_ops = { 478 .write_data_chunk = write_sparse_data_chunk, 479 .write_fill_chunk = write_sparse_fill_chunk, 480 .write_skip_chunk = write_sparse_skip_chunk, 481 .write_end_chunk = write_sparse_end_chunk, 482}; 483 484static int write_normal_data_chunk(struct output_file *out, unsigned int len, 485 void *data) 486{ 487 int ret; 488 unsigned int rnd_up_len = ALIGN(len, out->block_size); 489 490 ret = out->ops->write(out, data, len); 491 if (ret < 0) { 492 return ret; 493 } 494 495 if (rnd_up_len > len) { 496 ret = out->ops->skip(out, rnd_up_len - len); 497 } 498 499 return ret; 500} 501 502static int write_normal_fill_chunk(struct output_file *out, unsigned int len, 503 uint32_t fill_val) 504{ 505 int ret; 506 unsigned int i; 507 unsigned int write_len; 508 509 /* Initialize fill_buf with the fill_val */ 510 for (i = 0; i < out->block_size / sizeof(uint32_t); i++) { 511 out->fill_buf[i] = fill_val; 512 } 513 514 while (len) { 515 write_len = min(len, out->block_size); 516 ret = out->ops->write(out, out->fill_buf, write_len); 517 if (ret < 0) { 518 return ret; 519 } 520 521 len -= write_len; 522 } 523 524 return 0; 525} 526 527static int write_normal_skip_chunk(struct output_file *out, int64_t len) 528{ 529 return out->ops->skip(out, len); 530} 531 532int write_normal_end_chunk(struct output_file *out) 533{ 534 return out->ops->pad(out, out->len); 535} 536 537static struct sparse_file_ops normal_file_ops = { 538 .write_data_chunk = write_normal_data_chunk, 539 .write_fill_chunk = write_normal_fill_chunk, 540 .write_skip_chunk = write_normal_skip_chunk, 541 .write_end_chunk = write_normal_end_chunk, 542}; 543 544void output_file_close(struct output_file *out) 545{ 546 int ret; 547 548 out->sparse_ops->write_end_chunk(out); 549 out->ops->close(out); 550} 551 552static int output_file_init(struct output_file *out, int block_size, 553 int64_t len, bool sparse, int chunks, bool crc) 554{ 555 int ret; 556 557 out->len = len; 558 out->block_size = block_size; 559 out->cur_out_ptr = 0ll; 560 out->chunk_cnt = 0; 561 out->crc32 = 0; 562 out->use_crc = crc; 563 564 out->zero_buf = calloc(block_size, 1); 565 if (!out->zero_buf) { 566 error_errno("malloc zero_buf"); 567 return -ENOMEM; 568 } 569 570 out->fill_buf = calloc(block_size, 1); 571 if (!out->fill_buf) { 572 error_errno("malloc fill_buf"); 573 ret = -ENOMEM; 574 goto err_fill_buf; 575 } 576 577 if (sparse) { 578 out->sparse_ops = &sparse_file_ops; 579 } else { 580 out->sparse_ops = &normal_file_ops; 581 } 582 583 if (sparse) { 584 sparse_header_t sparse_header = { 585 .magic = SPARSE_HEADER_MAGIC, 586 .major_version = SPARSE_HEADER_MAJOR_VER, 587 .minor_version = SPARSE_HEADER_MINOR_VER, 588 .file_hdr_sz = SPARSE_HEADER_LEN, 589 .chunk_hdr_sz = CHUNK_HEADER_LEN, 590 .blk_sz = out->block_size, 591 .total_blks = out->len / out->block_size, 592 .total_chunks = chunks, 593 .image_checksum = 0 594 }; 595 596 if (out->use_crc) { 597 sparse_header.total_chunks++; 598 } 599 600 ret = out->ops->write(out, &sparse_header, sizeof(sparse_header)); 601 if (ret < 0) { 602 goto err_write; 603 } 604 } 605 606 return 0; 607 608err_write: 609 free(out->fill_buf); 610err_fill_buf: 611 free(out->zero_buf); 612 return ret; 613} 614 615static struct output_file *output_file_new_gz(void) 616{ 617 struct output_file_gz *outgz = calloc(1, sizeof(struct output_file_gz)); 618 if (!outgz) { 619 error_errno("malloc struct outgz"); 620 return NULL; 621 } 622 623 outgz->out.ops = &gz_file_ops; 624 625 return &outgz->out; 626} 627 628static struct output_file *output_file_new_normal(void) 629{ 630 struct output_file_normal *outn = calloc(1, sizeof(struct output_file_normal)); 631 if (!outn) { 632 error_errno("malloc struct outn"); 633 return NULL; 634 } 635 636 outn->out.ops = &file_ops; 637 638 return &outn->out; 639} 640 641struct output_file *output_file_open_callback(int (*write)(void *, const void *, int), 642 void *priv, unsigned int block_size, int64_t len, int gz, int sparse, 643 int chunks, int crc) 644{ 645 int ret; 646 struct output_file_callback *outc; 647 648 outc = calloc(1, sizeof(struct output_file_callback)); 649 if (!outc) { 650 error_errno("malloc struct outc"); 651 return NULL; 652 } 653 654 outc->out.ops = &callback_file_ops; 655 outc->priv = priv; 656 outc->write = write; 657 658 ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc); 659 if (ret < 0) { 660 free(outc); 661 return NULL; 662 } 663 664 return &outc->out; 665} 666 667struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len, 668 int gz, int sparse, int chunks, int crc) 669{ 670 int ret; 671 struct output_file *out; 672 673 if (gz) { 674 out = output_file_new_gz(); 675 } else { 676 out = output_file_new_normal(); 677 } 678 679 out->ops->open(out, fd); 680 681 ret = output_file_init(out, block_size, len, sparse, chunks, crc); 682 if (ret < 0) { 683 free(out); 684 return NULL; 685 } 686 687 return out; 688} 689 690/* Write a contiguous region of data blocks from a memory buffer */ 691int write_data_chunk(struct output_file *out, unsigned int len, void *data) 692{ 693 return out->sparse_ops->write_data_chunk(out, len, data); 694} 695 696/* Write a contiguous region of data blocks with a fill value */ 697int write_fill_chunk(struct output_file *out, unsigned int len, 698 uint32_t fill_val) 699{ 700 return out->sparse_ops->write_fill_chunk(out, len, fill_val); 701} 702 703int write_fd_chunk(struct output_file *out, unsigned int len, 704 int fd, int64_t offset) 705{ 706 int ret; 707 int64_t aligned_offset; 708 int aligned_diff; 709 int buffer_size; 710 char *ptr; 711 712 aligned_offset = offset & ~(4096 - 1); 713 aligned_diff = offset - aligned_offset; 714 buffer_size = len + aligned_diff; 715 716#ifndef USE_MINGW 717 char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd, 718 aligned_offset); 719 if (data == MAP_FAILED) { 720 return -errno; 721 } 722 ptr = data + aligned_diff; 723#else 724 off64_t pos; 725 char *data = malloc(len); 726 if (!data) { 727 return -errno; 728 } 729 pos = lseek64(fd, offset, SEEK_SET); 730 if (pos < 0) { 731 return -errno; 732 } 733 ret = read_all(fd, data, len); 734 if (ret < 0) { 735 return ret; 736 } 737 ptr = data; 738#endif 739 740 ret = out->sparse_ops->write_data_chunk(out, len, ptr); 741 742#ifndef USE_MINGW 743 munmap(data, buffer_size); 744#else 745 free(data); 746#endif 747 748 return ret; 749} 750 751/* Write a contiguous region of data blocks from a file */ 752int write_file_chunk(struct output_file *out, unsigned int len, 753 const char *file, int64_t offset) 754{ 755 int ret; 756 757 int file_fd = open(file, O_RDONLY | O_BINARY); 758 if (file_fd < 0) { 759 return -errno; 760 } 761 762 ret = write_fd_chunk(out, len, file_fd, offset); 763 764 close(file_fd); 765 766 return ret; 767} 768 769int write_skip_chunk(struct output_file *out, int64_t len) 770{ 771 return out->sparse_ops->write_skip_chunk(out, len); 772} 773