output_file.c revision ccecf1425412beb2bc3bb38d470293fdc244d6f1
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define _FILE_OFFSET_BITS 64 18#define _LARGEFILE64_SOURCE 1 19 20#include <fcntl.h> 21#include <inttypes.h> 22#include <limits.h> 23#include <stdbool.h> 24#include <stddef.h> 25#include <stdlib.h> 26#include <string.h> 27#include <sys/stat.h> 28#include <sys/types.h> 29#include <unistd.h> 30#include <zlib.h> 31 32#include "output_file.h" 33#include "sparse_format.h" 34#include "sparse_crc32.h" 35 36#ifndef USE_MINGW 37#include <sys/mman.h> 38#define O_BINARY 0 39#else 40#define ftruncate64 ftruncate 41#endif 42 43#if defined(__APPLE__) && defined(__MACH__) 44#define lseek64 lseek 45#define ftruncate64 ftruncate 46#define mmap64 mmap 47#define off64_t off_t 48#endif 49 50#define min(a, b) \ 51 ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; }) 52 53#define SPARSE_HEADER_MAJOR_VER 1 54#define SPARSE_HEADER_MINOR_VER 0 55#define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) 56#define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) 57 58#define container_of(inner, outer_t, elem) \ 59 ((outer_t *)((char *)inner - offsetof(outer_t, elem))) 60 61struct output_file_ops { 62 int (*open)(struct output_file *, int fd); 63 int (*skip)(struct output_file *, int64_t); 64 int (*pad)(struct output_file *, int64_t); 65 int (*write)(struct output_file *, void *, int); 66 void (*close)(struct output_file *); 67}; 68 69struct sparse_file_ops { 70 int (*write_data_chunk)(struct output_file *out, unsigned int len, 71 void *data); 72 int (*write_fill_chunk)(struct output_file *out, unsigned int len, 73 uint32_t fill_val); 74 int (*write_skip_chunk)(struct output_file *out, int64_t len); 75 int (*write_end_chunk)(struct output_file *out); 76}; 77 78struct output_file { 79 int64_t cur_out_ptr; 80 unsigned int chunk_cnt; 81 uint32_t crc32; 82 struct output_file_ops *ops; 83 struct sparse_file_ops *sparse_ops; 84 int use_crc; 85 unsigned int block_size; 86 int64_t len; 87 char *zero_buf; 88 uint32_t *fill_buf; 89 char *buf; 90}; 91 92struct output_file_gz { 93 struct output_file out; 94 gzFile gz_fd; 95}; 96 97#define to_output_file_gz(_o) \ 98 container_of((_o), struct output_file_gz, out) 99 100struct output_file_normal { 101 struct output_file out; 102 int fd; 103}; 104 105#define to_output_file_normal(_o) \ 106 container_of((_o), struct output_file_normal, out) 107 108struct output_file_callback { 109 struct output_file out; 110 void *priv; 111 int (*write)(void *priv, const void *buf, int len); 112}; 113 114#define to_output_file_callback(_o) \ 115 container_of((_o), struct output_file_callback, out) 116 117static int file_open(struct output_file *out, int fd) 118{ 119 struct output_file_normal *outn = to_output_file_normal(out); 120 121 outn->fd = fd; 122 return 0; 123} 124 125static int file_skip(struct output_file *out, int64_t cnt) 126{ 127 off64_t ret; 128 struct output_file_normal *outn = to_output_file_normal(out); 129 130 ret = lseek64(outn->fd, cnt, SEEK_CUR); 131 if (ret < 0) { 132 error_errno("lseek64"); 133 return -1; 134 } 135 return 0; 136} 137 138static int file_pad(struct output_file *out, int64_t len) 139{ 140 int ret; 141 struct output_file_normal *outn = to_output_file_normal(out); 142 143 ret = ftruncate64(outn->fd, len); 144 if (ret < 0) { 145 return -errno; 146 } 147 148 return 0; 149} 150 151static int file_write(struct output_file *out, void *data, int len) 152{ 153 int ret; 154 struct output_file_normal *outn = to_output_file_normal(out); 155 156 ret = write(outn->fd, data, len); 157 if (ret < 0) { 158 error_errno("write"); 159 return -1; 160 } else if (ret < len) { 161 error("incomplete write"); 162 return -1; 163 } 164 165 return 0; 166} 167 168static void file_close(struct output_file *out) 169{ 170 struct output_file_normal *outn = to_output_file_normal(out); 171 172 free(outn); 173} 174 175static struct output_file_ops file_ops = { 176 .open = file_open, 177 .skip = file_skip, 178 .pad = file_pad, 179 .write = file_write, 180 .close = file_close, 181}; 182 183static int gz_file_open(struct output_file *out, int fd) 184{ 185 struct output_file_gz *outgz = to_output_file_gz(out); 186 187 outgz->gz_fd = gzdopen(fd, "wb9"); 188 if (!outgz->gz_fd) { 189 error_errno("gzopen"); 190 return -errno; 191 } 192 193 return 0; 194} 195 196 197static int gz_file_skip(struct output_file *out, int64_t cnt) 198{ 199 off64_t ret; 200 struct output_file_gz *outgz = to_output_file_gz(out); 201 202 ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR); 203 if (ret < 0) { 204 error_errno("gzseek"); 205 return -1; 206 } 207 return 0; 208} 209 210static int gz_file_pad(struct output_file *out, int64_t len) 211{ 212 off64_t ret; 213 struct output_file_gz *outgz = to_output_file_gz(out); 214 215 ret = gztell(outgz->gz_fd); 216 if (ret < 0) { 217 return -1; 218 } 219 220 if (ret >= len) { 221 return 0; 222 } 223 224 ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET); 225 if (ret < 0) { 226 return -1; 227 } 228 229 gzwrite(outgz->gz_fd, "", 1); 230 231 return 0; 232} 233 234static int gz_file_write(struct output_file *out, void *data, int len) 235{ 236 int ret; 237 struct output_file_gz *outgz = to_output_file_gz(out); 238 239 ret = gzwrite(outgz->gz_fd, data, len); 240 if (ret < 0) { 241 error_errno("gzwrite"); 242 return -1; 243 } else if (ret < len) { 244 error("incomplete gzwrite"); 245 return -1; 246 } 247 248 return 0; 249} 250 251static void gz_file_close(struct output_file *out) 252{ 253 struct output_file_gz *outgz = to_output_file_gz(out); 254 255 gzclose(outgz->gz_fd); 256 free(outgz); 257} 258 259static struct output_file_ops gz_file_ops = { 260 .open = gz_file_open, 261 .skip = gz_file_skip, 262 .pad = gz_file_pad, 263 .write = gz_file_write, 264 .close = gz_file_close, 265}; 266 267static int callback_file_open(struct output_file *out, int fd) 268{ 269 return 0; 270} 271 272static int callback_file_skip(struct output_file *out, int64_t off) 273{ 274 struct output_file_callback *outc = to_output_file_callback(out); 275 int to_write; 276 int ret; 277 278 while (off > 0) { 279 to_write = min(off, (int64_t)INT_MAX); 280 ret = outc->write(outc->priv, NULL, to_write); 281 if (ret < 0) { 282 return ret; 283 } 284 off -= to_write; 285 } 286 287 return 0; 288} 289 290static int callback_file_pad(struct output_file *out, int64_t len) 291{ 292 return -1; 293} 294 295static int callback_file_write(struct output_file *out, void *data, int len) 296{ 297 int ret; 298 struct output_file_callback *outc = to_output_file_callback(out); 299 300 return outc->write(outc->priv, data, len); 301} 302 303static void callback_file_close(struct output_file *out) 304{ 305 struct output_file_callback *outc = to_output_file_callback(out); 306 307 free(outc); 308} 309 310static struct output_file_ops callback_file_ops = { 311 .open = callback_file_open, 312 .skip = callback_file_skip, 313 .pad = callback_file_pad, 314 .write = callback_file_write, 315 .close = callback_file_close, 316}; 317 318int read_all(int fd, void *buf, size_t len) 319{ 320 size_t total = 0; 321 int ret; 322 char *ptr = buf; 323 324 while (total < len) { 325 ret = read(fd, ptr, len - total); 326 327 if (ret < 0) 328 return -errno; 329 330 if (ret == 0) 331 return -EINVAL; 332 333 ptr += ret; 334 total += ret; 335 } 336 337 return 0; 338} 339 340static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len) 341{ 342 chunk_header_t chunk_header; 343 int ret, chunk; 344 345 if (skip_len % out->block_size) { 346 error("don't care size %"PRIi64" is not a multiple of the block size %u", 347 skip_len, out->block_size); 348 return -1; 349 } 350 351 /* We are skipping data, so emit a don't care chunk. */ 352 chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE; 353 chunk_header.reserved1 = 0; 354 chunk_header.chunk_sz = skip_len / out->block_size; 355 chunk_header.total_sz = CHUNK_HEADER_LEN; 356 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 357 if (ret < 0) 358 return -1; 359 360 out->cur_out_ptr += skip_len; 361 out->chunk_cnt++; 362 363 return 0; 364} 365 366static int write_sparse_fill_chunk(struct output_file *out, unsigned int len, 367 uint32_t fill_val) 368{ 369 chunk_header_t chunk_header; 370 int rnd_up_len, zero_len, count; 371 int ret; 372 unsigned int i; 373 374 /* Round up the fill length to a multiple of the block size */ 375 rnd_up_len = ALIGN(len, out->block_size); 376 377 /* Finally we can safely emit a chunk of data */ 378 chunk_header.chunk_type = CHUNK_TYPE_FILL; 379 chunk_header.reserved1 = 0; 380 chunk_header.chunk_sz = rnd_up_len / out->block_size; 381 chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val); 382 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 383 384 if (ret < 0) 385 return -1; 386 ret = out->ops->write(out, &fill_val, sizeof(fill_val)); 387 if (ret < 0) 388 return -1; 389 390 if (out->use_crc) { 391 count = out->block_size / sizeof(uint32_t); 392 while (count--) 393 out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t)); 394 } 395 396 out->cur_out_ptr += rnd_up_len; 397 out->chunk_cnt++; 398 399 return 0; 400} 401 402static int write_sparse_data_chunk(struct output_file *out, unsigned int len, 403 void *data) 404{ 405 chunk_header_t chunk_header; 406 int rnd_up_len, zero_len; 407 int ret; 408 409 /* Round up the data length to a multiple of the block size */ 410 rnd_up_len = ALIGN(len, out->block_size); 411 zero_len = rnd_up_len - len; 412 413 /* Finally we can safely emit a chunk of data */ 414 chunk_header.chunk_type = CHUNK_TYPE_RAW; 415 chunk_header.reserved1 = 0; 416 chunk_header.chunk_sz = rnd_up_len / out->block_size; 417 chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len; 418 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 419 420 if (ret < 0) 421 return -1; 422 ret = out->ops->write(out, data, len); 423 if (ret < 0) 424 return -1; 425 if (zero_len) { 426 ret = out->ops->write(out, out->zero_buf, zero_len); 427 if (ret < 0) 428 return -1; 429 } 430 431 if (out->use_crc) { 432 out->crc32 = sparse_crc32(out->crc32, data, len); 433 if (zero_len) 434 out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len); 435 } 436 437 out->cur_out_ptr += rnd_up_len; 438 out->chunk_cnt++; 439 440 return 0; 441} 442 443int write_sparse_end_chunk(struct output_file *out) 444{ 445 chunk_header_t chunk_header; 446 int ret; 447 448 if (out->use_crc) { 449 chunk_header.chunk_type = CHUNK_TYPE_CRC32; 450 chunk_header.reserved1 = 0; 451 chunk_header.chunk_sz = 0; 452 chunk_header.total_sz = CHUNK_HEADER_LEN + 4; 453 454 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 455 if (ret < 0) { 456 return ret; 457 } 458 out->ops->write(out, &out->crc32, 4); 459 if (ret < 0) { 460 return ret; 461 } 462 463 out->chunk_cnt++; 464 } 465 466 return 0; 467} 468 469static struct sparse_file_ops sparse_file_ops = { 470 .write_data_chunk = write_sparse_data_chunk, 471 .write_fill_chunk = write_sparse_fill_chunk, 472 .write_skip_chunk = write_sparse_skip_chunk, 473 .write_end_chunk = write_sparse_end_chunk, 474}; 475 476static int write_normal_data_chunk(struct output_file *out, unsigned int len, 477 void *data) 478{ 479 int ret; 480 unsigned int rnd_up_len = ALIGN(len, out->block_size); 481 482 ret = out->ops->write(out, data, len); 483 if (ret < 0) { 484 return ret; 485 } 486 487 if (rnd_up_len > len) { 488 ret = out->ops->skip(out, rnd_up_len - len); 489 } 490 491 return ret; 492} 493 494static int write_normal_fill_chunk(struct output_file *out, unsigned int len, 495 uint32_t fill_val) 496{ 497 int ret; 498 unsigned int i; 499 unsigned int write_len; 500 501 /* Initialize fill_buf with the fill_val */ 502 for (i = 0; i < out->block_size / sizeof(uint32_t); i++) { 503 out->fill_buf[i] = fill_val; 504 } 505 506 while (len) { 507 write_len = min(len, out->block_size); 508 ret = out->ops->write(out, out->fill_buf, write_len); 509 if (ret < 0) { 510 return ret; 511 } 512 513 len -= write_len; 514 } 515 516 return 0; 517} 518 519static int write_normal_skip_chunk(struct output_file *out, int64_t len) 520{ 521 return out->ops->skip(out, len); 522} 523 524int write_normal_end_chunk(struct output_file *out) 525{ 526 return out->ops->pad(out, out->len); 527} 528 529static struct sparse_file_ops normal_file_ops = { 530 .write_data_chunk = write_normal_data_chunk, 531 .write_fill_chunk = write_normal_fill_chunk, 532 .write_skip_chunk = write_normal_skip_chunk, 533 .write_end_chunk = write_normal_end_chunk, 534}; 535 536void output_file_close(struct output_file *out) 537{ 538 int ret; 539 540 out->sparse_ops->write_end_chunk(out); 541 out->ops->close(out); 542} 543 544static int output_file_init(struct output_file *out, int block_size, 545 int64_t len, bool sparse, int chunks, bool crc) 546{ 547 int ret; 548 549 out->len = len; 550 out->block_size = block_size; 551 out->cur_out_ptr = 0ll; 552 out->chunk_cnt = 0; 553 out->crc32 = 0; 554 out->use_crc = crc; 555 556 out->zero_buf = calloc(block_size, 1); 557 if (!out->zero_buf) { 558 error_errno("malloc zero_buf"); 559 return -ENOMEM; 560 } 561 562 out->fill_buf = calloc(block_size, 1); 563 if (!out->fill_buf) { 564 error_errno("malloc fill_buf"); 565 ret = -ENOMEM; 566 goto err_fill_buf; 567 } 568 569 if (sparse) { 570 out->sparse_ops = &sparse_file_ops; 571 } else { 572 out->sparse_ops = &normal_file_ops; 573 } 574 575 if (sparse) { 576 sparse_header_t sparse_header = { 577 .magic = SPARSE_HEADER_MAGIC, 578 .major_version = SPARSE_HEADER_MAJOR_VER, 579 .minor_version = SPARSE_HEADER_MINOR_VER, 580 .file_hdr_sz = SPARSE_HEADER_LEN, 581 .chunk_hdr_sz = CHUNK_HEADER_LEN, 582 .blk_sz = out->block_size, 583 .total_blks = out->len / out->block_size, 584 .total_chunks = chunks, 585 .image_checksum = 0 586 }; 587 588 if (out->use_crc) { 589 sparse_header.total_chunks++; 590 } 591 592 ret = out->ops->write(out, &sparse_header, sizeof(sparse_header)); 593 if (ret < 0) { 594 goto err_write; 595 } 596 } 597 598 return 0; 599 600err_write: 601 free(out->fill_buf); 602err_fill_buf: 603 free(out->zero_buf); 604 return ret; 605} 606 607static struct output_file *output_file_new_gz(void) 608{ 609 struct output_file_gz *outgz = calloc(1, sizeof(struct output_file_gz)); 610 if (!outgz) { 611 error_errno("malloc struct outgz"); 612 return NULL; 613 } 614 615 outgz->out.ops = &gz_file_ops; 616 617 return &outgz->out; 618} 619 620static struct output_file *output_file_new_normal(void) 621{ 622 struct output_file_normal *outn = calloc(1, sizeof(struct output_file_normal)); 623 if (!outn) { 624 error_errno("malloc struct outn"); 625 return NULL; 626 } 627 628 outn->out.ops = &file_ops; 629 630 return &outn->out; 631} 632 633struct output_file *output_file_open_callback(int (*write)(void *, const void *, int), 634 void *priv, unsigned int block_size, int64_t len, int gz, int sparse, 635 int chunks, int crc) 636{ 637 int ret; 638 struct output_file_callback *outc; 639 640 outc = calloc(1, sizeof(struct output_file_callback)); 641 if (!outc) { 642 error_errno("malloc struct outc"); 643 return NULL; 644 } 645 646 outc->out.ops = &callback_file_ops; 647 outc->priv = priv; 648 outc->write = write; 649 650 ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc); 651 if (ret < 0) { 652 free(outc); 653 return NULL; 654 } 655 656 return &outc->out; 657} 658 659struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len, 660 int gz, int sparse, int chunks, int crc) 661{ 662 int ret; 663 struct output_file *out; 664 665 if (gz) { 666 out = output_file_new_gz(); 667 } else { 668 out = output_file_new_normal(); 669 } 670 if (!out) { 671 return NULL; 672 } 673 674 out->ops->open(out, fd); 675 676 ret = output_file_init(out, block_size, len, sparse, chunks, crc); 677 if (ret < 0) { 678 free(out); 679 return NULL; 680 } 681 682 return out; 683} 684 685/* Write a contiguous region of data blocks from a memory buffer */ 686int write_data_chunk(struct output_file *out, unsigned int len, void *data) 687{ 688 return out->sparse_ops->write_data_chunk(out, len, data); 689} 690 691/* Write a contiguous region of data blocks with a fill value */ 692int write_fill_chunk(struct output_file *out, unsigned int len, 693 uint32_t fill_val) 694{ 695 return out->sparse_ops->write_fill_chunk(out, len, fill_val); 696} 697 698int write_fd_chunk(struct output_file *out, unsigned int len, 699 int fd, int64_t offset) 700{ 701 int ret; 702 int64_t aligned_offset; 703 int aligned_diff; 704 int buffer_size; 705 char *ptr; 706 707 aligned_offset = offset & ~(4096 - 1); 708 aligned_diff = offset - aligned_offset; 709 buffer_size = len + aligned_diff; 710 711#ifndef USE_MINGW 712 char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd, 713 aligned_offset); 714 if (data == MAP_FAILED) { 715 return -errno; 716 } 717 ptr = data + aligned_diff; 718#else 719 off64_t pos; 720 char *data = malloc(len); 721 if (!data) { 722 return -errno; 723 } 724 pos = lseek64(fd, offset, SEEK_SET); 725 if (pos < 0) { 726 free(data); 727 return -errno; 728 } 729 ret = read_all(fd, data, len); 730 if (ret < 0) { 731 free(data); 732 return ret; 733 } 734 ptr = data; 735#endif 736 737 ret = out->sparse_ops->write_data_chunk(out, len, ptr); 738 739#ifndef USE_MINGW 740 munmap(data, buffer_size); 741#else 742 free(data); 743#endif 744 745 return ret; 746} 747 748/* Write a contiguous region of data blocks from a file */ 749int write_file_chunk(struct output_file *out, unsigned int len, 750 const char *file, int64_t offset) 751{ 752 int ret; 753 754 int file_fd = open(file, O_RDONLY | O_BINARY); 755 if (file_fd < 0) { 756 return -errno; 757 } 758 759 ret = write_fd_chunk(out, len, file_fd, offset); 760 761 close(file_fd); 762 763 return ret; 764} 765 766int write_skip_chunk(struct output_file *out, int64_t len) 767{ 768 return out->sparse_ops->write_skip_chunk(out, len); 769} 770