output_file.c revision b4cd267db30c152245e6308598e0066d87c5c55d
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define _FILE_OFFSET_BITS 64 18#define _LARGEFILE64_SOURCE 1 19 20#include <fcntl.h> 21#include <stdbool.h> 22#include <stddef.h> 23#include <stdlib.h> 24#include <string.h> 25#include <sys/stat.h> 26#include <sys/types.h> 27#include <unistd.h> 28#include <zlib.h> 29 30#include "output_file.h" 31#include "sparse_format.h" 32#include "sparse_crc32.h" 33 34#ifndef USE_MINGW 35#include <sys/mman.h> 36#define O_BINARY 0 37#else 38#define ftruncate64 ftruncate 39#endif 40 41#if defined(__APPLE__) && defined(__MACH__) 42#define lseek64 lseek 43#define ftruncate64 ftruncate 44#define mmap64 mmap 45#define off64_t off_t 46#endif 47 48#ifdef __BIONIC__ 49extern void* __mmap2(void *, size_t, int, int, int, off_t); 50static inline void *mmap64(void *addr, size_t length, int prot, int flags, 51 int fd, off64_t offset) 52{ 53 return __mmap2(addr, length, prot, flags, fd, offset >> 12); 54} 55#endif 56 57#define min(a, b) \ 58 ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; }) 59 60#define SPARSE_HEADER_MAJOR_VER 1 61#define SPARSE_HEADER_MINOR_VER 0 62#define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) 63#define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) 64 65#define container_of(inner, outer_t, elem) \ 66 ((outer_t *)((char *)inner - offsetof(outer_t, elem))) 67 68struct output_file_ops { 69 int (*open)(struct output_file *, int fd); 70 int (*skip)(struct output_file *, int64_t); 71 int (*pad)(struct output_file *, int64_t); 72 int (*write)(struct output_file *, void *, int); 73 void (*close)(struct output_file *); 74}; 75 76struct sparse_file_ops { 77 int (*write_data_chunk)(struct output_file *out, unsigned int len, 78 void *data); 79 int (*write_fill_chunk)(struct output_file *out, unsigned int len, 80 uint32_t fill_val); 81 int (*write_skip_chunk)(struct output_file *out, int64_t len); 82 int (*write_end_chunk)(struct output_file *out); 83}; 84 85struct output_file { 86 int64_t cur_out_ptr; 87 unsigned int chunk_cnt; 88 uint32_t crc32; 89 struct output_file_ops *ops; 90 struct sparse_file_ops *sparse_ops; 91 int use_crc; 92 unsigned int block_size; 93 int64_t len; 94 char *zero_buf; 95 uint32_t *fill_buf; 96 char *buf; 97}; 98 99struct output_file_gz { 100 struct output_file out; 101 gzFile gz_fd; 102}; 103 104#define to_output_file_gz(_o) \ 105 container_of((_o), struct output_file_gz, out) 106 107struct output_file_normal { 108 struct output_file out; 109 int fd; 110}; 111 112#define to_output_file_normal(_o) \ 113 container_of((_o), struct output_file_normal, out) 114 115static int file_open(struct output_file *out, int fd) 116{ 117 struct output_file_normal *outn = to_output_file_normal(out); 118 119 outn->fd = fd; 120 return 0; 121} 122 123static int file_skip(struct output_file *out, int64_t cnt) 124{ 125 off64_t ret; 126 struct output_file_normal *outn = to_output_file_normal(out); 127 128 ret = lseek64(outn->fd, cnt, SEEK_CUR); 129 if (ret < 0) { 130 error_errno("lseek64"); 131 return -1; 132 } 133 return 0; 134} 135 136static int file_pad(struct output_file *out, int64_t len) 137{ 138 int ret; 139 struct output_file_normal *outn = to_output_file_normal(out); 140 141 ret = ftruncate64(outn->fd, len); 142 if (ret < 0) { 143 return -errno; 144 } 145 146 return 0; 147} 148 149static int file_write(struct output_file *out, void *data, int len) 150{ 151 int ret; 152 struct output_file_normal *outn = to_output_file_normal(out); 153 154 ret = write(outn->fd, data, len); 155 if (ret < 0) { 156 error_errno("write"); 157 return -1; 158 } else if (ret < len) { 159 error("incomplete write"); 160 return -1; 161 } 162 163 return 0; 164} 165 166static void file_close(struct output_file *out) 167{ 168 struct output_file_normal *outn = to_output_file_normal(out); 169 170 free(outn); 171} 172 173static struct output_file_ops file_ops = { 174 .open = file_open, 175 .skip = file_skip, 176 .pad = file_pad, 177 .write = file_write, 178 .close = file_close, 179}; 180 181static int gz_file_open(struct output_file *out, int fd) 182{ 183 struct output_file_gz *outgz = to_output_file_gz(out); 184 185 outgz->gz_fd = gzdopen(fd, "wb9"); 186 if (!outgz->gz_fd) { 187 error_errno("gzopen"); 188 return -errno; 189 } 190 191 return 0; 192} 193 194 195static int gz_file_skip(struct output_file *out, int64_t cnt) 196{ 197 off64_t ret; 198 struct output_file_gz *outgz = to_output_file_gz(out); 199 200 ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR); 201 if (ret < 0) { 202 error_errno("gzseek"); 203 return -1; 204 } 205 return 0; 206} 207 208static int gz_file_pad(struct output_file *out, int64_t len) 209{ 210 off64_t ret; 211 struct output_file_gz *outgz = to_output_file_gz(out); 212 213 ret = gztell(outgz->gz_fd); 214 if (ret < 0) { 215 return -1; 216 } 217 218 if (ret >= len) { 219 return 0; 220 } 221 222 ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET); 223 if (ret < 0) { 224 return -1; 225 } 226 227 gzwrite(outgz->gz_fd, "", 1); 228 229 return 0; 230} 231 232static int gz_file_write(struct output_file *out, void *data, int len) 233{ 234 int ret; 235 struct output_file_gz *outgz = to_output_file_gz(out); 236 237 ret = gzwrite(outgz->gz_fd, data, len); 238 if (ret < 0) { 239 error_errno("gzwrite"); 240 return -1; 241 } else if (ret < len) { 242 error("incomplete gzwrite"); 243 return -1; 244 } 245 246 return 0; 247} 248 249static void gz_file_close(struct output_file *out) 250{ 251 struct output_file_gz *outgz = to_output_file_gz(out); 252 253 gzclose(outgz->gz_fd); 254 free(outgz); 255} 256 257static struct output_file_ops gz_file_ops = { 258 .open = gz_file_open, 259 .skip = gz_file_skip, 260 .pad = gz_file_pad, 261 .write = gz_file_write, 262 .close = gz_file_close, 263}; 264 265int read_all(int fd, void *buf, size_t len) 266{ 267 size_t total = 0; 268 int ret; 269 char *ptr = buf; 270 271 while (total < len) { 272 ret = read(fd, ptr, len - total); 273 274 if (ret < 0) 275 return -errno; 276 277 if (ret == 0) 278 return -EINVAL; 279 280 ptr += ret; 281 total += ret; 282 } 283 284 return 0; 285} 286 287static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len) 288{ 289 chunk_header_t chunk_header; 290 int ret, chunk; 291 292 if (skip_len % out->block_size) { 293 error("don't care size %llu is not a multiple of the block size %u", 294 skip_len, out->block_size); 295 return -1; 296 } 297 298 /* We are skipping data, so emit a don't care chunk. */ 299 chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE; 300 chunk_header.reserved1 = 0; 301 chunk_header.chunk_sz = skip_len / out->block_size; 302 chunk_header.total_sz = CHUNK_HEADER_LEN; 303 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 304 if (ret < 0) 305 return -1; 306 307 out->cur_out_ptr += skip_len; 308 out->chunk_cnt++; 309 310 return 0; 311} 312 313static int write_sparse_fill_chunk(struct output_file *out, unsigned int len, 314 uint32_t fill_val) 315{ 316 chunk_header_t chunk_header; 317 int rnd_up_len, zero_len, count; 318 int ret; 319 unsigned int i; 320 321 /* Round up the fill length to a multiple of the block size */ 322 rnd_up_len = ALIGN(len, out->block_size); 323 324 /* Finally we can safely emit a chunk of data */ 325 chunk_header.chunk_type = CHUNK_TYPE_FILL; 326 chunk_header.reserved1 = 0; 327 chunk_header.chunk_sz = rnd_up_len / out->block_size; 328 chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val); 329 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 330 331 if (ret < 0) 332 return -1; 333 ret = out->ops->write(out, &fill_val, sizeof(fill_val)); 334 if (ret < 0) 335 return -1; 336 337 if (out->use_crc) { 338 count = out->block_size / sizeof(uint32_t); 339 while (count--) 340 out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t)); 341 } 342 343 out->cur_out_ptr += rnd_up_len; 344 out->chunk_cnt++; 345 346 return 0; 347} 348 349static int write_sparse_data_chunk(struct output_file *out, unsigned int len, 350 void *data) 351{ 352 chunk_header_t chunk_header; 353 int rnd_up_len, zero_len; 354 int ret; 355 356 /* Round up the data length to a multiple of the block size */ 357 rnd_up_len = ALIGN(len, out->block_size); 358 zero_len = rnd_up_len - len; 359 360 /* Finally we can safely emit a chunk of data */ 361 chunk_header.chunk_type = CHUNK_TYPE_RAW; 362 chunk_header.reserved1 = 0; 363 chunk_header.chunk_sz = rnd_up_len / out->block_size; 364 chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len; 365 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 366 367 if (ret < 0) 368 return -1; 369 ret = out->ops->write(out, data, len); 370 if (ret < 0) 371 return -1; 372 if (zero_len) { 373 ret = out->ops->write(out, out->zero_buf, zero_len); 374 if (ret < 0) 375 return -1; 376 } 377 378 if (out->use_crc) { 379 out->crc32 = sparse_crc32(out->crc32, data, len); 380 if (zero_len) 381 out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len); 382 } 383 384 out->cur_out_ptr += rnd_up_len; 385 out->chunk_cnt++; 386 387 return 0; 388} 389 390int write_sparse_end_chunk(struct output_file *out) 391{ 392 chunk_header_t chunk_header; 393 int ret; 394 395 if (out->use_crc) { 396 chunk_header.chunk_type = CHUNK_TYPE_CRC32; 397 chunk_header.reserved1 = 0; 398 chunk_header.chunk_sz = 0; 399 chunk_header.total_sz = CHUNK_HEADER_LEN + 4; 400 401 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 402 if (ret < 0) { 403 return ret; 404 } 405 out->ops->write(out, &out->crc32, 4); 406 if (ret < 0) { 407 return ret; 408 } 409 410 out->chunk_cnt++; 411 } 412 413 return 0; 414} 415 416static struct sparse_file_ops sparse_file_ops = { 417 .write_data_chunk = write_sparse_data_chunk, 418 .write_fill_chunk = write_sparse_fill_chunk, 419 .write_skip_chunk = write_sparse_skip_chunk, 420 .write_end_chunk = write_sparse_end_chunk, 421}; 422 423static int write_normal_data_chunk(struct output_file *out, unsigned int len, 424 void *data) 425{ 426 int ret; 427 unsigned int rnd_up_len = ALIGN(len, out->block_size); 428 429 ret = out->ops->write(out, data, len); 430 if (ret < 0) { 431 return ret; 432 } 433 434 if (rnd_up_len > len) { 435 ret = out->ops->skip(out, rnd_up_len - len); 436 } 437 438 return ret; 439} 440 441static int write_normal_fill_chunk(struct output_file *out, unsigned int len, 442 uint32_t fill_val) 443{ 444 int ret; 445 unsigned int i; 446 unsigned int write_len; 447 448 /* Initialize fill_buf with the fill_val */ 449 for (i = 0; i < out->block_size / sizeof(uint32_t); i++) { 450 out->fill_buf[i] = fill_val; 451 } 452 453 while (len) { 454 write_len = min(len, out->block_size); 455 ret = out->ops->write(out, out->fill_buf, write_len); 456 if (ret < 0) { 457 return ret; 458 } 459 460 len -= write_len; 461 } 462 463 return 0; 464} 465 466static int write_normal_skip_chunk(struct output_file *out, int64_t len) 467{ 468 return out->ops->skip(out, len); 469} 470 471int write_normal_end_chunk(struct output_file *out) 472{ 473 return out->ops->pad(out, out->len); 474} 475 476static struct sparse_file_ops normal_file_ops = { 477 .write_data_chunk = write_normal_data_chunk, 478 .write_fill_chunk = write_normal_fill_chunk, 479 .write_skip_chunk = write_normal_skip_chunk, 480 .write_end_chunk = write_normal_end_chunk, 481}; 482 483void close_output_file(struct output_file *out) 484{ 485 int ret; 486 487 out->sparse_ops->write_end_chunk(out); 488 out->ops->close(out); 489} 490 491static int output_file_init(struct output_file *out, int block_size, 492 int64_t len, bool sparse, int chunks, bool crc) 493{ 494 int ret; 495 496 out->len = len; 497 out->block_size = block_size; 498 out->cur_out_ptr = 0ll; 499 out->chunk_cnt = 0; 500 out->crc32 = 0; 501 out->use_crc = crc; 502 503 out->zero_buf = calloc(block_size, 1); 504 if (!out->zero_buf) { 505 error_errno("malloc zero_buf"); 506 return -ENOMEM; 507 } 508 509 out->fill_buf = calloc(block_size, 1); 510 if (!out->fill_buf) { 511 error_errno("malloc fill_buf"); 512 ret = -ENOMEM; 513 goto err_fill_buf; 514 } 515 516 if (sparse) { 517 out->sparse_ops = &sparse_file_ops; 518 } else { 519 out->sparse_ops = &normal_file_ops; 520 } 521 522 if (sparse) { 523 sparse_header_t sparse_header = { 524 .magic = SPARSE_HEADER_MAGIC, 525 .major_version = SPARSE_HEADER_MAJOR_VER, 526 .minor_version = SPARSE_HEADER_MINOR_VER, 527 .file_hdr_sz = SPARSE_HEADER_LEN, 528 .chunk_hdr_sz = CHUNK_HEADER_LEN, 529 .blk_sz = out->block_size, 530 .total_blks = out->len / out->block_size, 531 .total_chunks = chunks, 532 .image_checksum = 0 533 }; 534 535 if (out->use_crc) { 536 sparse_header.total_chunks++; 537 } 538 539 ret = out->ops->write(out, &sparse_header, sizeof(sparse_header)); 540 if (ret < 0) { 541 goto err_write; 542 } 543 } 544 545 return 0; 546 547err_write: 548 free(out->fill_buf); 549err_fill_buf: 550 free(out->zero_buf); 551 return ret; 552} 553 554static struct output_file *output_file_new_gz(void) 555{ 556 struct output_file_gz *outgz = calloc(1, sizeof(struct output_file_gz)); 557 if (!outgz) { 558 error_errno("malloc struct outgz"); 559 return NULL; 560 } 561 562 outgz->out.ops = &gz_file_ops; 563 564 return &outgz->out; 565} 566 567static struct output_file *output_file_new_normal(void) 568{ 569 struct output_file_normal *outn = calloc(1, sizeof(struct output_file_normal)); 570 if (!outn) { 571 error_errno("malloc struct outn"); 572 return NULL; 573 } 574 575 outn->out.ops = &file_ops; 576 577 return &outn->out; 578} 579 580struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, 581 int gz, int sparse, int chunks, int crc) 582{ 583 int ret; 584 struct output_file *out; 585 586 if (gz) { 587 out = output_file_new_gz(); 588 } else { 589 out = output_file_new_normal(); 590 } 591 592 out->ops->open(out, fd); 593 594 ret = output_file_init(out, block_size, len, sparse, chunks, crc); 595 if (ret < 0) { 596 free(out); 597 return NULL; 598 } 599 600 return out; 601} 602 603/* Write a contiguous region of data blocks from a memory buffer */ 604int write_data_chunk(struct output_file *out, unsigned int len, void *data) 605{ 606 return out->sparse_ops->write_data_chunk(out, len, data); 607} 608 609/* Write a contiguous region of data blocks with a fill value */ 610int write_fill_chunk(struct output_file *out, unsigned int len, 611 uint32_t fill_val) 612{ 613 return out->sparse_ops->write_fill_chunk(out, len, fill_val); 614} 615 616int write_fd_chunk(struct output_file *out, unsigned int len, 617 int fd, int64_t offset) 618{ 619 int ret; 620 int64_t aligned_offset; 621 int aligned_diff; 622 int buffer_size; 623 char *ptr; 624 625 aligned_offset = offset & ~(4096 - 1); 626 aligned_diff = offset - aligned_offset; 627 buffer_size = len + aligned_diff; 628 629#ifndef USE_MINGW 630 char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd, 631 aligned_offset); 632 if (data == MAP_FAILED) { 633 return -errno; 634 } 635 ptr = data + aligned_diff; 636#else 637 off64_t pos; 638 char *data = malloc(len); 639 if (!data) { 640 return -errno; 641 } 642 pos = lseek64(fd, offset, SEEK_SET); 643 if (pos < 0) { 644 return -errno; 645 } 646 ret = read_all(fd, data, len); 647 if (ret < 0) { 648 return ret; 649 } 650 ptr = data; 651#endif 652 653 ret = out->sparse_ops->write_data_chunk(out, len, ptr); 654 655#ifndef USE_MINGW 656 munmap(data, buffer_size); 657#else 658 free(data); 659#endif 660 661 return ret; 662} 663 664/* Write a contiguous region of data blocks from a file */ 665int write_file_chunk(struct output_file *out, unsigned int len, 666 const char *file, int64_t offset) 667{ 668 int ret; 669 670 int file_fd = open(file, O_RDONLY | O_BINARY); 671 if (file_fd < 0) { 672 return -errno; 673 } 674 675 ret = write_fd_chunk(out, len, file_fd, offset); 676 677 close(file_fd); 678 679 return ret; 680} 681 682int write_skip_chunk(struct output_file *out, int64_t len) 683{ 684 return out->sparse_ops->write_skip_chunk(out, len); 685} 686