output_file.c revision 13a560659381b34ce3edbfa8dbe6c0aa6c076f20
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define _FILE_OFFSET_BITS 64 18#define _LARGEFILE64_SOURCE 1 19 20#include <fcntl.h> 21#include <stdbool.h> 22#include <stdlib.h> 23#include <string.h> 24#include <sys/stat.h> 25#include <sys/types.h> 26#include <unistd.h> 27#include <zlib.h> 28 29#include "output_file.h" 30#include "sparse_format.h" 31#include "sparse_crc32.h" 32 33#ifndef USE_MINGW 34#include <sys/mman.h> 35#define O_BINARY 0 36#endif 37 38#if defined(__APPLE__) && defined(__MACH__) 39#define lseek64 lseek 40#define ftruncate64 ftruncate 41#define mmap64 mmap 42#define off64_t off_t 43#endif 44 45#ifdef __BIONIC__ 46extern void* __mmap2(void *, size_t, int, int, int, off_t); 47static inline void *mmap64(void *addr, size_t length, int prot, int flags, 48 int fd, off64_t offset) 49{ 50 return __mmap2(addr, length, prot, flags, fd, offset >> 12); 51} 52#endif 53 54#define min(a, b) \ 55 ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; }) 56 57#define SPARSE_HEADER_MAJOR_VER 1 58#define SPARSE_HEADER_MINOR_VER 0 59#define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) 60#define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) 61 62struct output_file_ops { 63 int (*skip)(struct output_file *, int64_t); 64 int (*write)(struct output_file *, void *, int); 65 void (*close)(struct output_file *); 66}; 67 68struct sparse_file_ops { 69 int (*write_data_chunk)(struct output_file *out, unsigned int len, 70 void *data); 71 int (*write_fill_chunk)(struct output_file *out, unsigned int len, 72 uint32_t fill_val); 73 int (*write_skip_chunk)(struct output_file *out, int64_t len); 74 int (*write_end_chunk)(struct output_file *out); 75}; 76 77struct output_file { 78 int fd; 79 gzFile gz_fd; 80 bool close_fd; 81 int64_t cur_out_ptr; 82 unsigned int chunk_cnt; 83 uint32_t crc32; 84 struct output_file_ops *ops; 85 struct sparse_file_ops *sparse_ops; 86 int use_crc; 87 unsigned int block_size; 88 int64_t len; 89 char *zero_buf; 90 uint32_t *fill_buf; 91}; 92 93static int file_skip(struct output_file *out, int64_t cnt) 94{ 95 off64_t ret; 96 97 ret = lseek64(out->fd, cnt, SEEK_CUR); 98 if (ret < 0) { 99 error_errno("lseek64"); 100 return -1; 101 } 102 return 0; 103} 104 105static int file_write(struct output_file *out, void *data, int len) 106{ 107 int ret; 108 ret = write(out->fd, data, len); 109 if (ret < 0) { 110 error_errno("write"); 111 return -1; 112 } else if (ret < len) { 113 error("incomplete write"); 114 return -1; 115 } 116 117 return 0; 118} 119 120static void file_close(struct output_file *out) 121{ 122 if (out->close_fd) { 123 close(out->fd); 124 } 125} 126 127static struct output_file_ops file_ops = { 128 .skip = file_skip, 129 .write = file_write, 130 .close = file_close, 131}; 132 133static int gz_file_skip(struct output_file *out, int64_t cnt) 134{ 135 off64_t ret; 136 137 ret = gzseek(out->gz_fd, cnt, SEEK_CUR); 138 if (ret < 0) { 139 error_errno("gzseek"); 140 return -1; 141 } 142 return 0; 143} 144 145static int gz_file_write(struct output_file *out, void *data, int len) 146{ 147 int ret; 148 ret = gzwrite(out->gz_fd, data, len); 149 if (ret < 0) { 150 error_errno("gzwrite"); 151 return -1; 152 } else if (ret < len) { 153 error("incomplete gzwrite"); 154 return -1; 155 } 156 157 return 0; 158} 159 160static void gz_file_close(struct output_file *out) 161{ 162 gzclose(out->gz_fd); 163} 164 165static struct output_file_ops gz_file_ops = { 166 .skip = gz_file_skip, 167 .write = gz_file_write, 168 .close = gz_file_close, 169}; 170 171int read_all(int fd, void *buf, size_t len) 172{ 173 size_t total = 0; 174 int ret; 175 char *ptr = buf; 176 177 while (total < len) { 178 ret = read(fd, ptr, len - total); 179 180 if (ret < 0) 181 return -errno; 182 183 if (ret == 0) 184 return -EINVAL; 185 186 ptr += ret; 187 total += ret; 188 } 189 190 return 0; 191} 192 193static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len) 194{ 195 chunk_header_t chunk_header; 196 int ret, chunk; 197 198 if (skip_len % out->block_size) { 199 error("don't care size %llu is not a multiple of the block size %u", 200 skip_len, out->block_size); 201 return -1; 202 } 203 204 /* We are skipping data, so emit a don't care chunk. */ 205 chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE; 206 chunk_header.reserved1 = 0; 207 chunk_header.chunk_sz = skip_len / out->block_size; 208 chunk_header.total_sz = CHUNK_HEADER_LEN; 209 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 210 if (ret < 0) 211 return -1; 212 213 out->cur_out_ptr += skip_len; 214 out->chunk_cnt++; 215 216 return 0; 217} 218 219static int write_sparse_fill_chunk(struct output_file *out, unsigned int len, 220 uint32_t fill_val) 221{ 222 chunk_header_t chunk_header; 223 int rnd_up_len, zero_len, count; 224 int ret; 225 unsigned int i; 226 227 /* Round up the fill length to a multiple of the block size */ 228 rnd_up_len = ALIGN(len, out->block_size); 229 230 /* Finally we can safely emit a chunk of data */ 231 chunk_header.chunk_type = CHUNK_TYPE_FILL; 232 chunk_header.reserved1 = 0; 233 chunk_header.chunk_sz = rnd_up_len / out->block_size; 234 chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val); 235 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 236 237 if (ret < 0) 238 return -1; 239 ret = out->ops->write(out, &fill_val, sizeof(fill_val)); 240 if (ret < 0) 241 return -1; 242 243 if (out->use_crc) { 244 count = out->block_size / sizeof(uint32_t); 245 while (count--) 246 out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t)); 247 } 248 249 out->cur_out_ptr += rnd_up_len; 250 out->chunk_cnt++; 251 252 return 0; 253} 254 255static int write_sparse_data_chunk(struct output_file *out, unsigned int len, 256 void *data) 257{ 258 chunk_header_t chunk_header; 259 int rnd_up_len, zero_len; 260 int ret; 261 262 /* Round up the data length to a multiple of the block size */ 263 rnd_up_len = ALIGN(len, out->block_size); 264 zero_len = rnd_up_len - len; 265 266 /* Finally we can safely emit a chunk of data */ 267 chunk_header.chunk_type = CHUNK_TYPE_RAW; 268 chunk_header.reserved1 = 0; 269 chunk_header.chunk_sz = rnd_up_len / out->block_size; 270 chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len; 271 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 272 273 if (ret < 0) 274 return -1; 275 ret = out->ops->write(out, data, len); 276 if (ret < 0) 277 return -1; 278 if (zero_len) { 279 ret = out->ops->write(out, out->zero_buf, zero_len); 280 if (ret < 0) 281 return -1; 282 } 283 284 if (out->use_crc) { 285 out->crc32 = sparse_crc32(out->crc32, data, len); 286 if (zero_len) 287 out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len); 288 } 289 290 out->cur_out_ptr += rnd_up_len; 291 out->chunk_cnt++; 292 293 return 0; 294} 295 296int write_sparse_end_chunk(struct output_file *out) 297{ 298 chunk_header_t chunk_header; 299 int ret; 300 301 if (out->use_crc) { 302 chunk_header.chunk_type = CHUNK_TYPE_CRC32; 303 chunk_header.reserved1 = 0; 304 chunk_header.chunk_sz = 0; 305 chunk_header.total_sz = CHUNK_HEADER_LEN + 4; 306 307 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 308 if (ret < 0) { 309 return ret; 310 } 311 out->ops->write(out, &out->crc32, 4); 312 if (ret < 0) { 313 return ret; 314 } 315 316 out->chunk_cnt++; 317 } 318 319 return 0; 320} 321 322static struct sparse_file_ops sparse_file_ops = { 323 .write_data_chunk = write_sparse_data_chunk, 324 .write_fill_chunk = write_sparse_fill_chunk, 325 .write_skip_chunk = write_sparse_skip_chunk, 326 .write_end_chunk = write_sparse_end_chunk, 327}; 328 329static int write_normal_data_chunk(struct output_file *out, unsigned int len, 330 void *data) 331{ 332 int ret; 333 unsigned int rnd_up_len = ALIGN(len, out->block_size); 334 335 ret = out->ops->write(out, data, len); 336 if (ret < 0) { 337 return ret; 338 } 339 340 if (rnd_up_len > len) { 341 ret = out->ops->skip(out, rnd_up_len - len); 342 } 343 344 return ret; 345} 346 347static int write_normal_fill_chunk(struct output_file *out, unsigned int len, 348 uint32_t fill_val) 349{ 350 int ret; 351 unsigned int i; 352 unsigned int write_len; 353 354 /* Initialize fill_buf with the fill_val */ 355 for (i = 0; i < out->block_size / sizeof(uint32_t); i++) { 356 out->fill_buf[i] = fill_val; 357 } 358 359 while (len) { 360 write_len = min(len, out->block_size); 361 ret = out->ops->write(out, out->fill_buf, write_len); 362 if (ret < 0) { 363 return ret; 364 } 365 366 len -= write_len; 367 } 368 369 return 0; 370} 371 372static int write_normal_skip_chunk(struct output_file *out, int64_t len) 373{ 374 int ret; 375 376 return out->ops->skip(out, len); 377} 378 379int write_normal_end_chunk(struct output_file *out) 380{ 381 int ret; 382 383 ret = ftruncate64(out->fd, out->len); 384 if (ret < 0) { 385 return -errno; 386 } 387 388 return 0; 389} 390 391static struct sparse_file_ops normal_file_ops = { 392 .write_data_chunk = write_normal_data_chunk, 393 .write_fill_chunk = write_normal_fill_chunk, 394 .write_skip_chunk = write_normal_skip_chunk, 395 .write_end_chunk = write_normal_end_chunk, 396}; 397 398void close_output_file(struct output_file *out) 399{ 400 int ret; 401 402 out->sparse_ops->write_end_chunk(out); 403 out->ops->close(out); 404 free(out); 405} 406 407struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, 408 int gz, int sparse, int chunks, int crc) 409{ 410 int ret; 411 struct output_file *out = malloc(sizeof(struct output_file)); 412 if (!out) { 413 error_errno("malloc struct out"); 414 return NULL; 415 } 416 out->zero_buf = calloc(block_size, 1); 417 if (!out->zero_buf) { 418 error_errno("malloc zero_buf"); 419 goto err_zero_buf; 420 } 421 422 out->fill_buf = calloc(block_size, 1); 423 if (!out->fill_buf) { 424 error_errno("malloc fill_buf"); 425 goto err_fill_buf; 426 } 427 428 if (gz) { 429 out->ops = &gz_file_ops; 430 out->gz_fd = gzdopen(fd, "wb9"); 431 if (!out->gz_fd) { 432 error_errno("gzopen"); 433 goto err_gzopen; 434 } 435 } else { 436 out->fd = fd; 437 out->ops = &file_ops; 438 } 439 440 if (sparse) { 441 out->sparse_ops = &sparse_file_ops; 442 } else { 443 out->sparse_ops = &normal_file_ops; 444 } 445 446 out->close_fd = false; 447 out->cur_out_ptr = 0ll; 448 out->chunk_cnt = 0; 449 450 /* Initialize the crc32 value */ 451 out->crc32 = 0; 452 out->use_crc = crc; 453 454 out->len = len; 455 out->block_size = block_size; 456 457 if (sparse) { 458 sparse_header_t sparse_header = { 459 .magic = SPARSE_HEADER_MAGIC, 460 .major_version = SPARSE_HEADER_MAJOR_VER, 461 .minor_version = SPARSE_HEADER_MINOR_VER, 462 .file_hdr_sz = SPARSE_HEADER_LEN, 463 .chunk_hdr_sz = CHUNK_HEADER_LEN, 464 .blk_sz = out->block_size, 465 .total_blks = out->len / out->block_size, 466 .total_chunks = chunks, 467 .image_checksum = 0 468 }; 469 470 if (out->use_crc) { 471 sparse_header.total_chunks++; 472 } 473 474 ret = out->ops->write(out, &sparse_header, sizeof(sparse_header)); 475 if (ret < 0) { 476 goto err_write; 477 } 478 } 479 480 return out; 481 482err_write: 483 if (gz) { 484 gzclose(out->gz_fd); 485 } 486err_gzopen: 487 free(out->fill_buf); 488err_fill_buf: 489 free(out->zero_buf); 490err_zero_buf: 491 free(out); 492 return NULL; 493} 494 495struct output_file *open_output_file(const char *filename, 496 unsigned int block_size, int64_t len, 497 int gz, int sparse, int chunks, int crc) 498{ 499 int fd; 500 struct output_file *file; 501 502 if (strcmp(filename, "-")) { 503 fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644); 504 if (fd < 0) { 505 error_errno("open"); 506 return NULL; 507 } 508 } else { 509 fd = STDOUT_FILENO; 510 } 511 512 file = open_output_fd(fd, block_size, len, gz, sparse, chunks, crc); 513 if (!file) { 514 close(fd); 515 return NULL; 516 } 517 518 file->close_fd = true; // we opened descriptor thus we responsible for closing it 519 520 return file; 521} 522 523/* Write a contiguous region of data blocks from a memory buffer */ 524int write_data_chunk(struct output_file *out, unsigned int len, void *data) 525{ 526 return out->sparse_ops->write_data_chunk(out, len, data); 527} 528 529/* Write a contiguous region of data blocks with a fill value */ 530int write_fill_chunk(struct output_file *out, unsigned int len, 531 uint32_t fill_val) 532{ 533 return out->sparse_ops->write_fill_chunk(out, len, fill_val); 534} 535 536int write_fd_chunk(struct output_file *out, unsigned int len, 537 int fd, int64_t offset) 538{ 539 int ret; 540 int64_t aligned_offset; 541 int aligned_diff; 542 int buffer_size; 543 char *ptr; 544 545 aligned_offset = offset & ~(4096 - 1); 546 aligned_diff = offset - aligned_offset; 547 buffer_size = len + aligned_diff; 548 549#ifndef USE_MINGW 550 char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd, 551 aligned_offset); 552 if (data == MAP_FAILED) { 553 return -errno; 554 } 555 ptr = data + aligned_diff; 556#else 557 off64_t pos; 558 char *data = malloc(len); 559 if (!data) { 560 return -errno; 561 } 562 pos = lseek64(fd, offset, SEEK_SET); 563 if (pos < 0) { 564 return -errno; 565 } 566 ret = read_all(fd, data, len); 567 if (ret < 0) { 568 return ret; 569 } 570 ptr = data; 571#endif 572 573 ret = out->sparse_ops->write_data_chunk(out, len, ptr); 574 575#ifndef USE_MINGW 576 munmap(data, buffer_size); 577#else 578 free(data); 579#endif 580 581 return ret; 582} 583 584/* Write a contiguous region of data blocks from a file */ 585int write_file_chunk(struct output_file *out, unsigned int len, 586 const char *file, int64_t offset) 587{ 588 int ret; 589 590 int file_fd = open(file, O_RDONLY | O_BINARY); 591 if (file_fd < 0) { 592 return -errno; 593 } 594 595 ret = write_fd_chunk(out, len, file_fd, offset); 596 597 close(file_fd); 598 599 return ret; 600} 601 602int write_skip_chunk(struct output_file *out, int64_t len) 603{ 604 return out->sparse_ops->write_skip_chunk(out, len); 605} 606