output_file.c revision 28fa5bc347390480fe190294c6c385b6a9f0d68b
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define _FILE_OFFSET_BITS 64 18#define _LARGEFILE64_SOURCE 1 19 20#include <fcntl.h> 21#include <stdbool.h> 22#include <stdlib.h> 23#include <string.h> 24#include <sys/stat.h> 25#include <sys/types.h> 26#include <unistd.h> 27#include <zlib.h> 28 29#include "output_file.h" 30#include "sparse_format.h" 31#include "sparse_crc32.h" 32 33#ifndef USE_MINGW 34#include <sys/mman.h> 35#define O_BINARY 0 36#endif 37 38#if defined(__APPLE__) && defined(__MACH__) 39#define lseek64 lseek 40#define ftruncate64 ftruncate 41#define mmap64 mmap 42#define off64_t off_t 43#endif 44 45#ifdef __BIONIC__ 46extern void* __mmap2(void *, size_t, int, int, int, off_t); 47static inline void *mmap64(void *addr, size_t length, int prot, int flags, 48 int fd, off64_t offset) 49{ 50 return __mmap2(addr, length, prot, flags, fd, offset >> 12); 51} 52#endif 53 54#define SPARSE_HEADER_MAJOR_VER 1 55#define SPARSE_HEADER_MINOR_VER 0 56#define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) 57#define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) 58 59struct output_file_ops { 60 int (*seek)(struct output_file *, int64_t); 61 int (*write)(struct output_file *, u8 *, int); 62 void (*close)(struct output_file *); 63}; 64 65struct output_file { 66 int fd; 67 gzFile gz_fd; 68 bool close_fd; 69 int sparse; 70 int64_t cur_out_ptr; 71 u32 chunk_cnt; 72 u32 crc32; 73 struct output_file_ops *ops; 74 int use_crc; 75 unsigned int block_size; 76 int64_t len; 77}; 78 79static int file_seek(struct output_file *out, int64_t off) 80{ 81 off64_t ret; 82 83 ret = lseek64(out->fd, off, SEEK_SET); 84 if (ret < 0) { 85 error_errno("lseek64"); 86 return -1; 87 } 88 return 0; 89} 90 91static int file_write(struct output_file *out, u8 *data, int len) 92{ 93 int ret; 94 ret = write(out->fd, data, len); 95 if (ret < 0) { 96 error_errno("write"); 97 return -1; 98 } else if (ret < len) { 99 error("incomplete write"); 100 return -1; 101 } 102 103 return 0; 104} 105 106static void file_close(struct output_file *out) 107{ 108 if (out->close_fd) { 109 close(out->fd); 110 } 111} 112 113 114static struct output_file_ops file_ops = { 115 .seek = file_seek, 116 .write = file_write, 117 .close = file_close, 118}; 119 120static int gz_file_seek(struct output_file *out, int64_t off) 121{ 122 off64_t ret; 123 124 ret = gzseek(out->gz_fd, off, SEEK_SET); 125 if (ret < 0) { 126 error_errno("gzseek"); 127 return -1; 128 } 129 return 0; 130} 131 132static int gz_file_write(struct output_file *out, u8 *data, int len) 133{ 134 int ret; 135 ret = gzwrite(out->gz_fd, data, len); 136 if (ret < 0) { 137 error_errno("gzwrite"); 138 return -1; 139 } else if (ret < len) { 140 error("incomplete gzwrite"); 141 return -1; 142 } 143 144 return 0; 145} 146 147static void gz_file_close(struct output_file *out) 148{ 149 gzclose(out->gz_fd); 150} 151 152static struct output_file_ops gz_file_ops = { 153 .seek = gz_file_seek, 154 .write = gz_file_write, 155 .close = gz_file_close, 156}; 157 158static sparse_header_t sparse_header = { 159 .magic = SPARSE_HEADER_MAGIC, 160 .major_version = SPARSE_HEADER_MAJOR_VER, 161 .minor_version = SPARSE_HEADER_MINOR_VER, 162 .file_hdr_sz = SPARSE_HEADER_LEN, 163 .chunk_hdr_sz = CHUNK_HEADER_LEN, 164 .blk_sz = 0, 165 .total_blks = 0, 166 .total_chunks = 0, 167 .image_checksum = 0 168}; 169 170static u8 *zero_buf; 171 172static int emit_skip_chunk(struct output_file *out, u64 skip_len) 173{ 174 chunk_header_t chunk_header; 175 int ret, chunk; 176 177 //DBG printf("skip chunk: 0x%llx bytes\n", skip_len); 178 179 if (skip_len % out->block_size) { 180 error("don't care size %llu is not a multiple of the block size %u", 181 skip_len, out->block_size); 182 return -1; 183 } 184 185 /* We are skipping data, so emit a don't care chunk. */ 186 chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE; 187 chunk_header.reserved1 = 0; 188 chunk_header.chunk_sz = skip_len / out->block_size; 189 chunk_header.total_sz = CHUNK_HEADER_LEN; 190 ret = out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); 191 if (ret < 0) 192 return -1; 193 194 out->cur_out_ptr += skip_len; 195 out->chunk_cnt++; 196 197 return 0; 198} 199 200static int write_chunk_fill(struct output_file *out, int64_t off, u32 fill_val, int len) 201{ 202 chunk_header_t chunk_header; 203 int rnd_up_len, zero_len, count; 204 int ret; 205 unsigned int i; 206 u32 fill_buf[4096/sizeof(u32)]; /* Maximum size of a block */ 207 208 /* We can assume that all the chunks to be written are in 209 * ascending order, block-size aligned, and non-overlapping. 210 * So, if the offset is less than the current output pointer, 211 * throw an error, and if there is a gap, emit a "don't care" 212 * chunk. The first write (of the super block) may not be 213 * blocksize aligned, so we need to deal with that too. 214 */ 215 //DBG printf("write chunk: offset 0x%llx, length 0x%x bytes\n", off, len); 216 217 if (off < out->cur_out_ptr) { 218 error("offset %llu is less than the current output offset %llu", 219 off, out->cur_out_ptr); 220 return -1; 221 } 222 223 if (off > out->cur_out_ptr) { 224 emit_skip_chunk(out, off - out->cur_out_ptr); 225 } 226 227 if (off % out->block_size) { 228 error("write chunk offset %llu is not a multiple of the block size %u", 229 off, out->block_size); 230 return -1; 231 } 232 233 if (off != out->cur_out_ptr) { 234 error("internal error, offset accounting screwy in write_chunk_raw()"); 235 return -1; 236 } 237 238 /* Round up the file length to a multiple of the block size */ 239 rnd_up_len = (len + (out->block_size - 1)) & (~(out->block_size -1)); 240 241 /* Finally we can safely emit a chunk of data */ 242 chunk_header.chunk_type = CHUNK_TYPE_FILL; 243 chunk_header.reserved1 = 0; 244 chunk_header.chunk_sz = rnd_up_len / out->block_size; 245 chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val); 246 ret = out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); 247 248 if (ret < 0) 249 return -1; 250 ret = out->ops->write(out, (u8 *)&fill_val, sizeof(fill_val)); 251 if (ret < 0) 252 return -1; 253 254 if (out->use_crc) { 255 /* Initialize fill_buf with the fill_val */ 256 for (i = 0; i < (out->block_size / sizeof(u32)); i++) { 257 fill_buf[i] = fill_val; 258 } 259 260 count = chunk_header.chunk_sz; 261 while (count) { 262 out->crc32 = sparse_crc32(out->crc32, fill_buf, out->block_size); 263 count--; 264 } 265 } 266 267 out->cur_out_ptr += rnd_up_len; 268 out->chunk_cnt++; 269 270 return 0; 271} 272 273static int write_chunk_raw(struct output_file *out, int64_t off, u8 *data, int len) 274{ 275 chunk_header_t chunk_header; 276 int rnd_up_len, zero_len; 277 int ret; 278 279 /* We can assume that all the chunks to be written are in 280 * ascending order, block-size aligned, and non-overlapping. 281 * So, if the offset is less than the current output pointer, 282 * throw an error, and if there is a gap, emit a "don't care" 283 * chunk. The first write (of the super block) may not be 284 * blocksize aligned, so we need to deal with that too. 285 */ 286 //DBG printf("write chunk: offset 0x%llx, length 0x%x bytes\n", off, len); 287 288 if (off < out->cur_out_ptr) { 289 error("offset %llu is less than the current output offset %llu", 290 off, out->cur_out_ptr); 291 return -1; 292 } 293 294 if (off > out->cur_out_ptr) { 295 emit_skip_chunk(out, off - out->cur_out_ptr); 296 } 297 298 if (off % out->block_size) { 299 error("write chunk offset %llu is not a multiple of the block size %u", 300 off, out->block_size); 301 return -1; 302 } 303 304 if (off != out->cur_out_ptr) { 305 error("internal error, offset accounting screwy in write_chunk_raw()"); 306 return -1; 307 } 308 309 /* Round up the file length to a multiple of the block size */ 310 rnd_up_len = (len + (out->block_size - 1)) & (~(out->block_size -1)); 311 zero_len = rnd_up_len - len; 312 313 /* Finally we can safely emit a chunk of data */ 314 chunk_header.chunk_type = CHUNK_TYPE_RAW; 315 chunk_header.reserved1 = 0; 316 chunk_header.chunk_sz = rnd_up_len / out->block_size; 317 chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len; 318 ret = out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); 319 320 if (ret < 0) 321 return -1; 322 ret = out->ops->write(out, data, len); 323 if (ret < 0) 324 return -1; 325 if (zero_len) { 326 ret = out->ops->write(out, zero_buf, zero_len); 327 if (ret < 0) 328 return -1; 329 } 330 331 if (out->use_crc) { 332 out->crc32 = sparse_crc32(out->crc32, data, len); 333 if (zero_len) 334 out->crc32 = sparse_crc32(out->crc32, zero_buf, zero_len); 335 } 336 337 out->cur_out_ptr += rnd_up_len; 338 out->chunk_cnt++; 339 340 return 0; 341} 342 343void close_output_file(struct output_file *out) 344{ 345 int ret; 346 chunk_header_t chunk_header; 347 348 if (out->sparse) { 349 if (out->use_crc) { 350 chunk_header.chunk_type = CHUNK_TYPE_CRC32; 351 chunk_header.reserved1 = 0; 352 chunk_header.chunk_sz = 0; 353 chunk_header.total_sz = CHUNK_HEADER_LEN + 4; 354 355 out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); 356 out->ops->write(out, (u8 *)&out->crc32, 4); 357 358 out->chunk_cnt++; 359 } 360 361 if (out->chunk_cnt != sparse_header.total_chunks) 362 error("sparse chunk count did not match: %d %d", out->chunk_cnt, 363 sparse_header.total_chunks); 364 } 365 out->ops->close(out); 366} 367 368struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, 369 int gz, int sparse, int chunks, int crc) 370{ 371 int ret; 372 struct output_file *out = malloc(sizeof(struct output_file)); 373 if (!out) { 374 error_errno("malloc struct out"); 375 return NULL; 376 } 377 zero_buf = malloc(out->block_size); 378 if (!zero_buf) { 379 error_errno("malloc zero_buf"); 380 free(out); 381 return NULL; 382 } 383 memset(zero_buf, '\0', out->block_size); 384 385 if (gz) { 386 out->ops = &gz_file_ops; 387 out->gz_fd = gzdopen(fd, "wb9"); 388 if (!out->gz_fd) { 389 error_errno("gzopen"); 390 free(out); 391 return NULL; 392 } 393 } else { 394 out->fd = fd; 395 out->ops = &file_ops; 396 } 397 out->close_fd = false; 398 out->sparse = sparse; 399 out->cur_out_ptr = 0ll; 400 out->chunk_cnt = 0; 401 402 /* Initialize the crc32 value */ 403 out->crc32 = 0; 404 out->use_crc = crc; 405 406 out->len = len; 407 out->block_size = block_size; 408 409 if (out->sparse) { 410 sparse_header.blk_sz = out->block_size, 411 sparse_header.total_blks = out->len / out->block_size, 412 sparse_header.total_chunks = chunks; 413 if (out->use_crc) 414 sparse_header.total_chunks++; 415 416 ret = out->ops->write(out, (u8 *)&sparse_header, sizeof(sparse_header)); 417 if (ret < 0) 418 return NULL; 419 } 420 421 return out; 422} 423 424struct output_file *open_output_file(const char *filename, 425 unsigned int block_size, int64_t len, 426 int gz, int sparse, int chunks, int crc) 427{ 428 int fd; 429 struct output_file *file; 430 431 if (strcmp(filename, "-")) { 432 fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644); 433 if (fd < 0) { 434 error_errno("open"); 435 return NULL; 436 } 437 } else { 438 fd = STDOUT_FILENO; 439 } 440 441 file = open_output_fd(fd, block_size, len, gz, sparse, chunks, crc); 442 if (!file) { 443 close(fd); 444 return NULL; 445 } 446 447 file->close_fd = true; // we opened descriptor thus we responsible for closing it 448 449 return file; 450} 451 452void pad_output_file(struct output_file *out, int64_t len) 453{ 454 int ret; 455 456 if (len > out->len) { 457 error("attempted to pad file %llu bytes past end of filesystem", 458 len - out->len); 459 return; 460 } 461 if (out->sparse) { 462 /* We need to emit a DONT_CARE chunk to pad out the file if the 463 * cur_out_ptr is not already at the end of the filesystem. 464 */ 465 if (len < out->cur_out_ptr) { 466 error("attempted to pad file %llu bytes less than the current output pointer", 467 out->cur_out_ptr - len); 468 return; 469 } 470 if (len > out->cur_out_ptr) { 471 emit_skip_chunk(out, len - out->cur_out_ptr); 472 } 473 } else { 474 //KEN TODO: Fixme. If the filesystem image needs no padding, 475 // this will overwrite the last byte in the file with 0 476 // The answer is to do accounting like the sparse image 477 // code does and know if there is already data there. 478 ret = out->ops->seek(out, len - 1); 479 if (ret < 0) 480 return; 481 482 ret = out->ops->write(out, (u8*)"", 1); 483 if (ret < 0) 484 return; 485 } 486} 487 488/* Write a contiguous region of data blocks from a memory buffer */ 489void write_data_block(struct output_file *out, int64_t off, void *data, int len) 490{ 491 int ret; 492 493 if (off + len > out->len) { 494 error("attempted to write block %llu past end of filesystem", 495 off + len - out->len); 496 return; 497 } 498 499 if (out->sparse) { 500 write_chunk_raw(out, off, data, len); 501 } else { 502 ret = out->ops->seek(out, off); 503 if (ret < 0) 504 return; 505 506 ret = out->ops->write(out, data, len); 507 if (ret < 0) 508 return; 509 } 510} 511 512/* Write a contiguous region of data blocks with a fill value */ 513void write_fill_block(struct output_file *out, int64_t off, unsigned int fill_val, int len) 514{ 515 int ret; 516 unsigned int i; 517 int write_len; 518 u32 fill_buf[4096/sizeof(u32)]; /* Maximum size of a block */ 519 520 if (off + len > out->len) { 521 error("attempted to write block %llu past end of filesystem", 522 off + len - out->len); 523 return; 524 } 525 526 if (out->sparse) { 527 write_chunk_fill(out, off, fill_val, len); 528 } else { 529 /* Initialize fill_buf with the fill_val */ 530 for (i = 0; i < sizeof(fill_buf)/sizeof(u32); i++) { 531 fill_buf[i] = fill_val; 532 } 533 534 ret = out->ops->seek(out, off); 535 if (ret < 0) 536 return; 537 538 while (len) { 539 write_len = (len > (int)sizeof(fill_buf) ? (int)sizeof(fill_buf) : len); 540 ret = out->ops->write(out, (u8 *)fill_buf, write_len); 541 if (ret < 0) { 542 return; 543 } else { 544 len -= write_len; 545 } 546 } 547 } 548} 549 550/* Write a contiguous region of data blocks from a file */ 551void write_data_file(struct output_file *out, int64_t off, const char *file, 552 int64_t offset, int len) 553{ 554 int ret; 555 int64_t aligned_offset; 556 int aligned_diff; 557 int buffer_size; 558 559 if (off + len >= out->len) { 560 error("attempted to write block %llu past end of filesystem", 561 off + len - out->len); 562 return; 563 } 564 565 int file_fd = open(file, O_RDONLY | O_BINARY); 566 if (file_fd < 0) { 567 error_errno("open"); 568 return; 569 } 570 571 aligned_offset = offset & ~(4096 - 1); 572 aligned_diff = offset - aligned_offset; 573 buffer_size = len + aligned_diff; 574 575#ifndef USE_MINGW 576 u8 *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, file_fd, 577 aligned_offset); 578 if (data == MAP_FAILED) { 579 error_errno("mmap64"); 580 close(file_fd); 581 return; 582 } 583#else 584 u8 *data = malloc(buffer_size); 585 if (!data) { 586 error_errno("malloc"); 587 close(file_fd); 588 return; 589 } 590 memset(data, 0, buffer_size); 591#endif 592 593 if (out->sparse) { 594 write_chunk_raw(out, off, data + aligned_diff, len); 595 } else { 596 ret = out->ops->seek(out, off); 597 if (ret < 0) 598 goto err; 599 600 ret = out->ops->write(out, data + aligned_diff, len); 601 if (ret < 0) 602 goto err; 603 } 604 605err: 606#ifndef USE_MINGW 607 munmap(data, buffer_size); 608#else 609 write(file_fd, data, buffer_size); 610 free(data); 611#endif 612 close(file_fd); 613} 614