output_file.c revision 13a560659381b34ce3edbfa8dbe6c0aa6c076f20 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define _FILE_OFFSET_BITS 64
18#define _LARGEFILE64_SOURCE 1
19
20#include <fcntl.h>
21#include <stdbool.h>
22#include <stdlib.h>
23#include <string.h>
24#include <sys/stat.h>
25#include <sys/types.h>
26#include <unistd.h>
27#include <zlib.h>
28
29#include "output_file.h"
30#include "sparse_format.h"
31#include "sparse_crc32.h"
32
33#ifndef USE_MINGW
34#include <sys/mman.h>
35#define O_BINARY 0
36#endif
37
38#if defined(__APPLE__) && defined(__MACH__)
39#define lseek64 lseek
40#define ftruncate64 ftruncate
41#define mmap64 mmap
42#define off64_t off_t
43#endif
44
45#ifdef __BIONIC__
46extern void*  __mmap2(void *, size_t, int, int, int, off_t);
47static inline void *mmap64(void *addr, size_t length, int prot, int flags,
48        int fd, off64_t offset)
49{
50    return __mmap2(addr, length, prot, flags, fd, offset >> 12);
51}
52#endif
53
54#define min(a, b) \
55	({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
56
57#define SPARSE_HEADER_MAJOR_VER 1
58#define SPARSE_HEADER_MINOR_VER 0
59#define SPARSE_HEADER_LEN       (sizeof(sparse_header_t))
60#define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
61
62struct output_file_ops {
63	int (*skip)(struct output_file *, int64_t);
64	int (*write)(struct output_file *, void *, int);
65	void (*close)(struct output_file *);
66};
67
68struct sparse_file_ops {
69	int (*write_data_chunk)(struct output_file *out, unsigned int len,
70			void *data);
71	int (*write_fill_chunk)(struct output_file *out, unsigned int len,
72			uint32_t fill_val);
73	int (*write_skip_chunk)(struct output_file *out, int64_t len);
74	int (*write_end_chunk)(struct output_file *out);
75};
76
77struct output_file {
78	int fd;
79	gzFile gz_fd;
80	bool close_fd;
81	int64_t cur_out_ptr;
82	unsigned int chunk_cnt;
83	uint32_t crc32;
84	struct output_file_ops *ops;
85	struct sparse_file_ops *sparse_ops;
86	int use_crc;
87	unsigned int block_size;
88	int64_t len;
89	char *zero_buf;
90	uint32_t *fill_buf;
91};
92
93static int file_skip(struct output_file *out, int64_t cnt)
94{
95	off64_t ret;
96
97	ret = lseek64(out->fd, cnt, SEEK_CUR);
98	if (ret < 0) {
99		error_errno("lseek64");
100		return -1;
101	}
102	return 0;
103}
104
105static int file_write(struct output_file *out, void *data, int len)
106{
107	int ret;
108	ret = write(out->fd, data, len);
109	if (ret < 0) {
110		error_errno("write");
111		return -1;
112	} else if (ret < len) {
113		error("incomplete write");
114		return -1;
115	}
116
117	return 0;
118}
119
120static void file_close(struct output_file *out)
121{
122	if (out->close_fd) {
123		close(out->fd);
124	}
125}
126
127static struct output_file_ops file_ops = {
128	.skip = file_skip,
129	.write = file_write,
130	.close = file_close,
131};
132
133static int gz_file_skip(struct output_file *out, int64_t cnt)
134{
135	off64_t ret;
136
137	ret = gzseek(out->gz_fd, cnt, SEEK_CUR);
138	if (ret < 0) {
139		error_errno("gzseek");
140		return -1;
141	}
142	return 0;
143}
144
145static int gz_file_write(struct output_file *out, void *data, int len)
146{
147	int ret;
148	ret = gzwrite(out->gz_fd, data, len);
149	if (ret < 0) {
150		error_errno("gzwrite");
151		return -1;
152	} else if (ret < len) {
153		error("incomplete gzwrite");
154		return -1;
155	}
156
157	return 0;
158}
159
160static void gz_file_close(struct output_file *out)
161{
162	gzclose(out->gz_fd);
163}
164
165static struct output_file_ops gz_file_ops = {
166	.skip = gz_file_skip,
167	.write = gz_file_write,
168	.close = gz_file_close,
169};
170
171int read_all(int fd, void *buf, size_t len)
172{
173	size_t total = 0;
174	int ret;
175	char *ptr = buf;
176
177	while (total < len) {
178		ret = read(fd, ptr, len - total);
179
180		if (ret < 0)
181			return -errno;
182
183		if (ret == 0)
184			return -EINVAL;
185
186		ptr += ret;
187		total += ret;
188	}
189
190	return 0;
191}
192
193static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len)
194{
195	chunk_header_t chunk_header;
196	int ret, chunk;
197
198	if (skip_len % out->block_size) {
199		error("don't care size %llu is not a multiple of the block size %u",
200				skip_len, out->block_size);
201		return -1;
202	}
203
204	/* We are skipping data, so emit a don't care chunk. */
205	chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE;
206	chunk_header.reserved1 = 0;
207	chunk_header.chunk_sz = skip_len / out->block_size;
208	chunk_header.total_sz = CHUNK_HEADER_LEN;
209	ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
210	if (ret < 0)
211		return -1;
212
213	out->cur_out_ptr += skip_len;
214	out->chunk_cnt++;
215
216	return 0;
217}
218
219static int write_sparse_fill_chunk(struct output_file *out, unsigned int len,
220		uint32_t fill_val)
221{
222	chunk_header_t chunk_header;
223	int rnd_up_len, zero_len, count;
224	int ret;
225	unsigned int i;
226
227	/* Round up the fill length to a multiple of the block size */
228	rnd_up_len = ALIGN(len, out->block_size);
229
230	/* Finally we can safely emit a chunk of data */
231	chunk_header.chunk_type = CHUNK_TYPE_FILL;
232	chunk_header.reserved1 = 0;
233	chunk_header.chunk_sz = rnd_up_len / out->block_size;
234	chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val);
235	ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
236
237	if (ret < 0)
238		return -1;
239	ret = out->ops->write(out, &fill_val, sizeof(fill_val));
240	if (ret < 0)
241		return -1;
242
243	if (out->use_crc) {
244		count = out->block_size / sizeof(uint32_t);
245		while (count--)
246			out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t));
247	}
248
249	out->cur_out_ptr += rnd_up_len;
250	out->chunk_cnt++;
251
252	return 0;
253}
254
255static int write_sparse_data_chunk(struct output_file *out, unsigned int len,
256		void *data)
257{
258	chunk_header_t chunk_header;
259	int rnd_up_len, zero_len;
260	int ret;
261
262	/* Round up the data length to a multiple of the block size */
263	rnd_up_len = ALIGN(len, out->block_size);
264	zero_len = rnd_up_len - len;
265
266	/* Finally we can safely emit a chunk of data */
267	chunk_header.chunk_type = CHUNK_TYPE_RAW;
268	chunk_header.reserved1 = 0;
269	chunk_header.chunk_sz = rnd_up_len / out->block_size;
270	chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len;
271	ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
272
273	if (ret < 0)
274		return -1;
275	ret = out->ops->write(out, data, len);
276	if (ret < 0)
277		return -1;
278	if (zero_len) {
279		ret = out->ops->write(out, out->zero_buf, zero_len);
280		if (ret < 0)
281			return -1;
282	}
283
284	if (out->use_crc) {
285		out->crc32 = sparse_crc32(out->crc32, data, len);
286		if (zero_len)
287			out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len);
288	}
289
290	out->cur_out_ptr += rnd_up_len;
291	out->chunk_cnt++;
292
293	return 0;
294}
295
296int write_sparse_end_chunk(struct output_file *out)
297{
298	chunk_header_t chunk_header;
299	int ret;
300
301	if (out->use_crc) {
302		chunk_header.chunk_type = CHUNK_TYPE_CRC32;
303		chunk_header.reserved1 = 0;
304		chunk_header.chunk_sz = 0;
305		chunk_header.total_sz = CHUNK_HEADER_LEN + 4;
306
307		ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
308		if (ret < 0) {
309			return ret;
310		}
311		out->ops->write(out, &out->crc32, 4);
312		if (ret < 0) {
313			return ret;
314		}
315
316		out->chunk_cnt++;
317	}
318
319	return 0;
320}
321
322static struct sparse_file_ops sparse_file_ops = {
323		.write_data_chunk = write_sparse_data_chunk,
324		.write_fill_chunk = write_sparse_fill_chunk,
325		.write_skip_chunk = write_sparse_skip_chunk,
326		.write_end_chunk = write_sparse_end_chunk,
327};
328
329static int write_normal_data_chunk(struct output_file *out, unsigned int len,
330		void *data)
331{
332	int ret;
333	unsigned int rnd_up_len = ALIGN(len, out->block_size);
334
335	ret = out->ops->write(out, data, len);
336	if (ret < 0) {
337		return ret;
338	}
339
340	if (rnd_up_len > len) {
341		ret = out->ops->skip(out, rnd_up_len - len);
342	}
343
344	return ret;
345}
346
347static int write_normal_fill_chunk(struct output_file *out, unsigned int len,
348		uint32_t fill_val)
349{
350	int ret;
351	unsigned int i;
352	unsigned int write_len;
353
354	/* Initialize fill_buf with the fill_val */
355	for (i = 0; i < out->block_size / sizeof(uint32_t); i++) {
356		out->fill_buf[i] = fill_val;
357	}
358
359	while (len) {
360		write_len = min(len, out->block_size);
361		ret = out->ops->write(out, out->fill_buf, write_len);
362		if (ret < 0) {
363			return ret;
364		}
365
366		len -= write_len;
367	}
368
369	return 0;
370}
371
372static int write_normal_skip_chunk(struct output_file *out, int64_t len)
373{
374	int ret;
375
376	return out->ops->skip(out, len);
377}
378
379int write_normal_end_chunk(struct output_file *out)
380{
381	int ret;
382
383	ret = ftruncate64(out->fd, out->len);
384	if (ret < 0) {
385		return -errno;
386	}
387
388	return 0;
389}
390
391static struct sparse_file_ops normal_file_ops = {
392		.write_data_chunk = write_normal_data_chunk,
393		.write_fill_chunk = write_normal_fill_chunk,
394		.write_skip_chunk = write_normal_skip_chunk,
395		.write_end_chunk = write_normal_end_chunk,
396};
397
398void close_output_file(struct output_file *out)
399{
400	int ret;
401
402	out->sparse_ops->write_end_chunk(out);
403	out->ops->close(out);
404	free(out);
405}
406
407struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len,
408		int gz, int sparse, int chunks, int crc)
409{
410	int ret;
411	struct output_file *out = malloc(sizeof(struct output_file));
412	if (!out) {
413		error_errno("malloc struct out");
414		return NULL;
415	}
416	out->zero_buf = calloc(block_size, 1);
417	if (!out->zero_buf) {
418		error_errno("malloc zero_buf");
419		goto err_zero_buf;
420	}
421
422	out->fill_buf = calloc(block_size, 1);
423	if (!out->fill_buf) {
424		error_errno("malloc fill_buf");
425		goto err_fill_buf;
426	}
427
428	if (gz) {
429		out->ops = &gz_file_ops;
430		out->gz_fd = gzdopen(fd, "wb9");
431		if (!out->gz_fd) {
432			error_errno("gzopen");
433			goto err_gzopen;
434		}
435	} else {
436		out->fd = fd;
437		out->ops = &file_ops;
438	}
439
440	if (sparse) {
441		out->sparse_ops = &sparse_file_ops;
442	} else {
443		out->sparse_ops = &normal_file_ops;
444	}
445
446	out->close_fd = false;
447	out->cur_out_ptr = 0ll;
448	out->chunk_cnt = 0;
449
450	/* Initialize the crc32 value */
451	out->crc32 = 0;
452	out->use_crc = crc;
453
454	out->len = len;
455	out->block_size = block_size;
456
457	if (sparse) {
458		sparse_header_t sparse_header = {
459				.magic = SPARSE_HEADER_MAGIC,
460				.major_version = SPARSE_HEADER_MAJOR_VER,
461				.minor_version = SPARSE_HEADER_MINOR_VER,
462				.file_hdr_sz = SPARSE_HEADER_LEN,
463				.chunk_hdr_sz = CHUNK_HEADER_LEN,
464				.blk_sz = out->block_size,
465				.total_blks = out->len / out->block_size,
466				.total_chunks = chunks,
467				.image_checksum = 0
468		};
469
470		if (out->use_crc) {
471			sparse_header.total_chunks++;
472		}
473
474		ret = out->ops->write(out, &sparse_header, sizeof(sparse_header));
475		if (ret < 0) {
476			goto err_write;
477		}
478	}
479
480	return out;
481
482err_write:
483	if (gz) {
484		gzclose(out->gz_fd);
485	}
486err_gzopen:
487	free(out->fill_buf);
488err_fill_buf:
489	free(out->zero_buf);
490err_zero_buf:
491	free(out);
492	return NULL;
493}
494
495struct output_file *open_output_file(const char *filename,
496		unsigned int block_size, int64_t len,
497		int gz, int sparse, int chunks, int crc)
498{
499	int fd;
500	struct output_file *file;
501
502	if (strcmp(filename, "-")) {
503		fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644);
504		if (fd < 0) {
505			error_errno("open");
506			return NULL;
507		}
508	} else {
509		fd = STDOUT_FILENO;
510	}
511
512	file = open_output_fd(fd, block_size, len, gz, sparse, chunks, crc);
513	if (!file) {
514		close(fd);
515		return NULL;
516	}
517
518	file->close_fd = true; // we opened descriptor thus we responsible for closing it
519
520	return file;
521}
522
523/* Write a contiguous region of data blocks from a memory buffer */
524int write_data_chunk(struct output_file *out, unsigned int len, void *data)
525{
526	return out->sparse_ops->write_data_chunk(out, len, data);
527}
528
529/* Write a contiguous region of data blocks with a fill value */
530int write_fill_chunk(struct output_file *out, unsigned int len,
531		uint32_t fill_val)
532{
533	return out->sparse_ops->write_fill_chunk(out, len, fill_val);
534}
535
536int write_fd_chunk(struct output_file *out, unsigned int len,
537		int fd, int64_t offset)
538{
539	int ret;
540	int64_t aligned_offset;
541	int aligned_diff;
542	int buffer_size;
543	char *ptr;
544
545	aligned_offset = offset & ~(4096 - 1);
546	aligned_diff = offset - aligned_offset;
547	buffer_size = len + aligned_diff;
548
549#ifndef USE_MINGW
550	char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd,
551			aligned_offset);
552	if (data == MAP_FAILED) {
553		return -errno;
554	}
555	ptr = data + aligned_diff;
556#else
557	off64_t pos;
558	char *data = malloc(len);
559	if (!data) {
560		return -errno;
561	}
562	pos = lseek64(fd, offset, SEEK_SET);
563	if (pos < 0) {
564		return -errno;
565	}
566	ret = read_all(fd, data, len);
567	if (ret < 0) {
568		return ret;
569	}
570	ptr = data;
571#endif
572
573	ret = out->sparse_ops->write_data_chunk(out, len, ptr);
574
575#ifndef USE_MINGW
576	munmap(data, buffer_size);
577#else
578	free(data);
579#endif
580
581	return ret;
582}
583
584/* Write a contiguous region of data blocks from a file */
585int write_file_chunk(struct output_file *out, unsigned int len,
586		const char *file, int64_t offset)
587{
588	int ret;
589
590	int file_fd = open(file, O_RDONLY | O_BINARY);
591	if (file_fd < 0) {
592		return -errno;
593	}
594
595	ret = write_fd_chunk(out, len, file_fd, offset);
596
597	close(file_fd);
598
599	return ret;
600}
601
602int write_skip_chunk(struct output_file *out, int64_t len)
603{
604	return out->sparse_ops->write_skip_chunk(out, len);
605}
606