sparse_read.c revision 0c4c47f88dfc15cada154a1cf9b4db88b49890f0
1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define _GNU_SOURCE
18#define _FILE_OFFSET_BITS 64
19#define _LARGEFILE64_SOURCE 1
20
21#include <fcntl.h>
22#include <stdarg.h>
23#include <stdbool.h>
24#include <stdint.h>
25#include <stdio.h>
26#include <stdlib.h>
27#include <string.h>
28#include <unistd.h>
29
30#include <sparse/sparse.h>
31
32#include "sparse_crc32.h"
33#include "sparse_file.h"
34#include "sparse_format.h"
35
36#if defined(__APPLE__) && defined(__MACH__)
37#define lseek64 lseek
38#define off64_t off_t
39#endif
40
41#define SPARSE_HEADER_MAJOR_VER 1
42#define SPARSE_HEADER_LEN       (sizeof(sparse_header_t))
43#define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
44
45#define COPY_BUF_SIZE (1024U*1024U)
46static char *copybuf;
47
48#define min(a, b) \
49	({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
50
51static void verbose_error(bool verbose, int err, const char *fmt, ...)
52{
53	char *s = "";
54	char *at = "";
55	if (fmt) {
56		va_list argp;
57		int size;
58
59		va_start(argp, fmt);
60		size = vsnprintf(NULL, 0, fmt, argp);
61		va_end(argp);
62
63		if (size < 0) {
64			return;
65		}
66
67		at = malloc(size + 1);
68		if (at == NULL) {
69			return;
70		}
71
72		va_start(argp, fmt);
73		vsnprintf(at, size, fmt, argp);
74		va_end(argp);
75		at[size] = 0;
76		s = " at ";
77	}
78	if (verbose) {
79#ifndef USE_MINGW
80		if (err == -EOVERFLOW) {
81			sparse_print_verbose("EOF while reading file%s%s\n", s, at);
82		} else
83#endif
84		if (err == -EINVAL) {
85			sparse_print_verbose("Invalid sparse file format%s%s\n", s, at);
86		} else if (err == -ENOMEM) {
87			sparse_print_verbose("Failed allocation while reading file%s%s\n",
88					s, at);
89		} else {
90			sparse_print_verbose("Unknown error %d%s%s\n", err, s, at);
91		}
92	}
93	if (fmt) {
94		free(at);
95	}
96}
97
98static int process_raw_chunk(struct sparse_file *s, unsigned int chunk_size,
99		int fd, int64_t offset, unsigned int blocks, unsigned int block,
100		uint32_t *crc32)
101{
102	int ret;
103	int chunk;
104	unsigned int len = blocks * s->block_size;
105
106	if (chunk_size % s->block_size != 0) {
107		return -EINVAL;
108	}
109
110	if (chunk_size / s->block_size != blocks) {
111		return -EINVAL;
112	}
113
114	ret = sparse_file_add_fd(s, fd, offset, len, block);
115	if (ret < 0) {
116		return ret;
117	}
118
119	if (crc32) {
120		while (len) {
121			chunk = min(len, COPY_BUF_SIZE);
122			ret = read_all(fd, copybuf, chunk);
123			if (ret < 0) {
124				return ret;
125			}
126			*crc32 = sparse_crc32(*crc32, copybuf, chunk);
127			len -= chunk;
128		}
129	} else {
130		lseek64(fd, len, SEEK_CUR);
131	}
132
133	return 0;
134}
135
136static int process_fill_chunk(struct sparse_file *s, unsigned int chunk_size,
137		int fd, unsigned int blocks, unsigned int block, uint32_t *crc32)
138{
139	int ret;
140	int chunk;
141	int64_t len = (int64_t)blocks * s->block_size;
142	uint32_t fill_val;
143	uint32_t *fillbuf;
144	unsigned int i;
145
146	if (chunk_size != sizeof(fill_val)) {
147		return -EINVAL;
148	}
149
150	ret = read_all(fd, &fill_val, sizeof(fill_val));
151	if (ret < 0) {
152		return ret;
153	}
154
155	ret = sparse_file_add_fill(s, fill_val, len, block);
156	if (ret < 0) {
157		return ret;
158	}
159
160	if (crc32) {
161		/* Fill copy_buf with the fill value */
162		fillbuf = (uint32_t *)copybuf;
163		for (i = 0; i < (COPY_BUF_SIZE / sizeof(fill_val)); i++) {
164			fillbuf[i] = fill_val;
165		}
166
167		while (len) {
168			chunk = min(len, COPY_BUF_SIZE);
169			*crc32 = sparse_crc32(*crc32, copybuf, chunk);
170			len -= chunk;
171		}
172	}
173
174	return 0;
175}
176
177static int process_skip_chunk(struct sparse_file *s, unsigned int chunk_size,
178		int fd, unsigned int blocks, unsigned int block, uint32_t *crc32)
179{
180	int ret;
181	int chunk;
182	int64_t len = (int64_t)blocks * s->block_size;
183	uint32_t fill_val;
184	uint32_t *fillbuf;
185	unsigned int i;
186
187	if (chunk_size != 0) {
188		return -EINVAL;
189	}
190
191	if (crc32) {
192		memset(copybuf, 0, COPY_BUF_SIZE);
193
194		while (len) {
195			chunk = min(len, COPY_BUF_SIZE);
196			*crc32 = sparse_crc32(*crc32, copybuf, chunk);
197			len -= chunk;
198		}
199	}
200
201	return 0;
202}
203
204static int process_crc32_chunk(int fd, unsigned int chunk_size, uint32_t crc32)
205{
206	uint32_t file_crc32;
207	int ret;
208
209	if (chunk_size != sizeof(file_crc32)) {
210		return -EINVAL;
211	}
212
213	ret = read_all(fd, &file_crc32, sizeof(file_crc32));
214	if (ret < 0) {
215		return ret;
216	}
217
218	if (file_crc32 != crc32) {
219		return -EINVAL;
220	}
221
222	return 0;
223}
224
225static int process_chunk(struct sparse_file *s, int fd, off64_t offset,
226		unsigned int chunk_hdr_sz, chunk_header_t *chunk_header,
227		unsigned int cur_block, uint32_t *crc_ptr)
228{
229	int ret;
230	unsigned int chunk_data_size;
231
232	chunk_data_size = chunk_header->total_sz - chunk_hdr_sz;
233
234	switch (chunk_header->chunk_type) {
235		case CHUNK_TYPE_RAW:
236			ret = process_raw_chunk(s, chunk_data_size, fd, offset,
237					chunk_header->chunk_sz, cur_block, crc_ptr);
238			if (ret < 0) {
239				verbose_error(s->verbose, ret, "data block at %lld", offset);
240				return ret;
241			}
242			return chunk_header->chunk_sz;
243		case CHUNK_TYPE_FILL:
244			ret = process_fill_chunk(s, chunk_data_size, fd,
245					chunk_header->chunk_sz, cur_block, crc_ptr);
246			if (ret < 0) {
247				verbose_error(s->verbose, ret, "fill block at %lld", offset);
248				return ret;
249			}
250			return chunk_header->chunk_sz;
251		case CHUNK_TYPE_DONT_CARE:
252			ret = process_skip_chunk(s, chunk_data_size, fd,
253					chunk_header->chunk_sz, cur_block, crc_ptr);
254			if (chunk_data_size != 0) {
255				if (ret < 0) {
256					verbose_error(s->verbose, ret, "skip block at %lld", offset);
257					return ret;
258				}
259			}
260			return chunk_header->chunk_sz;
261		case CHUNK_TYPE_CRC32:
262			ret = process_crc32_chunk(fd, chunk_data_size, *crc_ptr);
263			if (ret < 0) {
264				verbose_error(s->verbose, -EINVAL, "crc block at %lld",
265						offset);
266				return ret;
267			}
268			return 0;
269		default:
270			verbose_error(s->verbose, -EINVAL, "unknown block %04X at %lld",
271					chunk_header->chunk_type, offset);
272	}
273
274	return 0;
275}
276
277static int sparse_file_read_sparse(struct sparse_file *s, int fd, bool crc)
278{
279	int ret;
280	unsigned int i;
281	sparse_header_t sparse_header;
282	chunk_header_t chunk_header;
283	uint32_t crc32 = 0;
284	uint32_t *crc_ptr = 0;
285	unsigned int cur_block = 0;
286	off64_t offset;
287
288	if (!copybuf) {
289		copybuf = malloc(COPY_BUF_SIZE);
290	}
291
292	if (!copybuf) {
293		return -ENOMEM;
294	}
295
296	if (crc) {
297		crc_ptr = &crc32;
298	}
299
300	ret = read_all(fd, &sparse_header, sizeof(sparse_header));
301	if (ret < 0) {
302		return ret;
303	}
304
305	if (sparse_header.magic != SPARSE_HEADER_MAGIC) {
306		return -EINVAL;
307	}
308
309	if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) {
310		return -EINVAL;
311	}
312
313	if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) {
314		return -EINVAL;
315	}
316
317	if (sparse_header.chunk_hdr_sz < sizeof(chunk_header)) {
318		return -EINVAL;
319	}
320
321	if (sparse_header.file_hdr_sz > SPARSE_HEADER_LEN) {
322		/* Skip the remaining bytes in a header that is longer than
323		 * we expected.
324		 */
325		lseek64(fd, sparse_header.file_hdr_sz - SPARSE_HEADER_LEN, SEEK_CUR);
326	}
327
328	for (i = 0; i < sparse_header.total_chunks; i++) {
329		ret = read_all(fd, &chunk_header, sizeof(chunk_header));
330		if (ret < 0) {
331			return ret;
332		}
333
334		if (sparse_header.chunk_hdr_sz > CHUNK_HEADER_LEN) {
335			/* Skip the remaining bytes in a header that is longer than
336			 * we expected.
337			 */
338			lseek64(fd, sparse_header.chunk_hdr_sz - CHUNK_HEADER_LEN, SEEK_CUR);
339		}
340
341		offset = lseek64(fd, 0, SEEK_CUR);
342
343		ret = process_chunk(s, fd, offset, sparse_header.chunk_hdr_sz, &chunk_header,
344				cur_block, crc_ptr);
345		if (ret < 0) {
346			return ret;
347		}
348
349		cur_block += ret;
350	}
351
352	if (sparse_header.total_blks != cur_block) {
353		return -EINVAL;
354	}
355
356	return 0;
357}
358
359static int sparse_file_read_normal(struct sparse_file *s, int fd)
360{
361	int ret;
362	uint32_t *buf = malloc(s->block_size);
363	unsigned int block = 0;
364	int64_t remain = s->len;
365	int64_t offset = 0;
366	unsigned int to_read;
367	char *ptr;
368	unsigned int i;
369	bool sparse_block;
370
371	if (!buf) {
372		return -ENOMEM;
373	}
374
375	while (remain > 0) {
376		to_read = min(remain, s->block_size);
377		ret = read_all(fd, buf, to_read);
378		if (ret < 0) {
379			error("failed to read sparse file");
380			return ret;
381		}
382
383		if (to_read == s->block_size) {
384			sparse_block = true;
385			for (i = 1; i < s->block_size / sizeof(uint32_t); i++) {
386				if (buf[0] != buf[i]) {
387					sparse_block = false;
388					break;
389				}
390			}
391		} else {
392			sparse_block = false;
393		}
394
395		if (sparse_block) {
396			/* TODO: add flag to use skip instead of fill for buf[0] == 0 */
397			sparse_file_add_fill(s, buf[0], to_read, block);
398		} else {
399			sparse_file_add_fd(s, fd, offset, to_read, block);
400		}
401
402		remain -= to_read;
403		offset += to_read;
404		block++;
405	}
406
407	return 0;
408}
409
410int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc)
411{
412	if (crc && !sparse) {
413		return -EINVAL;
414	}
415
416	if (sparse) {
417		return sparse_file_read_sparse(s, fd, crc);
418	} else {
419		return sparse_file_read_normal(s, fd);
420	}
421}
422
423struct sparse_file *sparse_file_import(int fd, bool verbose, bool crc)
424{
425	int ret;
426	sparse_header_t sparse_header;
427	int64_t len;
428	struct sparse_file *s;
429
430	ret = read_all(fd, &sparse_header, sizeof(sparse_header));
431	if (ret < 0) {
432		verbose_error(verbose, ret, "header");
433		return NULL;
434	}
435
436	if (sparse_header.magic != SPARSE_HEADER_MAGIC) {
437		verbose_error(verbose, -EINVAL, "header magic");
438		return NULL;
439	}
440
441	if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) {
442		verbose_error(verbose, -EINVAL, "header major version");
443		return NULL;
444	}
445
446	if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) {
447		return NULL;
448	}
449
450	if (sparse_header.chunk_hdr_sz < sizeof(chunk_header_t)) {
451		return NULL;
452	}
453
454	len = (int64_t)sparse_header.total_blks * sparse_header.blk_sz;
455	s = sparse_file_new(sparse_header.blk_sz, len);
456	if (!s) {
457		verbose_error(verbose, -EINVAL, NULL);
458		return NULL;
459	}
460
461	ret = lseek64(fd, 0, SEEK_SET);
462	if (ret < 0) {
463		verbose_error(verbose, ret, "seeking");
464		sparse_file_destroy(s);
465		return NULL;
466	}
467
468	s->verbose = verbose;
469
470	ret = sparse_file_read(s, fd, true, crc);
471	if (ret < 0) {
472		sparse_file_destroy(s);
473		return NULL;
474	}
475
476	return s;
477}
478
479struct sparse_file *sparse_file_import_auto(int fd, bool crc)
480{
481	struct sparse_file *s;
482	int64_t len;
483	int ret;
484
485	s = sparse_file_import(fd, true, crc);
486	if (s) {
487		return s;
488	}
489
490	len = lseek64(fd, 0, SEEK_END);
491	if (len < 0) {
492		return NULL;
493	}
494
495	lseek64(fd, 0, SEEK_SET);
496
497	s = sparse_file_new(4096, len);
498	if (!s) {
499		return NULL;
500	}
501
502	ret = sparse_file_read_normal(s, fd);
503	if (ret < 0) {
504		sparse_file_destroy(s);
505		return NULL;
506	}
507
508	return s;
509}
510