1/* 2 * Copyright (c) 2010, 2011, 2012, 2013 3 * Phillip Lougher <phillip@squashfs.org.uk> 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License 7 * as published by the Free Software Foundation; either version 2, 8 * or (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18 * 19 * xz_wrapper.c 20 * 21 * Support for XZ (LZMA2) compression using XZ Utils liblzma 22 * http://tukaani.org/xz/ 23 */ 24 25#include <stdio.h> 26#include <string.h> 27#include <stdlib.h> 28#include <lzma.h> 29 30#include "squashfs_fs.h" 31#include "xz_wrapper.h" 32#include "compressor.h" 33 34static struct bcj bcj[] = { 35 { "x86", LZMA_FILTER_X86, 0 }, 36 { "powerpc", LZMA_FILTER_POWERPC, 0 }, 37 { "ia64", LZMA_FILTER_IA64, 0 }, 38 { "arm", LZMA_FILTER_ARM, 0 }, 39 { "armthumb", LZMA_FILTER_ARMTHUMB, 0 }, 40 { "sparc", LZMA_FILTER_SPARC, 0 }, 41 { NULL, LZMA_VLI_UNKNOWN, 0 } 42}; 43 44static int filter_count = 1; 45static int dictionary_size = 0; 46static float dictionary_percent = 0; 47 48 49/* 50 * This function is called by the options parsing code in mksquashfs.c 51 * to parse any -X compressor option. 52 * 53 * Two specific options are supported: 54 * -Xbcj 55 * -Xdict-size 56 * 57 * This function returns: 58 * >=0 (number of additional args parsed) on success 59 * -1 if the option was unrecognised, or 60 * -2 if the option was recognised, but otherwise bad in 61 * some way (e.g. invalid parameter) 62 * 63 * Note: this function sets internal compressor state, but does not 64 * pass back the results of the parsing other than success/failure. 65 * The xz_dump_options() function is called later to get the options in 66 * a format suitable for writing to the filesystem. 67 */ 68static int xz_options(char *argv[], int argc) 69{ 70 int i; 71 char *name; 72 73 if(strcmp(argv[0], "-Xbcj") == 0) { 74 if(argc < 2) { 75 fprintf(stderr, "xz: -Xbcj missing filter\n"); 76 goto failed; 77 } 78 79 name = argv[1]; 80 while(name[0] != '\0') { 81 for(i = 0; bcj[i].name; i++) { 82 int n = strlen(bcj[i].name); 83 if((strncmp(name, bcj[i].name, n) == 0) && 84 (name[n] == '\0' || 85 name[n] == ',')) { 86 if(bcj[i].selected == 0) { 87 bcj[i].selected = 1; 88 filter_count++; 89 } 90 name += name[n] == ',' ? n + 1 : n; 91 break; 92 } 93 } 94 if(bcj[i].name == NULL) { 95 fprintf(stderr, "xz: -Xbcj unrecognised " 96 "filter\n"); 97 goto failed; 98 } 99 } 100 101 return 1; 102 } else if(strcmp(argv[0], "-Xdict-size") == 0) { 103 char *b; 104 float size; 105 106 if(argc < 2) { 107 fprintf(stderr, "xz: -Xdict-size missing dict-size\n"); 108 goto failed; 109 } 110 111 size = strtof(argv[1], &b); 112 if(*b == '%') { 113 if(size <= 0 || size > 100) { 114 fprintf(stderr, "xz: -Xdict-size percentage " 115 "should be 0 < dict-size <= 100\n"); 116 goto failed; 117 } 118 119 dictionary_percent = size; 120 dictionary_size = 0; 121 } else { 122 if((float) ((int) size) != size) { 123 fprintf(stderr, "xz: -Xdict-size can't be " 124 "fractional unless a percentage of the" 125 " block size\n"); 126 goto failed; 127 } 128 129 dictionary_percent = 0; 130 dictionary_size = (int) size; 131 132 if(*b == 'k' || *b == 'K') 133 dictionary_size *= 1024; 134 else if(*b == 'm' || *b == 'M') 135 dictionary_size *= 1024 * 1024; 136 else if(*b != '\0') { 137 fprintf(stderr, "xz: -Xdict-size invalid " 138 "dict-size\n"); 139 goto failed; 140 } 141 } 142 143 return 1; 144 } 145 146 return -1; 147 148failed: 149 return -2; 150} 151 152 153/* 154 * This function is called after all options have been parsed. 155 * It is used to do post-processing on the compressor options using 156 * values that were not expected to be known at option parse time. 157 * 158 * In this case block_size may not be known until after -Xdict-size has 159 * been processed (in the case where -b is specified after -Xdict-size) 160 * 161 * This function returns 0 on successful post processing, or 162 * -1 on error 163 */ 164static int xz_options_post(int block_size) 165{ 166 /* 167 * if -Xdict-size has been specified use this to compute the datablock 168 * dictionary size 169 */ 170 if(dictionary_size || dictionary_percent) { 171 int n; 172 173 if(dictionary_size) { 174 if(dictionary_size > block_size) { 175 fprintf(stderr, "xz: -Xdict-size is larger than" 176 " block_size\n"); 177 goto failed; 178 } 179 } else 180 dictionary_size = block_size * dictionary_percent / 100; 181 182 if(dictionary_size < 8192) { 183 fprintf(stderr, "xz: -Xdict-size should be 8192 bytes " 184 "or larger\n"); 185 goto failed; 186 } 187 188 /* 189 * dictionary_size must be storable in xz header as either 190 * 2^n or as 2^n+2^(n+1) 191 */ 192 n = ffs(dictionary_size) - 1; 193 if(dictionary_size != (1 << n) && 194 dictionary_size != ((1 << n) + (1 << (n + 1)))) { 195 fprintf(stderr, "xz: -Xdict-size is an unsupported " 196 "value, dict-size must be storable in xz " 197 "header\n"); 198 fprintf(stderr, "as either 2^n or as 2^n+2^(n+1). " 199 "Example dict-sizes are 75%%, 50%%, 37.5%%, " 200 "25%%,\n"); 201 fprintf(stderr, "or 32K, 16K, 8K etc.\n"); 202 goto failed; 203 } 204 205 } else 206 /* No -Xdict-size specified, use defaults */ 207 dictionary_size = block_size; 208 209 return 0; 210 211failed: 212 return -1; 213} 214 215 216/* 217 * This function is called by mksquashfs to dump the parsed 218 * compressor options in a format suitable for writing to the 219 * compressor options field in the filesystem (stored immediately 220 * after the superblock). 221 * 222 * This function returns a pointer to the compression options structure 223 * to be stored (and the size), or NULL if there are no compression 224 * options 225 */ 226static void *xz_dump_options(int block_size, int *size) 227{ 228 static struct comp_opts comp_opts; 229 int flags = 0, i; 230 231 /* 232 * don't store compressor specific options in file system if the 233 * default options are being used - no compressor options in the 234 * file system means the default options are always assumed 235 * 236 * Defaults are: 237 * metadata dictionary size: SQUASHFS_METADATA_SIZE 238 * datablock dictionary size: block_size 239 * 1 filter 240 */ 241 if(dictionary_size == block_size && filter_count == 1) 242 return NULL; 243 244 for(i = 0; bcj[i].name; i++) 245 flags |= bcj[i].selected << i; 246 247 comp_opts.dictionary_size = dictionary_size; 248 comp_opts.flags = flags; 249 250 SQUASHFS_INSWAP_COMP_OPTS(&comp_opts); 251 252 *size = sizeof(comp_opts); 253 return &comp_opts; 254} 255 256 257/* 258 * This function is a helper specifically for the append mode of 259 * mksquashfs. Its purpose is to set the internal compressor state 260 * to the stored compressor options in the passed compressor options 261 * structure. 262 * 263 * In effect this function sets up the compressor options 264 * to the same state they were when the filesystem was originally 265 * generated, this is to ensure on appending, the compressor uses 266 * the same compression options that were used to generate the 267 * original filesystem. 268 * 269 * Note, even if there are no compressor options, this function is still 270 * called with an empty compressor structure (size == 0), to explicitly 271 * set the default options, this is to ensure any user supplied 272 * -X options on the appending mksquashfs command line are over-ridden 273 * 274 * This function returns 0 on sucessful extraction of options, and 275 * -1 on error 276 */ 277static int xz_extract_options(int block_size, void *buffer, int size) 278{ 279 struct comp_opts *comp_opts = buffer; 280 int flags, i, n; 281 282 if(size == 0) { 283 /* set defaults */ 284 dictionary_size = block_size; 285 flags = 0; 286 } else { 287 /* check passed comp opts struct is of the correct length */ 288 if(size != sizeof(struct comp_opts)) 289 goto failed; 290 291 SQUASHFS_INSWAP_COMP_OPTS(comp_opts); 292 293 dictionary_size = comp_opts->dictionary_size; 294 flags = comp_opts->flags; 295 296 /* 297 * check that the dictionary size seems correct - the dictionary 298 * size should 2^n or 2^n+2^(n+1) 299 */ 300 n = ffs(dictionary_size) - 1; 301 if(dictionary_size != (1 << n) && 302 dictionary_size != ((1 << n) + (1 << (n + 1)))) 303 goto failed; 304 } 305 306 filter_count = 1; 307 for(i = 0; bcj[i].name; i++) { 308 if((flags >> i) & 1) { 309 bcj[i].selected = 1; 310 filter_count ++; 311 } else 312 bcj[i].selected = 0; 313 } 314 315 return 0; 316 317failed: 318 fprintf(stderr, "xz: error reading stored compressor options from " 319 "filesystem!\n"); 320 321 return -1; 322} 323 324 325void xz_display_options(void *buffer, int size) 326{ 327 struct comp_opts *comp_opts = buffer; 328 int dictionary_size, flags, printed; 329 int i, n; 330 331 /* check passed comp opts struct is of the correct length */ 332 if(size != sizeof(struct comp_opts)) 333 goto failed; 334 335 SQUASHFS_INSWAP_COMP_OPTS(comp_opts); 336 337 dictionary_size = comp_opts->dictionary_size; 338 flags = comp_opts->flags; 339 340 /* 341 * check that the dictionary size seems correct - the dictionary 342 * size should 2^n or 2^n+2^(n+1) 343 */ 344 n = ffs(dictionary_size) - 1; 345 if(dictionary_size != (1 << n) && 346 dictionary_size != ((1 << n) + (1 << (n + 1)))) 347 goto failed; 348 349 printf("\tDictionary size %d\n", dictionary_size); 350 351 printed = 0; 352 for(i = 0; bcj[i].name; i++) { 353 if((flags >> i) & 1) { 354 if(printed) 355 printf(", "); 356 else 357 printf("\tFilters selected: "); 358 printf("%s", bcj[i].name); 359 printed = 1; 360 } 361 } 362 363 if(!printed) 364 printf("\tNo filters specified\n"); 365 else 366 printf("\n"); 367 368 return; 369 370failed: 371 fprintf(stderr, "xz: error reading stored compressor options from " 372 "filesystem!\n"); 373} 374 375 376/* 377 * This function is called by mksquashfs to initialise the 378 * compressor, before compress() is called. 379 * 380 * This function returns 0 on success, and 381 * -1 on error 382 */ 383static int xz_init(void **strm, int block_size, int datablock) 384{ 385 int i, j, filters = datablock ? filter_count : 1; 386 struct filter *filter = malloc(filters * sizeof(struct filter)); 387 struct xz_stream *stream; 388 389 if(filter == NULL) 390 goto failed; 391 392 stream = *strm = malloc(sizeof(struct xz_stream)); 393 if(stream == NULL) 394 goto failed2; 395 396 stream->filter = filter; 397 stream->filters = filters; 398 399 memset(filter, 0, filters * sizeof(struct filter)); 400 401 stream->dictionary_size = datablock ? dictionary_size : 402 SQUASHFS_METADATA_SIZE; 403 404 filter[0].filter[0].id = LZMA_FILTER_LZMA2; 405 filter[0].filter[0].options = &stream->opt; 406 filter[0].filter[1].id = LZMA_VLI_UNKNOWN; 407 408 for(i = 0, j = 1; datablock && bcj[i].name; i++) { 409 if(bcj[i].selected) { 410 filter[j].buffer = malloc(block_size); 411 if(filter[j].buffer == NULL) 412 goto failed3; 413 filter[j].filter[0].id = bcj[i].id; 414 filter[j].filter[1].id = LZMA_FILTER_LZMA2; 415 filter[j].filter[1].options = &stream->opt; 416 filter[j].filter[2].id = LZMA_VLI_UNKNOWN; 417 j++; 418 } 419 } 420 421 return 0; 422 423failed3: 424 for(i = 1; i < filters; i++) 425 free(filter[i].buffer); 426 free(stream); 427 428failed2: 429 free(filter); 430 431failed: 432 return -1; 433} 434 435 436static int xz_compress(void *strm, void *dest, void *src, int size, 437 int block_size, int *error) 438{ 439 int i; 440 lzma_ret res = 0; 441 struct xz_stream *stream = strm; 442 struct filter *selected = NULL; 443 444 stream->filter[0].buffer = dest; 445 446 for(i = 0; i < stream->filters; i++) { 447 struct filter *filter = &stream->filter[i]; 448 449 if(lzma_lzma_preset(&stream->opt, LZMA_PRESET_DEFAULT)) 450 goto failed; 451 452 stream->opt.dict_size = stream->dictionary_size; 453 454 filter->length = 0; 455 res = lzma_stream_buffer_encode(filter->filter, 456 LZMA_CHECK_CRC32, NULL, src, size, filter->buffer, 457 &filter->length, block_size); 458 459 if(res == LZMA_OK) { 460 if(!selected || selected->length > filter->length) 461 selected = filter; 462 } else if(res != LZMA_BUF_ERROR) 463 goto failed; 464 } 465 466 if(!selected) 467 /* 468 * Output buffer overflow. Return out of buffer space 469 */ 470 return 0; 471 472 if(selected->buffer != dest) 473 memcpy(dest, selected->buffer, selected->length); 474 475 return (int) selected->length; 476 477failed: 478 /* 479 * All other errors return failure, with the compressor 480 * specific error code in *error 481 */ 482 *error = res; 483 return -1; 484} 485 486 487static int xz_uncompress(void *dest, void *src, int size, int outsize, 488 int *error) 489{ 490 size_t src_pos = 0; 491 size_t dest_pos = 0; 492 uint64_t memlimit = MEMLIMIT; 493 494 lzma_ret res = lzma_stream_buffer_decode(&memlimit, 0, NULL, 495 src, &src_pos, size, dest, &dest_pos, outsize); 496 497 if(res == LZMA_OK && size == (int) src_pos) 498 return (int) dest_pos; 499 else { 500 *error = res; 501 return -1; 502 } 503} 504 505 506void xz_usage() 507{ 508 fprintf(stderr, "\t -Xbcj filter1,filter2,...,filterN\n"); 509 fprintf(stderr, "\t\tCompress using filter1,filter2,...,filterN in"); 510 fprintf(stderr, " turn\n\t\t(in addition to no filter), and choose"); 511 fprintf(stderr, " the best compression.\n"); 512 fprintf(stderr, "\t\tAvailable filters: x86, arm, armthumb,"); 513 fprintf(stderr, " powerpc, sparc, ia64\n"); 514 fprintf(stderr, "\t -Xdict-size <dict-size>\n"); 515 fprintf(stderr, "\t\tUse <dict-size> as the XZ dictionary size. The"); 516 fprintf(stderr, " dictionary size\n\t\tcan be specified as a"); 517 fprintf(stderr, " percentage of the block size, or as an\n\t\t"); 518 fprintf(stderr, "absolute value. The dictionary size must be less"); 519 fprintf(stderr, " than or equal\n\t\tto the block size and 8192 bytes"); 520 fprintf(stderr, " or larger. It must also be\n\t\tstorable in the xz"); 521 fprintf(stderr, " header as either 2^n or as 2^n+2^(n+1).\n\t\t"); 522 fprintf(stderr, "Example dict-sizes are 75%%, 50%%, 37.5%%, 25%%, or"); 523 fprintf(stderr, " 32K, 16K, 8K\n\t\tetc.\n"); 524} 525 526 527struct compressor xz_comp_ops = { 528 .init = xz_init, 529 .compress = xz_compress, 530 .uncompress = xz_uncompress, 531 .options = xz_options, 532 .options_post = xz_options_post, 533 .dump_options = xz_dump_options, 534 .extract_options = xz_extract_options, 535 .display_options = xz_display_options, 536 .usage = xz_usage, 537 .id = XZ_COMPRESSION, 538 .name = "xz", 539 .supported = 1 540}; 541