lz4io.c revision 45b0642bf54718d2b57dd61cb606b154afc0ab26
169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* 269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. LZ4io.c - LZ4 File/Stream Interface 3488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet Copyright (C) Yann Collet 2011-2015 445a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet 569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. GPL v2 License 669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. This program is free software; you can redistribute it and/or modify 869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. it under the terms of the GNU General Public License as published by 969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. the Free Software Foundation; either version 2 of the License, or 1069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. (at your option) any later version. 1169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 1269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. This program is distributed in the hope that it will be useful, 1369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. but WITHOUT ANY WARRANTY; without even the implied warranty of 1469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. GNU General Public License for more details. 1669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 1769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. You should have received a copy of the GNU General Public License along 1869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. with this program; if not, write to the Free Software Foundation, Inc., 1969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 2069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 2169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. You can contact the author at : 2245a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet - LZ4 source repository : https://github.com/Cyan4973/lz4 2369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 2469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.*/ 2569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* 2669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. Note : this is stand-alone program. 2769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. It is not part of LZ4 compression library, it is a user code of the LZ4 library. 2869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. - The license of LZ4 library is BSD. 2969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. - The license of xxHash library is BSD. 3069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. - The license of this source file is GPLv2. 3169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.*/ 3269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 338a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet/************************************** 348a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* Compiler Options 35d11ac4087254db4b9391039ce862e7305874fe97Yann Collet**************************************/ 3669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#ifdef _MSC_VER /* Visual Studio */ 3769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.# define _CRT_SECURE_NO_WARNINGS 388a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ 398a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 4069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#endif 4169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 428a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet#define _LARGE_FILES /* Large file support on 32-bits AIX */ 438a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet#define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */ 4469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 4569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 46d11ac4087254db4b9391039ce862e7305874fe97Yann Collet/***************************** 478a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* Includes 488a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet*****************************/ 49ceec6fa8492a5ff0ed163c96516716a3c2b09461Yann Collet#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout */ 508a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet#include <stdlib.h> /* malloc, free */ 518a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet#include <string.h> /* strcmp, strlen */ 528a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet#include <time.h> /* clock */ 5369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#include "lz4io.h" 548a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet#include "lz4.h" /* still required for legacy format */ 558a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet#include "lz4hc.h" /* still required for legacy format */ 566de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet#include "lz4frame.h" 5769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 5869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 59248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet/****************************** 608a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* OS-specific Includes 61248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet******************************/ 6269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) 638a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet# include <fcntl.h> /* _O_BINARY */ 64248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet# include <io.h> /* _setmode, _fileno, _get_osfhandle */ 65ceec6fa8492a5ff0ed163c96516716a3c2b09461Yann Collet# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) 6645b0642bf54718d2b57dd61cb606b154afc0ab26Yann Collet# include <Windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ 67248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } 6841b6ed3c5bc7f0c85f1bfe3abc940d9b74581e7dTakayuki MATSUOKA# if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ 6941b6ed3c5bc7f0c85f1bfe3abc940d9b74581e7dTakayuki MATSUOKA# define fseek _fseeki64 7041b6ed3c5bc7f0c85f1bfe3abc940d9b74581e7dTakayuki MATSUOKA# endif 7169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#else 7269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.# define SET_BINARY_MODE(file) 73248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet# define SET_SPARSE_FILE_MODE(file) 7469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#endif 7569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 7669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 77248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet/***************************** 788a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* Constants 798a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet*****************************/ 806de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet#define KB *(1 <<10) 816de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet#define MB *(1 <<20) 8269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define GB *(1U<<30) 8369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 8469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define _1BIT 0x01 8569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define _2BITS 0x03 8669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define _3BITS 0x07 8769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define _4BITS 0x0F 8869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define _8BITS 0xFF 8969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 9069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define MAGICNUMBER_SIZE 4 9169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define LZ4S_MAGICNUMBER 0x184D2204 9269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define LZ4S_SKIPPABLE0 0x184D2A50 9369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0 9469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define LEGACY_MAGICNUMBER 0x184C2102 9569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 9669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define CACHELINE 64 9769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define LEGACY_BLOCKSIZE (8 MB) 98d517d609d95bdbab665a6ddb6e018c450d1e5ae6Yann Collet#define MIN_STREAM_BUFSIZE (192 KB) 9969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define LZ4S_BLOCKSIZEID_DEFAULT 7 10069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define LZ4S_CHECKSUM_SEED 0 10169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define LZ4S_EOS 0 10269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define LZ4S_MAXHEADERSIZE (MAGICNUMBER_SIZE+2+8+4+1) 10369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 10469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 1058a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet/************************************** 1068a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* Macros 107d11ac4087254db4b9391039ce862e7305874fe97Yann Collet**************************************/ 10869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) 10912ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } 11012ab41571ef7fd11b8b2013aa943beae373cef8aYann Colletstatic int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ 11112ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet 11212ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ 11312ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet if ((LZ4IO_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \ 1146de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet { g_time = clock(); DISPLAY(__VA_ARGS__); \ 11512ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet if (g_displayLevel>=4) fflush(stdout); } } 1166de52c2a8c40b7381e364ad87af9e80a60d95229Yann Colletstatic const unsigned refreshRate = 150; 1176de52c2a8c40b7381e364ad87af9e80a60d95229Yann Colletstatic clock_t g_time = 0; 11869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 11969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 1208a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet/************************************** 1218a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* Local Parameters 122d11ac4087254db4b9391039ce862e7305874fe97Yann Collet**************************************/ 12312ab41571ef7fd11b8b2013aa943beae373cef8aYann Colletstatic int g_overwrite = 1; 12412ab41571ef7fd11b8b2013aa943beae373cef8aYann Colletstatic int g_blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; 12512ab41571ef7fd11b8b2013aa943beae373cef8aYann Colletstatic int g_blockChecksum = 0; 12612ab41571ef7fd11b8b2013aa943beae373cef8aYann Colletstatic int g_streamChecksum = 1; 12712ab41571ef7fd11b8b2013aa943beae373cef8aYann Colletstatic int g_blockIndependence = 1; 12812ab41571ef7fd11b8b2013aa943beae373cef8aYann Colletstatic int g_sparseFileSupport = 0; 12969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 13069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.static const int minBlockSizeID = 4; 13169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.static const int maxBlockSizeID = 7; 13269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 133c5decf7562a3b4065922ae6460b7785eb91366f8Yann Collet 1348a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet/************************************** 1358a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* Exceptions 1368a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet***************************************/ 13769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define DEBUG 0 13869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); 13969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define EXM_THROW(error, ...) \ 14069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ \ 14169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ 14269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(1, "Error %i : ", error); \ 14369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(1, __VA_ARGS__); \ 14469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(1, "\n"); \ 14569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. exit(error); \ 14669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 14769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 14869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 1498a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet/************************************** 1508a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* Version modifiers 1518a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet***************************************/ 15269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define EXTENDED_ARGUMENTS 15369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define EXTENDED_HELP 15469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define EXTENDED_FORMAT 15569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define DEFAULT_DECOMPRESSOR decodeLZ4S 15669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 15769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 15869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* ************************************************** */ 15969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* ****************** Parameters ******************** */ 16069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* ************************************************** */ 16169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 16269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* Default setting : overwrite = 1; return : overwrite mode (0/1) */ 16369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.int LZ4IO_setOverwrite(int yes) 16469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 16512ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet g_overwrite = (yes!=0); 16612ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet return g_overwrite; 16769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 16869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 16969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* blockSizeID : valid values : 4-5-6-7 */ 17069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.int LZ4IO_setBlockSizeID(int bsid) 17169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 17269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. static const int blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB }; 17369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return -1; 17412ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet g_blockSizeId = bsid; 17512ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet return blockSizeTable[g_blockSizeId-minBlockSizeID]; 17669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 17769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 1786de52c2a8c40b7381e364ad87af9e80a60d95229Yann Colletint LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode) 17969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 18012ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet g_blockIndependence = (blockMode == LZ4IO_blockIndependent); 18112ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet return g_blockIndependence; 18269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 18369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 18469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* Default setting : no checksum */ 18569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.int LZ4IO_setBlockChecksumMode(int xxhash) 18669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 18712ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet g_blockChecksum = (xxhash != 0); 18812ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet return g_blockChecksum; 18969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 19069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 19169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* Default setting : checksum enabled */ 19269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.int LZ4IO_setStreamChecksumMode(int xxhash) 19369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 19412ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet g_streamChecksum = (xxhash != 0); 19512ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet return g_streamChecksum; 19669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 19769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 19869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* Default setting : 0 (no notification) */ 19969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.int LZ4IO_setNotificationLevel(int level) 20069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 20112ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet g_displayLevel = level; 20212ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet return g_displayLevel; 20312ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet} 20412ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet 20512ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet/* Default setting : 0 (disabled) */ 20612ab41571ef7fd11b8b2013aa943beae373cef8aYann Colletint LZ4IO_setSparseFile(int enable) 20712ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet{ 20812ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet g_sparseFileSupport = enable; 20912ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet return g_sparseFileSupport; 21069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 21169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 2126de52c2a8c40b7381e364ad87af9e80a60d95229Yann Colletstatic unsigned LZ4IO_GetMilliSpan(clock_t nPrevious) 2136de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet{ 2146de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet clock_t nCurrent = clock(); 215c5decf7562a3b4065922ae6460b7785eb91366f8Yann Collet unsigned nSpan = (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC); 2166de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet return nSpan; 2176de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet} 21869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 21969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 220d11ac4087254db4b9391039ce862e7305874fe97Yann Collet/* ************************************************************************ ** 221d11ac4087254db4b9391039ce862e7305874fe97Yann Collet** ********************** LZ4 File / Pipe compression ********************* ** 222d11ac4087254db4b9391039ce862e7305874fe97Yann Collet** ************************************************************************ */ 22369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 22469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } 22569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; } 22669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 22769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 228488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Colletstatic int get_fileHandle(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput) 22969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 23069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 23169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (!strcmp (input_filename, stdinmark)) 23269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 23369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(4,"Using stdin for input\n"); 23469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. *pfinput = stdin; 23569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. SET_BINARY_MODE(stdin); 23669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 23769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. else 23869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 23969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. *pfinput = fopen(input_filename, "rb"); 24069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 24169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 24269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (!strcmp (output_filename, stdoutmark)) 24369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 24469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(4,"Using stdout for output\n"); 24569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. *pfoutput = stdout; 24669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. SET_BINARY_MODE(stdout); 24769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 24869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. else 24969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 2508a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Check if destination file already exists */ 25169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. *pfoutput=0; 25269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (output_filename != nulmark) *pfoutput = fopen( output_filename, "rb" ); 25369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (*pfoutput!=0) 25469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 25569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. fclose(*pfoutput); 25612ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet if (!g_overwrite) 25769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 25869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. char ch; 25969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename); 26069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2, "Overwrite ? (Y/N) : "); 26112ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet if (g_displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */ 26269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. ch = (char)getchar(); 26369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); 26469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 26569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 26669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. *pfoutput = fopen( output_filename, "wb" ); 26769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 26869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 26969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename); 27069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename); 27169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 27269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. return 0; 27369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 27469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 27569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 2766de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet 2776de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet 2786de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet/*************************************** 279d11ac4087254db4b9391039ce862e7305874fe97Yann Collet* Legacy Compression 280d11ac4087254db4b9391039ce862e7305874fe97Yann Collet***************************************/ 2816de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet 2826de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet/* unoptimized version; solves endianess & alignment issues */ 2836de52c2a8c40b7381e364ad87af9e80a60d95229Yann Colletstatic void LZ4IO_writeLE32 (void* p, unsigned value32) 2846de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet{ 285ceec6fa8492a5ff0ed163c96516716a3c2b09461Yann Collet unsigned char* dstPtr = (unsigned char*)p; 2866de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet dstPtr[0] = (unsigned char)value32; 2876de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet dstPtr[1] = (unsigned char)(value32 >> 8); 2886de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet dstPtr[2] = (unsigned char)(value32 >> 16); 2896de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet dstPtr[3] = (unsigned char)(value32 >> 24); 2906de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet} 2916de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet 2926de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet/* LZ4IO_compressFilename_Legacy : 2936de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet * This function is intentionally "hidden" (not published in .h) 2946de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet * It generates compressed streams using the old 'legacy' format */ 295488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Colletint LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel) 29669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 29769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. int (*compressionFunction)(const char*, char*, int); 29869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned long long filesize = 0; 29969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned long long compressedfilesize = MAGICNUMBER_SIZE; 30069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. char* in_buff; 30169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. char* out_buff; 30269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. FILE* finput; 30369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. FILE* foutput; 30469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. clock_t start, end; 30569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. size_t sizeCheck; 30669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 30769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3088a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Init */ 30969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. start = clock(); 3106de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC; 3116de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet 31269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. get_fileHandle(input_filename, output_filename, &finput, &foutput); 31312ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet if ((g_displayLevel==2) && (compressionlevel==1)) g_displayLevel=3; 31469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3158a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Allocate Memory */ 31669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. in_buff = (char*)malloc(LEGACY_BLOCKSIZE); 31769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); 31869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); 31969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3208a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Write Archive Header */ 3216de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER); 32269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput); 32369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); 32469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3258a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Main Loop */ 32669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. while (1) 32769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 32869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned int outSize; 3298a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Read Block */ 33069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); 33169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if( inSize<=0 ) break; 33269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. filesize += inSize; 33369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3348a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Compress Block */ 33569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. outSize = compressionFunction(in_buff, out_buff+4, inSize); 33669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. compressedfilesize += outSize+4; 3376de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet DISPLAYUPDATE(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); 33869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3398a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Write Block */ 3406de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet LZ4IO_writeLE32(out_buff, outSize); 34169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); 34269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(23, "Write error : cannot write compressed block"); 34369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 34469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3458a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Status */ 34669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. end = clock(); 34769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2, "\r%79s\r", ""); 34869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", 34969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); 35069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 35169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. double seconds = (double)(end - start)/CLOCKS_PER_SEC; 35269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); 35369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 35469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3558a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Close & Free */ 35669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. free(in_buff); 35769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. free(out_buff); 35869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. fclose(finput); 35969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. fclose(foutput); 36069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 36169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. return 0; 36269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 36369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 36469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 365d11ac4087254db4b9391039ce862e7305874fe97Yann Collet/********************************************* 366d11ac4087254db4b9391039ce862e7305874fe97Yann Collet* Compression using Frame format 367d11ac4087254db4b9391039ce862e7305874fe97Yann Collet*********************************************/ 36869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 369488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Colletint LZ4IO_compressFilename(const char* input_filename, const char* output_filename, int compressionLevel) 37069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 37169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned long long filesize = 0; 37269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned long long compressedfilesize = 0; 37369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. char* in_buff; 37469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. char* out_buff; 37569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. FILE* finput; 37669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. FILE* foutput; 37769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. clock_t start, end; 37869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. int blockSize; 3796de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet size_t sizeCheck, headerSize, readSize, outBuffSize; 3806de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet LZ4F_compressionContext_t ctx; 3816de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet LZ4F_errorCode_t errorCode; 382ceec6fa8492a5ff0ed163c96516716a3c2b09461Yann Collet LZ4F_preferences_t prefs; 38369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 38469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3858a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Init */ 38669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. start = clock(); 387ceec6fa8492a5ff0ed163c96516716a3c2b09461Yann Collet memset(&prefs, 0, sizeof(prefs)); 38812ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet if ((g_displayLevel==2) && (compressionLevel>=3)) g_displayLevel=3; 3896de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); 3906de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); 39169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. get_fileHandle(input_filename, output_filename, &finput, &foutput); 39212ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet blockSize = LZ4S_GetBlockSize_FromBlockId (g_blockSizeId); 39369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3948a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Set compression parameters */ 3956de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet prefs.autoFlush = 1; 3966de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet prefs.compressionLevel = compressionLevel; 39712ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet prefs.frameInfo.blockMode = (blockMode_t)g_blockIndependence; 39812ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet prefs.frameInfo.blockSizeID = (blockSizeID_t)g_blockSizeId; 39912ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_streamChecksum; 4006de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet 4018a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Allocate Memory */ 40269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. in_buff = (char*)malloc(blockSize); 4036de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet outBuffSize = LZ4F_compressBound(blockSize, &prefs); 4046de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet out_buff = (char*)malloc(outBuffSize); 4056de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory"); 40669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 4078a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Write Archive Header */ 4086de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet headerSize = LZ4F_compressBegin(ctx, out_buff, outBuffSize, &prefs); 4096de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); 4106de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet sizeCheck = fwrite(out_buff, 1, headerSize, foutput); 4116de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header"); 4126de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet compressedfilesize += headerSize; 41369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 4148a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* read first block */ 41569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); 4166de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet filesize += readSize; 41769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 4188a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Main Loop */ 41969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. while (readSize>0) 42069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 4216de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet size_t outSize; 42269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 4238a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Compress Block */ 4246de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet outSize = LZ4F_compressUpdate(ctx, out_buff, outBuffSize, in_buff, readSize, NULL); 4256de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize)); 4266de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet compressedfilesize += outSize; 4276de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet DISPLAYUPDATE(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); 42869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 4298a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Write Block */ 4306de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet sizeCheck = fwrite(out_buff, 1, outSize, foutput); 4316de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block"); 43269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 4338a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Read next block */ 43469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); 4356de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet filesize += readSize; 43669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 43769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 4388a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* End of Stream mark */ 4396de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet headerSize = LZ4F_compressEnd(ctx, out_buff, outBuffSize, NULL); 4406de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (LZ4F_isError(headerSize)) EXM_THROW(36, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); 4416de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet 4426de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet sizeCheck = fwrite(out_buff, 1, headerSize, foutput); 4436de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (sizeCheck!=headerSize) EXM_THROW(37, "Write error : cannot write end of stream"); 4446de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet compressedfilesize += headerSize; 44569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 4468a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Close & Free */ 44769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. free(in_buff); 44869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. free(out_buff); 44969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. fclose(finput); 45069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. fclose(foutput); 4516de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet errorCode = LZ4F_freeCompressionContext(ctx); 4526de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); 45369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 4548a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Final Status */ 45569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. end = clock(); 45669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2, "\r%79s\r", ""); 45769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", 45869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); 45969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 46069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. double seconds = (double)(end - start)/CLOCKS_PER_SEC; 46169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); 46269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 46369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 46469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. return 0; 46569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 46669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 46769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 46845b0642bf54718d2b57dd61cb606b154afc0ab26Yann Collet#define FNSPACE 30 469488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Colletint LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel) 470488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet{ 471488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet int i; 47245b0642bf54718d2b57dd61cb606b154afc0ab26Yann Collet char* outFileName = (char*)malloc(FNSPACE); 47345b0642bf54718d2b57dd61cb606b154afc0ab26Yann Collet size_t ofnSize = FNSPACE; 474488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet const size_t suffixSize = strlen(suffix); 475488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet 476488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet for (i=0; i<ifntSize; i++) 477488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet { 478488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet size_t ifnSize = strlen(inFileNamesTable[i]); 479ceec6fa8492a5ff0ed163c96516716a3c2b09461Yann Collet if (ofnSize <= ifnSize+suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); } 480488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet strcpy(outFileName, inFileNamesTable[i]); 481488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet strcat(outFileName, suffix); 482488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet LZ4IO_compressFilename(inFileNamesTable[i], outFileName, compressionlevel); 483488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet } 484488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet free(outFileName); 485488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet return 0; 486488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet} 487488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet 488488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet 489488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet 49069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* ********************************************************************* */ 491488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet/* ********************** LZ4 file-stream Decompression **************** */ 49269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* ********************************************************************* */ 49369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 4946de52c2a8c40b7381e364ad87af9e80a60d95229Yann Colletstatic unsigned LZ4IO_readLE32 (const void* s) 4956de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet{ 496ceec6fa8492a5ff0ed163c96516716a3c2b09461Yann Collet const unsigned char* srcPtr = (const unsigned char*)s; 4976de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet unsigned value32 = srcPtr[0]; 4986de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet value32 += (srcPtr[1]<<8); 4996de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet value32 += (srcPtr[2]<<16); 5006de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet value32 += (srcPtr[3]<<24); 5016de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet return value32; 5026de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet} 5036de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet 50469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) 50569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 50669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned long long filesize = 0; 50769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. char* in_buff; 50869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. char* out_buff; 50969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 5108a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Allocate Memory */ 51169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); 51269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. out_buff = (char*)malloc(LEGACY_BLOCKSIZE); 51369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory"); 51469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 5158a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Main Loop */ 51669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. while (1) 51769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 51869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. int decodeSize; 51969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. size_t sizeCheck; 520c5decf7562a3b4065922ae6460b7785eb91366f8Yann Collet unsigned int blockSize; 52169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 5228a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Block Size */ 5236de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet sizeCheck = fread(in_buff, 1, 4, finput); 5248a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet if (sizeCheck==0) break; /* Nothing to read : file read is completed */ 5258a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */ 52669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) 5278a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet { /* Cannot read next block : maybe new stream ? */ 52869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. fseek(finput, -4, SEEK_CUR); 52969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. break; 53069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 53169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 5328a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Read Block */ 53369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. sizeCheck = fread(in_buff, 1, blockSize, finput); 53445b0642bf54718d2b57dd61cb606b154afc0ab26Yann Collet if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); 53569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 5368a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Decode Block */ 53769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE); 53845b0642bf54718d2b57dd61cb606b154afc0ab26Yann Collet if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !"); 53969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. filesize += decodeSize; 54069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 5418a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Write Block */ 54269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. sizeCheck = fwrite(out_buff, 1, decodeSize, foutput); 54345b0642bf54718d2b57dd61cb606b154afc0ab26Yann Collet if (sizeCheck != (size_t)decodeSize) EXM_THROW(54, "Write error : cannot write decoded block into output\n"); 54469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 54569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 5468a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Free */ 54769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. free(in_buff); 54869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. free(out_buff); 54969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 55069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. return filesize; 55169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 55269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 55369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 55469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) 55569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 55669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned long long filesize = 0; 55712ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet void* inBuff; 55812ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet void* outBuff; 5596de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet# define HEADERMAX 20 5606de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet char headerBuff[HEADERMAX]; 5616de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet size_t sizeCheck, nextToRead, outBuffSize, inBuffSize; 5626de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet LZ4F_decompressionContext_t ctx; 5636de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet LZ4F_errorCode_t errorCode; 5646de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet LZ4F_frameInfo_t frameInfo; 56545a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet unsigned storedSkips = 0; 566d517d609d95bdbab665a6ddb6e018c450d1e5ae6Yann Collet 5678a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* init */ 5686de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); 5696de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (LZ4F_isError(errorCode)) EXM_THROW(60, "Allocation error : can't create context : %s", LZ4F_getErrorName(errorCode)); 5706de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet LZ4IO_writeLE32(headerBuff, LZ4S_MAGICNUMBER); /* regenerated here, as it was already read from finput */ 57169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 5728a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Decode stream descriptor */ 5736de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet outBuffSize = 0; inBuffSize = 0; sizeCheck = MAGICNUMBER_SIZE; 5746de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet nextToRead = LZ4F_decompress(ctx, NULL, &outBuffSize, headerBuff, &sizeCheck, NULL); 5756de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (LZ4F_isError(nextToRead)) EXM_THROW(61, "Decompression error : %s", LZ4F_getErrorName(nextToRead)); 5766de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (nextToRead > HEADERMAX) EXM_THROW(62, "Header too large (%i>%i)", (int)nextToRead, HEADERMAX); 5776de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet sizeCheck = fread(headerBuff, 1, nextToRead, finput); 5786de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (sizeCheck!=nextToRead) EXM_THROW(63, "Read error "); 5796de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet nextToRead = LZ4F_decompress(ctx, NULL, &outBuffSize, headerBuff, &sizeCheck, NULL); 5806de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet errorCode = LZ4F_getFrameInfo(ctx, &frameInfo, NULL, &inBuffSize); 5816de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (LZ4F_isError(errorCode)) EXM_THROW(64, "can't decode frame header : %s", LZ4F_getErrorName(errorCode)); 58269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 5838a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Allocate Memory */ 5846de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet outBuffSize = LZ4IO_setBlockSizeID(frameInfo.blockSizeID); 5856de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet inBuffSize = outBuffSize + 4; 58612ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet inBuff = malloc(inBuffSize); 58712ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet outBuff = malloc(outBuffSize); 5886de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (!inBuff || !outBuff) EXM_THROW(65, "Allocation error : not enough memory"); 58969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 5908a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Main Loop */ 5916de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet while (nextToRead != 0) 59269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 5936de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet size_t decodedBytes = outBuffSize; 59469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 5958a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Read Block */ 5966de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet sizeCheck = fread(inBuff, 1, nextToRead, finput); 5976de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (sizeCheck!=nextToRead) EXM_THROW(66, "Read error "); 598d517d609d95bdbab665a6ddb6e018c450d1e5ae6Yann Collet 5998a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Decode Block */ 6006de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet errorCode = LZ4F_decompress(ctx, outBuff, &decodedBytes, inBuff, &sizeCheck, NULL); 6016de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (LZ4F_isError(errorCode)) EXM_THROW(67, "Decompression error : %s", LZ4F_getErrorName(errorCode)); 6026de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (sizeCheck!=nextToRead) EXM_THROW(67, "Synchronization error"); 6036de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet nextToRead = errorCode; 6046de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet filesize += decodedBytes; 60569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 6068a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Write Block */ 60712ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet if (g_sparseFileSupport) 60812ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet { 609d11ac4087254db4b9391039ce862e7305874fe97Yann Collet char* const oBuffStart = (char*)outBuff; 610d11ac4087254db4b9391039ce862e7305874fe97Yann Collet char* oBuffPos = oBuffStart; 611d11ac4087254db4b9391039ce862e7305874fe97Yann Collet char* const oBuffEnd = oBuffStart + decodedBytes; 612d11ac4087254db4b9391039ce862e7305874fe97Yann Collet static const size_t zeroBlockSize = 32 KB; 613d11ac4087254db4b9391039ce862e7305874fe97Yann Collet while (oBuffPos < oBuffEnd) 61445a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet { 615248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet const size_t* sPtr = (const size_t*)oBuffPos; 616d11ac4087254db4b9391039ce862e7305874fe97Yann Collet size_t seg0Size = zeroBlockSize; 617d11ac4087254db4b9391039ce862e7305874fe97Yann Collet size_t nbSizeT; 618d11ac4087254db4b9391039ce862e7305874fe97Yann Collet size_t checked; 619d11ac4087254db4b9391039ce862e7305874fe97Yann Collet size_t skippedLength; 620d11ac4087254db4b9391039ce862e7305874fe97Yann Collet int seekResult; 621d11ac4087254db4b9391039ce862e7305874fe97Yann Collet if (seg0Size > decodedBytes) seg0Size = decodedBytes; 622d11ac4087254db4b9391039ce862e7305874fe97Yann Collet decodedBytes -= seg0Size; 623d11ac4087254db4b9391039ce862e7305874fe97Yann Collet nbSizeT = seg0Size / sizeof(size_t); 624d11ac4087254db4b9391039ce862e7305874fe97Yann Collet for (checked=0; (checked < nbSizeT) && (sPtr[checked] == 0); checked++) ; 625d11ac4087254db4b9391039ce862e7305874fe97Yann Collet skippedLength = checked * sizeof(size_t); 626d11ac4087254db4b9391039ce862e7305874fe97Yann Collet storedSkips += (unsigned)skippedLength; 627d11ac4087254db4b9391039ce862e7305874fe97Yann Collet if (storedSkips > 1 GB) 628d11ac4087254db4b9391039ce862e7305874fe97Yann Collet { 629d11ac4087254db4b9391039ce862e7305874fe97Yann Collet seekResult = fseek(foutput, 1 GB, SEEK_CUR); 630d11ac4087254db4b9391039ce862e7305874fe97Yann Collet if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)\n"); 631d11ac4087254db4b9391039ce862e7305874fe97Yann Collet storedSkips -= 1 GB; 632d11ac4087254db4b9391039ce862e7305874fe97Yann Collet } 633d11ac4087254db4b9391039ce862e7305874fe97Yann Collet if (skippedLength != seg0Size) 634d11ac4087254db4b9391039ce862e7305874fe97Yann Collet { 635d11ac4087254db4b9391039ce862e7305874fe97Yann Collet seekResult = fseek(foutput, storedSkips, SEEK_CUR); 636d11ac4087254db4b9391039ce862e7305874fe97Yann Collet if (seekResult != 0) EXM_THROW(68, "Skip error (sparse file)\n"); 637d11ac4087254db4b9391039ce862e7305874fe97Yann Collet storedSkips = 0; 638d11ac4087254db4b9391039ce862e7305874fe97Yann Collet seg0Size -= skippedLength; 639d11ac4087254db4b9391039ce862e7305874fe97Yann Collet oBuffPos += skippedLength; 640d11ac4087254db4b9391039ce862e7305874fe97Yann Collet sizeCheck = fwrite(oBuffPos, 1, seg0Size, foutput); 641d11ac4087254db4b9391039ce862e7305874fe97Yann Collet if (sizeCheck != seg0Size) EXM_THROW(68, "Write error : cannot write decoded block\n"); 642d11ac4087254db4b9391039ce862e7305874fe97Yann Collet } 643d11ac4087254db4b9391039ce862e7305874fe97Yann Collet oBuffPos += seg0Size; 64445a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet } 64512ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet } 64645a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet else 64745a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet { 64845a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); 64945a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n"); 65045a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet } 65145a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet } 65245a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet 65345a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet if ((g_sparseFileSupport) && (storedSkips>0)) 65445a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet { 65545a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet int seekResult; 65645a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet storedSkips --; 65745a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet seekResult = fseek(foutput, storedSkips, SEEK_CUR); 658d11ac4087254db4b9391039ce862e7305874fe97Yann Collet if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); 65945a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet memset(outBuff, 0, 1); 66045a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet sizeCheck = fwrite(outBuff, 1, 1, foutput); 661d11ac4087254db4b9391039ce862e7305874fe97Yann Collet if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); 66269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 66369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 6648a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Free */ 6656de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet free(inBuff); 6666de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet free(outBuff); 6676de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet errorCode = LZ4F_freeDecompressionContext(ctx); 6686de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); 66969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 67069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. return filesize; 67169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 67269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 67369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 67489d8b98d0dd896dccf6964fa3c6bb6ae6a2349b0Yann Collet#define ENDOFSTREAM ((unsigned long long)-1) 67569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.static unsigned long long selectDecoder( FILE* finput, FILE* foutput) 67669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 6776de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet unsigned char U32store[MAGICNUMBER_SIZE]; 6786de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet unsigned magicNumber, size; 67969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. int errorNb; 68069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. size_t nbReadBytes; 68169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 6828a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Check Archive Header */ 6836de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet nbReadBytes = fread(U32store, 1, MAGICNUMBER_SIZE, finput); 6848a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet if (nbReadBytes==0) return ENDOFSTREAM; /* EOF */ 6856de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); 6868a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet magicNumber = LZ4IO_readLE32(U32store); /* Little Endian format */ 6878a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; /* fold skippable magic numbers */ 68869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 68969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. switch(magicNumber) 69069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 69169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. case LZ4S_MAGICNUMBER: 69269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. return DEFAULT_DECOMPRESSOR(finput, foutput); 69369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. case LEGACY_MAGICNUMBER: 69469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(4, "Detected : Legacy format \n"); 69569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. return decodeLegacyStream(finput, foutput); 69669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. case LZ4S_SKIPPABLE0: 69769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(4, "Skipping detected skippable area \n"); 6986de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet nbReadBytes = fread(U32store, 1, 4, finput); 69969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); 7008a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet size = LZ4IO_readLE32(U32store); /* Little Endian format */ 70169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. errorNb = fseek(finput, size, SEEK_CUR); 70269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); 70369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. return selectDecoder(finput, foutput); 70469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. EXTENDED_FORMAT; 70569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. default: 7068a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */ 70769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2, "Stream followed by unrecognized data\n"); 70889d8b98d0dd896dccf6964fa3c6bb6ae6a2349b0Yann Collet return ENDOFSTREAM; 70969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 71069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 71169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 71269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 713488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Colletint LZ4IO_decompressFilename(const char* input_filename, const char* output_filename) 71469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 71569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned long long filesize = 0, decodedSize=0; 71669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. FILE* finput; 71769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. FILE* foutput; 71869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. clock_t start, end; 71969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 72069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 7218a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Init */ 72269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. start = clock(); 72369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. get_fileHandle(input_filename, output_filename, &finput, &foutput); 72469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 725248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet /* sparse file */ 72645b0642bf54718d2b57dd61cb606b154afc0ab26Yann Collet if (g_sparseFileSupport && foutput) 72745b0642bf54718d2b57dd61cb606b154afc0ab26Yann Collet { 72845b0642bf54718d2b57dd61cb606b154afc0ab26Yann Collet SET_SPARSE_FILE_MODE(foutput); 729248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet } 730248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet 7318a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Loop over multiple streams */ 73269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. do 73369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 73469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. decodedSize = selectDecoder(finput, foutput); 73589d8b98d0dd896dccf6964fa3c6bb6ae6a2349b0Yann Collet if (decodedSize != ENDOFSTREAM) 73689d8b98d0dd896dccf6964fa3c6bb6ae6a2349b0Yann Collet filesize += decodedSize; 73789d8b98d0dd896dccf6964fa3c6bb6ae6a2349b0Yann Collet } while (decodedSize != ENDOFSTREAM); 73869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 7398a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Final Status */ 74069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. end = clock(); 74169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2, "\r%79s\r", ""); 74269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize); 74369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 74469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. double seconds = (double)(end - start)/CLOCKS_PER_SEC; 74569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); 74669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 74769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 7488a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Close */ 74969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. fclose(finput); 75069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. fclose(foutput); 75169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 7528a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Error status = OK */ 75369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. return 0; 75469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 75569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 756