lz4io.c revision 14b40fb6d713211132524f6245d128bbb75a6f15
169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* 269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. LZ4io.c - LZ4 File/Stream Interface 3488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet Copyright (C) Yann Collet 2011-2015 445a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet 569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. GPL v2 License 669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. This program is free software; you can redistribute it and/or modify 869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. it under the terms of the GNU General Public License as published by 969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. the Free Software Foundation; either version 2 of the License, or 1069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. (at your option) any later version. 1169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 1269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. This program is distributed in the hope that it will be useful, 1369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. but WITHOUT ANY WARRANTY; without even the implied warranty of 1469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. GNU General Public License for more details. 1669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 1769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. You should have received a copy of the GNU General Public License along 1869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. with this program; if not, write to the Free Software Foundation, Inc., 1969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 2069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 2169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. You can contact the author at : 2245a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet - LZ4 source repository : https://github.com/Cyan4973/lz4 2369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 2469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.*/ 2569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* 2669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. Note : this is stand-alone program. 2769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. It is not part of LZ4 compression library, it is a user code of the LZ4 library. 2869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. - The license of LZ4 library is BSD. 2969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. - The license of xxHash library is BSD. 3069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. - The license of this source file is GPLv2. 3169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.*/ 3269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 338a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet/************************************** 348a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* Compiler Options 35d11ac4087254db4b9391039ce862e7305874fe97Yann Collet**************************************/ 3669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#ifdef _MSC_VER /* Visual Studio */ 3769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.# define _CRT_SECURE_NO_WARNINGS 388a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ 398a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 4069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#endif 4169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 428a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet#define _LARGE_FILES /* Large file support on 32-bits AIX */ 438a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet#define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */ 4469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 4569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 46d11ac4087254db4b9391039ce862e7305874fe97Yann Collet/***************************** 478a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* Includes 488a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet*****************************/ 49c64200dd85a85d9cc69a6fda682362d851be863bYann Collet#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ 507f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet#include <stdlib.h> /* malloc, free */ 517f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet#include <string.h> /* strcmp, strlen */ 527f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet#include <time.h> /* clock */ 537f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet#include <sys/types.h> /* stat64 */ 547f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet#include <sys/stat.h> /* stat64 */ 5569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#include "lz4io.h" 568b8e5efefe2dfdfbeb3ebc7591d2c345c34552f5Yann Collet#include "lz4.h" /* still required for legacy format */ 578b8e5efefe2dfdfbeb3ebc7591d2c345c34552f5Yann Collet#include "lz4hc.h" /* still required for legacy format */ 586de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet#include "lz4frame.h" 5969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 6069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 61248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet/****************************** 628a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* OS-specific Includes 63248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet******************************/ 641d3ab5d85d8c7536111d2c2e5d64d3b887b16b8amauro.persano#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) 658a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet# include <fcntl.h> /* _O_BINARY */ 66248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet# include <io.h> /* _setmode, _fileno, _get_osfhandle */ 67b2bbb2790d50f09b594f6588ebdade62966b5d5cLouis P. Santillan# if !defined(__DJGPP__) 68dfabec2d1817e08fda70c19ed9248819b6a1fb22Yann Collet# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } 6914b40fb6d713211132524f6245d128bbb75a6f15Kouhei Sutou# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ 70b2bbb2790d50f09b594f6588ebdade62966b5d5cLouis P. Santillan# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } 71b2bbb2790d50f09b594f6588ebdade62966b5d5cLouis P. Santillan# if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ 72b2bbb2790d50f09b594f6588ebdade62966b5d5cLouis P. Santillan# define fseek _fseeki64 73b2bbb2790d50f09b594f6588ebdade62966b5d5cLouis P. Santillan# endif 74b2bbb2790d50f09b594f6588ebdade62966b5d5cLouis P. Santillan# else 75b2bbb2790d50f09b594f6588ebdade62966b5d5cLouis P. Santillan# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) 76b2bbb2790d50f09b594f6588ebdade62966b5d5cLouis P. Santillan# define SET_SPARSE_FILE_MODE(file) 7741b6ed3c5bc7f0c85f1bfe3abc940d9b74581e7dTakayuki MATSUOKA# endif 7869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#else 7969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.# define SET_BINARY_MODE(file) 80248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet# define SET_SPARSE_FILE_MODE(file) 8169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#endif 8269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 833bba55c741293dd8162e5afb9eace5e03fcff023Yann Collet#if !defined(S_ISREG) 843bba55c741293dd8162e5afb9eace5e03fcff023Yann Collet# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) 853bba55c741293dd8162e5afb9eace5e03fcff023Yann Collet#endif 863bba55c741293dd8162e5afb9eace5e03fcff023Yann Collet 8769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 88248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet/***************************** 898a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* Constants 908a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet*****************************/ 916de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet#define KB *(1 <<10) 926de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet#define MB *(1 <<20) 9369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define GB *(1U<<30) 9469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 9569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define _1BIT 0x01 9669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define _2BITS 0x03 9769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define _3BITS 0x07 9869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define _4BITS 0x0F 9969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define _8BITS 0xFF 10069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 1017d87d43e619b0296bbe3f1674f10575b3f68c189Yann Collet#define MAGICNUMBER_SIZE 4 1027d87d43e619b0296bbe3f1674f10575b3f68c189Yann Collet#define LZ4IO_MAGICNUMBER 0x184D2204 1037d87d43e619b0296bbe3f1674f10575b3f68c189Yann Collet#define LZ4IO_SKIPPABLE0 0x184D2A50 1047d87d43e619b0296bbe3f1674f10575b3f68c189Yann Collet#define LZ4IO_SKIPPABLEMASK 0xFFFFFFF0 1057d87d43e619b0296bbe3f1674f10575b3f68c189Yann Collet#define LEGACY_MAGICNUMBER 0x184C2102 10669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 10769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define CACHELINE 64 10869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define LEGACY_BLOCKSIZE (8 MB) 109d517d609d95bdbab665a6ddb6e018c450d1e5ae6Yann Collet#define MIN_STREAM_BUFSIZE (192 KB) 1107d87d43e619b0296bbe3f1674f10575b3f68c189Yann Collet#define LZ4IO_BLOCKSIZEID_DEFAULT 7 1117d87d43e619b0296bbe3f1674f10575b3f68c189Yann Collet 1127d87d43e619b0296bbe3f1674f10575b3f68c189Yann Collet#define sizeT sizeof(size_t) 1137d87d43e619b0296bbe3f1674f10575b3f68c189Yann Collet#define maskT (sizeT - 1) 11469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 11569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 1168a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet/************************************** 1178a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* Macros 118d11ac4087254db4b9391039ce862e7305874fe97Yann Collet**************************************/ 11969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) 12012ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } 12112ab41571ef7fd11b8b2013aa943beae373cef8aYann Colletstatic int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ 12212ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet 12312ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ 12412ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet if ((LZ4IO_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \ 1256de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet { g_time = clock(); DISPLAY(__VA_ARGS__); \ 12612ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet if (g_displayLevel>=4) fflush(stdout); } } 1276de52c2a8c40b7381e364ad87af9e80a60d95229Yann Colletstatic const unsigned refreshRate = 150; 1286de52c2a8c40b7381e364ad87af9e80a60d95229Yann Colletstatic clock_t g_time = 0; 12969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 13069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 1318a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet/************************************** 1328a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* Local Parameters 133d11ac4087254db4b9391039ce862e7305874fe97Yann Collet**************************************/ 13412ab41571ef7fd11b8b2013aa943beae373cef8aYann Colletstatic int g_overwrite = 1; 1357d87d43e619b0296bbe3f1674f10575b3f68c189Yann Colletstatic int g_blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT; 13612ab41571ef7fd11b8b2013aa943beae373cef8aYann Colletstatic int g_blockChecksum = 0; 13712ab41571ef7fd11b8b2013aa943beae373cef8aYann Colletstatic int g_streamChecksum = 1; 13812ab41571ef7fd11b8b2013aa943beae373cef8aYann Colletstatic int g_blockIndependence = 1; 1399fd4f1f9f780f1be38ad6695d62a9ba2ea426d6dYann Colletstatic int g_sparseFileSupport = 1; 1407f436a1215f11b0fb872c34f088b8b5888d0630dYann Colletstatic int g_contentSizeFlag = 0; 14169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 14269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.static const int minBlockSizeID = 4; 14369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.static const int maxBlockSizeID = 7; 14469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 145c5decf7562a3b4065922ae6460b7785eb91366f8Yann Collet 1468a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet/************************************** 1478a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* Exceptions 1488a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet***************************************/ 14969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define DEBUG 0 15069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); 15169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define EXM_THROW(error, ...) \ 15269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ \ 15369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ 15469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(1, "Error %i : ", error); \ 15569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(1, __VA_ARGS__); \ 15669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(1, "\n"); \ 15769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. exit(error); \ 15869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 15969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 16069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 1618a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet/************************************** 1628a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet* Version modifiers 1637d87d43e619b0296bbe3f1674f10575b3f68c189Yann Collet**************************************/ 16469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define EXTENDED_ARGUMENTS 16569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define EXTENDED_HELP 16669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.#define EXTENDED_FORMAT 167a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet#define DEFAULT_DECOMPRESSOR LZ4IO_decompressLZ4F 16869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 16969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 17069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* ************************************************** */ 17169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* ****************** Parameters ******************** */ 17269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* ************************************************** */ 17369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 17469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* Default setting : overwrite = 1; return : overwrite mode (0/1) */ 17569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.int LZ4IO_setOverwrite(int yes) 17669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 17712ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet g_overwrite = (yes!=0); 17812ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet return g_overwrite; 17969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 18069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 18169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* blockSizeID : valid values : 4-5-6-7 */ 18269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.int LZ4IO_setBlockSizeID(int bsid) 18369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 18469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. static const int blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB }; 18569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return -1; 18612ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet g_blockSizeId = bsid; 18712ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet return blockSizeTable[g_blockSizeId-minBlockSizeID]; 18869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 18969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 1906de52c2a8c40b7381e364ad87af9e80a60d95229Yann Colletint LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode) 19169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 19212ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet g_blockIndependence = (blockMode == LZ4IO_blockIndependent); 19312ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet return g_blockIndependence; 19469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 19569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 19669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* Default setting : no checksum */ 19769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.int LZ4IO_setBlockChecksumMode(int xxhash) 19869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 19912ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet g_blockChecksum = (xxhash != 0); 20012ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet return g_blockChecksum; 20169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 20269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 20369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* Default setting : checksum enabled */ 20469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.int LZ4IO_setStreamChecksumMode(int xxhash) 20569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 20612ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet g_streamChecksum = (xxhash != 0); 20712ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet return g_streamChecksum; 20869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 20969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 21069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* Default setting : 0 (no notification) */ 21169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.int LZ4IO_setNotificationLevel(int level) 21269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 21312ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet g_displayLevel = level; 21412ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet return g_displayLevel; 21512ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet} 21612ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet 21712ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet/* Default setting : 0 (disabled) */ 21812ab41571ef7fd11b8b2013aa943beae373cef8aYann Colletint LZ4IO_setSparseFile(int enable) 21912ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet{ 220e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet g_sparseFileSupport = (enable!=0); 22112ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet return g_sparseFileSupport; 22269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 22369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 2247f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet/* Default setting : 0 (disabled) */ 2257f436a1215f11b0fb872c34f088b8b5888d0630dYann Colletint LZ4IO_setContentSize(int enable) 2267f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet{ 2277f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet g_contentSizeFlag = (enable!=0); 2287f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet return g_contentSizeFlag; 2297f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet} 2307f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet 2316de52c2a8c40b7381e364ad87af9e80a60d95229Yann Colletstatic unsigned LZ4IO_GetMilliSpan(clock_t nPrevious) 2326de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet{ 2336de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet clock_t nCurrent = clock(); 234c5decf7562a3b4065922ae6460b7785eb91366f8Yann Collet unsigned nSpan = (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC); 2356de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet return nSpan; 2366de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet} 23769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 2387f436a1215f11b0fb872c34f088b8b5888d0630dYann Colletstatic unsigned long long LZ4IO_GetFileSize(const char* infilename) 2397f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet{ 2407f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet int r; 2417f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet#if defined(_MSC_VER) 2427f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet struct _stat64 statbuf; 2437f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet r = _stat64(infilename, &statbuf); 2447f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet#else 2457f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet struct stat statbuf; 2467f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet r = stat(infilename, &statbuf); 2477f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet#endif 248113b150f0aeb3512b2850a65a824ad87f605187eYann Collet if (r || !S_ISREG(statbuf.st_mode)) return 0; /* failure, or is not a regular file */ 2497f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet return (unsigned long long)statbuf.st_size; 2507f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet} 2517f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet 25269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 253d11ac4087254db4b9391039ce862e7305874fe97Yann Collet/* ************************************************************************ ** 254d11ac4087254db4b9391039ce862e7305874fe97Yann Collet** ********************** LZ4 File / Pipe compression ********************* ** 255d11ac4087254db4b9391039ce862e7305874fe97Yann Collet** ************************************************************************ */ 25669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 2577d87d43e619b0296bbe3f1674f10575b3f68c189Yann Colletstatic int LZ4IO_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } 2587d87d43e619b0296bbe3f1674f10575b3f68c189Yann Colletstatic int LZ4IO_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0; } 25969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 26069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 261c64200dd85a85d9cc69a6fda682362d851be863bYann Colletstatic int LZ4IO_getFiles(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput) 26269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 26369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 26469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (!strcmp (input_filename, stdinmark)) 26569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 26669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(4,"Using stdin for input\n"); 26769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. *pfinput = stdin; 26869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. SET_BINARY_MODE(stdin); 26969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 27069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. else 27169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 27269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. *pfinput = fopen(input_filename, "rb"); 27369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 27469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 275c64200dd85a85d9cc69a6fda682362d851be863bYann Collet if ( *pfinput==0 ) 276c64200dd85a85d9cc69a6fda682362d851be863bYann Collet { 277bce2eeb9dfbca883003f21681e21df657fac90c4Yann Collet DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", input_filename); 278c64200dd85a85d9cc69a6fda682362d851be863bYann Collet return 1; 279c64200dd85a85d9cc69a6fda682362d851be863bYann Collet } 280c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 28169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (!strcmp (output_filename, stdoutmark)) 28269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 28369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(4,"Using stdout for output\n"); 28469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. *pfoutput = stdout; 28569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. SET_BINARY_MODE(stdout); 286acae59a73969f436f95ed31130efe0ee9f7b3e48Yann Collet if (g_sparseFileSupport==1) 287acae59a73969f436f95ed31130efe0ee9f7b3e48Yann Collet { 288acae59a73969f436f95ed31130efe0ee9f7b3e48Yann Collet g_sparseFileSupport = 0; 289acae59a73969f436f95ed31130efe0ee9f7b3e48Yann Collet DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); 290acae59a73969f436f95ed31130efe0ee9f7b3e48Yann Collet } 29169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 29269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. else 29369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 2948a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Check if destination file already exists */ 29569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. *pfoutput=0; 29669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (output_filename != nulmark) *pfoutput = fopen( output_filename, "rb" ); 29769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (*pfoutput!=0) 29869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 29969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. fclose(*pfoutput); 30012ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet if (!g_overwrite) 30169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 3029e92bee0443b45efa3de6b71f51863e8d66ead20Yann Collet int ch = 'Y'; 30369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename); 304c64200dd85a85d9cc69a6fda682362d851be863bYann Collet if ((g_displayLevel <= 1) || (*pfinput == stdin)) 305c64200dd85a85d9cc69a6fda682362d851be863bYann Collet EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */ 306c64200dd85a85d9cc69a6fda682362d851be863bYann Collet DISPLAYLEVEL(2, "Overwrite ? (Y/n) : "); 3079e92bee0443b45efa3de6b71f51863e8d66ead20Yann Collet while((ch = getchar()) != '\n' && ch != EOF) /* flush integrated */ 308c64200dd85a85d9cc69a6fda682362d851be863bYann Collet if ((ch!='Y') && (ch!='y')) EXM_THROW(12, "No. Operation aborted : %s already exists", output_filename); 30969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 31069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 31169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. *pfoutput = fopen( output_filename, "wb" ); 31269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 31369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 314a9ff13a8bb9932a4784b9cbb654075dfff7652feYann Collet if (*pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename); 31569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 31669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. return 0; 31769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 31869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 31969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3206de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet 3216de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet/*************************************** 322d11ac4087254db4b9391039ce862e7305874fe97Yann Collet* Legacy Compression 323d11ac4087254db4b9391039ce862e7305874fe97Yann Collet***************************************/ 3246de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet 3256de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet/* unoptimized version; solves endianess & alignment issues */ 3266de52c2a8c40b7381e364ad87af9e80a60d95229Yann Colletstatic void LZ4IO_writeLE32 (void* p, unsigned value32) 3276de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet{ 328ceec6fa8492a5ff0ed163c96516716a3c2b09461Yann Collet unsigned char* dstPtr = (unsigned char*)p; 3296de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet dstPtr[0] = (unsigned char)value32; 3306de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet dstPtr[1] = (unsigned char)(value32 >> 8); 3316de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet dstPtr[2] = (unsigned char)(value32 >> 16); 3326de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet dstPtr[3] = (unsigned char)(value32 >> 24); 3336de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet} 3346de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet 335b805d581b97be95fcc000134a54aa7c591b3ef09Yann Colletstatic int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSize, int cLevel) 336b805d581b97be95fcc000134a54aa7c591b3ef09Yann Collet{ 337b805d581b97be95fcc000134a54aa7c591b3ef09Yann Collet (void)cLevel; 3381b17bf2ab8cf66dd2b740eca376e2d46f7ad7041Yann Collet return LZ4_compress_fast(src, dst, srcSize, dstSize, 1); 339b805d581b97be95fcc000134a54aa7c591b3ef09Yann Collet} 340b805d581b97be95fcc000134a54aa7c591b3ef09Yann Collet 3416de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet/* LZ4IO_compressFilename_Legacy : 3426de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet * This function is intentionally "hidden" (not published in .h) 3436de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet * It generates compressed streams using the old 'legacy' format */ 344488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Colletint LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel) 34569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 346b805d581b97be95fcc000134a54aa7c591b3ef09Yann Collet int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); 34769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned long long filesize = 0; 34869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned long long compressedfilesize = MAGICNUMBER_SIZE; 34969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. char* in_buff; 35069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. char* out_buff; 351f11afafe234ea45a6d80f8b21df1d603ace4e025Yann Collet const int outBuffSize = LZ4_compressBound(LEGACY_BLOCKSIZE); 35269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. FILE* finput; 35369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. FILE* foutput; 35469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. clock_t start, end; 35569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. size_t sizeCheck; 35669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 35769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3588a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Init */ 35969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. start = clock(); 360e05088d0eb500d8d673e081929620e538df3d718Yann Collet if (compressionlevel < 3) compressionFunction = LZ4IO_LZ4_compress; else compressionFunction = LZ4_compress_HC; 3616de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet 362c64200dd85a85d9cc69a6fda682362d851be863bYann Collet if (LZ4IO_getFiles(input_filename, output_filename, &finput, &foutput)) 363c64200dd85a85d9cc69a6fda682362d851be863bYann Collet EXM_THROW(20, "File error"); 36469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3658a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Allocate Memory */ 36669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. in_buff = (char*)malloc(LEGACY_BLOCKSIZE); 367f11afafe234ea45a6d80f8b21df1d603ace4e025Yann Collet out_buff = (char*)malloc(outBuffSize); 36869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); 36969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3708a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Write Archive Header */ 3716de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER); 37269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput); 37369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); 37469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3758a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Main Loop */ 37669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. while (1) 37769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 37869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned int outSize; 3798a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Read Block */ 38069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); 38169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if( inSize<=0 ) break; 38269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. filesize += inSize; 38369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3848a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Compress Block */ 385b805d581b97be95fcc000134a54aa7c591b3ef09Yann Collet outSize = compressionFunction(in_buff, out_buff+4, inSize, outBuffSize, compressionlevel); 38669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. compressedfilesize += outSize+4; 387f11afafe234ea45a6d80f8b21df1d603ace4e025Yann Collet DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); 38869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3898a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Write Block */ 3906de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet LZ4IO_writeLE32(out_buff, outSize); 39169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); 39269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(23, "Write error : cannot write compressed block"); 39369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 39469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 3958a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Status */ 39669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. end = clock(); 39769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2, "\r%79s\r", ""); 39862ed15319570d80690915f0d0dba0dc2e9478631Yann Collet filesize += !filesize; /* avoid divide by zero */ 39969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", 40069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); 40169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 40269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. double seconds = (double)(end - start)/CLOCKS_PER_SEC; 40369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); 40469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 40569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 4068a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Close & Free */ 40769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. free(in_buff); 40869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. free(out_buff); 40969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. fclose(finput); 41069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. fclose(foutput); 41169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 41269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. return 0; 41369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 41469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 41569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 416d11ac4087254db4b9391039ce862e7305874fe97Yann Collet/********************************************* 417d11ac4087254db4b9391039ce862e7305874fe97Yann Collet* Compression using Frame format 418d11ac4087254db4b9391039ce862e7305874fe97Yann Collet*********************************************/ 41969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 420c64200dd85a85d9cc69a6fda682362d851be863bYann Collettypedef struct { 421348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet void* srcBuffer; 422c64200dd85a85d9cc69a6fda682362d851be863bYann Collet size_t srcBufferSize; 423348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet void* dstBuffer; 424c64200dd85a85d9cc69a6fda682362d851be863bYann Collet size_t dstBufferSize; 425c64200dd85a85d9cc69a6fda682362d851be863bYann Collet LZ4F_compressionContext_t ctx; 426348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet} cRess_t; 427c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 428348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Colletstatic cRess_t LZ4IO_createCResources(void) 429348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet{ 430348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); 431348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet cRess_t ress; 432348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet LZ4F_errorCode_t errorCode; 433c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 434348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION); 435348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); 436348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet 437348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet /* Allocate Memory */ 438348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet ress.srcBuffer = malloc(blockSize); 439348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet ress.srcBufferSize = blockSize; 440e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet ress.dstBufferSize = LZ4F_compressFrameBound(blockSize, NULL); /* cover worst case */ 441348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet ress.dstBuffer = malloc(ress.dstBufferSize); 442348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory"); 443348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet 444348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet return ress; 445348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet} 446348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet 447348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Colletstatic void LZ4IO_freeCResources(cRess_t ress) 448348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet{ 449348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet LZ4F_errorCode_t errorCode; 450348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet free(ress.srcBuffer); 451348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet free(ress.dstBuffer); 452348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet errorCode = LZ4F_freeCompressionContext(ress.ctx); 453348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); 454348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet} 455348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet 456d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet/* 457d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet * LZ4IO_compressFilename_extRess() 458d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet * result : 0 : compression completed correctly 459d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet * 1 : missing or pb opening srcFileName 460d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet */ 461e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Colletstatic int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel) 46269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 46369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned long long filesize = 0; 46469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned long long compressedfilesize = 0; 465348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet FILE* srcFile; 466348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet FILE* dstFile; 467c64200dd85a85d9cc69a6fda682362d851be863bYann Collet void* const srcBuffer = ress.srcBuffer; 468c64200dd85a85d9cc69a6fda682362d851be863bYann Collet void* const dstBuffer = ress.dstBuffer; 469c64200dd85a85d9cc69a6fda682362d851be863bYann Collet const size_t dstBufferSize = ress.dstBufferSize; 470c64200dd85a85d9cc69a6fda682362d851be863bYann Collet const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); 471c64200dd85a85d9cc69a6fda682362d851be863bYann Collet size_t sizeCheck, headerSize, readSize; 472c64200dd85a85d9cc69a6fda682362d851be863bYann Collet LZ4F_compressionContext_t ctx = ress.ctx; /* just a pointer */ 473ceec6fa8492a5ff0ed163c96516716a3c2b09461Yann Collet LZ4F_preferences_t prefs; 47469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 47569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 4768a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Init */ 477ceec6fa8492a5ff0ed163c96516716a3c2b09461Yann Collet memset(&prefs, 0, sizeof(prefs)); 47869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 479348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet /* File check */ 480d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet if (LZ4IO_getFiles(srcFileName, dstFileName, &srcFile, &dstFile)) return 1; 481348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet 4828a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Set compression parameters */ 4836de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet prefs.autoFlush = 1; 484348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet prefs.compressionLevel = compressionLevel; 485081bcca33bba4f6ac3e913b3cf2bc2ba58b6b697Takayuki MATSUOKA prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence; 486081bcca33bba4f6ac3e913b3cf2bc2ba58b6b697Takayuki MATSUOKA prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId; 487081bcca33bba4f6ac3e913b3cf2bc2ba58b6b697Takayuki MATSUOKA prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum; 4887f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet if (g_contentSizeFlag) 4897f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet { 490348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet unsigned long long fileSize = LZ4IO_GetFileSize(srcFileName); 491d5da787c1bd92a22c87b6428321668548155995fYann Collet prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */ 49290c0104c4eaf8f06932990aacf07e60dd21685b9Yann Collet if (fileSize==0) 49390c0104c4eaf8f06932990aacf07e60dd21685b9Yann Collet DISPLAYLEVEL(3, "Warning : cannot determine uncompressed frame content size \n"); 4947f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet } 4956de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet 4968a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* read first block */ 497e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile); 4986de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet filesize += readSize; 49969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 500e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet /* single-block file */ 501e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet if (readSize < blockSize) 50269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 503e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet /* Compress in single pass */ 504e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet size_t cSize = LZ4F_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs); 505e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet if (LZ4F_isError(cSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(cSize)); 506e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet compressedfilesize += cSize; 5072ed9dcc900a8a662a52d11968baf27f61a9bedd3Yann Collet DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", 5082ed9dcc900a8a662a52d11968baf27f61a9bedd3Yann Collet (unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */ 50969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 5108a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Write Block */ 511e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile); 512e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet if (sizeCheck!=cSize) EXM_THROW(35, "Write error : cannot write compressed block"); 51369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 51469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 515e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet else 516e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet 517e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet /* multiple-blocks file */ 518e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet { 519e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet /* Write Archive Header */ 520e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs); 521e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); 522e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); 523e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header"); 524e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet compressedfilesize += headerSize; 525e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet 526e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet /* Main Loop */ 527e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet while (readSize>0) 528e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet { 529e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet size_t outSize; 530e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet 531e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet /* Compress Block */ 532e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); 533e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize)); 534e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet compressedfilesize += outSize; 535e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); 536e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet 537e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet /* Write Block */ 538e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile); 539e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block"); 540e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet 541e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet /* Read next block */ 542e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile); 543e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet filesize += readSize; 544e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet } 545e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet 546e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet /* End of Stream mark */ 547e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); 548e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet if (LZ4F_isError(headerSize)) EXM_THROW(36, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); 5496de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet 550e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); 551e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet if (sizeCheck!=headerSize) EXM_THROW(37, "Write error : cannot write end of stream"); 552e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet compressedfilesize += headerSize; 553e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet } 55469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 555348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet /* Release files */ 556348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet fclose (srcFile); 557348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet fclose (dstFile); 558348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet 5598a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Final Status */ 56069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2, "\r%79s\r", ""); 561d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", 562d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet filesize, compressedfilesize, (double)compressedfilesize/(filesize + !filesize)*100); /* avoid division by zero */ 56369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 56469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. return 0; 56569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 56669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 56769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 568348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Colletint LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel) 569488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet{ 570c64200dd85a85d9cc69a6fda682362d851be863bYann Collet clock_t start, end; 571348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet cRess_t ress; 572348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet int issueWithSrcFile = 0; 573c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 574c64200dd85a85d9cc69a6fda682362d851be863bYann Collet /* Init */ 575c64200dd85a85d9cc69a6fda682362d851be863bYann Collet start = clock(); 576348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet ress = LZ4IO_createCResources(); 577c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 578348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet /* Compress File */ 579e328d41ef4124870c31cba1fe1f4eed282b8e53eYann Collet issueWithSrcFile += LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel); 580c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 581348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet /* Free resources */ 582348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet LZ4IO_freeCResources(ress); 583c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 584c64200dd85a85d9cc69a6fda682362d851be863bYann Collet /* Final Status */ 585c64200dd85a85d9cc69a6fda682362d851be863bYann Collet end = clock(); 586488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet { 587c64200dd85a85d9cc69a6fda682362d851be863bYann Collet double seconds = (double)(end - start) / CLOCKS_PER_SEC; 588c64200dd85a85d9cc69a6fda682362d851be863bYann Collet DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); 589488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet } 590c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 591348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet return issueWithSrcFile; 592488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet} 593488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet 594c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 595c64200dd85a85d9cc69a6fda682362d851be863bYann Collet#define FNSPACE 30 596c64200dd85a85d9cc69a6fda682362d851be863bYann Colletint LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel) 5970169502b49dfa5eb8878f5c85be3270012266fc3Kyle J Harper{ 5980169502b49dfa5eb8878f5c85be3270012266fc3Kyle J Harper int i; 599d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet int missed_files = 0; 600d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet char* dstFileName = (char*)malloc(FNSPACE); 6010169502b49dfa5eb8878f5c85be3270012266fc3Kyle J Harper size_t ofnSize = FNSPACE; 602cc2412401400fbacd7530b8dcc4af0b0d7c734ceYann Collet const size_t suffixSize = strlen(suffix); 603348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet cRess_t ress; 604c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 605c64200dd85a85d9cc69a6fda682362d851be863bYann Collet /* init */ 606348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet ress = LZ4IO_createCResources(); 607c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 608c64200dd85a85d9cc69a6fda682362d851be863bYann Collet /* loop on each file */ 6090169502b49dfa5eb8878f5c85be3270012266fc3Kyle J Harper for (i=0; i<ifntSize; i++) 6100169502b49dfa5eb8878f5c85be3270012266fc3Kyle J Harper { 611348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet size_t ifnSize = strlen(inFileNamesTable[i]); 612d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); } 613d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet strcpy(dstFileName, inFileNamesTable[i]); 614d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet strcat(dstFileName, suffix); 615c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 616d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet missed_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], dstFileName, compressionLevel); 6170169502b49dfa5eb8878f5c85be3270012266fc3Kyle J Harper } 618c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 619c64200dd85a85d9cc69a6fda682362d851be863bYann Collet /* Close & Free */ 620348f5099e42c23393ec9c8e6b403f4bca99bbb41Yann Collet LZ4IO_freeCResources(ress); 621d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet free(dstFileName); 622c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 623d6dc0a410d0be93551f247eeb871603d2db11c17Yann Collet return missed_files; 6240169502b49dfa5eb8878f5c85be3270012266fc3Kyle J Harper} 625488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet 626c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 62769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* ********************************************************************* */ 628488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3fYann Collet/* ********************** LZ4 file-stream Decompression **************** */ 62969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet./* ********************************************************************* */ 63069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 6316de52c2a8c40b7381e364ad87af9e80a60d95229Yann Colletstatic unsigned LZ4IO_readLE32 (const void* s) 6326de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet{ 633ceec6fa8492a5ff0ed163c96516716a3c2b09461Yann Collet const unsigned char* srcPtr = (const unsigned char*)s; 6346de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet unsigned value32 = srcPtr[0]; 6356de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet value32 += (srcPtr[1]<<8); 6366de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet value32 += (srcPtr[2]<<16); 637c64200dd85a85d9cc69a6fda682362d851be863bYann Collet value32 += ((unsigned)srcPtr[3])<<24; 6386de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet return value32; 6396de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet} 6406de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet 641113b150f0aeb3512b2850a65a824ad87f605187eYann Colletstatic unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) 642113b150f0aeb3512b2850a65a824ad87f605187eYann Collet{ 6431c3e633c48766c58df949887297dc5838170a33fYann Collet const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ 6441c3e633c48766c58df949887297dc5838170a33fYann Collet const size_t* ptrT = bufferT; 645113b150f0aeb3512b2850a65a824ad87f605187eYann Collet size_t bufferSizeT = bufferSize / sizeT; 6461c3e633c48766c58df949887297dc5838170a33fYann Collet const size_t* const bufferTEnd = bufferT + bufferSizeT; 647113b150f0aeb3512b2850a65a824ad87f605187eYann Collet static const size_t segmentSizeT = (32 KB) / sizeT; 648113b150f0aeb3512b2850a65a824ad87f605187eYann Collet 649113b150f0aeb3512b2850a65a824ad87f605187eYann Collet if (!g_sparseFileSupport) /* normal write */ 650113b150f0aeb3512b2850a65a824ad87f605187eYann Collet { 651113b150f0aeb3512b2850a65a824ad87f605187eYann Collet size_t sizeCheck = fwrite(buffer, 1, bufferSize, file); 65258b5aadb1fc5d937e81c8f33e0e8290f2097c6bbYann Collet if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block"); 653113b150f0aeb3512b2850a65a824ad87f605187eYann Collet return 0; 654113b150f0aeb3512b2850a65a824ad87f605187eYann Collet } 655113b150f0aeb3512b2850a65a824ad87f605187eYann Collet 656113b150f0aeb3512b2850a65a824ad87f605187eYann Collet /* avoid int overflow */ 657113b150f0aeb3512b2850a65a824ad87f605187eYann Collet if (storedSkips > 1 GB) 658113b150f0aeb3512b2850a65a824ad87f605187eYann Collet { 659113b150f0aeb3512b2850a65a824ad87f605187eYann Collet int seekResult = fseek(file, 1 GB, SEEK_CUR); 66058b5aadb1fc5d937e81c8f33e0e8290f2097c6bbYann Collet if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)"); 661113b150f0aeb3512b2850a65a824ad87f605187eYann Collet storedSkips -= 1 GB; 662113b150f0aeb3512b2850a65a824ad87f605187eYann Collet } 663113b150f0aeb3512b2850a65a824ad87f605187eYann Collet 664113b150f0aeb3512b2850a65a824ad87f605187eYann Collet while (ptrT < bufferTEnd) 665113b150f0aeb3512b2850a65a824ad87f605187eYann Collet { 666113b150f0aeb3512b2850a65a824ad87f605187eYann Collet size_t seg0SizeT = segmentSizeT; 667113b150f0aeb3512b2850a65a824ad87f605187eYann Collet size_t nb0T; 668113b150f0aeb3512b2850a65a824ad87f605187eYann Collet int seekResult; 669113b150f0aeb3512b2850a65a824ad87f605187eYann Collet 670113b150f0aeb3512b2850a65a824ad87f605187eYann Collet /* count leading zeros */ 671113b150f0aeb3512b2850a65a824ad87f605187eYann Collet if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; 672113b150f0aeb3512b2850a65a824ad87f605187eYann Collet bufferSizeT -= seg0SizeT; 673113b150f0aeb3512b2850a65a824ad87f605187eYann Collet for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; 674113b150f0aeb3512b2850a65a824ad87f605187eYann Collet storedSkips += (unsigned)(nb0T * sizeT); 675113b150f0aeb3512b2850a65a824ad87f605187eYann Collet 676113b150f0aeb3512b2850a65a824ad87f605187eYann Collet if (nb0T != seg0SizeT) /* not all 0s */ 677113b150f0aeb3512b2850a65a824ad87f605187eYann Collet { 678113b150f0aeb3512b2850a65a824ad87f605187eYann Collet size_t sizeCheck; 679113b150f0aeb3512b2850a65a824ad87f605187eYann Collet seekResult = fseek(file, storedSkips, SEEK_CUR); 680acae59a73969f436f95ed31130efe0ee9f7b3e48Yann Collet if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); 681113b150f0aeb3512b2850a65a824ad87f605187eYann Collet storedSkips = 0; 682113b150f0aeb3512b2850a65a824ad87f605187eYann Collet seg0SizeT -= nb0T; 683113b150f0aeb3512b2850a65a824ad87f605187eYann Collet ptrT += nb0T; 684113b150f0aeb3512b2850a65a824ad87f605187eYann Collet sizeCheck = fwrite(ptrT, sizeT, seg0SizeT, file); 68558b5aadb1fc5d937e81c8f33e0e8290f2097c6bbYann Collet if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block"); 686113b150f0aeb3512b2850a65a824ad87f605187eYann Collet } 687113b150f0aeb3512b2850a65a824ad87f605187eYann Collet ptrT += seg0SizeT; 688113b150f0aeb3512b2850a65a824ad87f605187eYann Collet } 689113b150f0aeb3512b2850a65a824ad87f605187eYann Collet 690113b150f0aeb3512b2850a65a824ad87f605187eYann Collet if (bufferSize & maskT) /* size not multiple of sizeT : implies end of block */ 691113b150f0aeb3512b2850a65a824ad87f605187eYann Collet { 6921c3e633c48766c58df949887297dc5838170a33fYann Collet const char* const restStart = (const char*)bufferTEnd; 693113b150f0aeb3512b2850a65a824ad87f605187eYann Collet const char* restPtr = restStart; 694113b150f0aeb3512b2850a65a824ad87f605187eYann Collet size_t restSize = bufferSize & maskT; 695113b150f0aeb3512b2850a65a824ad87f605187eYann Collet const char* const restEnd = restStart + restSize; 696113b150f0aeb3512b2850a65a824ad87f605187eYann Collet for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; 697113b150f0aeb3512b2850a65a824ad87f605187eYann Collet storedSkips += (unsigned) (restPtr - restStart); 698113b150f0aeb3512b2850a65a824ad87f605187eYann Collet if (restPtr != restEnd) 699113b150f0aeb3512b2850a65a824ad87f605187eYann Collet { 700113b150f0aeb3512b2850a65a824ad87f605187eYann Collet size_t sizeCheck; 701113b150f0aeb3512b2850a65a824ad87f605187eYann Collet int seekResult = fseek(file, storedSkips, SEEK_CUR); 702acae59a73969f436f95ed31130efe0ee9f7b3e48Yann Collet if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse"); 703113b150f0aeb3512b2850a65a824ad87f605187eYann Collet storedSkips = 0; 704113b150f0aeb3512b2850a65a824ad87f605187eYann Collet sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); 70558b5aadb1fc5d937e81c8f33e0e8290f2097c6bbYann Collet if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block"); 706113b150f0aeb3512b2850a65a824ad87f605187eYann Collet } 707113b150f0aeb3512b2850a65a824ad87f605187eYann Collet } 708113b150f0aeb3512b2850a65a824ad87f605187eYann Collet 709113b150f0aeb3512b2850a65a824ad87f605187eYann Collet return storedSkips; 710113b150f0aeb3512b2850a65a824ad87f605187eYann Collet} 711113b150f0aeb3512b2850a65a824ad87f605187eYann Collet 712113b150f0aeb3512b2850a65a824ad87f605187eYann Colletstatic void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips) 713113b150f0aeb3512b2850a65a824ad87f605187eYann Collet{ 714113b150f0aeb3512b2850a65a824ad87f605187eYann Collet char lastZeroByte[1] = { 0 }; 715113b150f0aeb3512b2850a65a824ad87f605187eYann Collet 716113b150f0aeb3512b2850a65a824ad87f605187eYann Collet if (storedSkips>0) /* implies g_sparseFileSupport */ 717113b150f0aeb3512b2850a65a824ad87f605187eYann Collet { 718113b150f0aeb3512b2850a65a824ad87f605187eYann Collet int seekResult; 719113b150f0aeb3512b2850a65a824ad87f605187eYann Collet size_t sizeCheck; 720113b150f0aeb3512b2850a65a824ad87f605187eYann Collet storedSkips --; 721113b150f0aeb3512b2850a65a824ad87f605187eYann Collet seekResult = fseek(file, storedSkips, SEEK_CUR); 722113b150f0aeb3512b2850a65a824ad87f605187eYann Collet if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); 723113b150f0aeb3512b2850a65a824ad87f605187eYann Collet sizeCheck = fwrite(lastZeroByte, 1, 1, file); 724113b150f0aeb3512b2850a65a824ad87f605187eYann Collet if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); 725113b150f0aeb3512b2850a65a824ad87f605187eYann Collet } 726113b150f0aeb3512b2850a65a824ad87f605187eYann Collet} 727113b150f0aeb3512b2850a65a824ad87f605187eYann Collet 728113b150f0aeb3512b2850a65a824ad87f605187eYann Collet 72962ed15319570d80690915f0d0dba0dc2e9478631Yann Colletstatic unsigned g_magicRead = 0; 730113b150f0aeb3512b2850a65a824ad87f605187eYann Colletstatic unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput) 73169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 73269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned long long filesize = 0; 73369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. char* in_buff; 73469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. char* out_buff; 735113b150f0aeb3512b2850a65a824ad87f605187eYann Collet unsigned storedSkips = 0; 73669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 7378a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Allocate Memory */ 73869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); 73969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. out_buff = (char*)malloc(LEGACY_BLOCKSIZE); 74069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory"); 74169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 7428a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Main Loop */ 74369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. while (1) 74469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 74569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. int decodeSize; 74669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. size_t sizeCheck; 747c5decf7562a3b4065922ae6460b7785eb91366f8Yann Collet unsigned int blockSize; 74869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 7498a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Block Size */ 7506de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet sizeCheck = fread(in_buff, 1, 4, finput); 7518a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet if (sizeCheck==0) break; /* Nothing to read : file read is completed */ 7528a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */ 75369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) 7548a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet { /* Cannot read next block : maybe new stream ? */ 75562ed15319570d80690915f0d0dba0dc2e9478631Yann Collet g_magicRead = blockSize; 75669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. break; 75769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 75869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 7598a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Read Block */ 76069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. sizeCheck = fread(in_buff, 1, blockSize, finput); 76145b0642bf54718d2b57dd61cb606b154afc0ab26Yann Collet if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); 76269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 7638a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Decode Block */ 76469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE); 76545b0642bf54718d2b57dd61cb606b154afc0ab26Yann Collet if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !"); 76669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. filesize += decodeSize; 76769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 7688a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Write Block */ 769113b150f0aeb3512b2850a65a824ad87f605187eYann Collet storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips); 77069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 77169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 772113b150f0aeb3512b2850a65a824ad87f605187eYann Collet LZ4IO_fwriteSparseEnd(foutput, storedSkips); 773113b150f0aeb3512b2850a65a824ad87f605187eYann Collet 7748a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Free */ 77569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. free(in_buff); 77669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. free(out_buff); 77769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 77869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. return filesize; 77969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 78069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 78169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 782113b150f0aeb3512b2850a65a824ad87f605187eYann Collet 783113b150f0aeb3512b2850a65a824ad87f605187eYann Collettypedef struct { 784113b150f0aeb3512b2850a65a824ad87f605187eYann Collet void* srcBuffer; 785113b150f0aeb3512b2850a65a824ad87f605187eYann Collet size_t srcBufferSize; 786113b150f0aeb3512b2850a65a824ad87f605187eYann Collet void* dstBuffer; 787113b150f0aeb3512b2850a65a824ad87f605187eYann Collet size_t dstBufferSize; 788113b150f0aeb3512b2850a65a824ad87f605187eYann Collet LZ4F_decompressionContext_t dCtx; 789a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet} dRess_t; 790113b150f0aeb3512b2850a65a824ad87f605187eYann Collet 791a430b859cf552aa43215fe082e15a95c8bda5d6fYann Colletstatic const size_t LZ4IO_dBufferSize = 64 KB; 792a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 793a430b859cf552aa43215fe082e15a95c8bda5d6fYann Colletstatic dRess_t LZ4IO_createDResources(void) 794a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet{ 795a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet dRess_t ress; 796a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet LZ4F_errorCode_t errorCode; 797a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 798a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet /* init */ 799a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet errorCode = LZ4F_createDecompressionContext(&ress.dCtx, LZ4F_VERSION); 800a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); 801a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 802a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet /* Allocate Memory */ 803a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet ress.srcBufferSize = LZ4IO_dBufferSize; 804a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet ress.srcBuffer = malloc(ress.srcBufferSize); 805a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet ress.dstBufferSize = LZ4IO_dBufferSize; 806a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet ress.dstBuffer = malloc(ress.dstBufferSize); 807a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); 808a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 809a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet return ress; 810a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet} 811a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 812a430b859cf552aa43215fe082e15a95c8bda5d6fYann Colletstatic void LZ4IO_freeDResources(dRess_t ress) 813a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet{ 814a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(ress.dCtx); 815a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); 816a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet free(ress.srcBuffer); 817a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet free(ress.dstBuffer); 818a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet} 819a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 820a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 821a430b859cf552aa43215fe082e15a95c8bda5d6fYann Colletstatic unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE* dstFile) 82269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 82369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned long long filesize = 0; 824197982ec6cf449734f78d849ed4a845e075b2cf4Yann Collet LZ4F_errorCode_t nextToLoad; 82545a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet unsigned storedSkips = 0; 826d517d609d95bdbab665a6ddb6e018c450d1e5ae6Yann Collet 827113b150f0aeb3512b2850a65a824ad87f605187eYann Collet /* Init feed with magic number (already consumed from FILE* sFile) */ 82869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 829113b150f0aeb3512b2850a65a824ad87f605187eYann Collet size_t inSize = MAGICNUMBER_SIZE; 830113b150f0aeb3512b2850a65a824ad87f605187eYann Collet size_t outSize= 0; 831113b150f0aeb3512b2850a65a824ad87f605187eYann Collet LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER); 832197982ec6cf449734f78d849ed4a845e075b2cf4Yann Collet nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, NULL); 833197982ec6cf449734f78d849ed4a845e075b2cf4Yann Collet if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad)); 8347f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet } 83569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 8367f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet /* Main Loop */ 837cbcdd88ccb97632015cf3732b46f8800e62e337bYann Collet for (;nextToLoad;) 8387f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet { 8397f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet size_t readSize; 8407f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet size_t pos = 0; 841197982ec6cf449734f78d849ed4a845e075b2cf4Yann Collet size_t decodedBytes = ress.dstBufferSize; 84269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 8437f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet /* Read input */ 844cbcdd88ccb97632015cf3732b46f8800e62e337bYann Collet if (nextToLoad > ress.srcBufferSize) nextToLoad = ress.srcBufferSize; 845cbcdd88ccb97632015cf3732b46f8800e62e337bYann Collet readSize = fread(ress.srcBuffer, 1, nextToLoad, srcFile); 846197982ec6cf449734f78d849ed4a845e075b2cf4Yann Collet if (!readSize) 847197982ec6cf449734f78d849ed4a845e075b2cf4Yann Collet break; /* empty file or stream */ 8487f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet 849cbcdd88ccb97632015cf3732b46f8800e62e337bYann Collet while ((pos < readSize) || (decodedBytes == ress.dstBufferSize)) /* still to read, or still to flush */ 85012ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet { 8517f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet /* Decode Input (at least partially) */ 8527f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet size_t remaining = readSize - pos; 853197982ec6cf449734f78d849ed4a845e075b2cf4Yann Collet decodedBytes = ress.dstBufferSize; 854197982ec6cf449734f78d849ed4a845e075b2cf4Yann Collet nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL); 855197982ec6cf449734f78d849ed4a845e075b2cf4Yann Collet if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); 8567f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet pos += remaining; 8577f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet 8587f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet if (decodedBytes) 85945a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet { 8607f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet /* Write Block */ 8617f436a1215f11b0fb872c34f088b8b5888d0630dYann Collet filesize += decodedBytes; 86290c0104c4eaf8f06932990aacf07e60dd21685b9Yann Collet DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); 863a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips); 86445a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet } 86526065c3f47381c6ebe56ea114015b3c2b1c2a615Yann Collet 86626065c3f47381c6ebe56ea114015b3c2b1c2a615Yann Collet if (!nextToLoad) break; 86712ab41571ef7fd11b8b2013aa943beae373cef8aYann Collet } 86845a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet } 86945a357fd1704e9c6d2d8037277bda62e8c86308eYann Collet 870a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet LZ4IO_fwriteSparseEnd(dstFile, storedSkips); 87169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 872197982ec6cf449734f78d849ed4a845e075b2cf4Yann Collet if (nextToLoad!=0) 873197982ec6cf449734f78d849ed4a845e075b2cf4Yann Collet EXM_THROW(67, "Unfinished stream"); 874197982ec6cf449734f78d849ed4a845e075b2cf4Yann Collet 87569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. return filesize; 87669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 87769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 87869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 8791098d897a0c410306f0ac547c4b92e02922f8ddbYann Collet#define PTSIZE (64 KB) 8801098d897a0c410306f0ac547c4b92e02922f8ddbYann Collet#define PTSIZET (PTSIZE / sizeof(size_t)) 881a430b859cf552aa43215fe082e15a95c8bda5d6fYann Colletstatic unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE]) 882e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet{ 8831098d897a0c410306f0ac547c4b92e02922f8ddbYann Collet size_t buffer[PTSIZET]; 884e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet size_t read = 1, sizeCheck; 885e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet unsigned long long total = MAGICNUMBER_SIZE; 886113b150f0aeb3512b2850a65a824ad87f605187eYann Collet unsigned storedSkips = 0; 887e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet 888a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput); 889a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error"); 890e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet 891e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet while (read) 892e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet { 8931098d897a0c410306f0ac547c4b92e02922f8ddbYann Collet read = fread(buffer, 1, PTSIZE, finput); 894e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet total += read; 895113b150f0aeb3512b2850a65a824ad87f605187eYann Collet storedSkips = LZ4IO_fwriteSparse(foutput, buffer, read, storedSkips); 896e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet } 897e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet 898113b150f0aeb3512b2850a65a824ad87f605187eYann Collet LZ4IO_fwriteSparseEnd(foutput, storedSkips); 899e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet return total; 900e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet} 901e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet 902e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet 90389d8b98d0dd896dccf6964fa3c6bb6ae6a2349b0Yann Collet#define ENDOFSTREAM ((unsigned long long)-1) 904a430b859cf552aa43215fe082e15a95c8bda5d6fYann Colletstatic unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput) 90569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 906a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet unsigned char MNstore[MAGICNUMBER_SIZE]; 9076de52c2a8c40b7381e364ad87af9e80a60d95229Yann Collet unsigned magicNumber, size; 90869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. int errorNb; 90969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. size_t nbReadBytes; 910e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet static unsigned nbCalls = 0; 911e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet 912e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet /* init */ 913e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet nbCalls++; 91469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 9158a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Check Archive Header */ 91662ed15319570d80690915f0d0dba0dc2e9478631Yann Collet if (g_magicRead) 91762ed15319570d80690915f0d0dba0dc2e9478631Yann Collet { 91862ed15319570d80690915f0d0dba0dc2e9478631Yann Collet magicNumber = g_magicRead; 91962ed15319570d80690915f0d0dba0dc2e9478631Yann Collet g_magicRead = 0; 92062ed15319570d80690915f0d0dba0dc2e9478631Yann Collet } 92162ed15319570d80690915f0d0dba0dc2e9478631Yann Collet else 92262ed15319570d80690915f0d0dba0dc2e9478631Yann Collet { 923a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput); 92462ed15319570d80690915f0d0dba0dc2e9478631Yann Collet if (nbReadBytes==0) return ENDOFSTREAM; /* EOF */ 92562ed15319570d80690915f0d0dba0dc2e9478631Yann Collet if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); 926a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */ 92762ed15319570d80690915f0d0dba0dc2e9478631Yann Collet } 9287d87d43e619b0296bbe3f1674f10575b3f68c189Yann Collet if (LZ4IO_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */ 92969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 93069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. switch(magicNumber) 93169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 9327d87d43e619b0296bbe3f1674f10575b3f68c189Yann Collet case LZ4IO_MAGICNUMBER: 933a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet return LZ4IO_decompressLZ4F(ress, finput, foutput); 93469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. case LEGACY_MAGICNUMBER: 93569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(4, "Detected : Legacy format \n"); 936113b150f0aeb3512b2850a65a824ad87f605187eYann Collet return LZ4IO_decodeLegacyStream(finput, foutput); 9377d87d43e619b0296bbe3f1674f10575b3f68c189Yann Collet case LZ4IO_SKIPPABLE0: 93869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(4, "Skipping detected skippable area \n"); 939a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet nbReadBytes = fread(MNstore, 1, 4, finput); 94069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); 941a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet size = LZ4IO_readLE32(MNstore); /* Little Endian format */ 94269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. errorNb = fseek(finput, size, SEEK_CUR); 94369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); 944a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet return selectDecoder(ress, finput, foutput); 94569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. EXTENDED_FORMAT; 94669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. default: 947e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet if (nbCalls == 1) /* just started */ 948e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet { 949e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet if (g_overwrite) 950a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet return LZ4IO_passThrough(finput, foutput, MNstore); 951e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */ 952e7fb4d151d104764e5e8f988c679fdff8807683eYann Collet } 95369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2, "Stream followed by unrecognized data\n"); 95489d8b98d0dd896dccf6964fa3c6bb6ae6a2349b0Yann Collet return ENDOFSTREAM; 95569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. } 95669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 95769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 95869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 959a430b859cf552aa43215fe082e15a95c8bda5d6fYann Colletstatic int LZ4IO_decompressFile_extRess(dRess_t ress, const char* input_filename, const char* output_filename) 96069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.{ 96169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. unsigned long long filesize = 0, decodedSize=0; 96269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. FILE* finput; 96369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. FILE* foutput; 96469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 96569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 9668a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Init */ 967c64200dd85a85d9cc69a6fda682362d851be863bYann Collet if (LZ4IO_getFiles(input_filename, output_filename, &finput, &foutput)) 968a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet return 1; 96969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 970248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet /* sparse file */ 97162ed15319570d80690915f0d0dba0dc2e9478631Yann Collet if (g_sparseFileSupport) { SET_SPARSE_FILE_MODE(foutput); } 972248b761f55b92a34eacbfe2ac5d6f1b94e69d7bfYann Collet 9738a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Loop over multiple streams */ 97469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. do 97569dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. { 976a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet decodedSize = selectDecoder(ress, finput, foutput); 97789d8b98d0dd896dccf6964fa3c6bb6ae6a2349b0Yann Collet if (decodedSize != ENDOFSTREAM) 97889d8b98d0dd896dccf6964fa3c6bb6ae6a2349b0Yann Collet filesize += decodedSize; 97989d8b98d0dd896dccf6964fa3c6bb6ae6a2349b0Yann Collet } while (decodedSize != ENDOFSTREAM); 98069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 9818a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Final Status */ 98269dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2, "\r%79s\r", ""); 98369dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize); 98469dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 9858a9fb8cf3229c9a704c982667c63ac440b8487baYann Collet /* Close */ 98669dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. fclose(finput); 98769dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. fclose(foutput); 98869dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 98969dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. return 0; 99069dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet.} 99169dc85b8abe78246bea91a5ba1205e4c07b96a97yann.collet. 992c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 993a430b859cf552aa43215fe082e15a95c8bda5d6fYann Colletint LZ4IO_decompressFilename(const char* input_filename, const char* output_filename) 994a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet{ 995a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet dRess_t ress; 996a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet clock_t start, end; 997a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet int missingFiles = 0; 998a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 999a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet start = clock(); 1000a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 1001a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet ress = LZ4IO_createDResources(); 1002a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet missingFiles += LZ4IO_decompressFile_extRess(ress, input_filename, output_filename); 1003a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet LZ4IO_freeDResources(ress); 1004a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 1005a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet end = clock(); 1006a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet if (end==start) end=start+1; 1007a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet { 1008a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet double seconds = (double)(end - start)/CLOCKS_PER_SEC; 1009a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds); 1010a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet } 1011a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 1012a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet return missingFiles; 1013a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet} 1014a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 1015a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 10168901dcdc9db3ea5d47900be903dfdb5070dc6b21Yann Collet#define MAXSUFFIXSIZE 8 1017c64200dd85a85d9cc69a6fda682362d851be863bYann Colletint LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix) 1018c64200dd85a85d9cc69a6fda682362d851be863bYann Collet{ 1019c64200dd85a85d9cc69a6fda682362d851be863bYann Collet int i; 1020a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet int skippedFiles = 0; 1021a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet int missingFiles = 0; 1022c64200dd85a85d9cc69a6fda682362d851be863bYann Collet char* outFileName = (char*)malloc(FNSPACE); 1023c64200dd85a85d9cc69a6fda682362d851be863bYann Collet size_t ofnSize = FNSPACE; 1024c64200dd85a85d9cc69a6fda682362d851be863bYann Collet const size_t suffixSize = strlen(suffix); 10258901dcdc9db3ea5d47900be903dfdb5070dc6b21Yann Collet const char* suffixPtr; 1026a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet dRess_t ress; 1027a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 10288901dcdc9db3ea5d47900be903dfdb5070dc6b21Yann Collet if (outFileName==NULL) exit(1); /* not enough memory */ 1029a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet ress = LZ4IO_createDResources(); 1030c64200dd85a85d9cc69a6fda682362d851be863bYann Collet 1031c64200dd85a85d9cc69a6fda682362d851be863bYann Collet for (i=0; i<ifntSize; i++) 1032c64200dd85a85d9cc69a6fda682362d851be863bYann Collet { 1033a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet size_t ifnSize = strlen(inFileNamesTable[i]); 10348901dcdc9db3ea5d47900be903dfdb5070dc6b21Yann Collet suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize; 10358901dcdc9db3ea5d47900be903dfdb5070dc6b21Yann Collet if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) exit(1); } 10368901dcdc9db3ea5d47900be903dfdb5070dc6b21Yann Collet if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) 1037c64200dd85a85d9cc69a6fda682362d851be863bYann Collet { 1038bce2eeb9dfbca883003f21681e21df657fac90c4Yann Collet DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]); 1039a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet skippedFiles++; 1040c64200dd85a85d9cc69a6fda682362d851be863bYann Collet continue; 1041c64200dd85a85d9cc69a6fda682362d851be863bYann Collet } 1042c64200dd85a85d9cc69a6fda682362d851be863bYann Collet memcpy(outFileName, inFileNamesTable[i], ifnSize - suffixSize); 1043c64200dd85a85d9cc69a6fda682362d851be863bYann Collet outFileName[ifnSize-suffixSize] = '\0'; 1044a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 1045a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet missingFiles += LZ4IO_decompressFile_extRess(ress, inFileNamesTable[i], outFileName); 1046c64200dd85a85d9cc69a6fda682362d851be863bYann Collet } 1047a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet 1048a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet LZ4IO_freeDResources(ress); 1049c64200dd85a85d9cc69a6fda682362d851be863bYann Collet free(outFileName); 1050a430b859cf552aa43215fe082e15a95c8bda5d6fYann Collet return missingFiles + skippedFiles; 1051c64200dd85a85d9cc69a6fda682362d851be863bYann Collet} 1052