frametest.c revision 9546ba62d01a9618aab91eafe77929120653d275
1/* 2 frameTest - test tool for lz4frame 3 Copyright (C) Yann Collet 2014-2016 4 5 GPL v2 License 6 7 This program is free software; you can redistribute it and/or modify 8 it under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 2 of the License, or 10 (at your option) any later version. 11 12 This program is distributed in the hope that it will be useful, 13 but WITHOUT ANY WARRANTY; without even the implied warranty of 14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 GNU General Public License for more details. 16 17 You should have received a copy of the GNU General Public License along 18 with this program; if not, write to the Free Software Foundation, Inc., 19 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 20 21 You can contact the author at : 22 - LZ4 homepage : http://www.lz4.org 23 - LZ4 source repository : https://github.com/lz4/lz4 24*/ 25 26/*-************************************ 27* Compiler specific 28**************************************/ 29#ifdef _MSC_VER /* Visual Studio */ 30# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ 31#endif 32 33 34/*-************************************ 35* Includes 36**************************************/ 37#include "platform.h" /* Compiler options */ 38#include "util.h" /* U32 */ 39#include <stdlib.h> /* malloc, free */ 40#include <stdio.h> /* fprintf */ 41#include <string.h> /* strcmp */ 42#include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */ 43#include "lz4frame_static.h" 44#include "lz4.h" /* LZ4_VERSION_STRING */ 45#define XXH_STATIC_LINKING_ONLY 46#include "xxhash.h" /* XXH64 */ 47 48 49/* unoptimized version; solves endianess & alignment issues */ 50static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32) 51{ 52 BYTE* dstPtr = (BYTE*)dstVoidPtr; 53 dstPtr[0] = (BYTE)value32; 54 dstPtr[1] = (BYTE)(value32 >> 8); 55 dstPtr[2] = (BYTE)(value32 >> 16); 56 dstPtr[3] = (BYTE)(value32 >> 24); 57} 58 59 60/*-************************************ 61* Constants 62**************************************/ 63#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U 64 65#define KB *(1U<<10) 66#define MB *(1U<<20) 67#define GB *(1U<<30) 68 69static const U32 nbTestsDefault = 256 KB; 70#define COMPRESSIBLE_NOISE_LENGTH (2 MB) 71#define FUZ_COMPRESSIBILITY_DEFAULT 50 72static const U32 prime1 = 2654435761U; 73static const U32 prime2 = 2246822519U; 74 75 76 77/*-************************************ 78* Macros 79**************************************/ 80#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) 81#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } 82#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \ 83 if ((FUZ_GetClockSpan(g_clockTime) > refreshRate) || (displayLevel>=4)) \ 84 { g_clockTime = clock(); DISPLAY(__VA_ARGS__); \ 85 if (displayLevel>=4) fflush(stdout); } } 86static const clock_t refreshRate = CLOCKS_PER_SEC / 6; 87static clock_t g_clockTime = 0; 88 89 90/*-*************************************** 91* Local Parameters 92*****************************************/ 93static U32 no_prompt = 0; 94static U32 displayLevel = 2; 95static U32 pause = 0; 96 97 98/*-******************************************************* 99* Fuzzer functions 100*********************************************************/ 101#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) 102#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) 103 104static clock_t FUZ_GetClockSpan(clock_t clockStart) 105{ 106 return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */ 107} 108 109 110#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) 111unsigned int FUZ_rand(unsigned int* src) 112{ 113 U32 rand32 = *src; 114 rand32 *= prime1; 115 rand32 += prime2; 116 rand32 = FUZ_rotl32(rand32, 13); 117 *src = rand32; 118 return rand32 >> 5; 119} 120 121 122#define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF) 123#define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) 124static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed) 125{ 126 BYTE* BBuffer = (BYTE*)buffer; 127 size_t pos = 0; 128 U32 P32 = (U32)(32768 * proba); 129 130 /* First Byte */ 131 BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); 132 133 while (pos < bufferSize) { 134 /* Select : Literal (noise) or copy (within 64K) */ 135 if (FUZ_RAND15BITS < P32) { 136 /* Copy (within 64K) */ 137 size_t const lengthRand = FUZ_RANDLENGTH + 4; 138 size_t const length = MIN(lengthRand, bufferSize - pos); 139 size_t const end = pos + length; 140 size_t const offsetRand = FUZ_RAND15BITS + 1; 141 size_t const offset = MIN(offsetRand, pos); 142 size_t match = pos - offset; 143 while (pos < end) BBuffer[pos++] = BBuffer[match++]; 144 } else { 145 /* Literal (noise) */ 146 size_t const lengthRand = FUZ_RANDLENGTH + 4; 147 size_t const length = MIN(lengthRand, bufferSize - pos); 148 size_t const end = pos + length; 149 while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); 150 } 151 } 152} 153 154 155static unsigned FUZ_highbit(U32 v32) 156{ 157 unsigned nbBits = 0; 158 if (v32==0) return 0; 159 while (v32) v32 >>= 1, nbBits ++; 160 return nbBits; 161} 162 163 164/*-******************************************************* 165* Tests 166*********************************************************/ 167int basicTests(U32 seed, double compressibility) 168{ 169 int testResult = 0; 170 void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); 171 size_t const cBuffSize = LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL); 172 void* const compressedBuffer = malloc(cBuffSize); 173 void* const decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); 174 U32 randState = seed; 175 size_t cSize, testSize; 176 LZ4F_decompressionContext_t dCtx = NULL; 177 LZ4F_compressionContext_t cctx = NULL; 178 U64 crcOrig; 179 180 LZ4F_preferences_t prefs; 181 memset(&prefs, 0, sizeof(prefs)); 182 if (!CNBuffer || !compressedBuffer || !decodedBuffer) { 183 DISPLAY("allocation error, not enough memory to start fuzzer tests \n"); 184 goto _output_error; 185 } 186 FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); 187 crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); 188 189 /* Special case : null-content frame */ 190 testSize = 0; 191 DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : \n"); 192 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); 193 if (LZ4F_isError(cSize)) goto _output_error; 194 DISPLAYLEVEL(3, "Compressed null content into a %i bytes frame \n", (int)cSize); 195 196 DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n"); 197 { LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 198 if (LZ4F_isError(errorCode)) goto _output_error; } 199 200 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n"); 201 { size_t avail_in = cSize; 202 LZ4F_frameInfo_t frame_info; 203 LZ4F_errorCode_t const errorCode = LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in); 204 if (LZ4F_isError(errorCode)) goto _output_error; 205 } 206 207 DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n"); 208 { LZ4F_errorCode_t const errorCode = LZ4F_freeDecompressionContext(dCtx); 209 if (LZ4F_isError(errorCode)) goto _output_error; } 210 dCtx = NULL; 211 212 /* test one-pass frame compression */ 213 testSize = COMPRESSIBLE_NOISE_LENGTH; 214 DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : \n"); 215 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); 216 if (LZ4F_isError(cSize)) goto _output_error; 217 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); 218 219 DISPLAYLEVEL(3, "Decompression test : \n"); 220 { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; 221 size_t compressedBufferSize = cSize; 222 BYTE* ip = (BYTE*)compressedBuffer; 223 BYTE* const iend = (BYTE*)compressedBuffer + cSize; 224 225 LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 226 if (LZ4F_isError(errorCode)) goto _output_error; 227 228 DISPLAYLEVEL(3, "Single Pass decompression : \n"); 229 { size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL); 230 if (LZ4F_isError(decompressError)) goto _output_error; } 231 { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); 232 if (crcDest != crcOrig) goto _output_error; } 233 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize); 234 235 DISPLAYLEVEL(3, "Reusing decompression context \n"); 236 { size_t const missingBytes = 4; 237 size_t iSize = compressedBufferSize - missingBytes; 238 const BYTE* cBuff = (const BYTE*) compressedBuffer; 239 BYTE* const ostart = (BYTE*)decodedBuffer; 240 BYTE* op = ostart; 241 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; 242 size_t decResult, oSize = COMPRESSIBLE_NOISE_LENGTH; 243 DISPLAYLEVEL(3, "Missing last %u bytes : ", (U32)missingBytes); 244 decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL); 245 if (LZ4F_isError(decResult)) goto _output_error; 246 if (decResult != missingBytes) { 247 DISPLAY("%u bytes missing != %u bytes requested \n", (U32)missingBytes, (U32)decResult); 248 goto _output_error; 249 } 250 DISPLAYLEVEL(3, "indeed, requests %u bytes \n", (unsigned)decResult); 251 cBuff += iSize; 252 iSize = decResult; 253 op += oSize; 254 oSize = (size_t)(oend-op); 255 decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL); 256 if (decResult != 0) goto _output_error; /* should finish now */ 257 op += oSize; 258 if (op>oend) { DISPLAY("decompression write overflow \n"); goto _output_error; } 259 { U64 const crcDest = XXH64(decodedBuffer, op-ostart, 1); 260 if (crcDest != crcOrig) goto _output_error; 261 } } 262 263 { size_t oSize = 0; 264 size_t iSize = 0; 265 LZ4F_frameInfo_t fi; 266 267 DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : "); 268 errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL); 269 if (LZ4F_isError(errorCode)) goto _output_error; 270 DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode); 271 272 DISPLAYLEVEL(3, "get FrameInfo on null input : "); 273 errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); 274 if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error; 275 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode)); 276 277 DISPLAYLEVEL(3, "get FrameInfo on not enough input : "); 278 iSize = 6; 279 errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); 280 if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error; 281 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode)); 282 ip += iSize; 283 284 DISPLAYLEVEL(3, "get FrameInfo on enough input : "); 285 iSize = 15 - iSize; 286 errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); 287 if (LZ4F_isError(errorCode)) goto _output_error; 288 DISPLAYLEVEL(3, " correctly decoded \n"); 289 ip += iSize; 290 } 291 292 DISPLAYLEVEL(3, "Byte after byte : \n"); 293 { BYTE* const ostart = (BYTE*)decodedBuffer; 294 BYTE* op = ostart; 295 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; 296 while (ip < iend) { 297 size_t oSize = oend-op; 298 size_t iSize = 1; 299 errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); 300 if (LZ4F_isError(errorCode)) goto _output_error; 301 op += oSize; 302 ip += iSize; 303 } 304 { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); 305 if (crcDest != crcOrig) goto _output_error; } 306 DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), COMPRESSIBLE_NOISE_LENGTH); 307 } 308 309 errorCode = LZ4F_freeDecompressionContext(dCtx); 310 if (LZ4F_isError(errorCode)) goto _output_error; 311 dCtx = NULL; 312 } 313 314 DISPLAYLEVEL(3, "Using 64 KB block : \n"); 315 prefs.frameInfo.blockSizeID = LZ4F_max64KB; 316 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; 317 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); 318 if (LZ4F_isError(cSize)) goto _output_error; 319 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 320 321 DISPLAYLEVEL(3, "without checksum : \n"); 322 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; 323 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); 324 if (LZ4F_isError(cSize)) goto _output_error; 325 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 326 327 DISPLAYLEVEL(3, "Using 256 KB block : \n"); 328 prefs.frameInfo.blockSizeID = LZ4F_max256KB; 329 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; 330 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); 331 if (LZ4F_isError(cSize)) goto _output_error; 332 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 333 334 DISPLAYLEVEL(3, "Decompression test : \n"); 335 { size_t const decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; 336 unsigned const maxBits = FUZ_highbit((U32)decodedBufferSize); 337 BYTE* const ostart = (BYTE*)decodedBuffer; 338 BYTE* op = ostart; 339 BYTE* const oend = ostart + COMPRESSIBLE_NOISE_LENGTH; 340 const BYTE* ip = (const BYTE*)compressedBuffer; 341 const BYTE* const iend = (const BYTE*)compressedBuffer + cSize; 342 343 { LZ4F_errorCode_t const createError = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 344 if (LZ4F_isError(createError)) goto _output_error; } 345 346 DISPLAYLEVEL(3, "random segment sizes : \n"); 347 while (ip < iend) { 348 unsigned const nbBits = FUZ_rand(&randState) % maxBits; 349 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; 350 size_t oSize = oend-op; 351 if (iSize > (size_t)(iend-ip)) iSize = iend-ip; 352 { size_t const decompressError = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); 353 if (LZ4F_isError(decompressError)) goto _output_error; } 354 op += oSize; 355 ip += iSize; 356 } 357 { size_t const decodedSize = (size_t)(op - ostart); 358 U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1); 359 if (crcDest != crcOrig) goto _output_error; 360 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); 361 } 362 363 { LZ4F_errorCode_t const freeError = LZ4F_freeDecompressionContext(dCtx); 364 if (LZ4F_isError(freeError)) goto _output_error; } 365 dCtx = NULL; 366 } 367 368 DISPLAYLEVEL(3, "without checksum : \n"); 369 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; 370 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); 371 if (LZ4F_isError(cSize)) goto _output_error; 372 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 373 374 DISPLAYLEVEL(3, "Using 1 MB block : \n"); 375 prefs.frameInfo.blockSizeID = LZ4F_max1MB; 376 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; 377 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); 378 if (LZ4F_isError(cSize)) goto _output_error; 379 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 380 381 DISPLAYLEVEL(3, "without checksum : \n"); 382 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; 383 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); 384 if (LZ4F_isError(cSize)) goto _output_error; 385 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 386 387 DISPLAYLEVEL(3, "Using 4 MB block : \n"); 388 prefs.frameInfo.blockSizeID = LZ4F_max4MB; 389 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; 390 { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs); 391 DISPLAYLEVEL(4, "dstCapacity = %u \n", (U32)dstCapacity) 392 cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs); 393 if (LZ4F_isError(cSize)) goto _output_error; 394 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); 395 } 396 397 DISPLAYLEVEL(3, "without checksum : \n"); 398 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; 399 { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs); 400 DISPLAYLEVEL(4, "dstCapacity = %u \n", (U32)dstCapacity) 401 cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs); 402 if (LZ4F_isError(cSize)) goto _output_error; 403 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); 404 } 405 406 { size_t errorCode; 407 BYTE* const ostart = (BYTE*)compressedBuffer; 408 BYTE* op = ostart; 409 errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION); 410 if (LZ4F_isError(errorCode)) goto _output_error; 411 412 DISPLAYLEVEL(3, "compress without frameSize : \n"); 413 memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo)); 414 errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); 415 if (LZ4F_isError(errorCode)) goto _output_error; 416 op += errorCode; 417 errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); 418 if (LZ4F_isError(errorCode)) goto _output_error; 419 op += errorCode; 420 errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL); 421 if (LZ4F_isError(errorCode)) goto _output_error; 422 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); 423 424 DISPLAYLEVEL(3, "compress with frameSize : \n"); 425 prefs.frameInfo.contentSize = testSize; 426 op = ostart; 427 errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); 428 if (LZ4F_isError(errorCode)) goto _output_error; 429 op += errorCode; 430 errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); 431 if (LZ4F_isError(errorCode)) goto _output_error; 432 op += errorCode; 433 errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL); 434 if (LZ4F_isError(errorCode)) goto _output_error; 435 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); 436 437 DISPLAYLEVEL(3, "compress with wrong frameSize : \n"); 438 prefs.frameInfo.contentSize = testSize+1; 439 op = ostart; 440 errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); 441 if (LZ4F_isError(errorCode)) goto _output_error; 442 op += errorCode; 443 errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); 444 if (LZ4F_isError(errorCode)) goto _output_error; 445 op += errorCode; 446 errorCode = LZ4F_compressEnd(cctx, op, testSize, NULL); 447 if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(errorCode)); } 448 else 449 goto _output_error; 450 451 errorCode = LZ4F_freeCompressionContext(cctx); 452 if (LZ4F_isError(errorCode)) goto _output_error; 453 cctx = NULL; 454 } 455 456 DISPLAYLEVEL(3, "Skippable frame test : \n"); 457 { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; 458 unsigned maxBits = FUZ_highbit((U32)decodedBufferSize); 459 BYTE* op = (BYTE*)decodedBuffer; 460 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; 461 BYTE* ip = (BYTE*)compressedBuffer; 462 BYTE* iend = (BYTE*)compressedBuffer + cSize + 8; 463 464 LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 465 if (LZ4F_isError(errorCode)) goto _output_error; 466 467 /* generate skippable frame */ 468 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START); 469 FUZ_writeLE32(ip+4, (U32)cSize); 470 471 DISPLAYLEVEL(3, "random segment sizes : \n"); 472 while (ip < iend) { 473 unsigned nbBits = FUZ_rand(&randState) % maxBits; 474 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; 475 size_t oSize = oend-op; 476 if (iSize > (size_t)(iend-ip)) iSize = iend-ip; 477 errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); 478 if (LZ4F_isError(errorCode)) goto _output_error; 479 op += oSize; 480 ip += iSize; 481 } 482 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize); 483 484 /* generate zero-size skippable frame */ 485 DISPLAYLEVEL(3, "zero-size skippable frame\n"); 486 ip = (BYTE*)compressedBuffer; 487 op = (BYTE*)decodedBuffer; 488 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+1); 489 FUZ_writeLE32(ip+4, 0); 490 iend = ip+8; 491 492 while (ip < iend) { 493 unsigned nbBits = FUZ_rand(&randState) % maxBits; 494 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; 495 size_t oSize = oend-op; 496 if (iSize > (size_t)(iend-ip)) iSize = iend-ip; 497 errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); 498 if (LZ4F_isError(errorCode)) goto _output_error; 499 op += oSize; 500 ip += iSize; 501 } 502 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); 503 504 DISPLAYLEVEL(3, "Skippable frame header complete in first call \n"); 505 ip = (BYTE*)compressedBuffer; 506 op = (BYTE*)decodedBuffer; 507 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+2); 508 FUZ_writeLE32(ip+4, 10); 509 iend = ip+18; 510 while (ip < iend) { 511 size_t iSize = 10; 512 size_t oSize = 10; 513 if (iSize > (size_t)(iend-ip)) iSize = iend-ip; 514 errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); 515 if (LZ4F_isError(errorCode)) goto _output_error; 516 op += oSize; 517 ip += iSize; 518 } 519 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); 520 } 521 522 DISPLAY("Basic tests completed \n"); 523_end: 524 free(CNBuffer); 525 free(compressedBuffer); 526 free(decodedBuffer); 527 LZ4F_freeDecompressionContext(dCtx); dCtx = NULL; 528 LZ4F_freeCompressionContext(cctx); cctx = NULL; 529 return testResult; 530 531_output_error: 532 testResult = 1; 533 DISPLAY("Error detected ! \n"); 534 goto _end; 535} 536 537 538static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous) 539{ 540 int p=0; 541 const BYTE* b1=(const BYTE*)buff1; 542 const BYTE* b2=(const BYTE*)buff2; 543 if (nonContiguous) { 544 DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size); 545 return; 546 } 547 while (b1[p]==b2[p]) p++; 548 DISPLAY("Error at pos %i/%i : %02X != %02X \n", p, (int)size, b1[p], b2[p]); 549} 550 551 552int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s) 553{ 554 unsigned testResult = 0; 555 unsigned testNb = 0; 556 size_t const srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ 557 void* srcBuffer = NULL; 558 size_t const compressedBufferSize = LZ4F_compressFrameBound(srcDataLength, NULL); 559 void* compressedBuffer = NULL; 560 void* decodedBuffer = NULL; 561 U32 coreRand = seed; 562 LZ4F_decompressionContext_t dCtx = NULL; 563 LZ4F_compressionContext_t cCtx = NULL; 564 size_t result; 565 clock_t const startClock = clock(); 566 clock_t const clockDuration = duration_s * CLOCKS_PER_SEC; 567# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ 568 DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } 569 570 /* Create buffers */ 571 result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 572 CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); 573 result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION); 574 CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); 575 srcBuffer = malloc(srcDataLength); 576 CHECK(srcBuffer==NULL, "srcBuffer Allocation failed"); 577 compressedBuffer = malloc(compressedBufferSize); 578 CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed"); 579 decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ 580 CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed"); 581 FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand); 582 583 /* jump to requested testNb */ 584 for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); /* sync randomizer */ 585 586 /* main fuzzer test loop */ 587 for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) { 588 U32 randState = coreRand ^ prime1; 589 unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(srcDataLength-1)) - 1)) + 1; 590 size_t const srcSize = (FUZ_rand(&randState) & ((1<<srcBits)-1)) + 1; 591 size_t const srcStartId = FUZ_rand(&randState) % (srcDataLength - srcSize); 592 const BYTE* const srcStart = (const BYTE*)srcBuffer + srcStartId; 593 unsigned const neverFlush = (FUZ_rand(&randState) & 15) == 1; 594 U64 const crcOrig = XXH64(srcStart, srcSize, 1); 595 LZ4F_preferences_t prefs; 596 const LZ4F_preferences_t* prefsPtr = &prefs; 597 size_t cSize; 598 599 (void)FUZ_rand(&coreRand); /* update seed */ 600 memset(&prefs, 0, sizeof(prefs)); 601 prefs.frameInfo.blockMode = (LZ4F_blockMode_t)(FUZ_rand(&randState) & 1); 602 prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)(4 + (FUZ_rand(&randState) & 3)); 603 prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)(FUZ_rand(&randState) & 1); 604 prefs.frameInfo.contentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0; 605 prefs.autoFlush = neverFlush ? 0 : (FUZ_rand(&randState) & 7) == 2; 606 prefs.compressionLevel = FUZ_rand(&randState) % 5; 607 if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL; 608 609 DISPLAYUPDATE(2, "\r%5u ", testNb); 610 611 if ((FUZ_rand(&randState) & 0xFFF) == 0) { 612 /* create a skippable frame (rare case) */ 613 BYTE* op = (BYTE*)compressedBuffer; 614 FUZ_writeLE32(op, LZ4F_MAGIC_SKIPPABLE_START + (FUZ_rand(&randState) & 15)); 615 FUZ_writeLE32(op+4, (U32)srcSize); 616 cSize = srcSize+8; 617 } else if ((FUZ_rand(&randState) & 0xF) == 2) { /* single pass compression (simple) */ 618 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), srcStart, srcSize, prefsPtr); 619 CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize)); 620 } else { /* multi-segments compression */ 621 const BYTE* ip = srcStart; 622 const BYTE* const iend = srcStart + srcSize; 623 BYTE* op = (BYTE*)compressedBuffer; 624 BYTE* const oend = op + (neverFlush ? LZ4F_compressFrameBound(srcSize, prefsPtr) : compressedBufferSize); /* when flushes are possible, can't guarantee a max compressed size */ 625 unsigned const maxBits = FUZ_highbit((U32)srcSize); 626 LZ4F_compressOptions_t cOptions; 627 memset(&cOptions, 0, sizeof(cOptions)); 628 result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr); 629 CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result); 630 op += result; 631 while (ip < iend) { 632 unsigned const nbBitsSeg = FUZ_rand(&randState) % maxBits; 633 size_t const sampleMax = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1; 634 size_t const iSize = MIN(sampleMax, (size_t)(iend-ip)); 635 size_t const oSize = LZ4F_compressBound(iSize, prefsPtr); 636 cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1); 637 638 result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); 639 CHECK(LZ4F_isError(result), "Compression failed (error %i : %s)", (int)result, LZ4F_getErrorName(result)); 640 op += result; 641 ip += iSize; 642 643 { unsigned const forceFlush = neverFlush ? 0 : ((FUZ_rand(&randState) & 3) == 1); 644 if (forceFlush) { 645 result = LZ4F_flush(cCtx, op, oend-op, &cOptions); 646 CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); 647 op += result; 648 } } 649 } 650 CHECK(op>=oend, "LZ4F_compressFrameBound overflow"); 651 result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions); 652 CHECK(LZ4F_isError(result), "Compression completion failed (error %i : %s)", (int)result, LZ4F_getErrorName(result)); 653 op += result; 654 cSize = op-(BYTE*)compressedBuffer; 655 DISPLAYLEVEL(5, "\nCompressed %u bytes into %u \n", (U32)srcSize, (U32)cSize); 656 } 657 658 /* multi-segments decompression */ 659 { const BYTE* ip = (const BYTE*)compressedBuffer; 660 const BYTE* const iend = ip + cSize; 661 BYTE* op = (BYTE*)decodedBuffer; 662 BYTE* const oend = op + srcDataLength; 663 unsigned const suggestedBits = FUZ_highbit((U32)cSize); 664 unsigned const maxBits = MAX(3, suggestedBits); 665 unsigned const nonContiguousDst = FUZ_rand(&randState) % 3; /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */ 666 size_t totalOut = 0; 667 XXH64_state_t xxh64; 668 XXH64_reset(&xxh64, 1); 669 while (ip < iend) { 670 unsigned const nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; 671 unsigned const nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1; 672 size_t const iSizeMax = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1; 673 size_t iSize = MIN(iSizeMax, (size_t)(iend-ip)); 674 size_t const oSizeMax = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2; 675 size_t oSize = MIN(oSizeMax, (size_t)(oend-op)); 676 LZ4F_decompressOptions_t dOptions; 677 memset(&dOptions, 0, sizeof(dOptions)); 678 dOptions.stableDst = FUZ_rand(&randState) & 1; 679 if (nonContiguousDst==2) dOptions.stableDst = 0; /* overwrite mode */ 680 result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); 681 if (LZ4F_getErrorCode(result) == LZ4F_ERROR_contentChecksum_invalid) locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst); 682 CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName(result)); 683 XXH64_update(&xxh64, op, (U32)oSize); 684 totalOut += oSize; 685 op += oSize; 686 ip += iSize; 687 op += nonContiguousDst; 688 if (nonContiguousDst==2) op = (BYTE*)decodedBuffer; /* overwritten destination */ 689 } 690 CHECK(result != 0, "Frame decompression failed (error %i)", (int)result); 691 if (totalOut) { /* otherwise, it's a skippable frame */ 692 U64 const crcDecoded = XXH64_digest(&xxh64); 693 if (crcDecoded != crcOrig) locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst); 694 CHECK(crcDecoded != crcOrig, "Decompression corruption"); 695 } 696 } 697 } 698 699 DISPLAYLEVEL(2, "\rAll tests completed \n"); 700 701_end: 702 LZ4F_freeDecompressionContext(dCtx); 703 LZ4F_freeCompressionContext(cCtx); 704 free(srcBuffer); 705 free(compressedBuffer); 706 free(decodedBuffer); 707 708 if (pause) { 709 DISPLAY("press enter to finish \n"); 710 (void)getchar(); 711 } 712 return testResult; 713 714_output_error: 715 testResult = 1; 716 goto _end; 717} 718 719 720int FUZ_usage(const char* programName) 721{ 722 DISPLAY( "Usage :\n"); 723 DISPLAY( " %s [args]\n", programName); 724 DISPLAY( "\n"); 725 DISPLAY( "Arguments :\n"); 726 DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault); 727 DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); 728 DISPLAY( " -s# : Select seed (default:prompt user)\n"); 729 DISPLAY( " -t# : Select starting test number (default:0)\n"); 730 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); 731 DISPLAY( " -v : verbose\n"); 732 DISPLAY( " -h : display help and exit\n"); 733 return 0; 734} 735 736 737int main(int argc, const char** argv) 738{ 739 U32 seed=0; 740 int seedset=0; 741 int argNb; 742 int nbTests = nbTestsDefault; 743 int testNb = 0; 744 int proba = FUZ_COMPRESSIBILITY_DEFAULT; 745 int result=0; 746 U32 duration=0; 747 const char* const programName = argv[0]; 748 749 /* Check command line */ 750 for (argNb=1; argNb<argc; argNb++) { 751 const char* argument = argv[argNb]; 752 753 if(!argument) continue; /* Protection if argument empty */ 754 755 /* Decode command (note : aggregated short commands are allowed) */ 756 if (argument[0]=='-') { 757 if (!strcmp(argument, "--no-prompt")) { 758 no_prompt=1; 759 seedset=1; 760 displayLevel=1; 761 continue; 762 } 763 argument++; 764 765 while (*argument!=0) { 766 switch(*argument) 767 { 768 case 'h': 769 return FUZ_usage(programName); 770 case 'v': 771 argument++; 772 displayLevel++; 773 break; 774 case 'q': 775 argument++; 776 displayLevel--; 777 break; 778 case 'p': /* pause at the end */ 779 argument++; 780 pause = 1; 781 break; 782 783 case 'i': 784 argument++; 785 nbTests=0; duration=0; 786 while ((*argument>='0') && (*argument<='9')) { 787 nbTests *= 10; 788 nbTests += *argument - '0'; 789 argument++; 790 } 791 break; 792 793 case 'T': 794 argument++; 795 nbTests = 0; duration = 0; 796 for (;;) { 797 switch(*argument) 798 { 799 case 'm': duration *= 60; argument++; continue; 800 case 's': 801 case 'n': argument++; continue; 802 case '0': 803 case '1': 804 case '2': 805 case '3': 806 case '4': 807 case '5': 808 case '6': 809 case '7': 810 case '8': 811 case '9': duration *= 10; duration += *argument++ - '0'; continue; 812 } 813 break; 814 } 815 break; 816 817 case 's': 818 argument++; 819 seed=0; 820 seedset=1; 821 while ((*argument>='0') && (*argument<='9')) { 822 seed *= 10; 823 seed += *argument - '0'; 824 argument++; 825 } 826 break; 827 case 't': 828 argument++; 829 testNb=0; 830 while ((*argument>='0') && (*argument<='9')) { 831 testNb *= 10; 832 testNb += *argument - '0'; 833 argument++; 834 } 835 break; 836 case 'P': /* compressibility % */ 837 argument++; 838 proba=0; 839 while ((*argument>='0') && (*argument<='9')) { 840 proba *= 10; 841 proba += *argument - '0'; 842 argument++; 843 } 844 if (proba<0) proba=0; 845 if (proba>100) proba=100; 846 break; 847 default: 848 ; 849 return FUZ_usage(programName); 850 } 851 } 852 } 853 } 854 855 /* Get Seed */ 856 DISPLAY("Starting lz4frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); 857 858 if (!seedset) { 859 time_t const t = time(NULL); 860 U32 const h = XXH32(&t, sizeof(t), 1); 861 seed = h % 10000; 862 } 863 DISPLAY("Seed = %u\n", seed); 864 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba); 865 866 if (nbTests<=0) nbTests=1; 867 868 if (testNb==0) result = basicTests(seed, ((double)proba) / 100); 869 if (result) return 1; 870 return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration); 871} 872