1/* XzEnc.c -- Xz Encode 22009-06-04 : Igor Pavlov : Public domain */ 3 4#include <stdlib.h> 5#include <string.h> 6 7#include "7zCrc.h" 8#include "Alloc.h" 9#include "Bra.h" 10#include "CpuArch.h" 11#ifdef USE_SUBBLOCK 12#include "SbEnc.h" 13#endif 14 15#include "XzEnc.h" 16 17static void *SzBigAlloc(void *p, size_t size) { p = p; return BigAlloc(size); } 18static void SzBigFree(void *p, void *address) { p = p; BigFree(address); } 19static ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; 20 21static void *SzAlloc(void *p, size_t size) { p = p; return MyAlloc(size); } 22static void SzFree(void *p, void *address) { p = p; MyFree(address); } 23static ISzAlloc g_Alloc = { SzAlloc, SzFree }; 24 25#define XzBlock_ClearFlags(p) (p)->flags = 0; 26#define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1); 27#define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE; 28#define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE; 29 30static SRes WriteBytes(ISeqOutStream *s, const void *buf, UInt32 size) 31{ 32 return (s->Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE; 33} 34 35static SRes WriteBytesAndCrc(ISeqOutStream *s, const void *buf, UInt32 size, UInt32 *crc) 36{ 37 *crc = CrcUpdate(*crc, buf, size); 38 return WriteBytes(s, buf, size); 39} 40 41SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s) 42{ 43 UInt32 crc; 44 Byte header[XZ_STREAM_HEADER_SIZE]; 45 memcpy(header, XZ_SIG, XZ_SIG_SIZE); 46 header[XZ_SIG_SIZE] = (Byte)(f >> 8); 47 header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF); 48 crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE); 49 SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc); 50 return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE); 51} 52 53SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s) 54{ 55 Byte header[XZ_BLOCK_HEADER_SIZE_MAX]; 56 57 unsigned pos = 1; 58 int numFilters, i; 59 header[pos++] = p->flags; 60 61 if (XzBlock_HasPackSize(p)) pos += Xz_WriteVarInt(header + pos, p->packSize); 62 if (XzBlock_HasUnpackSize(p)) pos += Xz_WriteVarInt(header + pos, p->unpackSize); 63 numFilters = XzBlock_GetNumFilters(p); 64 for (i = 0; i < numFilters; i++) 65 { 66 const CXzFilter *f = &p->filters[i]; 67 pos += Xz_WriteVarInt(header + pos, f->id); 68 pos += Xz_WriteVarInt(header + pos, f->propsSize); 69 memcpy(header + pos, f->props, f->propsSize); 70 pos += f->propsSize; 71 } 72 while((pos & 3) != 0) 73 header[pos++] = 0; 74 header[0] = (Byte)(pos >> 2); 75 SetUi32(header + pos, CrcCalc(header, pos)); 76 return WriteBytes(s, header, pos + 4); 77} 78 79SRes Xz_WriteFooter(CXzStream *p, ISeqOutStream *s) 80{ 81 Byte buf[32]; 82 UInt64 globalPos; 83 { 84 UInt32 crc = CRC_INIT_VAL; 85 unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks); 86 size_t i; 87 88 globalPos = pos; 89 buf[0] = 0; 90 RINOK(WriteBytesAndCrc(s, buf, pos, &crc)); 91 for (i = 0; i < p->numBlocks; i++) 92 { 93 const CXzBlockSizes *block = &p->blocks[i]; 94 pos = Xz_WriteVarInt(buf, block->totalSize); 95 pos += Xz_WriteVarInt(buf + pos, block->unpackSize); 96 globalPos += pos; 97 RINOK(WriteBytesAndCrc(s, buf, pos, &crc)); 98 } 99 pos = ((unsigned)globalPos & 3); 100 if (pos != 0) 101 { 102 buf[0] = buf[1] = buf[2] = 0; 103 RINOK(WriteBytesAndCrc(s, buf, 4 - pos, &crc)); 104 globalPos += 4 - pos; 105 } 106 { 107 SetUi32(buf, CRC_GET_DIGEST(crc)); 108 RINOK(WriteBytes(s, buf, 4)); 109 globalPos += 4; 110 } 111 } 112 113 { 114 UInt32 indexSize = (UInt32)((globalPos >> 2) - 1); 115 SetUi32(buf + 4, indexSize); 116 buf[8] = (Byte)(p->flags >> 8); 117 buf[9] = (Byte)(p->flags & 0xFF); 118 SetUi32(buf, CrcCalc(buf + 4, 6)); 119 memcpy(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE); 120 return WriteBytes(s, buf, 12); 121 } 122} 123 124SRes Xz_AddIndexRecord(CXzStream *p, UInt64 unpackSize, UInt64 totalSize, ISzAlloc *alloc) 125{ 126 if (p->blocks == 0 || p->numBlocksAllocated == p->numBlocks) 127 { 128 size_t num = (p->numBlocks + 1) * 2; 129 size_t newSize = sizeof(CXzBlockSizes) * num; 130 CXzBlockSizes *blocks; 131 if (newSize / sizeof(CXzBlockSizes) != num) 132 return SZ_ERROR_MEM; 133 blocks = alloc->Alloc(alloc, newSize); 134 if (blocks == 0) 135 return SZ_ERROR_MEM; 136 if (p->numBlocks != 0) 137 { 138 memcpy(blocks, p->blocks, p->numBlocks * sizeof(CXzBlockSizes)); 139 Xz_Free(p, alloc); 140 } 141 p->blocks = blocks; 142 p->numBlocksAllocated = num; 143 } 144 { 145 CXzBlockSizes *block = &p->blocks[p->numBlocks++]; 146 block->totalSize = totalSize; 147 block->unpackSize = unpackSize; 148 } 149 return SZ_OK; 150} 151 152/* ---------- CSeqCheckInStream ---------- */ 153 154typedef struct 155{ 156 ISeqInStream p; 157 ISeqInStream *realStream; 158 UInt64 processed; 159 CXzCheck check; 160} CSeqCheckInStream; 161 162void SeqCheckInStream_Init(CSeqCheckInStream *p, int mode) 163{ 164 p->processed = 0; 165 XzCheck_Init(&p->check, mode); 166} 167 168void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest) 169{ 170 XzCheck_Final(&p->check, digest); 171} 172 173static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size) 174{ 175 CSeqCheckInStream *p = (CSeqCheckInStream *)pp; 176 SRes res = p->realStream->Read(p->realStream, data, size); 177 XzCheck_Update(&p->check, data, *size); 178 p->processed += *size; 179 return res; 180} 181 182/* ---------- CSeqSizeOutStream ---------- */ 183 184typedef struct 185{ 186 ISeqOutStream p; 187 ISeqOutStream *realStream; 188 UInt64 processed; 189} CSeqSizeOutStream; 190 191static size_t MyWrite(void *pp, const void *data, size_t size) 192{ 193 CSeqSizeOutStream *p = (CSeqSizeOutStream *)pp; 194 size = p->realStream->Write(p->realStream, data, size); 195 p->processed += size; 196 return size; 197} 198 199/* ---------- CSeqInFilter ---------- */ 200 201/* 202typedef struct _IFilter 203{ 204 void *p; 205 void (*Free)(void *p, ISzAlloc *alloc); 206 SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAlloc *alloc); 207 void (*Init)(void *p); 208 size_t (*Filter)(void *p, Byte *data, SizeT destLen); 209} IFilter; 210 211#define FILT_BUF_SIZE (1 << 19) 212 213typedef struct 214{ 215 ISeqInStream p; 216 ISeqInStream *realStream; 217 UInt32 x86State; 218 UInt32 ip; 219 UInt64 processed; 220 CXzCheck check; 221 Byte buf[FILT_BUF_SIZE]; 222 UInt32 bufferPos; 223 UInt32 convertedPosBegin; 224 UInt32 convertedPosEnd; 225 IFilter *filter; 226} CSeqInFilter; 227 228static SRes SeqInFilter_Read(void *pp, void *data, size_t *size) 229{ 230 CSeqInFilter *p = (CSeqInFilter *)pp; 231 size_t remSize = *size; 232 *size = 0; 233 234 while (remSize > 0) 235 { 236 int i; 237 if (p->convertedPosBegin != p->convertedPosEnd) 238 { 239 UInt32 sizeTemp = p->convertedPosEnd - p->convertedPosBegin; 240 if (remSize < sizeTemp) 241 sizeTemp = (UInt32)remSize; 242 memmove(data, p->buf + p->convertedPosBegin, sizeTemp); 243 p->convertedPosBegin += sizeTemp; 244 data = (void *)((Byte *)data + sizeTemp); 245 remSize -= sizeTemp; 246 *size += sizeTemp; 247 break; 248 } 249 for (i = 0; p->convertedPosEnd + i < p->bufferPos; i++) 250 p->buf[i] = p->buf[i + p->convertedPosEnd]; 251 p->bufferPos = i; 252 p->convertedPosBegin = p->convertedPosEnd = 0; 253 { 254 size_t processedSizeTemp = FILT_BUF_SIZE - p->bufferPos; 255 RINOK(p->realStream->Read(p->realStream, p->buf + p->bufferPos, &processedSizeTemp)); 256 p->bufferPos = p->bufferPos + (UInt32)processedSizeTemp; 257 } 258 p->convertedPosEnd = (UInt32)p->filter->Filter(p->filter->p, p->buf, p->bufferPos); 259 if (p->convertedPosEnd == 0) 260 { 261 if (p->bufferPos == 0) 262 break; 263 else 264 { 265 p->convertedPosEnd = p->bufferPos; 266 continue; 267 } 268 } 269 if (p->convertedPosEnd > p->bufferPos) 270 { 271 for (; p->bufferPos < p->convertedPosEnd; p->bufferPos++) 272 p->buf[p->bufferPos] = 0; 273 p->convertedPosEnd = (UInt32)p->filter->Filter(p->filter->p, p->buf, p->bufferPos); 274 } 275 } 276 return SZ_OK; 277} 278*/ 279 280/* 281typedef struct 282{ 283 ISeqInStream p; 284 ISeqInStream *realStream; 285 CMixCoder mixCoder; 286 Byte buf[FILT_BUF_SIZE]; 287 UInt32 bufPos; 288 UInt32 bufSize; 289} CMixCoderSeqInStream; 290 291static SRes CMixCoderSeqInStream_Read(void *pp, void *data, size_t *size) 292{ 293 CMixCoderSeqInStream *p = (CMixCoderSeqInStream *)pp; 294 SRes res = SZ_OK; 295 size_t remSize = *size; 296 *size = 0; 297 while (remSize > 0) 298 { 299 if (p->bufPos == p->bufSize) 300 { 301 size_t curSize; 302 p->bufPos = p->bufSize = 0; 303 if (*size != 0) 304 break; 305 curSize = FILT_BUF_SIZE; 306 RINOK(p->realStream->Read(p->realStream, p->buf, &curSize)); 307 p->bufSize = (UInt32)curSize; 308 } 309 { 310 SizeT destLen = remSize; 311 SizeT srcLen = p->bufSize - p->bufPos; 312 res = MixCoder_Code(&p->mixCoder, data, &destLen, p->buf + p->bufPos, &srcLen, 0); 313 data = (void *)((Byte *)data + destLen); 314 remSize -= destLen; 315 *size += destLen; 316 p->bufPos += srcLen; 317 } 318 } 319 return res; 320} 321*/ 322 323#ifdef USE_SUBBLOCK 324typedef struct 325{ 326 ISeqInStream p; 327 CSubblockEnc sb; 328 UInt64 processed; 329} CSbEncInStream; 330 331void SbEncInStream_Init(CSbEncInStream *p) 332{ 333 p->processed = 0; 334 SubblockEnc_Init(&p->sb); 335} 336 337static SRes SbEncInStream_Read(void *pp, void *data, size_t *size) 338{ 339 CSbEncInStream *p = (CSbEncInStream *)pp; 340 SRes res = SubblockEnc_Read(&p->sb, data, size); 341 p->processed += *size; 342 return res; 343} 344#endif 345 346typedef struct 347{ 348 /* CMixCoderSeqInStream inStream; */ 349 CLzma2EncHandle lzma2; 350 #ifdef USE_SUBBLOCK 351 CSbEncInStream sb; 352 #endif 353 ISzAlloc *alloc; 354 ISzAlloc *bigAlloc; 355} CLzma2WithFilters; 356 357 358static void Lzma2WithFilters_Construct(CLzma2WithFilters *p, ISzAlloc *alloc, ISzAlloc *bigAlloc) 359{ 360 p->alloc = alloc; 361 p->bigAlloc = bigAlloc; 362 p->lzma2 = NULL; 363 #ifdef USE_SUBBLOCK 364 p->sb.p.Read = SbEncInStream_Read; 365 SubblockEnc_Construct(&p->sb.sb, p->alloc); 366 #endif 367} 368 369static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p) 370{ 371 p->lzma2 = Lzma2Enc_Create(p->alloc, p->bigAlloc); 372 if (p->lzma2 == 0) 373 return SZ_ERROR_MEM; 374 return SZ_OK; 375} 376 377static void Lzma2WithFilters_Free(CLzma2WithFilters *p) 378{ 379 #ifdef USE_SUBBLOCK 380 SubblockEnc_Free(&p->sb.sb); 381 #endif 382 if (p->lzma2) 383 { 384 Lzma2Enc_Destroy(p->lzma2); 385 p->lzma2 = NULL; 386 } 387} 388 389static SRes Xz_Compress(CXzStream *xz, 390 CLzma2WithFilters *lzmaf, 391 ISeqOutStream *outStream, 392 ISeqInStream *inStream, 393 const CLzma2EncProps *lzma2Props, 394 Bool useSubblock, 395 ICompressProgress *progress) 396{ 397 xz->flags = XZ_CHECK_CRC32; 398 399 RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, lzma2Props)); 400 RINOK(Xz_WriteHeader(xz->flags, outStream)); 401 402 { 403 CSeqCheckInStream checkInStream; 404 CSeqSizeOutStream seqSizeOutStream; 405 CXzBlock block; 406 int filterIndex = 0; 407 408 XzBlock_ClearFlags(&block); 409 XzBlock_SetNumFilters(&block, 1 + (useSubblock ? 1 : 0)); 410 411 if (useSubblock) 412 { 413 CXzFilter *f = &block.filters[filterIndex++]; 414 f->id = XZ_ID_Subblock; 415 f->propsSize = 0; 416 } 417 418 { 419 CXzFilter *f = &block.filters[filterIndex++]; 420 f->id = XZ_ID_LZMA2; 421 f->propsSize = 1; 422 f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2); 423 } 424 425 seqSizeOutStream.p.Write = MyWrite; 426 seqSizeOutStream.realStream = outStream; 427 seqSizeOutStream.processed = 0; 428 429 RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.p)); 430 431 checkInStream.p.Read = SeqCheckInStream_Read; 432 checkInStream.realStream = inStream; 433 SeqCheckInStream_Init(&checkInStream, XzFlags_GetCheckType(xz->flags)); 434 435 #ifdef USE_SUBBLOCK 436 if (useSubblock) 437 { 438 lzmaf->sb.sb.inStream = &checkInStream.p; 439 SubblockEnc_Init(&lzmaf->sb.sb); 440 } 441 #endif 442 443 { 444 UInt64 packPos = seqSizeOutStream.processed; 445 SRes res = Lzma2Enc_Encode(lzmaf->lzma2, &seqSizeOutStream.p, 446 #ifdef USE_SUBBLOCK 447 useSubblock ? &lzmaf->sb.p: 448 #endif 449 &checkInStream.p, 450 progress); 451 RINOK(res); 452 block.unpackSize = checkInStream.processed; 453 block.packSize = seqSizeOutStream.processed - packPos; 454 } 455 456 { 457 unsigned padSize = 0; 458 Byte buf[128]; 459 while((((unsigned)block.packSize + padSize) & 3) != 0) 460 buf[padSize++] = 0; 461 SeqCheckInStream_GetDigest(&checkInStream, buf + padSize); 462 RINOK(WriteBytes(&seqSizeOutStream.p, buf, padSize + XzFlags_GetCheckSize(xz->flags))); 463 RINOK(Xz_AddIndexRecord(xz, block.unpackSize, seqSizeOutStream.processed - padSize, &g_Alloc)); 464 } 465 } 466 return Xz_WriteFooter(xz, outStream); 467} 468 469SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, 470 const CLzma2EncProps *lzma2Props, Bool useSubblock, 471 ICompressProgress *progress) 472{ 473 SRes res; 474 CXzStream xz; 475 CLzma2WithFilters lzmaf; 476 Xz_Construct(&xz); 477 Lzma2WithFilters_Construct(&lzmaf, &g_Alloc, &g_BigAlloc); 478 res = Lzma2WithFilters_Create(&lzmaf); 479 if (res == SZ_OK) 480 res = Xz_Compress(&xz, &lzmaf, outStream, inStream, 481 lzma2Props, useSubblock, progress); 482 Lzma2WithFilters_Free(&lzmaf); 483 Xz_Free(&xz, &g_Alloc); 484 return res; 485} 486 487SRes Xz_EncodeEmpty(ISeqOutStream *outStream) 488{ 489 SRes res; 490 CXzStream xz; 491 Xz_Construct(&xz); 492 res = Xz_WriteHeader(xz.flags, outStream); 493 if (res == SZ_OK) 494 res = Xz_WriteFooter(&xz, outStream); 495 Xz_Free(&xz, &g_Alloc); 496 return res; 497} 498