1/* XzEnc.c -- Xz Encode
22009-06-04 : Igor Pavlov : Public domain */
3
4#include <stdlib.h>
5#include <string.h>
6
7#include "7zCrc.h"
8#include "Alloc.h"
9#include "Bra.h"
10#include "CpuArch.h"
11#ifdef USE_SUBBLOCK
12#include "SbEnc.h"
13#endif
14
15#include "XzEnc.h"
16
17static void *SzBigAlloc(void *p, size_t size) { p = p; return BigAlloc(size); }
18static void SzBigFree(void *p, void *address) { p = p; BigFree(address); }
19static ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree };
20
21static void *SzAlloc(void *p, size_t size) { p = p; return MyAlloc(size); }
22static void SzFree(void *p, void *address) { p = p; MyFree(address); }
23static ISzAlloc g_Alloc = { SzAlloc, SzFree };
24
25#define XzBlock_ClearFlags(p)       (p)->flags = 0;
26#define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1);
27#define XzBlock_SetHasPackSize(p)   (p)->flags |= XZ_BF_PACK_SIZE;
28#define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE;
29
30static SRes WriteBytes(ISeqOutStream *s, const void *buf, UInt32 size)
31{
32  return (s->Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE;
33}
34
35static SRes WriteBytesAndCrc(ISeqOutStream *s, const void *buf, UInt32 size, UInt32 *crc)
36{
37  *crc = CrcUpdate(*crc, buf, size);
38  return WriteBytes(s, buf, size);
39}
40
41SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s)
42{
43  UInt32 crc;
44  Byte header[XZ_STREAM_HEADER_SIZE];
45  memcpy(header, XZ_SIG, XZ_SIG_SIZE);
46  header[XZ_SIG_SIZE] = (Byte)(f >> 8);
47  header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF);
48  crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE);
49  SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc);
50  return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE);
51}
52
53SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s)
54{
55  Byte header[XZ_BLOCK_HEADER_SIZE_MAX];
56
57  unsigned pos = 1;
58  int numFilters, i;
59  header[pos++] = p->flags;
60
61  if (XzBlock_HasPackSize(p)) pos += Xz_WriteVarInt(header + pos, p->packSize);
62  if (XzBlock_HasUnpackSize(p)) pos += Xz_WriteVarInt(header + pos, p->unpackSize);
63  numFilters = XzBlock_GetNumFilters(p);
64  for (i = 0; i < numFilters; i++)
65  {
66    const CXzFilter *f = &p->filters[i];
67    pos += Xz_WriteVarInt(header + pos, f->id);
68    pos += Xz_WriteVarInt(header + pos, f->propsSize);
69    memcpy(header + pos, f->props, f->propsSize);
70    pos += f->propsSize;
71  }
72  while((pos & 3) != 0)
73    header[pos++] = 0;
74  header[0] = (Byte)(pos >> 2);
75  SetUi32(header + pos, CrcCalc(header, pos));
76  return WriteBytes(s, header, pos + 4);
77}
78
79SRes Xz_WriteFooter(CXzStream *p, ISeqOutStream *s)
80{
81  Byte buf[32];
82  UInt64 globalPos;
83  {
84    UInt32 crc = CRC_INIT_VAL;
85    unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks);
86    size_t i;
87
88    globalPos = pos;
89    buf[0] = 0;
90    RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
91    for (i = 0; i < p->numBlocks; i++)
92    {
93      const CXzBlockSizes *block = &p->blocks[i];
94      pos = Xz_WriteVarInt(buf, block->totalSize);
95      pos += Xz_WriteVarInt(buf + pos, block->unpackSize);
96      globalPos += pos;
97      RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
98    }
99    pos = ((unsigned)globalPos & 3);
100    if (pos != 0)
101    {
102      buf[0] = buf[1] = buf[2] = 0;
103      RINOK(WriteBytesAndCrc(s, buf, 4 - pos, &crc));
104      globalPos += 4 - pos;
105    }
106    {
107      SetUi32(buf, CRC_GET_DIGEST(crc));
108      RINOK(WriteBytes(s, buf, 4));
109      globalPos += 4;
110    }
111  }
112
113  {
114    UInt32 indexSize = (UInt32)((globalPos >> 2) - 1);
115    SetUi32(buf + 4, indexSize);
116    buf[8] = (Byte)(p->flags >> 8);
117    buf[9] = (Byte)(p->flags & 0xFF);
118    SetUi32(buf, CrcCalc(buf + 4, 6));
119    memcpy(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE);
120    return WriteBytes(s, buf, 12);
121  }
122}
123
124SRes Xz_AddIndexRecord(CXzStream *p, UInt64 unpackSize, UInt64 totalSize, ISzAlloc *alloc)
125{
126  if (p->blocks == 0 || p->numBlocksAllocated == p->numBlocks)
127  {
128    size_t num = (p->numBlocks + 1) * 2;
129    size_t newSize = sizeof(CXzBlockSizes) * num;
130    CXzBlockSizes *blocks;
131    if (newSize / sizeof(CXzBlockSizes) != num)
132      return SZ_ERROR_MEM;
133    blocks = alloc->Alloc(alloc, newSize);
134    if (blocks == 0)
135      return SZ_ERROR_MEM;
136    if (p->numBlocks != 0)
137    {
138      memcpy(blocks, p->blocks, p->numBlocks * sizeof(CXzBlockSizes));
139      Xz_Free(p, alloc);
140    }
141    p->blocks = blocks;
142    p->numBlocksAllocated = num;
143  }
144  {
145    CXzBlockSizes *block = &p->blocks[p->numBlocks++];
146    block->totalSize = totalSize;
147    block->unpackSize = unpackSize;
148  }
149  return SZ_OK;
150}
151
152/* ---------- CSeqCheckInStream ---------- */
153
154typedef struct
155{
156  ISeqInStream p;
157  ISeqInStream *realStream;
158  UInt64 processed;
159  CXzCheck check;
160} CSeqCheckInStream;
161
162void SeqCheckInStream_Init(CSeqCheckInStream *p, int mode)
163{
164  p->processed = 0;
165  XzCheck_Init(&p->check, mode);
166}
167
168void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest)
169{
170  XzCheck_Final(&p->check, digest);
171}
172
173static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size)
174{
175  CSeqCheckInStream *p = (CSeqCheckInStream *)pp;
176  SRes res = p->realStream->Read(p->realStream, data, size);
177  XzCheck_Update(&p->check, data, *size);
178  p->processed += *size;
179  return res;
180}
181
182/* ---------- CSeqSizeOutStream ---------- */
183
184typedef struct
185{
186  ISeqOutStream p;
187  ISeqOutStream *realStream;
188  UInt64 processed;
189} CSeqSizeOutStream;
190
191static size_t MyWrite(void *pp, const void *data, size_t size)
192{
193  CSeqSizeOutStream *p = (CSeqSizeOutStream *)pp;
194  size = p->realStream->Write(p->realStream, data, size);
195  p->processed += size;
196  return size;
197}
198
199/* ---------- CSeqInFilter ---------- */
200
201/*
202typedef struct _IFilter
203{
204  void *p;
205  void (*Free)(void *p, ISzAlloc *alloc);
206  SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAlloc *alloc);
207  void (*Init)(void *p);
208  size_t (*Filter)(void *p, Byte *data, SizeT destLen);
209} IFilter;
210
211#define FILT_BUF_SIZE (1 << 19)
212
213typedef struct
214{
215  ISeqInStream p;
216  ISeqInStream *realStream;
217  UInt32 x86State;
218  UInt32 ip;
219  UInt64 processed;
220  CXzCheck check;
221  Byte buf[FILT_BUF_SIZE];
222  UInt32 bufferPos;
223  UInt32 convertedPosBegin;
224  UInt32 convertedPosEnd;
225  IFilter *filter;
226} CSeqInFilter;
227
228static SRes SeqInFilter_Read(void *pp, void *data, size_t *size)
229{
230  CSeqInFilter *p = (CSeqInFilter *)pp;
231  size_t remSize = *size;
232  *size = 0;
233
234  while (remSize > 0)
235  {
236    int i;
237    if (p->convertedPosBegin != p->convertedPosEnd)
238    {
239      UInt32 sizeTemp = p->convertedPosEnd - p->convertedPosBegin;
240      if (remSize < sizeTemp)
241        sizeTemp = (UInt32)remSize;
242      memmove(data, p->buf + p->convertedPosBegin, sizeTemp);
243      p->convertedPosBegin += sizeTemp;
244      data = (void *)((Byte *)data + sizeTemp);
245      remSize -= sizeTemp;
246      *size += sizeTemp;
247      break;
248    }
249    for (i = 0; p->convertedPosEnd + i < p->bufferPos; i++)
250      p->buf[i] = p->buf[i + p->convertedPosEnd];
251    p->bufferPos = i;
252    p->convertedPosBegin = p->convertedPosEnd = 0;
253    {
254      size_t processedSizeTemp = FILT_BUF_SIZE - p->bufferPos;
255      RINOK(p->realStream->Read(p->realStream, p->buf + p->bufferPos, &processedSizeTemp));
256      p->bufferPos = p->bufferPos + (UInt32)processedSizeTemp;
257    }
258    p->convertedPosEnd = (UInt32)p->filter->Filter(p->filter->p, p->buf, p->bufferPos);
259    if (p->convertedPosEnd == 0)
260    {
261      if (p->bufferPos == 0)
262        break;
263      else
264      {
265        p->convertedPosEnd = p->bufferPos;
266        continue;
267      }
268    }
269    if (p->convertedPosEnd > p->bufferPos)
270    {
271      for (; p->bufferPos < p->convertedPosEnd; p->bufferPos++)
272        p->buf[p->bufferPos] = 0;
273      p->convertedPosEnd = (UInt32)p->filter->Filter(p->filter->p, p->buf, p->bufferPos);
274    }
275  }
276  return SZ_OK;
277}
278*/
279
280/*
281typedef struct
282{
283  ISeqInStream p;
284  ISeqInStream *realStream;
285  CMixCoder mixCoder;
286  Byte buf[FILT_BUF_SIZE];
287  UInt32 bufPos;
288  UInt32 bufSize;
289} CMixCoderSeqInStream;
290
291static SRes CMixCoderSeqInStream_Read(void *pp, void *data, size_t *size)
292{
293  CMixCoderSeqInStream *p = (CMixCoderSeqInStream *)pp;
294  SRes res = SZ_OK;
295  size_t remSize = *size;
296  *size = 0;
297  while (remSize > 0)
298  {
299    if (p->bufPos == p->bufSize)
300    {
301      size_t curSize;
302      p->bufPos = p->bufSize = 0;
303      if (*size != 0)
304        break;
305      curSize = FILT_BUF_SIZE;
306      RINOK(p->realStream->Read(p->realStream, p->buf, &curSize));
307      p->bufSize = (UInt32)curSize;
308    }
309    {
310      SizeT destLen = remSize;
311      SizeT srcLen = p->bufSize - p->bufPos;
312      res = MixCoder_Code(&p->mixCoder, data, &destLen, p->buf + p->bufPos, &srcLen, 0);
313      data = (void *)((Byte *)data + destLen);
314      remSize -= destLen;
315      *size += destLen;
316      p->bufPos += srcLen;
317    }
318  }
319  return res;
320}
321*/
322
323#ifdef USE_SUBBLOCK
324typedef struct
325{
326  ISeqInStream p;
327  CSubblockEnc sb;
328  UInt64 processed;
329} CSbEncInStream;
330
331void SbEncInStream_Init(CSbEncInStream *p)
332{
333  p->processed = 0;
334  SubblockEnc_Init(&p->sb);
335}
336
337static SRes SbEncInStream_Read(void *pp, void *data, size_t *size)
338{
339  CSbEncInStream *p = (CSbEncInStream *)pp;
340  SRes res = SubblockEnc_Read(&p->sb, data, size);
341  p->processed += *size;
342  return res;
343}
344#endif
345
346typedef struct
347{
348  /* CMixCoderSeqInStream inStream; */
349  CLzma2EncHandle lzma2;
350  #ifdef USE_SUBBLOCK
351  CSbEncInStream sb;
352  #endif
353  ISzAlloc *alloc;
354  ISzAlloc *bigAlloc;
355} CLzma2WithFilters;
356
357
358static void Lzma2WithFilters_Construct(CLzma2WithFilters *p, ISzAlloc *alloc, ISzAlloc *bigAlloc)
359{
360  p->alloc = alloc;
361  p->bigAlloc = bigAlloc;
362  p->lzma2 = NULL;
363  #ifdef USE_SUBBLOCK
364  p->sb.p.Read = SbEncInStream_Read;
365  SubblockEnc_Construct(&p->sb.sb, p->alloc);
366  #endif
367}
368
369static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p)
370{
371  p->lzma2 = Lzma2Enc_Create(p->alloc, p->bigAlloc);
372  if (p->lzma2 == 0)
373    return SZ_ERROR_MEM;
374  return SZ_OK;
375}
376
377static void Lzma2WithFilters_Free(CLzma2WithFilters *p)
378{
379  #ifdef USE_SUBBLOCK
380  SubblockEnc_Free(&p->sb.sb);
381  #endif
382  if (p->lzma2)
383  {
384    Lzma2Enc_Destroy(p->lzma2);
385    p->lzma2 = NULL;
386  }
387}
388
389static SRes Xz_Compress(CXzStream *xz,
390    CLzma2WithFilters *lzmaf,
391    ISeqOutStream *outStream,
392    ISeqInStream *inStream,
393    const CLzma2EncProps *lzma2Props,
394    Bool useSubblock,
395    ICompressProgress *progress)
396{
397  xz->flags = XZ_CHECK_CRC32;
398
399  RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, lzma2Props));
400  RINOK(Xz_WriteHeader(xz->flags, outStream));
401
402  {
403    CSeqCheckInStream checkInStream;
404    CSeqSizeOutStream seqSizeOutStream;
405    CXzBlock block;
406    int filterIndex = 0;
407
408    XzBlock_ClearFlags(&block);
409    XzBlock_SetNumFilters(&block, 1 + (useSubblock ? 1 : 0));
410
411    if (useSubblock)
412    {
413      CXzFilter *f = &block.filters[filterIndex++];
414      f->id = XZ_ID_Subblock;
415      f->propsSize = 0;
416    }
417
418    {
419      CXzFilter *f = &block.filters[filterIndex++];
420      f->id = XZ_ID_LZMA2;
421      f->propsSize = 1;
422      f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2);
423    }
424
425    seqSizeOutStream.p.Write = MyWrite;
426    seqSizeOutStream.realStream = outStream;
427    seqSizeOutStream.processed = 0;
428
429    RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.p));
430
431    checkInStream.p.Read = SeqCheckInStream_Read;
432    checkInStream.realStream = inStream;
433    SeqCheckInStream_Init(&checkInStream, XzFlags_GetCheckType(xz->flags));
434
435    #ifdef USE_SUBBLOCK
436    if (useSubblock)
437    {
438      lzmaf->sb.sb.inStream = &checkInStream.p;
439      SubblockEnc_Init(&lzmaf->sb.sb);
440    }
441    #endif
442
443    {
444      UInt64 packPos = seqSizeOutStream.processed;
445      SRes res = Lzma2Enc_Encode(lzmaf->lzma2, &seqSizeOutStream.p,
446        #ifdef USE_SUBBLOCK
447        useSubblock ? &lzmaf->sb.p:
448        #endif
449        &checkInStream.p,
450        progress);
451      RINOK(res);
452      block.unpackSize = checkInStream.processed;
453      block.packSize = seqSizeOutStream.processed - packPos;
454    }
455
456    {
457      unsigned padSize = 0;
458      Byte buf[128];
459      while((((unsigned)block.packSize + padSize) & 3) != 0)
460        buf[padSize++] = 0;
461      SeqCheckInStream_GetDigest(&checkInStream, buf + padSize);
462      RINOK(WriteBytes(&seqSizeOutStream.p, buf, padSize + XzFlags_GetCheckSize(xz->flags)));
463      RINOK(Xz_AddIndexRecord(xz, block.unpackSize, seqSizeOutStream.processed - padSize, &g_Alloc));
464    }
465  }
466  return Xz_WriteFooter(xz, outStream);
467}
468
469SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream,
470    const CLzma2EncProps *lzma2Props, Bool useSubblock,
471    ICompressProgress *progress)
472{
473  SRes res;
474  CXzStream xz;
475  CLzma2WithFilters lzmaf;
476  Xz_Construct(&xz);
477  Lzma2WithFilters_Construct(&lzmaf, &g_Alloc, &g_BigAlloc);
478  res = Lzma2WithFilters_Create(&lzmaf);
479  if (res == SZ_OK)
480    res = Xz_Compress(&xz, &lzmaf, outStream, inStream,
481        lzma2Props, useSubblock, progress);
482  Lzma2WithFilters_Free(&lzmaf);
483  Xz_Free(&xz, &g_Alloc);
484  return res;
485}
486
487SRes Xz_EncodeEmpty(ISeqOutStream *outStream)
488{
489  SRes res;
490  CXzStream xz;
491  Xz_Construct(&xz);
492  res = Xz_WriteHeader(xz.flags, outStream);
493  if (res == SZ_OK)
494    res = Xz_WriteFooter(&xz, outStream);
495  Xz_Free(&xz, &g_Alloc);
496  return res;
497}
498