1// 7zIn.cpp
2
3#include "StdAfx.h"
4
5#include "../../../../C/7zCrc.h"
6#include "../../../../C/CpuArch.h"
7
8#include "../../Common/StreamObjects.h"
9#include "../../Common/StreamUtils.h"
10
11#include "7zDecode.h"
12#include "7zIn.h"
13
14#define Get16(p) GetUi16(p)
15#define Get32(p) GetUi32(p)
16#define Get64(p) GetUi64(p)
17
18// define FORMAT_7Z_RECOVERY if you want to recover multivolume archives with empty StartHeader
19#ifndef _SFX
20#define FORMAT_7Z_RECOVERY
21#endif
22
23namespace NArchive {
24namespace N7z {
25
26static void BoolVector_Fill_False(CBoolVector &v, int size)
27{
28  v.Clear();
29  v.Reserve(size);
30  for (int i = 0; i < size; i++)
31    v.Add(false);
32}
33
34static bool BoolVector_GetAndSet(CBoolVector &v, UInt32 index)
35{
36  if (index >= (UInt32)v.Size())
37    return true;
38  bool res = v[index];
39  v[index] = true;
40  return res;
41}
42
43bool CFolder::CheckStructure() const
44{
45  const int kNumCodersMax = sizeof(UInt32) * 8; // don't change it
46  const int kMaskSize = sizeof(UInt32) * 8; // it must be >= kNumCodersMax
47  const int kNumBindsMax = 32;
48
49  if (Coders.Size() > kNumCodersMax || BindPairs.Size() > kNumBindsMax)
50    return false;
51
52  {
53    CBoolVector v;
54    BoolVector_Fill_False(v, BindPairs.Size() + PackStreams.Size());
55
56    int i;
57    for (i = 0; i < BindPairs.Size(); i++)
58      if (BoolVector_GetAndSet(v, BindPairs[i].InIndex))
59        return false;
60    for (i = 0; i < PackStreams.Size(); i++)
61      if (BoolVector_GetAndSet(v, PackStreams[i]))
62        return false;
63
64    BoolVector_Fill_False(v, UnpackSizes.Size());
65    for (i = 0; i < BindPairs.Size(); i++)
66      if (BoolVector_GetAndSet(v, BindPairs[i].OutIndex))
67        return false;
68  }
69
70  UInt32 mask[kMaskSize];
71  int i;
72  for (i = 0; i < kMaskSize; i++)
73    mask[i] = 0;
74
75  {
76    CIntVector inStreamToCoder, outStreamToCoder;
77    for (i = 0; i < Coders.Size(); i++)
78    {
79      CNum j;
80      const CCoderInfo &coder = Coders[i];
81      for (j = 0; j < coder.NumInStreams; j++)
82        inStreamToCoder.Add(i);
83      for (j = 0; j < coder.NumOutStreams; j++)
84        outStreamToCoder.Add(i);
85    }
86
87    for (i = 0; i < BindPairs.Size(); i++)
88    {
89      const CBindPair &bp = BindPairs[i];
90      mask[inStreamToCoder[bp.InIndex]] |= (1 << outStreamToCoder[bp.OutIndex]);
91    }
92  }
93
94  for (i = 0; i < kMaskSize; i++)
95    for (int j = 0; j < kMaskSize; j++)
96      if (((1 << j) & mask[i]) != 0)
97        mask[i] |= mask[j];
98
99  for (i = 0; i < kMaskSize; i++)
100    if (((1 << i) & mask[i]) != 0)
101      return false;
102
103  return true;
104}
105
106class CInArchiveException {};
107
108static void ThrowException() { throw CInArchiveException(); }
109static inline void ThrowEndOfData()   { ThrowException(); }
110static inline void ThrowUnsupported() { ThrowException(); }
111static inline void ThrowIncorrect()   { ThrowException(); }
112static inline void ThrowUnsupportedVersion() { ThrowException(); }
113
114/*
115class CInArchiveException
116{
117public:
118  enum CCauseType
119  {
120    kUnsupportedVersion = 0,
121    kUnsupported,
122    kIncorrect,
123    kEndOfData
124  } Cause;
125  CInArchiveException(CCauseType cause): Cause(cause) {};
126};
127
128static void ThrowException(CInArchiveException::CCauseType c) { throw CInArchiveException(c); }
129static void ThrowEndOfData()   { ThrowException(CInArchiveException::kEndOfData); }
130static void ThrowUnsupported() { ThrowException(CInArchiveException::kUnsupported); }
131static void ThrowIncorrect()   { ThrowException(CInArchiveException::kIncorrect); }
132static void ThrowUnsupportedVersion() { ThrowException(CInArchiveException::kUnsupportedVersion); }
133*/
134
135class CStreamSwitch
136{
137  CInArchive *_archive;
138  bool _needRemove;
139public:
140  CStreamSwitch(): _needRemove(false) {}
141  ~CStreamSwitch() { Remove(); }
142  void Remove();
143  void Set(CInArchive *archive, const Byte *data, size_t size);
144  void Set(CInArchive *archive, const CByteBuffer &byteBuffer);
145  void Set(CInArchive *archive, const CObjectVector<CByteBuffer> *dataVector);
146};
147
148void CStreamSwitch::Remove()
149{
150  if (_needRemove)
151  {
152    _archive->DeleteByteStream();
153    _needRemove = false;
154  }
155}
156
157void CStreamSwitch::Set(CInArchive *archive, const Byte *data, size_t size)
158{
159  Remove();
160  _archive = archive;
161  _archive->AddByteStream(data, size);
162  _needRemove = true;
163}
164
165void CStreamSwitch::Set(CInArchive *archive, const CByteBuffer &byteBuffer)
166{
167  Set(archive, byteBuffer, byteBuffer.GetCapacity());
168}
169
170void CStreamSwitch::Set(CInArchive *archive, const CObjectVector<CByteBuffer> *dataVector)
171{
172  Remove();
173  Byte external = archive->ReadByte();
174  if (external != 0)
175  {
176    int dataIndex = (int)archive->ReadNum();
177    if (dataIndex < 0 || dataIndex >= dataVector->Size())
178      ThrowIncorrect();
179    Set(archive, (*dataVector)[dataIndex]);
180  }
181}
182
183Byte CInByte2::ReadByte()
184{
185  if (_pos >= _size)
186    ThrowEndOfData();
187  return _buffer[_pos++];
188}
189
190void CInByte2::ReadBytes(Byte *data, size_t size)
191{
192  if (size > _size - _pos)
193    ThrowEndOfData();
194  for (size_t i = 0; i < size; i++)
195    data[i] = _buffer[_pos++];
196}
197
198void CInByte2::SkipData(UInt64 size)
199{
200  if (size > _size - _pos)
201    ThrowEndOfData();
202  _pos += (size_t)size;
203}
204
205void CInByte2::SkipData()
206{
207  SkipData(ReadNumber());
208}
209
210UInt64 CInByte2::ReadNumber()
211{
212  if (_pos >= _size)
213    ThrowEndOfData();
214  Byte firstByte = _buffer[_pos++];
215  Byte mask = 0x80;
216  UInt64 value = 0;
217  for (int i = 0; i < 8; i++)
218  {
219    if ((firstByte & mask) == 0)
220    {
221      UInt64 highPart = firstByte & (mask - 1);
222      value += (highPart << (i * 8));
223      return value;
224    }
225    if (_pos >= _size)
226      ThrowEndOfData();
227    value |= ((UInt64)_buffer[_pos++] << (8 * i));
228    mask >>= 1;
229  }
230  return value;
231}
232
233CNum CInByte2::ReadNum()
234{
235  UInt64 value = ReadNumber();
236  if (value > kNumMax)
237    ThrowUnsupported();
238  return (CNum)value;
239}
240
241UInt32 CInByte2::ReadUInt32()
242{
243  if (_pos + 4 > _size)
244    ThrowEndOfData();
245  UInt32 res = Get32(_buffer + _pos);
246  _pos += 4;
247  return res;
248}
249
250UInt64 CInByte2::ReadUInt64()
251{
252  if (_pos + 8 > _size)
253    ThrowEndOfData();
254  UInt64 res = Get64(_buffer + _pos);
255  _pos += 8;
256  return res;
257}
258
259void CInByte2::ReadString(UString &s)
260{
261  const Byte *buf = _buffer + _pos;
262  size_t rem = (_size - _pos) / 2 * 2;
263  {
264    size_t i;
265    for (i = 0; i < rem; i += 2)
266      if (buf[i] == 0 && buf[i + 1] == 0)
267        break;
268    if (i == rem)
269      ThrowEndOfData();
270    rem = i;
271  }
272  int len = (int)(rem / 2);
273  if (len < 0 || (size_t)len * 2 != rem)
274    ThrowUnsupported();
275  wchar_t *p = s.GetBuffer(len);
276  int i;
277  for (i = 0; i < len; i++, buf += 2)
278    p[i] = (wchar_t)Get16(buf);
279  s.ReleaseBuffer(len);
280  _pos += rem + 2;
281}
282
283static inline bool TestSignature(const Byte *p)
284{
285  for (int i = 0; i < kSignatureSize; i++)
286    if (p[i] != kSignature[i])
287      return false;
288  return CrcCalc(p + 12, 20) == GetUi32(p + 8);
289}
290
291#ifdef FORMAT_7Z_RECOVERY
292static inline bool TestSignature2(const Byte *p)
293{
294  int i;
295  for (i = 0; i < kSignatureSize; i++)
296    if (p[i] != kSignature[i])
297      return false;
298  if (CrcCalc(p + 12, 20) == GetUi32(p + 8))
299    return true;
300  for (i = 8; i < kHeaderSize; i++)
301    if (p[i] != 0)
302      return false;
303  return (p[6] != 0 || p[7] != 0);
304}
305#else
306#define TestSignature2(p) TestSignature(p)
307#endif
308
309HRESULT CInArchive::FindAndReadSignature(IInStream *stream, const UInt64 *searchHeaderSizeLimit)
310{
311  RINOK(ReadStream_FALSE(stream, _header, kHeaderSize));
312
313  if (TestSignature2(_header))
314    return S_OK;
315
316  CByteBuffer byteBuffer;
317  const UInt32 kBufferSize = (1 << 16);
318  byteBuffer.SetCapacity(kBufferSize);
319  Byte *buffer = byteBuffer;
320  UInt32 numPrevBytes = kHeaderSize;
321  memcpy(buffer, _header, kHeaderSize);
322  UInt64 curTestPos = _arhiveBeginStreamPosition;
323  for (;;)
324  {
325    if (searchHeaderSizeLimit != NULL)
326      if (curTestPos - _arhiveBeginStreamPosition > *searchHeaderSizeLimit)
327        break;
328    do
329    {
330      UInt32 numReadBytes = kBufferSize - numPrevBytes;
331      UInt32 processedSize;
332      RINOK(stream->Read(buffer + numPrevBytes, numReadBytes, &processedSize));
333      numPrevBytes += processedSize;
334      if (processedSize == 0)
335        return S_FALSE;
336    }
337    while (numPrevBytes <= kHeaderSize);
338    UInt32 numTests = numPrevBytes - kHeaderSize;
339    for (UInt32 pos = 0; pos < numTests; pos++)
340    {
341      for (; buffer[pos] != '7' && pos < numTests; pos++);
342      if (pos == numTests)
343        break;
344      if (TestSignature(buffer + pos))
345      {
346        memcpy(_header, buffer + pos, kHeaderSize);
347        curTestPos += pos;
348        _arhiveBeginStreamPosition = curTestPos;
349        return stream->Seek(curTestPos + kHeaderSize, STREAM_SEEK_SET, NULL);
350      }
351    }
352    curTestPos += numTests;
353    numPrevBytes -= numTests;
354    memmove(buffer, buffer + numTests, numPrevBytes);
355  }
356  return S_FALSE;
357}
358
359// S_FALSE means that file is not archive
360HRESULT CInArchive::Open(IInStream *stream, const UInt64 *searchHeaderSizeLimit)
361{
362  HeadersSize = 0;
363  Close();
364  RINOK(stream->Seek(0, STREAM_SEEK_CUR, &_arhiveBeginStreamPosition))
365  RINOK(FindAndReadSignature(stream, searchHeaderSizeLimit));
366  _stream = stream;
367  return S_OK;
368}
369
370void CInArchive::Close()
371{
372  _stream.Release();
373}
374
375void CInArchive::ReadArchiveProperties(CInArchiveInfo & /* archiveInfo */)
376{
377  for (;;)
378  {
379    if (ReadID() == NID::kEnd)
380      break;
381    SkipData();
382  }
383}
384
385void CInArchive::GetNextFolderItem(CFolder &folder)
386{
387  CNum numCoders = ReadNum();
388
389  folder.Coders.Clear();
390  folder.Coders.Reserve((int)numCoders);
391  CNum numInStreams = 0;
392  CNum numOutStreams = 0;
393  CNum i;
394  for (i = 0; i < numCoders; i++)
395  {
396    folder.Coders.Add(CCoderInfo());
397    CCoderInfo &coder = folder.Coders.Back();
398
399    {
400      Byte mainByte = ReadByte();
401      int idSize = (mainByte & 0xF);
402      Byte longID[15];
403      ReadBytes(longID, idSize);
404      if (idSize > 8)
405        ThrowUnsupported();
406      UInt64 id = 0;
407      for (int j = 0; j < idSize; j++)
408        id |= (UInt64)longID[idSize - 1 - j] << (8 * j);
409      coder.MethodID = id;
410
411      if ((mainByte & 0x10) != 0)
412      {
413        coder.NumInStreams = ReadNum();
414        coder.NumOutStreams = ReadNum();
415      }
416      else
417      {
418        coder.NumInStreams = 1;
419        coder.NumOutStreams = 1;
420      }
421      if ((mainByte & 0x20) != 0)
422      {
423        CNum propsSize = ReadNum();
424        coder.Props.SetCapacity((size_t)propsSize);
425        ReadBytes((Byte *)coder.Props, (size_t)propsSize);
426      }
427      if ((mainByte & 0x80) != 0)
428        ThrowUnsupported();
429    }
430    numInStreams += coder.NumInStreams;
431    numOutStreams += coder.NumOutStreams;
432  }
433
434  CNum numBindPairs = numOutStreams - 1;
435  folder.BindPairs.Clear();
436  folder.BindPairs.Reserve(numBindPairs);
437  for (i = 0; i < numBindPairs; i++)
438  {
439    CBindPair bp;
440    bp.InIndex = ReadNum();
441    bp.OutIndex = ReadNum();
442    folder.BindPairs.Add(bp);
443  }
444
445  if (numInStreams < numBindPairs)
446    ThrowUnsupported();
447  CNum numPackStreams = numInStreams - numBindPairs;
448  folder.PackStreams.Reserve(numPackStreams);
449  if (numPackStreams == 1)
450  {
451    for (i = 0; i < numInStreams; i++)
452      if (folder.FindBindPairForInStream(i) < 0)
453      {
454        folder.PackStreams.Add(i);
455        break;
456      }
457    if (folder.PackStreams.Size() != 1)
458      ThrowUnsupported();
459  }
460  else
461    for (i = 0; i < numPackStreams; i++)
462      folder.PackStreams.Add(ReadNum());
463}
464
465void CInArchive::WaitAttribute(UInt64 attribute)
466{
467  for (;;)
468  {
469    UInt64 type = ReadID();
470    if (type == attribute)
471      return;
472    if (type == NID::kEnd)
473      ThrowIncorrect();
474    SkipData();
475  }
476}
477
478void CInArchive::ReadHashDigests(int numItems,
479    CBoolVector &digestsDefined,
480    CRecordVector<UInt32> &digests)
481{
482  ReadBoolVector2(numItems, digestsDefined);
483  digests.Clear();
484  digests.Reserve(numItems);
485  for (int i = 0; i < numItems; i++)
486  {
487    UInt32 crc = 0;
488    if (digestsDefined[i])
489      crc = ReadUInt32();
490    digests.Add(crc);
491  }
492}
493
494void CInArchive::ReadPackInfo(
495    UInt64 &dataOffset,
496    CRecordVector<UInt64> &packSizes,
497    CBoolVector &packCRCsDefined,
498    CRecordVector<UInt32> &packCRCs)
499{
500  dataOffset = ReadNumber();
501  CNum numPackStreams = ReadNum();
502
503  WaitAttribute(NID::kSize);
504  packSizes.Clear();
505  packSizes.Reserve(numPackStreams);
506  for (CNum i = 0; i < numPackStreams; i++)
507    packSizes.Add(ReadNumber());
508
509  UInt64 type;
510  for (;;)
511  {
512    type = ReadID();
513    if (type == NID::kEnd)
514      break;
515    if (type == NID::kCRC)
516    {
517      ReadHashDigests(numPackStreams, packCRCsDefined, packCRCs);
518      continue;
519    }
520    SkipData();
521  }
522  if (packCRCsDefined.IsEmpty())
523  {
524    BoolVector_Fill_False(packCRCsDefined, numPackStreams);
525    packCRCs.Reserve(numPackStreams);
526    packCRCs.Clear();
527    for (CNum i = 0; i < numPackStreams; i++)
528      packCRCs.Add(0);
529  }
530}
531
532void CInArchive::ReadUnpackInfo(
533    const CObjectVector<CByteBuffer> *dataVector,
534    CObjectVector<CFolder> &folders)
535{
536  WaitAttribute(NID::kFolder);
537  CNum numFolders = ReadNum();
538
539  {
540    CStreamSwitch streamSwitch;
541    streamSwitch.Set(this, dataVector);
542    folders.Clear();
543    folders.Reserve(numFolders);
544    for (CNum i = 0; i < numFolders; i++)
545    {
546      folders.Add(CFolder());
547      GetNextFolderItem(folders.Back());
548    }
549  }
550
551  WaitAttribute(NID::kCodersUnpackSize);
552
553  CNum i;
554  for (i = 0; i < numFolders; i++)
555  {
556    CFolder &folder = folders[i];
557    CNum numOutStreams = folder.GetNumOutStreams();
558    folder.UnpackSizes.Reserve(numOutStreams);
559    for (CNum j = 0; j < numOutStreams; j++)
560      folder.UnpackSizes.Add(ReadNumber());
561  }
562
563  for (;;)
564  {
565    UInt64 type = ReadID();
566    if (type == NID::kEnd)
567      return;
568    if (type == NID::kCRC)
569    {
570      CBoolVector crcsDefined;
571      CRecordVector<UInt32> crcs;
572      ReadHashDigests(numFolders, crcsDefined, crcs);
573      for (i = 0; i < numFolders; i++)
574      {
575        CFolder &folder = folders[i];
576        folder.UnpackCRCDefined = crcsDefined[i];
577        folder.UnpackCRC = crcs[i];
578      }
579      continue;
580    }
581    SkipData();
582  }
583}
584
585void CInArchive::ReadSubStreamsInfo(
586    const CObjectVector<CFolder> &folders,
587    CRecordVector<CNum> &numUnpackStreamsInFolders,
588    CRecordVector<UInt64> &unpackSizes,
589    CBoolVector &digestsDefined,
590    CRecordVector<UInt32> &digests)
591{
592  numUnpackStreamsInFolders.Clear();
593  numUnpackStreamsInFolders.Reserve(folders.Size());
594  UInt64 type;
595  for (;;)
596  {
597    type = ReadID();
598    if (type == NID::kNumUnpackStream)
599    {
600      for (int i = 0; i < folders.Size(); i++)
601        numUnpackStreamsInFolders.Add(ReadNum());
602      continue;
603    }
604    if (type == NID::kCRC || type == NID::kSize)
605      break;
606    if (type == NID::kEnd)
607      break;
608    SkipData();
609  }
610
611  if (numUnpackStreamsInFolders.IsEmpty())
612    for (int i = 0; i < folders.Size(); i++)
613      numUnpackStreamsInFolders.Add(1);
614
615  int i;
616  for (i = 0; i < numUnpackStreamsInFolders.Size(); i++)
617  {
618    // v3.13 incorrectly worked with empty folders
619    // v4.07: we check that folder is empty
620    CNum numSubstreams = numUnpackStreamsInFolders[i];
621    if (numSubstreams == 0)
622      continue;
623    UInt64 sum = 0;
624    for (CNum j = 1; j < numSubstreams; j++)
625      if (type == NID::kSize)
626      {
627        UInt64 size = ReadNumber();
628        unpackSizes.Add(size);
629        sum += size;
630      }
631    unpackSizes.Add(folders[i].GetUnpackSize() - sum);
632  }
633  if (type == NID::kSize)
634    type = ReadID();
635
636  int numDigests = 0;
637  int numDigestsTotal = 0;
638  for (i = 0; i < folders.Size(); i++)
639  {
640    CNum numSubstreams = numUnpackStreamsInFolders[i];
641    if (numSubstreams != 1 || !folders[i].UnpackCRCDefined)
642      numDigests += numSubstreams;
643    numDigestsTotal += numSubstreams;
644  }
645
646  for (;;)
647  {
648    if (type == NID::kCRC)
649    {
650      CBoolVector digestsDefined2;
651      CRecordVector<UInt32> digests2;
652      ReadHashDigests(numDigests, digestsDefined2, digests2);
653      int digestIndex = 0;
654      for (i = 0; i < folders.Size(); i++)
655      {
656        CNum numSubstreams = numUnpackStreamsInFolders[i];
657        const CFolder &folder = folders[i];
658        if (numSubstreams == 1 && folder.UnpackCRCDefined)
659        {
660          digestsDefined.Add(true);
661          digests.Add(folder.UnpackCRC);
662        }
663        else
664          for (CNum j = 0; j < numSubstreams; j++, digestIndex++)
665          {
666            digestsDefined.Add(digestsDefined2[digestIndex]);
667            digests.Add(digests2[digestIndex]);
668          }
669      }
670    }
671    else if (type == NID::kEnd)
672    {
673      if (digestsDefined.IsEmpty())
674      {
675        BoolVector_Fill_False(digestsDefined, numDigestsTotal);
676        digests.Clear();
677        for (int i = 0; i < numDigestsTotal; i++)
678          digests.Add(0);
679      }
680      return;
681    }
682    else
683      SkipData();
684    type = ReadID();
685  }
686}
687
688void CInArchive::ReadStreamsInfo(
689    const CObjectVector<CByteBuffer> *dataVector,
690    UInt64 &dataOffset,
691    CRecordVector<UInt64> &packSizes,
692    CBoolVector &packCRCsDefined,
693    CRecordVector<UInt32> &packCRCs,
694    CObjectVector<CFolder> &folders,
695    CRecordVector<CNum> &numUnpackStreamsInFolders,
696    CRecordVector<UInt64> &unpackSizes,
697    CBoolVector &digestsDefined,
698    CRecordVector<UInt32> &digests)
699{
700  for (;;)
701  {
702    UInt64 type = ReadID();
703    if (type > ((UInt32)1 << 30))
704      ThrowIncorrect();
705    switch((UInt32)type)
706    {
707      case NID::kEnd:
708        return;
709      case NID::kPackInfo:
710      {
711        ReadPackInfo(dataOffset, packSizes, packCRCsDefined, packCRCs);
712        break;
713      }
714      case NID::kUnpackInfo:
715      {
716        ReadUnpackInfo(dataVector, folders);
717        break;
718      }
719      case NID::kSubStreamsInfo:
720      {
721        ReadSubStreamsInfo(folders, numUnpackStreamsInFolders,
722            unpackSizes, digestsDefined, digests);
723        break;
724      }
725      default:
726        ThrowIncorrect();
727    }
728  }
729}
730
731void CInArchive::ReadBoolVector(int numItems, CBoolVector &v)
732{
733  v.Clear();
734  v.Reserve(numItems);
735  Byte b = 0;
736  Byte mask = 0;
737  for (int i = 0; i < numItems; i++)
738  {
739    if (mask == 0)
740    {
741      b = ReadByte();
742      mask = 0x80;
743    }
744    v.Add((b & mask) != 0);
745    mask >>= 1;
746  }
747}
748
749void CInArchive::ReadBoolVector2(int numItems, CBoolVector &v)
750{
751  Byte allAreDefined = ReadByte();
752  if (allAreDefined == 0)
753  {
754    ReadBoolVector(numItems, v);
755    return;
756  }
757  v.Clear();
758  v.Reserve(numItems);
759  for (int i = 0; i < numItems; i++)
760    v.Add(true);
761}
762
763void CInArchive::ReadUInt64DefVector(const CObjectVector<CByteBuffer> &dataVector,
764    CUInt64DefVector &v, int numFiles)
765{
766  ReadBoolVector2(numFiles, v.Defined);
767
768  CStreamSwitch streamSwitch;
769  streamSwitch.Set(this, &dataVector);
770  v.Values.Reserve(numFiles);
771
772  for (int i = 0; i < numFiles; i++)
773  {
774    UInt64 t = 0;
775    if (v.Defined[i])
776      t = ReadUInt64();
777    v.Values.Add(t);
778  }
779}
780
781HRESULT CInArchive::ReadAndDecodePackedStreams(
782    DECL_EXTERNAL_CODECS_LOC_VARS
783    UInt64 baseOffset,
784    UInt64 &dataOffset, CObjectVector<CByteBuffer> &dataVector
785    #ifndef _NO_CRYPTO
786    , ICryptoGetTextPassword *getTextPassword, bool &passwordIsDefined
787    #endif
788    )
789{
790  CRecordVector<UInt64> packSizes;
791  CBoolVector packCRCsDefined;
792  CRecordVector<UInt32> packCRCs;
793  CObjectVector<CFolder> folders;
794
795  CRecordVector<CNum> numUnpackStreamsInFolders;
796  CRecordVector<UInt64> unpackSizes;
797  CBoolVector digestsDefined;
798  CRecordVector<UInt32> digests;
799
800  ReadStreamsInfo(NULL,
801    dataOffset,
802    packSizes,
803    packCRCsDefined,
804    packCRCs,
805    folders,
806    numUnpackStreamsInFolders,
807    unpackSizes,
808    digestsDefined,
809    digests);
810
811  // db.ArchiveInfo.DataStartPosition2 += db.ArchiveInfo.StartPositionAfterHeader;
812
813  CNum packIndex = 0;
814  CDecoder decoder(
815    #ifdef _ST_MODE
816    false
817    #else
818    true
819    #endif
820    );
821  UInt64 dataStartPos = baseOffset + dataOffset;
822  for (int i = 0; i < folders.Size(); i++)
823  {
824    const CFolder &folder = folders[i];
825    dataVector.Add(CByteBuffer());
826    CByteBuffer &data = dataVector.Back();
827    UInt64 unpackSize64 = folder.GetUnpackSize();
828    size_t unpackSize = (size_t)unpackSize64;
829    if (unpackSize != unpackSize64)
830      ThrowUnsupported();
831    data.SetCapacity(unpackSize);
832
833    CBufPtrSeqOutStream *outStreamSpec = new CBufPtrSeqOutStream;
834    CMyComPtr<ISequentialOutStream> outStream = outStreamSpec;
835    outStreamSpec->Init(data, unpackSize);
836
837    HRESULT result = decoder.Decode(
838      EXTERNAL_CODECS_LOC_VARS
839      _stream, dataStartPos,
840      &packSizes[packIndex], folder, outStream, NULL
841      #ifndef _NO_CRYPTO
842      , getTextPassword, passwordIsDefined
843      #endif
844      #if !defined(_7ZIP_ST) && !defined(_SFX)
845      , false, 1
846      #endif
847      );
848    RINOK(result);
849
850    if (folder.UnpackCRCDefined)
851      if (CrcCalc(data, unpackSize) != folder.UnpackCRC)
852        ThrowIncorrect();
853    for (int j = 0; j < folder.PackStreams.Size(); j++)
854    {
855      UInt64 packSize = packSizes[packIndex++];
856      dataStartPos += packSize;
857      HeadersSize += packSize;
858    }
859  }
860  return S_OK;
861}
862
863HRESULT CInArchive::ReadHeader(
864    DECL_EXTERNAL_CODECS_LOC_VARS
865    CArchiveDatabaseEx &db
866    #ifndef _NO_CRYPTO
867    , ICryptoGetTextPassword *getTextPassword, bool &passwordIsDefined
868    #endif
869    )
870{
871  UInt64 type = ReadID();
872
873  if (type == NID::kArchiveProperties)
874  {
875    ReadArchiveProperties(db.ArchiveInfo);
876    type = ReadID();
877  }
878
879  CObjectVector<CByteBuffer> dataVector;
880
881  if (type == NID::kAdditionalStreamsInfo)
882  {
883    HRESULT result = ReadAndDecodePackedStreams(
884        EXTERNAL_CODECS_LOC_VARS
885        db.ArchiveInfo.StartPositionAfterHeader,
886        db.ArchiveInfo.DataStartPosition2,
887        dataVector
888        #ifndef _NO_CRYPTO
889        , getTextPassword, passwordIsDefined
890        #endif
891        );
892    RINOK(result);
893    db.ArchiveInfo.DataStartPosition2 += db.ArchiveInfo.StartPositionAfterHeader;
894    type = ReadID();
895  }
896
897  CRecordVector<UInt64> unpackSizes;
898  CBoolVector digestsDefined;
899  CRecordVector<UInt32> digests;
900
901  if (type == NID::kMainStreamsInfo)
902  {
903    ReadStreamsInfo(&dataVector,
904        db.ArchiveInfo.DataStartPosition,
905        db.PackSizes,
906        db.PackCRCsDefined,
907        db.PackCRCs,
908        db.Folders,
909        db.NumUnpackStreamsVector,
910        unpackSizes,
911        digestsDefined,
912        digests);
913    db.ArchiveInfo.DataStartPosition += db.ArchiveInfo.StartPositionAfterHeader;
914    type = ReadID();
915  }
916  else
917  {
918    for (int i = 0; i < db.Folders.Size(); i++)
919    {
920      db.NumUnpackStreamsVector.Add(1);
921      CFolder &folder = db.Folders[i];
922      unpackSizes.Add(folder.GetUnpackSize());
923      digestsDefined.Add(folder.UnpackCRCDefined);
924      digests.Add(folder.UnpackCRC);
925    }
926  }
927
928  db.Files.Clear();
929
930  if (type == NID::kEnd)
931    return S_OK;
932  if (type != NID::kFilesInfo)
933    ThrowIncorrect();
934
935  CNum numFiles = ReadNum();
936  db.Files.Reserve(numFiles);
937  CNum i;
938  for (i = 0; i < numFiles; i++)
939    db.Files.Add(CFileItem());
940
941  db.ArchiveInfo.FileInfoPopIDs.Add(NID::kSize);
942  if (!db.PackSizes.IsEmpty())
943    db.ArchiveInfo.FileInfoPopIDs.Add(NID::kPackInfo);
944  if (numFiles > 0  && !digests.IsEmpty())
945    db.ArchiveInfo.FileInfoPopIDs.Add(NID::kCRC);
946
947  CBoolVector emptyStreamVector;
948  BoolVector_Fill_False(emptyStreamVector, (int)numFiles);
949  CBoolVector emptyFileVector;
950  CBoolVector antiFileVector;
951  CNum numEmptyStreams = 0;
952
953  for (;;)
954  {
955    UInt64 type = ReadID();
956    if (type == NID::kEnd)
957      break;
958    UInt64 size = ReadNumber();
959    size_t ppp = _inByteBack->_pos;
960    bool addPropIdToList = true;
961    bool isKnownType = true;
962    if (type > ((UInt32)1 << 30))
963      isKnownType = false;
964    else switch((UInt32)type)
965    {
966      case NID::kName:
967      {
968        CStreamSwitch streamSwitch;
969        streamSwitch.Set(this, &dataVector);
970        for (int i = 0; i < db.Files.Size(); i++)
971          _inByteBack->ReadString(db.Files[i].Name);
972        break;
973      }
974      case NID::kWinAttributes:
975      {
976        CBoolVector boolVector;
977        ReadBoolVector2(db.Files.Size(), boolVector);
978        CStreamSwitch streamSwitch;
979        streamSwitch.Set(this, &dataVector);
980        for (i = 0; i < numFiles; i++)
981        {
982          CFileItem &file = db.Files[i];
983          file.AttribDefined = boolVector[i];
984          if (file.AttribDefined)
985            file.Attrib = ReadUInt32();
986        }
987        break;
988      }
989      case NID::kEmptyStream:
990      {
991        ReadBoolVector(numFiles, emptyStreamVector);
992        for (i = 0; i < (CNum)emptyStreamVector.Size(); i++)
993          if (emptyStreamVector[i])
994            numEmptyStreams++;
995
996        BoolVector_Fill_False(emptyFileVector, numEmptyStreams);
997        BoolVector_Fill_False(antiFileVector, numEmptyStreams);
998
999        break;
1000      }
1001      case NID::kEmptyFile:  ReadBoolVector(numEmptyStreams, emptyFileVector); break;
1002      case NID::kAnti:  ReadBoolVector(numEmptyStreams, antiFileVector); break;
1003      case NID::kStartPos:  ReadUInt64DefVector(dataVector, db.StartPos, (int)numFiles); break;
1004      case NID::kCTime:  ReadUInt64DefVector(dataVector, db.CTime, (int)numFiles); break;
1005      case NID::kATime:  ReadUInt64DefVector(dataVector, db.ATime, (int)numFiles); break;
1006      case NID::kMTime:  ReadUInt64DefVector(dataVector, db.MTime, (int)numFiles); break;
1007      case NID::kDummy:
1008      {
1009        for (UInt64 j = 0; j < size; j++)
1010          if (ReadByte() != 0)
1011            ThrowIncorrect();
1012        addPropIdToList = false;
1013        break;
1014      }
1015      default:
1016        addPropIdToList = isKnownType = false;
1017    }
1018    if (isKnownType)
1019    {
1020      if(addPropIdToList)
1021        db.ArchiveInfo.FileInfoPopIDs.Add(type);
1022    }
1023    else
1024      SkipData(size);
1025    bool checkRecordsSize = (db.ArchiveInfo.Version.Major > 0 ||
1026        db.ArchiveInfo.Version.Minor > 2);
1027    if (checkRecordsSize && _inByteBack->_pos - ppp != size)
1028      ThrowIncorrect();
1029  }
1030
1031  CNum emptyFileIndex = 0;
1032  CNum sizeIndex = 0;
1033
1034  CNum numAntiItems = 0;
1035  for (i = 0; i < numEmptyStreams; i++)
1036    if (antiFileVector[i])
1037      numAntiItems++;
1038
1039  for (i = 0; i < numFiles; i++)
1040  {
1041    CFileItem &file = db.Files[i];
1042    bool isAnti;
1043    file.HasStream = !emptyStreamVector[i];
1044    if (file.HasStream)
1045    {
1046      file.IsDir = false;
1047      isAnti = false;
1048      file.Size = unpackSizes[sizeIndex];
1049      file.Crc = digests[sizeIndex];
1050      file.CrcDefined = digestsDefined[sizeIndex];
1051      sizeIndex++;
1052    }
1053    else
1054    {
1055      file.IsDir = !emptyFileVector[emptyFileIndex];
1056      isAnti = antiFileVector[emptyFileIndex];
1057      emptyFileIndex++;
1058      file.Size = 0;
1059      file.CrcDefined = false;
1060    }
1061    if (numAntiItems != 0)
1062      db.IsAnti.Add(isAnti);
1063  }
1064  return S_OK;
1065}
1066
1067
1068void CArchiveDatabaseEx::FillFolderStartPackStream()
1069{
1070  FolderStartPackStreamIndex.Clear();
1071  FolderStartPackStreamIndex.Reserve(Folders.Size());
1072  CNum startPos = 0;
1073  for (int i = 0; i < Folders.Size(); i++)
1074  {
1075    FolderStartPackStreamIndex.Add(startPos);
1076    startPos += (CNum)Folders[i].PackStreams.Size();
1077  }
1078}
1079
1080void CArchiveDatabaseEx::FillStartPos()
1081{
1082  PackStreamStartPositions.Clear();
1083  PackStreamStartPositions.Reserve(PackSizes.Size());
1084  UInt64 startPos = 0;
1085  for (int i = 0; i < PackSizes.Size(); i++)
1086  {
1087    PackStreamStartPositions.Add(startPos);
1088    startPos += PackSizes[i];
1089  }
1090}
1091
1092void CArchiveDatabaseEx::FillFolderStartFileIndex()
1093{
1094  FolderStartFileIndex.Clear();
1095  FolderStartFileIndex.Reserve(Folders.Size());
1096  FileIndexToFolderIndexMap.Clear();
1097  FileIndexToFolderIndexMap.Reserve(Files.Size());
1098
1099  int folderIndex = 0;
1100  CNum indexInFolder = 0;
1101  for (int i = 0; i < Files.Size(); i++)
1102  {
1103    const CFileItem &file = Files[i];
1104    bool emptyStream = !file.HasStream;
1105    if (emptyStream && indexInFolder == 0)
1106    {
1107      FileIndexToFolderIndexMap.Add(kNumNoIndex);
1108      continue;
1109    }
1110    if (indexInFolder == 0)
1111    {
1112      // v3.13 incorrectly worked with empty folders
1113      // v4.07: Loop for skipping empty folders
1114      for (;;)
1115      {
1116        if (folderIndex >= Folders.Size())
1117          ThrowIncorrect();
1118        FolderStartFileIndex.Add(i); // check it
1119        if (NumUnpackStreamsVector[folderIndex] != 0)
1120          break;
1121        folderIndex++;
1122      }
1123    }
1124    FileIndexToFolderIndexMap.Add(folderIndex);
1125    if (emptyStream)
1126      continue;
1127    indexInFolder++;
1128    if (indexInFolder >= NumUnpackStreamsVector[folderIndex])
1129    {
1130      folderIndex++;
1131      indexInFolder = 0;
1132    }
1133  }
1134}
1135
1136HRESULT CInArchive::ReadDatabase2(
1137    DECL_EXTERNAL_CODECS_LOC_VARS
1138    CArchiveDatabaseEx &db
1139    #ifndef _NO_CRYPTO
1140    , ICryptoGetTextPassword *getTextPassword, bool &passwordIsDefined
1141    #endif
1142    )
1143{
1144  db.Clear();
1145  db.ArchiveInfo.StartPosition = _arhiveBeginStreamPosition;
1146
1147  db.ArchiveInfo.Version.Major = _header[6];
1148  db.ArchiveInfo.Version.Minor = _header[7];
1149
1150  if (db.ArchiveInfo.Version.Major != kMajorVersion)
1151    ThrowUnsupportedVersion();
1152
1153  UInt32 crcFromArchive = Get32(_header + 8);
1154  UInt64 nextHeaderOffset = Get64(_header + 0xC);
1155  UInt64 nextHeaderSize = Get64(_header + 0x14);
1156  UInt32 nextHeaderCRC = Get32(_header + 0x1C);
1157  UInt32 crc = CrcCalc(_header + 0xC, 20);
1158
1159  #ifdef FORMAT_7Z_RECOVERY
1160  if (crcFromArchive == 0 && nextHeaderOffset == 0 && nextHeaderSize == 0 && nextHeaderCRC == 0)
1161  {
1162    UInt64 cur, cur2;
1163    RINOK(_stream->Seek(0, STREAM_SEEK_CUR, &cur));
1164    const int kCheckSize = 500;
1165    Byte buf[kCheckSize];
1166    RINOK(_stream->Seek(0, STREAM_SEEK_END, &cur2));
1167    int checkSize = kCheckSize;
1168    if (cur2 - cur < kCheckSize)
1169      checkSize = (int)(cur2 - cur);
1170    RINOK(_stream->Seek(-checkSize, STREAM_SEEK_END, &cur2));
1171
1172    RINOK(ReadStream_FALSE(_stream, buf, (size_t)checkSize));
1173
1174    int i;
1175    for (i = (int)checkSize - 2; i >= 0; i--)
1176      if (buf[i] == 0x17 && buf[i + 1] == 0x6 || buf[i] == 0x01 && buf[i + 1] == 0x04)
1177        break;
1178    if (i < 0)
1179      return S_FALSE;
1180    nextHeaderSize = checkSize - i;
1181    nextHeaderOffset = cur2 - cur + i;
1182    nextHeaderCRC = CrcCalc(buf + i, (size_t)nextHeaderSize);
1183    RINOK(_stream->Seek(cur, STREAM_SEEK_SET, NULL));
1184  }
1185  else
1186  #endif
1187  {
1188    if (crc != crcFromArchive)
1189      ThrowIncorrect();
1190  }
1191
1192  db.ArchiveInfo.StartPositionAfterHeader = _arhiveBeginStreamPosition + kHeaderSize;
1193
1194  if (nextHeaderSize == 0)
1195    return S_OK;
1196
1197  if (nextHeaderSize > (UInt64)0xFFFFFFFF)
1198    return S_FALSE;
1199
1200  if ((Int64)nextHeaderOffset < 0)
1201    return S_FALSE;
1202
1203  RINOK(_stream->Seek(nextHeaderOffset, STREAM_SEEK_CUR, NULL));
1204
1205  CByteBuffer buffer2;
1206  buffer2.SetCapacity((size_t)nextHeaderSize);
1207
1208  RINOK(ReadStream_FALSE(_stream, buffer2, (size_t)nextHeaderSize));
1209  HeadersSize += kHeaderSize + nextHeaderSize;
1210  db.PhySize = kHeaderSize + nextHeaderOffset + nextHeaderSize;
1211
1212  if (CrcCalc(buffer2, (UInt32)nextHeaderSize) != nextHeaderCRC)
1213    ThrowIncorrect();
1214
1215  CStreamSwitch streamSwitch;
1216  streamSwitch.Set(this, buffer2);
1217
1218  CObjectVector<CByteBuffer> dataVector;
1219
1220  UInt64 type = ReadID();
1221  if (type != NID::kHeader)
1222  {
1223    if (type != NID::kEncodedHeader)
1224      ThrowIncorrect();
1225    HRESULT result = ReadAndDecodePackedStreams(
1226        EXTERNAL_CODECS_LOC_VARS
1227        db.ArchiveInfo.StartPositionAfterHeader,
1228        db.ArchiveInfo.DataStartPosition2,
1229        dataVector
1230        #ifndef _NO_CRYPTO
1231        , getTextPassword, passwordIsDefined
1232        #endif
1233        );
1234    RINOK(result);
1235    if (dataVector.Size() == 0)
1236      return S_OK;
1237    if (dataVector.Size() > 1)
1238      ThrowIncorrect();
1239    streamSwitch.Remove();
1240    streamSwitch.Set(this, dataVector.Front());
1241    if (ReadID() != NID::kHeader)
1242      ThrowIncorrect();
1243  }
1244
1245  db.HeadersSize = HeadersSize;
1246
1247  return ReadHeader(
1248    EXTERNAL_CODECS_LOC_VARS
1249    db
1250    #ifndef _NO_CRYPTO
1251    , getTextPassword, passwordIsDefined
1252    #endif
1253    );
1254}
1255
1256HRESULT CInArchive::ReadDatabase(
1257    DECL_EXTERNAL_CODECS_LOC_VARS
1258    CArchiveDatabaseEx &db
1259    #ifndef _NO_CRYPTO
1260    , ICryptoGetTextPassword *getTextPassword, bool &passwordIsDefined
1261    #endif
1262    )
1263{
1264  try
1265  {
1266    return ReadDatabase2(
1267      EXTERNAL_CODECS_LOC_VARS db
1268      #ifndef _NO_CRYPTO
1269      , getTextPassword, passwordIsDefined
1270      #endif
1271      );
1272  }
1273  catch(CInArchiveException &) { return S_FALSE; }
1274}
1275
1276}}
1277