1// 7zUpdate.cpp
2
3#include "StdAfx.h"
4
5#include "../../../../C/CpuArch.h"
6
7#include "../../Common/LimitedStreams.h"
8#include "../../Common/ProgressUtils.h"
9
10#include "../../Common/CreateCoder.h"
11
12#include "../../Compress/CopyCoder.h"
13
14#include "../Common/ItemNameUtils.h"
15#include "../Common/OutStreamWithCRC.h"
16
17#include "7zDecode.h"
18#include "7zEncode.h"
19#include "7zFolderInStream.h"
20#include "7zHandler.h"
21#include "7zOut.h"
22#include "7zUpdate.h"
23
24namespace NArchive {
25namespace N7z {
26
27static const UInt64 k_LZMA = 0x030101;
28static const UInt64 k_BCJ  = 0x03030103;
29static const UInt64 k_BCJ2 = 0x0303011B;
30
31static const wchar_t *kMatchFinderForBCJ2_LZMA = L"BT2";
32static const UInt32 kDictionaryForBCJ2_LZMA = 1 << 20;
33static const UInt32 kAlgorithmForBCJ2_LZMA = 1;
34static const UInt32 kNumFastBytesForBCJ2_LZMA = 64;
35
36#ifdef MY_CPU_X86_OR_AMD64
37#define USE_86_FILTER
38#endif
39
40static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream,
41    UInt64 position, UInt64 size, ICompressProgressInfo *progress)
42{
43  RINOK(inStream->Seek(position, STREAM_SEEK_SET, 0));
44  CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream;
45  CMyComPtr<CLimitedSequentialInStream> inStreamLimited(streamSpec);
46  streamSpec->SetStream(inStream);
47  streamSpec->Init(size);
48
49  NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder;
50  CMyComPtr<ICompressCoder> copyCoder = copyCoderSpec;
51  RINOK(copyCoder->Code(inStreamLimited, outStream, NULL, NULL, progress));
52  return (copyCoderSpec->TotalSize == size ? S_OK : E_FAIL);
53}
54
55static int GetReverseSlashPos(const UString &name)
56{
57  int slashPos = name.ReverseFind(L'/');
58  #ifdef _WIN32
59  int slash1Pos = name.ReverseFind(L'\\');
60  slashPos = MyMax(slashPos, slash1Pos);
61  #endif
62  return slashPos;
63}
64
65int CUpdateItem::GetExtensionPos() const
66{
67  int slashPos = GetReverseSlashPos(Name);
68  int dotPos = Name.ReverseFind(L'.');
69  if (dotPos < 0 || (dotPos < slashPos && slashPos >= 0))
70    return Name.Length();
71  return dotPos + 1;
72}
73
74UString CUpdateItem::GetExtension() const
75{
76  return Name.Mid(GetExtensionPos());
77}
78
79#define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; }
80
81#define RINOZ_COMP(a, b) RINOZ(MyCompare(a, b))
82
83static int CompareBuffers(const CByteBuffer &a1, const CByteBuffer &a2)
84{
85  size_t c1 = a1.GetCapacity();
86  size_t c2 = a2.GetCapacity();
87  RINOZ_COMP(c1, c2);
88  for (size_t i = 0; i < c1; i++)
89    RINOZ_COMP(a1[i], a2[i]);
90  return 0;
91}
92
93static int CompareCoders(const CCoderInfo &c1, const CCoderInfo &c2)
94{
95  RINOZ_COMP(c1.NumInStreams, c2.NumInStreams);
96  RINOZ_COMP(c1.NumOutStreams, c2.NumOutStreams);
97  RINOZ_COMP(c1.MethodID, c2.MethodID);
98  return CompareBuffers(c1.Props, c2.Props);
99}
100
101static int CompareBindPairs(const CBindPair &b1, const CBindPair &b2)
102{
103  RINOZ_COMP(b1.InIndex, b2.InIndex);
104  return MyCompare(b1.OutIndex, b2.OutIndex);
105}
106
107static int CompareFolders(const CFolder &f1, const CFolder &f2)
108{
109  int s1 = f1.Coders.Size();
110  int s2 = f2.Coders.Size();
111  RINOZ_COMP(s1, s2);
112  int i;
113  for (i = 0; i < s1; i++)
114    RINOZ(CompareCoders(f1.Coders[i], f2.Coders[i]));
115  s1 = f1.BindPairs.Size();
116  s2 = f2.BindPairs.Size();
117  RINOZ_COMP(s1, s2);
118  for (i = 0; i < s1; i++)
119    RINOZ(CompareBindPairs(f1.BindPairs[i], f2.BindPairs[i]));
120  return 0;
121}
122
123/*
124static int CompareFiles(const CFileItem &f1, const CFileItem &f2)
125{
126  return MyStringCompareNoCase(f1.Name, f2.Name);
127}
128*/
129
130struct CFolderRepack
131{
132  int FolderIndex;
133  int Group;
134  CNum NumCopyFiles;
135};
136
137static int CompareFolderRepacks(const CFolderRepack *p1, const CFolderRepack *p2, void *param)
138{
139  RINOZ_COMP(p1->Group, p2->Group);
140  int i1 = p1->FolderIndex;
141  int i2 = p2->FolderIndex;
142  const CArchiveDatabaseEx &db = *(const CArchiveDatabaseEx *)param;
143  RINOZ(CompareFolders(
144      db.Folders[i1],
145      db.Folders[i2]));
146  return MyCompare(i1, i2);
147  /*
148  RINOZ_COMP(
149      db.NumUnpackStreamsVector[i1],
150      db.NumUnpackStreamsVector[i2]);
151  if (db.NumUnpackStreamsVector[i1] == 0)
152    return 0;
153  return CompareFiles(
154      db.Files[db.FolderStartFileIndex[i1]],
155      db.Files[db.FolderStartFileIndex[i2]]);
156  */
157}
158
159////////////////////////////////////////////////////////////
160
161static int CompareEmptyItems(const int *p1, const int *p2, void *param)
162{
163  const CObjectVector<CUpdateItem> &updateItems = *(const CObjectVector<CUpdateItem> *)param;
164  const CUpdateItem &u1 = updateItems[*p1];
165  const CUpdateItem &u2 = updateItems[*p2];
166  if (u1.IsDir != u2.IsDir)
167    return (u1.IsDir) ? 1 : -1;
168  if (u1.IsDir)
169  {
170    if (u1.IsAnti != u2.IsAnti)
171      return (u1.IsAnti ? 1 : -1);
172    int n = MyStringCompareNoCase(u1.Name, u2.Name);
173    return -n;
174  }
175  if (u1.IsAnti != u2.IsAnti)
176    return (u1.IsAnti ? 1 : -1);
177  return MyStringCompareNoCase(u1.Name, u2.Name);
178}
179
180static const char *g_Exts =
181  " lzma 7z ace arc arj bz bz2 deb lzo lzx gz pak rpm sit tgz tbz tbz2 tgz cab ha lha lzh rar zoo"
182  " zip jar ear war msi"
183  " 3gp avi mov mpeg mpg mpe wmv"
184  " aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav"
185  " swf "
186  " chm hxi hxs"
187  " gif jpeg jpg jp2 png tiff  bmp ico psd psp"
188  " awg ps eps cgm dxf svg vrml wmf emf ai md"
189  " cad dwg pps key sxi"
190  " max 3ds"
191  " iso bin nrg mdf img pdi tar cpio xpi"
192  " vfd vhd vud vmc vsv"
193  " vmdk dsk nvram vmem vmsd vmsn vmss vmtm"
194  " inl inc idl acf asa h hpp hxx c cpp cxx rc java cs pas bas vb cls ctl frm dlg def"
195  " f77 f f90 f95"
196  " asm sql manifest dep "
197  " mak clw csproj vcproj sln dsp dsw "
198  " class "
199  " bat cmd"
200  " xml xsd xsl xslt hxk hxc htm html xhtml xht mht mhtml htw asp aspx css cgi jsp shtml"
201  " awk sed hta js php php3 php4 php5 phptml pl pm py pyo rb sh tcl vbs"
202  " text txt tex ans asc srt reg ini doc docx mcw dot rtf hlp xls xlr xlt xlw ppt pdf"
203  " sxc sxd sxi sxg sxw stc sti stw stm odt ott odg otg odp otp ods ots odf"
204  " abw afp cwk lwp wpd wps wpt wrf wri"
205  " abf afm bdf fon mgf otf pcf pfa snf ttf"
206  " dbf mdb nsf ntf wdb db fdb gdb"
207  " exe dll ocx vbx sfx sys tlb awx com obj lib out o so "
208  " pdb pch idb ncb opt";
209
210int GetExtIndex(const char *ext)
211{
212  int extIndex = 1;
213  const char *p = g_Exts;
214  for (;;)
215  {
216    char c = *p++;
217    if (c == 0)
218      return extIndex;
219    if (c == ' ')
220      continue;
221    int pos = 0;
222    for (;;)
223    {
224      char c2 = ext[pos++];
225      if (c2 == 0 && (c == 0 || c == ' '))
226        return extIndex;
227      if (c != c2)
228        break;
229      c = *p++;
230    }
231    extIndex++;
232    for (;;)
233    {
234      if (c == 0)
235        return extIndex;
236      if (c == ' ')
237        break;
238      c = *p++;
239    }
240  }
241}
242
243struct CRefItem
244{
245  const CUpdateItem *UpdateItem;
246  UInt32 Index;
247  UInt32 ExtensionPos;
248  UInt32 NamePos;
249  int ExtensionIndex;
250  CRefItem(UInt32 index, const CUpdateItem &ui, bool sortByType):
251    UpdateItem(&ui),
252    Index(index),
253    ExtensionPos(0),
254    NamePos(0),
255    ExtensionIndex(0)
256  {
257    if (sortByType)
258    {
259      int slashPos = GetReverseSlashPos(ui.Name);
260      NamePos = ((slashPos >= 0) ? (slashPos + 1) : 0);
261      int dotPos = ui.Name.ReverseFind(L'.');
262      if (dotPos < 0 || (dotPos < slashPos && slashPos >= 0))
263        ExtensionPos = ui.Name.Length();
264      else
265      {
266        ExtensionPos = dotPos + 1;
267        UString us = ui.Name.Mid(ExtensionPos);
268        if (!us.IsEmpty())
269        {
270          us.MakeLower();
271          int i;
272          AString s;
273          for (i = 0; i < us.Length(); i++)
274          {
275            wchar_t c = us[i];
276            if (c >= 0x80)
277              break;
278            s += (char)c;
279          }
280          if (i == us.Length())
281            ExtensionIndex = GetExtIndex(s);
282          else
283            ExtensionIndex = 0;
284        }
285      }
286    }
287  }
288};
289
290static int CompareUpdateItems(const CRefItem *p1, const CRefItem *p2, void *param)
291{
292  const CRefItem &a1 = *p1;
293  const CRefItem &a2 = *p2;
294  const CUpdateItem &u1 = *a1.UpdateItem;
295  const CUpdateItem &u2 = *a2.UpdateItem;
296  int n;
297  if (u1.IsDir != u2.IsDir)
298    return (u1.IsDir) ? 1 : -1;
299  if (u1.IsDir)
300  {
301    if (u1.IsAnti != u2.IsAnti)
302      return (u1.IsAnti ? 1 : -1);
303    n = MyStringCompareNoCase(u1.Name, u2.Name);
304    return -n;
305  }
306  bool sortByType = *(bool *)param;
307  if (sortByType)
308  {
309    RINOZ_COMP(a1.ExtensionIndex, a2.ExtensionIndex);
310    RINOZ(MyStringCompareNoCase(u1.Name + a1.ExtensionPos, u2.Name + a2.ExtensionPos));
311    RINOZ(MyStringCompareNoCase(u1.Name + a1.NamePos, u2.Name + a2.NamePos));
312    if (!u1.MTimeDefined && u2.MTimeDefined) return 1;
313    if (u1.MTimeDefined && !u2.MTimeDefined) return -1;
314    if (u1.MTimeDefined && u2.MTimeDefined) RINOZ_COMP(u1.MTime, u2.MTime);
315    RINOZ_COMP(u1.Size, u2.Size);
316  }
317  return MyStringCompareNoCase(u1.Name, u2.Name);
318}
319
320struct CSolidGroup
321{
322  CRecordVector<UInt32> Indices;
323};
324
325static wchar_t *g_ExeExts[] =
326{
327  L"dll",
328  L"exe",
329  L"ocx",
330  L"sfx",
331  L"sys"
332};
333
334static bool IsExeExt(const UString &ext)
335{
336  for (int i = 0; i < sizeof(g_ExeExts) / sizeof(g_ExeExts[0]); i++)
337    if (ext.CompareNoCase(g_ExeExts[i]) == 0)
338      return true;
339  return false;
340}
341
342#ifdef USE_86_FILTER
343
344static inline void GetMethodFull(UInt64 methodID, UInt32 numInStreams, CMethodFull &methodResult)
345{
346  methodResult.Id = methodID;
347  methodResult.NumInStreams = numInStreams;
348  methodResult.NumOutStreams = 1;
349}
350
351static void MakeExeMethod(const CCompressionMethodMode &method,
352    bool bcj2Filter, CCompressionMethodMode &exeMethod)
353{
354  exeMethod = method;
355  if (bcj2Filter)
356  {
357    CMethodFull methodFull;
358    GetMethodFull(k_BCJ2, 4, methodFull);
359    exeMethod.Methods.Insert(0, methodFull);
360    GetMethodFull(k_LZMA, 1, methodFull);
361    {
362      CProp prop;
363      prop.Id = NCoderPropID::kAlgorithm;
364      prop.Value = kAlgorithmForBCJ2_LZMA;
365      methodFull.Props.Add(prop);
366    }
367    {
368      CProp prop;
369      prop.Id = NCoderPropID::kMatchFinder;
370      prop.Value = kMatchFinderForBCJ2_LZMA;
371      methodFull.Props.Add(prop);
372    }
373    {
374      CProp prop;
375      prop.Id = NCoderPropID::kDictionarySize;
376      prop.Value = kDictionaryForBCJ2_LZMA;
377      methodFull.Props.Add(prop);
378    }
379    {
380      CProp prop;
381      prop.Id = NCoderPropID::kNumFastBytes;
382      prop.Value = kNumFastBytesForBCJ2_LZMA;
383      methodFull.Props.Add(prop);
384    }
385    {
386      CProp prop;
387      prop.Id = NCoderPropID::kNumThreads;
388      prop.Value = (UInt32)1;
389      methodFull.Props.Add(prop);
390    }
391
392    exeMethod.Methods.Add(methodFull);
393    exeMethod.Methods.Add(methodFull);
394    CBind bind;
395
396    bind.OutCoder = 0;
397    bind.InStream = 0;
398
399    bind.InCoder = 1;
400    bind.OutStream = 0;
401    exeMethod.Binds.Add(bind);
402
403    bind.InCoder = 2;
404    bind.OutStream = 1;
405    exeMethod.Binds.Add(bind);
406
407    bind.InCoder = 3;
408    bind.OutStream = 2;
409    exeMethod.Binds.Add(bind);
410  }
411  else
412  {
413    CMethodFull methodFull;
414    GetMethodFull(k_BCJ, 1, methodFull);
415    exeMethod.Methods.Insert(0, methodFull);
416    CBind bind;
417    bind.OutCoder = 0;
418    bind.InStream = 0;
419    bind.InCoder = 1;
420    bind.OutStream = 0;
421    exeMethod.Binds.Add(bind);
422  }
423}
424
425#endif
426
427static void FromUpdateItemToFileItem(const CUpdateItem &ui,
428    CFileItem &file, CFileItem2 &file2)
429{
430  file.Name = NItemName::MakeLegalName(ui.Name);
431  if (ui.AttribDefined)
432    file.SetAttrib(ui.Attrib);
433
434  file2.CTime = ui.CTime;  file2.CTimeDefined = ui.CTimeDefined;
435  file2.ATime = ui.ATime;  file2.ATimeDefined = ui.ATimeDefined;
436  file2.MTime = ui.MTime;  file2.MTimeDefined = ui.MTimeDefined;
437  file2.IsAnti = ui.IsAnti;
438  file2.StartPosDefined = false;
439
440  file.Size = ui.Size;
441  file.IsDir = ui.IsDir;
442  file.HasStream = ui.HasStream();
443}
444
445class CFolderOutStream2:
446  public ISequentialOutStream,
447  public CMyUnknownImp
448{
449  COutStreamWithCRC *_crcStreamSpec;
450  CMyComPtr<ISequentialOutStream> _crcStream;
451  const CArchiveDatabaseEx *_db;
452  const CBoolVector *_extractStatuses;
453  CMyComPtr<ISequentialOutStream> _outStream;
454  UInt32 _startIndex;
455  int _currentIndex;
456  bool _fileIsOpen;
457  UInt64 _rem;
458
459  void OpenFile();
460  void CloseFile();
461  HRESULT CloseFileAndSetResult();
462  HRESULT ProcessEmptyFiles();
463public:
464  MY_UNKNOWN_IMP
465
466  CFolderOutStream2()
467  {
468    _crcStreamSpec = new COutStreamWithCRC;
469    _crcStream = _crcStreamSpec;
470  }
471
472  HRESULT Init(const CArchiveDatabaseEx *db, UInt32 startIndex,
473      const CBoolVector *extractStatuses, ISequentialOutStream *outStream);
474  void ReleaseOutStream();
475  HRESULT CheckFinishedState() const { return (_currentIndex == _extractStatuses->Size()) ? S_OK: E_FAIL; }
476
477  STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize);
478};
479
480HRESULT CFolderOutStream2::Init(const CArchiveDatabaseEx *db, UInt32 startIndex,
481    const CBoolVector *extractStatuses, ISequentialOutStream *outStream)
482{
483  _db = db;
484  _startIndex = startIndex;
485  _extractStatuses = extractStatuses;
486  _outStream = outStream;
487
488  _currentIndex = 0;
489  _fileIsOpen = false;
490  return ProcessEmptyFiles();
491}
492
493void CFolderOutStream2::ReleaseOutStream()
494{
495  _outStream.Release();
496  _crcStreamSpec->ReleaseStream();
497}
498
499void CFolderOutStream2::OpenFile()
500{
501  _crcStreamSpec->SetStream((*_extractStatuses)[_currentIndex] ? _outStream : NULL);
502  _crcStreamSpec->Init(true);
503  _fileIsOpen = true;
504  _rem = _db->Files[_startIndex + _currentIndex].Size;
505}
506
507void CFolderOutStream2::CloseFile()
508{
509  _crcStreamSpec->ReleaseStream();
510  _fileIsOpen = false;
511  _currentIndex++;
512}
513
514HRESULT CFolderOutStream2::CloseFileAndSetResult()
515{
516  const CFileItem &file = _db->Files[_startIndex + _currentIndex];
517  CloseFile();
518  return (file.IsDir || !file.CrcDefined || file.Crc == _crcStreamSpec->GetCRC()) ? S_OK: S_FALSE;
519}
520
521HRESULT CFolderOutStream2::ProcessEmptyFiles()
522{
523  while (_currentIndex < _extractStatuses->Size() && _db->Files[_startIndex + _currentIndex].Size == 0)
524  {
525    OpenFile();
526    RINOK(CloseFileAndSetResult());
527  }
528  return S_OK;
529}
530
531STDMETHODIMP CFolderOutStream2::Write(const void *data, UInt32 size, UInt32 *processedSize)
532{
533  if (processedSize != NULL)
534    *processedSize = 0;
535  while (size != 0)
536  {
537    if (_fileIsOpen)
538    {
539      UInt32 cur = size < _rem ? size : (UInt32)_rem;
540      RINOK(_crcStream->Write(data, cur, &cur));
541      if (cur == 0)
542        break;
543      data = (const Byte *)data + cur;
544      size -= cur;
545      _rem -= cur;
546      if (processedSize != NULL)
547        *processedSize += cur;
548      if (_rem == 0)
549      {
550        RINOK(CloseFileAndSetResult());
551        RINOK(ProcessEmptyFiles());
552        continue;
553      }
554    }
555    else
556    {
557      RINOK(ProcessEmptyFiles());
558      if (_currentIndex == _extractStatuses->Size())
559      {
560        // we don't support partial extracting
561        return E_FAIL;
562      }
563      OpenFile();
564    }
565  }
566  return S_OK;
567}
568
569class CThreadDecoder: public CVirtThread
570{
571public:
572  HRESULT Result;
573  CMyComPtr<IInStream> InStream;
574
575  CFolderOutStream2 *FosSpec;
576  CMyComPtr<ISequentialOutStream> Fos;
577
578  UInt64 StartPos;
579  const UInt64 *PackSizes;
580  const CFolder *Folder;
581  #ifndef _NO_CRYPTO
582  CMyComPtr<ICryptoGetTextPassword> GetTextPassword;
583  #endif
584
585  DECL_EXTERNAL_CODECS_VARS
586  CDecoder Decoder;
587
588  #ifndef _7ZIP_ST
589  bool MtMode;
590  UInt32 NumThreads;
591  #endif
592
593  CThreadDecoder():
594    Decoder(true)
595  {
596    #ifndef _7ZIP_ST
597    MtMode = false;
598    NumThreads = 1;
599    #endif
600    FosSpec = new CFolderOutStream2;
601    Fos = FosSpec;
602    Result = E_FAIL;
603  }
604  virtual void Execute();
605};
606
607void CThreadDecoder::Execute()
608{
609  try
610  {
611    #ifndef _NO_CRYPTO
612    bool passwordIsDefined;
613    #endif
614    Result = Decoder.Decode(
615      EXTERNAL_CODECS_VARS
616      InStream,
617      StartPos,
618      PackSizes,
619      *Folder,
620      Fos,
621      NULL
622      #ifndef _NO_CRYPTO
623      , GetTextPassword, passwordIsDefined
624      #endif
625      #ifndef _7ZIP_ST
626      , MtMode, NumThreads
627      #endif
628      );
629  }
630  catch(...)
631  {
632    Result = E_FAIL;
633  }
634  if (Result == S_OK)
635    Result = FosSpec->CheckFinishedState();
636  FosSpec->ReleaseOutStream();
637}
638
639bool static Is86FilteredFolder(const CFolder &f)
640{
641  for (int i = 0; i < f.Coders.Size(); i++)
642  {
643    CMethodId m = f.Coders[i].MethodID;
644    if (m == k_BCJ || m == k_BCJ2)
645      return true;
646  }
647  return false;
648}
649
650#ifndef _NO_CRYPTO
651
652class CCryptoGetTextPassword:
653  public ICryptoGetTextPassword,
654  public CMyUnknownImp
655{
656public:
657  UString Password;
658
659  MY_UNKNOWN_IMP
660  STDMETHOD(CryptoGetTextPassword)(BSTR *password);
661};
662
663STDMETHODIMP CCryptoGetTextPassword::CryptoGetTextPassword(BSTR *password)
664{
665  return StringToBstr(Password, password);
666}
667
668#endif
669
670static const int kNumGroupsMax = 4;
671
672#ifdef USE_86_FILTER
673static bool Is86Group(int group) { return (group & 1) != 0; }
674#endif
675static bool IsEncryptedGroup(int group) { return (group & 2) != 0; }
676static int GetGroupIndex(bool encrypted, int bcjFiltered)
677  { return (encrypted ? 2 : 0) + (bcjFiltered ? 1 : 0); }
678
679HRESULT Update(
680    DECL_EXTERNAL_CODECS_LOC_VARS
681    IInStream *inStream,
682    const CArchiveDatabaseEx *db,
683    const CObjectVector<CUpdateItem> &updateItems,
684    COutArchive &archive,
685    CArchiveDatabase &newDatabase,
686    ISequentialOutStream *seqOutStream,
687    IArchiveUpdateCallback *updateCallback,
688    const CUpdateOptions &options
689    #ifndef _NO_CRYPTO
690    , ICryptoGetTextPassword *getDecoderPassword
691    #endif
692    )
693{
694  UInt64 numSolidFiles = options.NumSolidFiles;
695  if (numSolidFiles == 0)
696    numSolidFiles = 1;
697  /*
698  CMyComPtr<IOutStream> outStream;
699  RINOK(seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStream));
700  if (!outStream)
701    return E_NOTIMPL;
702  */
703
704  UInt64 startBlockSize = db != 0 ? db->ArchiveInfo.StartPosition: 0;
705  if (startBlockSize > 0 && !options.RemoveSfxBlock)
706  {
707    RINOK(WriteRange(inStream, seqOutStream, 0, startBlockSize, NULL));
708  }
709
710  CRecordVector<int> fileIndexToUpdateIndexMap;
711  CRecordVector<CFolderRepack> folderRefs;
712  UInt64 complexity = 0;
713  UInt64 inSizeForReduce2 = 0;
714  bool needEncryptedRepack = false;
715  if (db != 0)
716  {
717    fileIndexToUpdateIndexMap.Reserve(db->Files.Size());
718    int i;
719    for (i = 0; i < db->Files.Size(); i++)
720      fileIndexToUpdateIndexMap.Add(-1);
721
722    for (i = 0; i < updateItems.Size(); i++)
723    {
724      int index = updateItems[i].IndexInArchive;
725      if (index != -1)
726        fileIndexToUpdateIndexMap[index] = i;
727    }
728
729    for (i = 0; i < db->Folders.Size(); i++)
730    {
731      CNum indexInFolder = 0;
732      CNum numCopyItems = 0;
733      CNum numUnpackStreams = db->NumUnpackStreamsVector[i];
734      UInt64 repackSize = 0;
735      for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++)
736      {
737        const CFileItem &file = db->Files[fi];
738        if (file.HasStream)
739        {
740          indexInFolder++;
741          int updateIndex = fileIndexToUpdateIndexMap[fi];
742          if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
743          {
744            numCopyItems++;
745            repackSize += file.Size;
746          }
747        }
748      }
749
750      if (numCopyItems == 0)
751        continue;
752
753      CFolderRepack rep;
754      rep.FolderIndex = i;
755      rep.NumCopyFiles = numCopyItems;
756      const CFolder &f = db->Folders[i];
757      bool isEncrypted = f.IsEncrypted();
758      rep.Group = GetGroupIndex(isEncrypted, Is86FilteredFolder(f));
759      folderRefs.Add(rep);
760      if (numCopyItems == numUnpackStreams)
761        complexity += db->GetFolderFullPackSize(i);
762      else
763      {
764        complexity += repackSize;
765        if (repackSize > inSizeForReduce2)
766          inSizeForReduce2 = repackSize;
767        if (isEncrypted)
768          needEncryptedRepack = true;
769      }
770    }
771    folderRefs.Sort(CompareFolderRepacks, (void *)db);
772  }
773
774  UInt64 inSizeForReduce = 0;
775  int i;
776  for (i = 0; i < updateItems.Size(); i++)
777  {
778    const CUpdateItem &ui = updateItems[i];
779    if (ui.NewData)
780    {
781      complexity += ui.Size;
782      if (numSolidFiles != 1)
783        inSizeForReduce += ui.Size;
784      else if (ui.Size > inSizeForReduce)
785        inSizeForReduce = ui.Size;
786    }
787  }
788
789  if (inSizeForReduce2 > inSizeForReduce)
790    inSizeForReduce = inSizeForReduce2;
791
792  const UInt32 kMinReduceSize = (1 << 16);
793  if (inSizeForReduce < kMinReduceSize)
794    inSizeForReduce = kMinReduceSize;
795
796  RINOK(updateCallback->SetTotal(complexity));
797
798  CLocalProgress *lps = new CLocalProgress;
799  CMyComPtr<ICompressProgressInfo> progress = lps;
800  lps->Init(updateCallback, true);
801
802  CThreadDecoder threadDecoder;
803  if (!folderRefs.IsEmpty())
804  {
805    #ifdef EXTERNAL_CODECS
806    threadDecoder._codecsInfo = codecsInfo;
807    threadDecoder._externalCodecs = *externalCodecs;
808    #endif
809    RINOK(threadDecoder.Create());
810  }
811
812  CObjectVector<CSolidGroup> groups;
813  for (i = 0; i < kNumGroupsMax; i++)
814    groups.Add(CSolidGroup());
815
816  {
817    // ---------- Split files to 2 groups ----------
818
819    bool useFilters = options.UseFilters;
820    const CCompressionMethodMode &method = *options.Method;
821    if (method.Methods.Size() != 1 || method.Binds.Size() != 0)
822      useFilters = false;
823    for (i = 0; i < updateItems.Size(); i++)
824    {
825      const CUpdateItem &ui = updateItems[i];
826      if (!ui.NewData || !ui.HasStream())
827        continue;
828      bool filteredGroup = false;
829      if (useFilters)
830      {
831        int dotPos = ui.Name.ReverseFind(L'.');
832        if (dotPos >= 0)
833          filteredGroup = IsExeExt(ui.Name.Mid(dotPos + 1));
834      }
835      groups[GetGroupIndex(method.PasswordIsDefined, filteredGroup)].Indices.Add(i);
836    }
837  }
838
839  #ifndef _NO_CRYPTO
840
841  CCryptoGetTextPassword *getPasswordSpec = NULL;
842  if (needEncryptedRepack)
843  {
844    getPasswordSpec = new CCryptoGetTextPassword;
845    threadDecoder.GetTextPassword = getPasswordSpec;
846
847    if (options.Method->PasswordIsDefined)
848      getPasswordSpec->Password = options.Method->Password;
849    else
850    {
851      if (!getDecoderPassword)
852        return E_NOTIMPL;
853      CMyComBSTR password;
854      RINOK(getDecoderPassword->CryptoGetTextPassword(&password));
855      getPasswordSpec->Password = password;
856    }
857  }
858
859  #endif
860
861  // ---------- Compress ----------
862
863  RINOK(archive.Create(seqOutStream, false));
864  RINOK(archive.SkipPrefixArchiveHeader());
865
866  int folderRefIndex = 0;
867  lps->ProgressOffset = 0;
868
869  for (int groupIndex = 0; groupIndex < kNumGroupsMax; groupIndex++)
870  {
871    const CSolidGroup &group = groups[groupIndex];
872
873    CCompressionMethodMode method;
874    #ifdef USE_86_FILTER
875    if (Is86Group(groupIndex))
876      MakeExeMethod(*options.Method, options.MaxFilter, method);
877    else
878    #endif
879      method = *options.Method;
880
881    if (IsEncryptedGroup(groupIndex))
882    {
883      if (!method.PasswordIsDefined)
884      {
885        #ifndef _NO_CRYPTO
886        if (getPasswordSpec)
887          method.Password = getPasswordSpec->Password;
888        #endif
889        method.PasswordIsDefined = true;
890      }
891    }
892    else
893    {
894      method.PasswordIsDefined = false;
895      method.Password.Empty();
896    }
897
898    CEncoder encoder(method);
899
900    for (; folderRefIndex < folderRefs.Size(); folderRefIndex++)
901    {
902      const CFolderRepack &rep = folderRefs[folderRefIndex];
903      if (rep.Group != groupIndex)
904        break;
905      int folderIndex = rep.FolderIndex;
906
907      if (rep.NumCopyFiles == db->NumUnpackStreamsVector[folderIndex])
908      {
909        UInt64 packSize = db->GetFolderFullPackSize(folderIndex);
910        RINOK(WriteRange(inStream, archive.SeqStream,
911          db->GetFolderStreamPos(folderIndex, 0), packSize, progress));
912        lps->ProgressOffset += packSize;
913
914        const CFolder &folder = db->Folders[folderIndex];
915        CNum startIndex = db->FolderStartPackStreamIndex[folderIndex];
916        for (int j = 0; j < folder.PackStreams.Size(); j++)
917        {
918          newDatabase.PackSizes.Add(db->PackSizes[startIndex + j]);
919          // newDatabase.PackCRCsDefined.Add(db.PackCRCsDefined[startIndex + j]);
920          // newDatabase.PackCRCs.Add(db.PackCRCs[startIndex + j]);
921        }
922        newDatabase.Folders.Add(folder);
923      }
924      else
925      {
926        CStreamBinder sb;
927        RINOK(sb.CreateEvents());
928        CMyComPtr<ISequentialOutStream> sbOutStream;
929        CMyComPtr<ISequentialInStream> sbInStream;
930        sb.CreateStreams(&sbInStream, &sbOutStream);
931        CBoolVector extractStatuses;
932
933        CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
934        CNum indexInFolder = 0;
935
936        for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
937        {
938          bool needExtract = false;
939          if (db->Files[fi].HasStream)
940          {
941            indexInFolder++;
942            int updateIndex = fileIndexToUpdateIndexMap[fi];
943            if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
944              needExtract = true;
945          }
946          extractStatuses.Add(needExtract);
947        }
948
949        RINOK(threadDecoder.FosSpec->Init(db, db->FolderStartFileIndex[folderIndex], &extractStatuses, sbOutStream));
950        sbOutStream.Release();
951
952        threadDecoder.InStream = inStream;
953        threadDecoder.Folder = &db->Folders[folderIndex];
954        threadDecoder.StartPos = db->GetFolderStreamPos(folderIndex, 0);
955        threadDecoder.PackSizes = &db->PackSizes[db->FolderStartPackStreamIndex[folderIndex]];
956
957        threadDecoder.Start();
958
959        int startPackIndex = newDatabase.PackSizes.Size();
960        CFolder newFolder;
961        RINOK(encoder.Encode(
962          EXTERNAL_CODECS_LOC_VARS
963          sbInStream, NULL, &inSizeForReduce, newFolder,
964          archive.SeqStream, newDatabase.PackSizes, progress));
965
966        threadDecoder.WaitFinish();
967
968        RINOK(threadDecoder.Result);
969
970        for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
971          lps->OutSize += newDatabase.PackSizes[startPackIndex];
972        lps->InSize += newFolder.GetUnpackSize();
973
974        newDatabase.Folders.Add(newFolder);
975      }
976
977      newDatabase.NumUnpackStreamsVector.Add(rep.NumCopyFiles);
978
979      CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
980
981      CNum indexInFolder = 0;
982      for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
983      {
984        CFileItem file;
985        CFileItem2 file2;
986        db->GetFile(fi, file, file2);
987        if (file.HasStream)
988        {
989          indexInFolder++;
990          int updateIndex = fileIndexToUpdateIndexMap[fi];
991          if (updateIndex >= 0)
992          {
993            const CUpdateItem &ui = updateItems[updateIndex];
994            if (ui.NewData)
995              continue;
996            if (ui.NewProps)
997            {
998              CFileItem uf;
999              FromUpdateItemToFileItem(ui, uf, file2);
1000              uf.Size = file.Size;
1001              uf.Crc = file.Crc;
1002              uf.CrcDefined = file.CrcDefined;
1003              uf.HasStream = file.HasStream;
1004              file = uf;
1005            }
1006            newDatabase.AddFile(file, file2);
1007          }
1008        }
1009      }
1010    }
1011
1012    int numFiles = group.Indices.Size();
1013    if (numFiles == 0)
1014      continue;
1015    CRecordVector<CRefItem> refItems;
1016    refItems.Reserve(numFiles);
1017    bool sortByType = (numSolidFiles > 1);
1018    for (i = 0; i < numFiles; i++)
1019      refItems.Add(CRefItem(group.Indices[i], updateItems[group.Indices[i]], sortByType));
1020    refItems.Sort(CompareUpdateItems, (void *)&sortByType);
1021
1022    CRecordVector<UInt32> indices;
1023    indices.Reserve(numFiles);
1024
1025    for (i = 0; i < numFiles; i++)
1026    {
1027      UInt32 index = refItems[i].Index;
1028      indices.Add(index);
1029      /*
1030      const CUpdateItem &ui = updateItems[index];
1031      CFileItem file;
1032      if (ui.NewProps)
1033        FromUpdateItemToFileItem(ui, file);
1034      else
1035        file = db.Files[ui.IndexInArchive];
1036      if (file.IsAnti || file.IsDir)
1037        return E_FAIL;
1038      newDatabase.Files.Add(file);
1039      */
1040    }
1041
1042    for (i = 0; i < numFiles;)
1043    {
1044      UInt64 totalSize = 0;
1045      int numSubFiles;
1046      UString prevExtension;
1047      for (numSubFiles = 0; i + numSubFiles < numFiles &&
1048          numSubFiles < numSolidFiles; numSubFiles++)
1049      {
1050        const CUpdateItem &ui = updateItems[indices[i + numSubFiles]];
1051        totalSize += ui.Size;
1052        if (totalSize > options.NumSolidBytes)
1053          break;
1054        if (options.SolidExtension)
1055        {
1056          UString ext = ui.GetExtension();
1057          if (numSubFiles == 0)
1058            prevExtension = ext;
1059          else
1060            if (ext.CompareNoCase(prevExtension) != 0)
1061              break;
1062        }
1063      }
1064      if (numSubFiles < 1)
1065        numSubFiles = 1;
1066
1067      CFolderInStream *inStreamSpec = new CFolderInStream;
1068      CMyComPtr<ISequentialInStream> solidInStream(inStreamSpec);
1069      inStreamSpec->Init(updateCallback, &indices[i], numSubFiles);
1070
1071      CFolder folderItem;
1072
1073      int startPackIndex = newDatabase.PackSizes.Size();
1074      RINOK(encoder.Encode(
1075          EXTERNAL_CODECS_LOC_VARS
1076          solidInStream, NULL, &inSizeForReduce, folderItem,
1077          archive.SeqStream, newDatabase.PackSizes, progress));
1078
1079      for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
1080        lps->OutSize += newDatabase.PackSizes[startPackIndex];
1081
1082      lps->InSize += folderItem.GetUnpackSize();
1083      // for ()
1084      // newDatabase.PackCRCsDefined.Add(false);
1085      // newDatabase.PackCRCs.Add(0);
1086
1087      newDatabase.Folders.Add(folderItem);
1088
1089      CNum numUnpackStreams = 0;
1090      for (int subIndex = 0; subIndex < numSubFiles; subIndex++)
1091      {
1092        const CUpdateItem &ui = updateItems[indices[i + subIndex]];
1093        CFileItem file;
1094        CFileItem2 file2;
1095        if (ui.NewProps)
1096          FromUpdateItemToFileItem(ui, file, file2);
1097        else
1098          db->GetFile(ui.IndexInArchive, file, file2);
1099        if (file2.IsAnti || file.IsDir)
1100          return E_FAIL;
1101
1102        /*
1103        CFileItem &file = newDatabase.Files[
1104              startFileIndexInDatabase + i + subIndex];
1105        */
1106        if (!inStreamSpec->Processed[subIndex])
1107        {
1108          continue;
1109          // file.Name += L".locked";
1110        }
1111
1112        file.Crc = inStreamSpec->CRCs[subIndex];
1113        file.Size = inStreamSpec->Sizes[subIndex];
1114        if (file.Size != 0)
1115        {
1116          file.CrcDefined = true;
1117          file.HasStream = true;
1118          numUnpackStreams++;
1119        }
1120        else
1121        {
1122          file.CrcDefined = false;
1123          file.HasStream = false;
1124        }
1125        newDatabase.AddFile(file, file2);
1126      }
1127      // numUnpackStreams = 0 is very bad case for locked files
1128      // v3.13 doesn't understand it.
1129      newDatabase.NumUnpackStreamsVector.Add(numUnpackStreams);
1130      i += numSubFiles;
1131    }
1132  }
1133
1134  if (folderRefIndex != folderRefs.Size())
1135    return E_FAIL;
1136
1137  /*
1138  folderRefs.ClearAndFree();
1139  fileIndexToUpdateIndexMap.ClearAndFree();
1140  groups.ClearAndFree();
1141  */
1142
1143  {
1144    // ---------- Write Folders & Empty Files ----------
1145
1146    CRecordVector<int> emptyRefs;
1147    for (i = 0; i < updateItems.Size(); i++)
1148    {
1149      const CUpdateItem &ui = updateItems[i];
1150      if (ui.NewData)
1151      {
1152        if (ui.HasStream())
1153          continue;
1154      }
1155      else if (ui.IndexInArchive != -1 && db->Files[ui.IndexInArchive].HasStream)
1156        continue;
1157      emptyRefs.Add(i);
1158    }
1159    emptyRefs.Sort(CompareEmptyItems, (void *)&updateItems);
1160    for (i = 0; i < emptyRefs.Size(); i++)
1161    {
1162      const CUpdateItem &ui = updateItems[emptyRefs[i]];
1163      CFileItem file;
1164      CFileItem2 file2;
1165      if (ui.NewProps)
1166        FromUpdateItemToFileItem(ui, file, file2);
1167      else
1168        db->GetFile(ui.IndexInArchive, file, file2);
1169      newDatabase.AddFile(file, file2);
1170    }
1171  }
1172
1173  newDatabase.ReserveDown();
1174  return S_OK;
1175}
1176
1177}}
1178