1// 7zUpdate.cpp
2
3#include "StdAfx.h"
4
5#include "../../../../C/CpuArch.h"
6
7#include "../../../Common/Wildcard.h"
8
9#include "../../Common/CreateCoder.h"
10#include "../../Common/LimitedStreams.h"
11#include "../../Common/ProgressUtils.h"
12
13#include "../../Compress/CopyCoder.h"
14
15#include "../Common/ItemNameUtils.h"
16#include "../Common/OutStreamWithCRC.h"
17
18#include "7zDecode.h"
19#include "7zEncode.h"
20#include "7zFolderInStream.h"
21#include "7zHandler.h"
22#include "7zOut.h"
23#include "7zUpdate.h"
24
25namespace NArchive {
26namespace N7z {
27
28#ifdef MY_CPU_X86_OR_AMD64
29#define USE_86_FILTER
30#endif
31
32static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream,
33    UInt64 position, UInt64 size, ICompressProgressInfo *progress)
34{
35  RINOK(inStream->Seek(position, STREAM_SEEK_SET, 0));
36  CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream;
37  CMyComPtr<CLimitedSequentialInStream> inStreamLimited(streamSpec);
38  streamSpec->SetStream(inStream);
39  streamSpec->Init(size);
40
41  NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder;
42  CMyComPtr<ICompressCoder> copyCoder = copyCoderSpec;
43  RINOK(copyCoder->Code(inStreamLimited, outStream, NULL, NULL, progress));
44  return (copyCoderSpec->TotalSize == size ? S_OK : E_FAIL);
45}
46
47static int GetReverseSlashPos(const UString &name)
48{
49  int slashPos = name.ReverseFind(L'/');
50  #ifdef _WIN32
51  int slash1Pos = name.ReverseFind(L'\\');
52  slashPos = MyMax(slashPos, slash1Pos);
53  #endif
54  return slashPos;
55}
56
57int CUpdateItem::GetExtensionPos() const
58{
59  int slashPos = GetReverseSlashPos(Name);
60  int dotPos = Name.ReverseFind(L'.');
61  if (dotPos < 0 || (dotPos < slashPos && slashPos >= 0))
62    return Name.Len();
63  return dotPos + 1;
64}
65
66UString CUpdateItem::GetExtension() const
67{
68  return Name.Ptr(GetExtensionPos());
69}
70
71#define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; }
72
73#define RINOZ_COMP(a, b) RINOZ(MyCompare(a, b))
74
75/*
76static int CompareBuffers(const CByteBuffer &a1, const CByteBuffer &a2)
77{
78  size_t c1 = a1.GetCapacity();
79  size_t c2 = a2.GetCapacity();
80  RINOZ_COMP(c1, c2);
81  for (size_t i = 0; i < c1; i++)
82    RINOZ_COMP(a1[i], a2[i]);
83  return 0;
84}
85
86static int CompareCoders(const CCoderInfo &c1, const CCoderInfo &c2)
87{
88  RINOZ_COMP(c1.NumInStreams, c2.NumInStreams);
89  RINOZ_COMP(c1.NumOutStreams, c2.NumOutStreams);
90  RINOZ_COMP(c1.MethodID, c2.MethodID);
91  return CompareBuffers(c1.Props, c2.Props);
92}
93
94static int CompareBindPairs(const CBindPair &b1, const CBindPair &b2)
95{
96  RINOZ_COMP(b1.InIndex, b2.InIndex);
97  return MyCompare(b1.OutIndex, b2.OutIndex);
98}
99
100static int CompareFolders(const CFolder &f1, const CFolder &f2)
101{
102  int s1 = f1.Coders.Size();
103  int s2 = f2.Coders.Size();
104  RINOZ_COMP(s1, s2);
105  int i;
106  for (i = 0; i < s1; i++)
107    RINOZ(CompareCoders(f1.Coders[i], f2.Coders[i]));
108  s1 = f1.BindPairs.Size();
109  s2 = f2.BindPairs.Size();
110  RINOZ_COMP(s1, s2);
111  for (i = 0; i < s1; i++)
112    RINOZ(CompareBindPairs(f1.BindPairs[i], f2.BindPairs[i]));
113  return 0;
114}
115*/
116
117/*
118static int CompareFiles(const CFileItem &f1, const CFileItem &f2)
119{
120  return CompareFileNames(f1.Name, f2.Name);
121}
122*/
123
124struct CFolderRepack
125{
126  int FolderIndex;
127  int Group;
128  CNum NumCopyFiles;
129};
130
131static int CompareFolderRepacks(const CFolderRepack *p1, const CFolderRepack *p2, void * /* param */)
132{
133  RINOZ_COMP(p1->Group, p2->Group);
134  int i1 = p1->FolderIndex;
135  int i2 = p2->FolderIndex;
136  /*
137  // In that version we don't want to parse folders here, so we don't compare folders
138  // probably it must be improved in future
139  const CDbEx &db = *(const CDbEx *)param;
140  RINOZ(CompareFolders(
141      db.Folders[i1],
142      db.Folders[i2]));
143  */
144  return MyCompare(i1, i2);
145  /*
146  RINOZ_COMP(
147      db.NumUnpackStreamsVector[i1],
148      db.NumUnpackStreamsVector[i2]);
149  if (db.NumUnpackStreamsVector[i1] == 0)
150    return 0;
151  return CompareFiles(
152      db.Files[db.FolderStartFileIndex[i1]],
153      db.Files[db.FolderStartFileIndex[i2]]);
154  */
155}
156
157/*
158  we sort empty files and dirs in such order:
159  - Dir.NonAnti   (name sorted)
160  - File.NonAnti  (name sorted)
161  - File.Anti     (name sorted)
162  - Dir.Anti (reverse name sorted)
163*/
164
165static int CompareEmptyItems(const int *p1, const int *p2, void *param)
166{
167  const CObjectVector<CUpdateItem> &updateItems = *(const CObjectVector<CUpdateItem> *)param;
168  const CUpdateItem &u1 = updateItems[*p1];
169  const CUpdateItem &u2 = updateItems[*p2];
170  // NonAnti < Anti
171  if (u1.IsAnti != u2.IsAnti)
172    return (u1.IsAnti ? 1 : -1);
173  if (u1.IsDir != u2.IsDir)
174  {
175    // Dir.NonAnti < File < Dir.Anti
176    if (u1.IsDir)
177      return (u1.IsAnti ? 1 : -1);
178    return (u2.IsAnti ? -1 : 1);
179  }
180  int n = CompareFileNames(u1.Name, u2.Name);
181  return (u1.IsDir && u1.IsAnti) ? -n : n;
182}
183
184static const char *g_Exts =
185  " lzma 7z ace arc arj bz bz2 deb lzo lzx gz pak rpm sit tgz tbz tbz2 tgz cab ha lha lzh rar zoo"
186  " zip jar ear war msi"
187  " 3gp avi mov mpeg mpg mpe wmv"
188  " aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav"
189  " swf "
190  " chm hxi hxs"
191  " gif jpeg jpg jp2 png tiff  bmp ico psd psp"
192  " awg ps eps cgm dxf svg vrml wmf emf ai md"
193  " cad dwg pps key sxi"
194  " max 3ds"
195  " iso bin nrg mdf img pdi tar cpio xpi"
196  " vfd vhd vud vmc vsv"
197  " vmdk dsk nvram vmem vmsd vmsn vmss vmtm"
198  " inl inc idl acf asa h hpp hxx c cpp cxx rc java cs pas bas vb cls ctl frm dlg def"
199  " f77 f f90 f95"
200  " asm sql manifest dep "
201  " mak clw csproj vcproj sln dsp dsw "
202  " class "
203  " bat cmd"
204  " xml xsd xsl xslt hxk hxc htm html xhtml xht mht mhtml htw asp aspx css cgi jsp shtml"
205  " awk sed hta js php php3 php4 php5 phptml pl pm py pyo rb sh tcl vbs"
206  " text txt tex ans asc srt reg ini doc docx mcw dot rtf hlp xls xlr xlt xlw ppt pdf"
207  " sxc sxd sxi sxg sxw stc sti stw stm odt ott odg otg odp otp ods ots odf"
208  " abw afp cwk lwp wpd wps wpt wrf wri"
209  " abf afm bdf fon mgf otf pcf pfa snf ttf"
210  " dbf mdb nsf ntf wdb db fdb gdb"
211  " exe dll ocx vbx sfx sys tlb awx com obj lib out o so "
212  " pdb pch idb ncb opt";
213
214static int GetExtIndex(const char *ext)
215{
216  int extIndex = 1;
217  const char *p = g_Exts;
218  for (;;)
219  {
220    char c = *p++;
221    if (c == 0)
222      return extIndex;
223    if (c == ' ')
224      continue;
225    int pos = 0;
226    for (;;)
227    {
228      char c2 = ext[pos++];
229      if (c2 == 0 && (c == 0 || c == ' '))
230        return extIndex;
231      if (c != c2)
232        break;
233      c = *p++;
234    }
235    extIndex++;
236    for (;;)
237    {
238      if (c == 0)
239        return extIndex;
240      if (c == ' ')
241        break;
242      c = *p++;
243    }
244  }
245}
246
247struct CRefItem
248{
249  const CUpdateItem *UpdateItem;
250  UInt32 Index;
251  UInt32 ExtensionPos;
252  UInt32 NamePos;
253  unsigned ExtensionIndex;
254
255  CRefItem() {};
256  CRefItem(UInt32 index, const CUpdateItem &ui, bool sortByType):
257    UpdateItem(&ui),
258    Index(index),
259    ExtensionPos(0),
260    NamePos(0),
261    ExtensionIndex(0)
262  {
263    if (sortByType)
264    {
265      int slashPos = GetReverseSlashPos(ui.Name);
266      NamePos = slashPos + 1;
267      int dotPos = ui.Name.ReverseFind(L'.');
268      if (dotPos < 0 || dotPos < slashPos)
269        ExtensionPos = ui.Name.Len();
270      else
271      {
272        ExtensionPos = dotPos + 1;
273        if (ExtensionPos != ui.Name.Len())
274        {
275          AString s;
276          for (unsigned pos = ExtensionPos;; pos++)
277          {
278            wchar_t c = ui.Name[pos];
279            if (c >= 0x80)
280              break;
281            if (c == 0)
282            {
283              ExtensionIndex = GetExtIndex(s);
284              break;
285            }
286            s += (char)MyCharLower_Ascii((char)c);
287          }
288        }
289      }
290    }
291  }
292};
293
294struct CSortParam
295{
296  // const CObjectVector<CTreeFolder> *TreeFolders;
297  bool SortByType;
298};
299
300/*
301  we sort files in such order:
302  - Dir.NonAnti   (name sorted)
303  - alt streams
304  - Dirs
305  - Dir.Anti (reverse name sorted)
306*/
307
308
309static int CompareUpdateItems(const CRefItem *p1, const CRefItem *p2, void *param)
310{
311  const CRefItem &a1 = *p1;
312  const CRefItem &a2 = *p2;
313  const CUpdateItem &u1 = *a1.UpdateItem;
314  const CUpdateItem &u2 = *a2.UpdateItem;
315
316  /*
317  if (u1.IsAltStream != u2.IsAltStream)
318    return u1.IsAltStream ? 1 : -1;
319  */
320
321  // Actually there are no dirs that time. They were stored in other steps
322  // So that code is unused?
323  if (u1.IsDir != u2.IsDir)
324    return u1.IsDir ? 1 : -1;
325  if (u1.IsDir)
326  {
327    if (u1.IsAnti != u2.IsAnti)
328      return (u1.IsAnti ? 1 : -1);
329    int n = CompareFileNames(u1.Name, u2.Name);
330    return -n;
331  }
332
333  // bool sortByType = *(bool *)param;
334  const CSortParam *sortParam = (const CSortParam *)param;
335  bool sortByType = sortParam->SortByType;
336  if (sortByType)
337  {
338    RINOZ_COMP(a1.ExtensionIndex, a2.ExtensionIndex);
339    RINOZ(CompareFileNames(u1.Name.Ptr(a1.ExtensionPos), u2.Name.Ptr(a2.ExtensionPos)));
340    RINOZ(CompareFileNames(u1.Name.Ptr(a1.NamePos), u2.Name.Ptr(a2.NamePos)));
341    if (!u1.MTimeDefined && u2.MTimeDefined) return 1;
342    if (u1.MTimeDefined && !u2.MTimeDefined) return -1;
343    if (u1.MTimeDefined && u2.MTimeDefined) RINOZ_COMP(u1.MTime, u2.MTime);
344    RINOZ_COMP(u1.Size, u2.Size);
345  }
346  /*
347  int par1 = a1.UpdateItem->ParentFolderIndex;
348  int par2 = a2.UpdateItem->ParentFolderIndex;
349  const CTreeFolder &tf1 = (*sortParam->TreeFolders)[par1];
350  const CTreeFolder &tf2 = (*sortParam->TreeFolders)[par2];
351
352  int b1 = tf1.SortIndex, e1 = tf1.SortIndexEnd;
353  int b2 = tf2.SortIndex, e2 = tf2.SortIndexEnd;
354  if (b1 < b2)
355  {
356    if (e1 <= b2)
357      return -1;
358    // p2 in p1
359    int par = par2;
360    for (;;)
361    {
362      const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
363      par = tf.Parent;
364      if (par == par1)
365      {
366        RINOZ(CompareFileNames(u1.Name, tf.Name));
367        break;
368      }
369    }
370  }
371  else if (b2 < b1)
372  {
373    if (e2 <= b1)
374      return 1;
375    // p1 in p2
376    int par = par1;
377    for (;;)
378    {
379      const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
380      par = tf.Parent;
381      if (par == par2)
382      {
383        RINOZ(CompareFileNames(tf.Name, u2.Name));
384        break;
385      }
386    }
387  }
388  */
389  // RINOZ_COMP(a1.UpdateItem->ParentSortIndex, a2.UpdateItem->ParentSortIndex);
390  RINOK(CompareFileNames(u1.Name, u2.Name));
391  RINOZ_COMP(a1.UpdateItem->IndexInClient, a2.UpdateItem->IndexInClient);
392  RINOZ_COMP(a1.UpdateItem->IndexInArchive, a2.UpdateItem->IndexInArchive);
393  return 0;
394}
395
396struct CSolidGroup
397{
398  CRecordVector<UInt32> Indices;
399};
400
401static const wchar_t *g_ExeExts[] =
402{
403    L"dll"
404  , L"exe"
405  , L"ocx"
406  , L"sfx"
407  , L"sys"
408};
409
410static bool IsExeExt(const wchar_t *ext)
411{
412  for (int i = 0; i < ARRAY_SIZE(g_ExeExts); i++)
413    if (MyStringCompareNoCase(ext, g_ExeExts[i]) == 0)
414      return true;
415  return false;
416}
417
418
419static inline void GetMethodFull(UInt64 methodID, UInt32 numInStreams, CMethodFull &m)
420{
421  m.Id = methodID;
422  m.NumInStreams = numInStreams;
423  m.NumOutStreams = 1;
424}
425
426static void AddBcj2Methods(CCompressionMethodMode &mode)
427{
428  CMethodFull m;
429  GetMethodFull(k_LZMA, 1, m);
430
431  m.AddProp32(NCoderPropID::kDictionarySize, 1 << 20);
432  m.AddProp32(NCoderPropID::kNumFastBytes, 128);
433  m.AddProp32(NCoderPropID::kNumThreads, 1);
434  m.AddProp32(NCoderPropID::kLitPosBits, 2);
435  m.AddProp32(NCoderPropID::kLitContextBits, 0);
436  // m.AddPropString(NCoderPropID::kMatchFinder, L"BT2");
437
438  mode.Methods.Add(m);
439  mode.Methods.Add(m);
440
441  CBind bind;
442  bind.OutCoder = 0;
443  bind.InStream = 0;
444  bind.InCoder = 1;  bind.OutStream = 0;  mode.Binds.Add(bind);
445  bind.InCoder = 2;  bind.OutStream = 1;  mode.Binds.Add(bind);
446  bind.InCoder = 3;  bind.OutStream = 2;  mode.Binds.Add(bind);
447}
448
449static void MakeExeMethod(CCompressionMethodMode &mode,
450    bool useFilters, bool addFilter, bool bcj2Filter)
451{
452  if (!mode.Binds.IsEmpty() || !useFilters || mode.Methods.Size() > 2)
453    return;
454  if (mode.Methods.Size() == 2)
455  {
456    if (mode.Methods[0].Id == k_BCJ2)
457      AddBcj2Methods(mode);
458    return;
459  }
460  if (!addFilter)
461    return;
462  bcj2Filter = bcj2Filter;
463  #ifdef USE_86_FILTER
464  if (bcj2Filter)
465  {
466    CMethodFull m;
467    GetMethodFull(k_BCJ2, 4, m);
468    mode.Methods.Insert(0, m);
469    AddBcj2Methods(mode);
470  }
471  else
472  {
473    CMethodFull m;
474    GetMethodFull(k_BCJ, 1, m);
475    mode.Methods.Insert(0, m);
476    CBind bind;
477    bind.OutCoder = 0;
478    bind.InStream = 0;
479    bind.InCoder = 1;
480    bind.OutStream = 0;
481    mode.Binds.Add(bind);
482  }
483  #endif
484}
485
486
487static void FromUpdateItemToFileItem(const CUpdateItem &ui,
488    CFileItem &file, CFileItem2 &file2)
489{
490  if (ui.AttribDefined)
491    file.SetAttrib(ui.Attrib);
492
493  file2.CTime = ui.CTime;  file2.CTimeDefined = ui.CTimeDefined;
494  file2.ATime = ui.ATime;  file2.ATimeDefined = ui.ATimeDefined;
495  file2.MTime = ui.MTime;  file2.MTimeDefined = ui.MTimeDefined;
496  file2.IsAnti = ui.IsAnti;
497  // file2.IsAux = false;
498  file2.StartPosDefined = false;
499
500  file.Size = ui.Size;
501  file.IsDir = ui.IsDir;
502  file.HasStream = ui.HasStream();
503  // file.IsAltStream = ui.IsAltStream;
504}
505
506class CFolderOutStream2:
507  public ISequentialOutStream,
508  public CMyUnknownImp
509{
510  COutStreamWithCRC *_crcStreamSpec;
511  CMyComPtr<ISequentialOutStream> _crcStream;
512  const CDbEx *_db;
513  const CBoolVector *_extractStatuses;
514  CMyComPtr<ISequentialOutStream> _outStream;
515  UInt32 _startIndex;
516  unsigned _currentIndex;
517  bool _fileIsOpen;
518  UInt64 _rem;
519
520  void OpenFile();
521  void CloseFile();
522  HRESULT CloseFileAndSetResult();
523  HRESULT ProcessEmptyFiles();
524public:
525  MY_UNKNOWN_IMP
526
527  CFolderOutStream2()
528  {
529    _crcStreamSpec = new COutStreamWithCRC;
530    _crcStream = _crcStreamSpec;
531  }
532
533  HRESULT Init(const CDbEx *db, UInt32 startIndex,
534      const CBoolVector *extractStatuses, ISequentialOutStream *outStream);
535  void ReleaseOutStream();
536  HRESULT CheckFinishedState() const { return (_currentIndex == _extractStatuses->Size()) ? S_OK: E_FAIL; }
537
538  STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize);
539};
540
541HRESULT CFolderOutStream2::Init(const CDbEx *db, UInt32 startIndex,
542    const CBoolVector *extractStatuses, ISequentialOutStream *outStream)
543{
544  _db = db;
545  _startIndex = startIndex;
546  _extractStatuses = extractStatuses;
547  _outStream = outStream;
548
549  _currentIndex = 0;
550  _fileIsOpen = false;
551  return ProcessEmptyFiles();
552}
553
554void CFolderOutStream2::ReleaseOutStream()
555{
556  _outStream.Release();
557  _crcStreamSpec->ReleaseStream();
558}
559
560void CFolderOutStream2::OpenFile()
561{
562  _crcStreamSpec->SetStream((*_extractStatuses)[_currentIndex] ? _outStream : NULL);
563  _crcStreamSpec->Init(true);
564  _fileIsOpen = true;
565  _rem = _db->Files[_startIndex + _currentIndex].Size;
566}
567
568void CFolderOutStream2::CloseFile()
569{
570  _crcStreamSpec->ReleaseStream();
571  _fileIsOpen = false;
572  _currentIndex++;
573}
574
575HRESULT CFolderOutStream2::CloseFileAndSetResult()
576{
577  const CFileItem &file = _db->Files[_startIndex + _currentIndex];
578  CloseFile();
579  return (file.IsDir || !file.CrcDefined || file.Crc == _crcStreamSpec->GetCRC()) ? S_OK: S_FALSE;
580}
581
582HRESULT CFolderOutStream2::ProcessEmptyFiles()
583{
584  while (_currentIndex < _extractStatuses->Size() && _db->Files[_startIndex + _currentIndex].Size == 0)
585  {
586    OpenFile();
587    RINOK(CloseFileAndSetResult());
588  }
589  return S_OK;
590}
591
592STDMETHODIMP CFolderOutStream2::Write(const void *data, UInt32 size, UInt32 *processedSize)
593{
594  if (processedSize != NULL)
595    *processedSize = 0;
596  while (size != 0)
597  {
598    if (_fileIsOpen)
599    {
600      UInt32 cur = size < _rem ? size : (UInt32)_rem;
601      RINOK(_crcStream->Write(data, cur, &cur));
602      if (cur == 0)
603        break;
604      data = (const Byte *)data + cur;
605      size -= cur;
606      _rem -= cur;
607      if (processedSize != NULL)
608        *processedSize += cur;
609      if (_rem == 0)
610      {
611        RINOK(CloseFileAndSetResult());
612        RINOK(ProcessEmptyFiles());
613        continue;
614      }
615    }
616    else
617    {
618      RINOK(ProcessEmptyFiles());
619      if (_currentIndex == _extractStatuses->Size())
620      {
621        // we don't support partial extracting
622        return E_FAIL;
623      }
624      OpenFile();
625    }
626  }
627  return S_OK;
628}
629
630class CThreadDecoder: public CVirtThread
631{
632public:
633  HRESULT Result;
634  CMyComPtr<IInStream> InStream;
635
636  CFolderOutStream2 *FosSpec;
637  CMyComPtr<ISequentialOutStream> Fos;
638
639  UInt64 StartPos;
640  const CFolders *Folders;
641  int FolderIndex;
642  #ifndef _NO_CRYPTO
643  CMyComPtr<ICryptoGetTextPassword> getTextPassword;
644  #endif
645
646  DECL_EXTERNAL_CODECS_LOC_VARS2;
647  CDecoder Decoder;
648
649  #ifndef _7ZIP_ST
650  bool MtMode;
651  UInt32 NumThreads;
652  #endif
653
654  CThreadDecoder():
655    Decoder(true)
656  {
657    #ifndef _7ZIP_ST
658    MtMode = false;
659    NumThreads = 1;
660    #endif
661    FosSpec = new CFolderOutStream2;
662    Fos = FosSpec;
663    Result = E_FAIL;
664  }
665  ~CThreadDecoder() { CVirtThread::WaitThreadFinish(); }
666  virtual void Execute();
667};
668
669void CThreadDecoder::Execute()
670{
671  try
672  {
673    #ifndef _NO_CRYPTO
674      bool isEncrypted = false;
675      bool passwordIsDefined = false;
676    #endif
677
678    Result = Decoder.Decode(
679      EXTERNAL_CODECS_LOC_VARS
680      InStream,
681      StartPos,
682      *Folders, FolderIndex,
683      Fos,
684      NULL
685      _7Z_DECODER_CRYPRO_VARS
686      #ifndef _7ZIP_ST
687        , MtMode, NumThreads
688      #endif
689      );
690  }
691  catch(...)
692  {
693    Result = E_FAIL;
694  }
695  if (Result == S_OK)
696    Result = FosSpec->CheckFinishedState();
697  FosSpec->ReleaseOutStream();
698}
699
700bool static Is86FilteredFolder(const CFolder &f)
701{
702  FOR_VECTOR(i, f.Coders)
703  {
704    CMethodId m = f.Coders[i].MethodID;
705    if (m == k_BCJ || m == k_BCJ2)
706      return true;
707  }
708  return false;
709}
710
711#ifndef _NO_CRYPTO
712
713class CCryptoGetTextPassword:
714  public ICryptoGetTextPassword,
715  public CMyUnknownImp
716{
717public:
718  UString Password;
719
720  MY_UNKNOWN_IMP
721  STDMETHOD(CryptoGetTextPassword)(BSTR *password);
722};
723
724STDMETHODIMP CCryptoGetTextPassword::CryptoGetTextPassword(BSTR *password)
725{
726  return StringToBstr(Password, password);
727}
728
729#endif
730
731static const int kNumGroupsMax = 4;
732
733static bool Is86Group(int group) { return (group & 1) != 0; }
734static bool IsEncryptedGroup(int group) { return (group & 2) != 0; }
735static int GetGroupIndex(bool encrypted, int bcjFiltered)
736  { return (encrypted ? 2 : 0) + (bcjFiltered ? 1 : 0); }
737
738static void GetFile(const CDatabase &inDb, int index, CFileItem &file, CFileItem2 &file2)
739{
740  file = inDb.Files[index];
741  file2.CTimeDefined = inDb.CTime.GetItem(index, file2.CTime);
742  file2.ATimeDefined = inDb.ATime.GetItem(index, file2.ATime);
743  file2.MTimeDefined = inDb.MTime.GetItem(index, file2.MTime);
744  file2.StartPosDefined = inDb.StartPos.GetItem(index, file2.StartPos);
745  file2.IsAnti = inDb.IsItemAnti(index);
746  // file2.IsAux = inDb.IsItemAux(index);
747}
748
749HRESULT Update(
750    DECL_EXTERNAL_CODECS_LOC_VARS
751    IInStream *inStream,
752    const CDbEx *db,
753    const CObjectVector<CUpdateItem> &updateItems,
754    // const CObjectVector<CTreeFolder> &treeFolders,
755    // const CUniqBlocks &secureBlocks,
756    COutArchive &archive,
757    CArchiveDatabaseOut &newDatabase,
758    ISequentialOutStream *seqOutStream,
759    IArchiveUpdateCallback *updateCallback,
760    const CUpdateOptions &options
761    #ifndef _NO_CRYPTO
762    , ICryptoGetTextPassword *getDecoderPassword
763    #endif
764    )
765{
766  UInt64 numSolidFiles = options.NumSolidFiles;
767  if (numSolidFiles == 0)
768    numSolidFiles = 1;
769
770  // size_t totalSecureDataSize = (size_t)secureBlocks.GetTotalSizeInBytes();
771
772  /*
773  CMyComPtr<IOutStream> outStream;
774  RINOK(seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStream));
775  if (!outStream)
776    return E_NOTIMPL;
777  */
778
779  UInt64 startBlockSize = db != 0 ? db->ArcInfo.StartPosition: 0;
780  if (startBlockSize > 0 && !options.RemoveSfxBlock)
781  {
782    RINOK(WriteRange(inStream, seqOutStream, 0, startBlockSize, NULL));
783  }
784
785  CIntArr fileIndexToUpdateIndexMap;
786  CRecordVector<CFolderRepack> folderRefs;
787  UInt64 complexity = 0;
788  UInt64 inSizeForReduce2 = 0;
789  bool needEncryptedRepack = false;
790  if (db != 0)
791  {
792    fileIndexToUpdateIndexMap.Alloc(db->Files.Size());
793    unsigned i;
794    for (i = 0; i < db->Files.Size(); i++)
795      fileIndexToUpdateIndexMap[i] = -1;
796
797    for (i = 0; i < updateItems.Size(); i++)
798    {
799      int index = updateItems[i].IndexInArchive;
800      if (index != -1)
801        fileIndexToUpdateIndexMap[index] = i;
802    }
803
804    for (i = 0; i < (int)db->NumFolders; i++)
805    {
806      CNum indexInFolder = 0;
807      CNum numCopyItems = 0;
808      CNum numUnpackStreams = db->NumUnpackStreamsVector[i];
809      UInt64 repackSize = 0;
810      for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++)
811      {
812        const CFileItem &file = db->Files[fi];
813        if (file.HasStream)
814        {
815          indexInFolder++;
816          int updateIndex = fileIndexToUpdateIndexMap[fi];
817          if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
818          {
819            numCopyItems++;
820            repackSize += file.Size;
821          }
822        }
823      }
824
825      if (numCopyItems == 0)
826        continue;
827
828      CFolderRepack rep;
829      rep.FolderIndex = i;
830      rep.NumCopyFiles = numCopyItems;
831      CFolder f;
832      db->ParseFolderInfo(i, f);
833      bool isEncrypted = f.IsEncrypted();
834      rep.Group = GetGroupIndex(isEncrypted, Is86FilteredFolder(f));
835      folderRefs.Add(rep);
836      if (numCopyItems == numUnpackStreams)
837        complexity += db->GetFolderFullPackSize(i);
838      else
839      {
840        complexity += repackSize;
841        if (repackSize > inSizeForReduce2)
842          inSizeForReduce2 = repackSize;
843        if (isEncrypted)
844          needEncryptedRepack = true;
845      }
846    }
847    folderRefs.Sort(CompareFolderRepacks, (void *)db);
848  }
849
850  UInt64 inSizeForReduce = 0;
851  unsigned i;
852  for (i = 0; i < updateItems.Size(); i++)
853  {
854    const CUpdateItem &ui = updateItems[i];
855    if (ui.NewData)
856    {
857      complexity += ui.Size;
858      if (numSolidFiles != 1)
859        inSizeForReduce += ui.Size;
860      else if (ui.Size > inSizeForReduce)
861        inSizeForReduce = ui.Size;
862    }
863  }
864
865  if (inSizeForReduce2 > inSizeForReduce)
866    inSizeForReduce = inSizeForReduce2;
867
868  RINOK(updateCallback->SetTotal(complexity));
869
870  CLocalProgress *lps = new CLocalProgress;
871  CMyComPtr<ICompressProgressInfo> progress = lps;
872  lps->Init(updateCallback, true);
873
874  CStreamBinder sb;
875  RINOK(sb.CreateEvents());
876
877  CThreadDecoder threadDecoder;
878  if (!folderRefs.IsEmpty())
879  {
880    #ifdef EXTERNAL_CODECS
881    threadDecoder.__externalCodecs = __externalCodecs;
882    #endif
883    RINOK(threadDecoder.Create());
884  }
885
886  CObjectVector<CSolidGroup> groups;
887  for (i = 0; i < kNumGroupsMax; i++)
888    groups.AddNew();
889
890  {
891    // ---------- Split files to groups ----------
892
893    bool useFilters = options.UseFilters;
894    const CCompressionMethodMode &method = *options.Method;
895    if (method.Methods.Size() != 1 || method.Binds.Size() != 0)
896      useFilters = false;
897    for (i = 0; i < updateItems.Size(); i++)
898    {
899      const CUpdateItem &ui = updateItems[i];
900      if (!ui.NewData || !ui.HasStream())
901        continue;
902      bool filteredGroup = false;
903      if (useFilters)
904      {
905        int dotPos = ui.Name.ReverseFind(L'.');
906        if (dotPos >= 0)
907          filteredGroup = IsExeExt(ui.Name.Ptr(dotPos + 1));
908      }
909      groups[GetGroupIndex(method.PasswordIsDefined, filteredGroup)].Indices.Add(i);
910    }
911  }
912
913  #ifndef _NO_CRYPTO
914
915  CCryptoGetTextPassword *getPasswordSpec = NULL;
916  if (needEncryptedRepack)
917  {
918    getPasswordSpec = new CCryptoGetTextPassword;
919    threadDecoder.getTextPassword = getPasswordSpec;
920
921    if (options.Method->PasswordIsDefined)
922      getPasswordSpec->Password = options.Method->Password;
923    else
924    {
925      if (!getDecoderPassword)
926        return E_NOTIMPL;
927      CMyComBSTR password;
928      RINOK(getDecoderPassword->CryptoGetTextPassword(&password));
929      if ((BSTR)password)
930        getPasswordSpec->Password = password;
931    }
932  }
933
934  #endif
935
936
937  // ---------- Compress ----------
938
939  RINOK(archive.Create(seqOutStream, false));
940  RINOK(archive.SkipPrefixArchiveHeader());
941
942  /*
943  CIntVector treeFolderToArcIndex;
944  treeFolderToArcIndex.Reserve(treeFolders.Size());
945  for (i = 0; i < treeFolders.Size(); i++)
946    treeFolderToArcIndex.Add(-1);
947  // ---------- Write Tree (only AUX dirs) ----------
948  for (i = 1; i < treeFolders.Size(); i++)
949  {
950    const CTreeFolder &treeFolder = treeFolders[i];
951    CFileItem file;
952    CFileItem2 file2;
953    file2.Init();
954    int secureID = 0;
955    if (treeFolder.UpdateItemIndex < 0)
956    {
957      // we can store virtual dir item wuthout attrib, but we want all items have attrib.
958      file.SetAttrib(FILE_ATTRIBUTE_DIRECTORY);
959      file2.IsAux = true;
960    }
961    else
962    {
963      const CUpdateItem &ui = updateItems[treeFolder.UpdateItemIndex];
964      // if item is not dir, then it's parent for alt streams.
965      // we will write such items later
966      if (!ui.IsDir)
967        continue;
968      secureID = ui.SecureIndex;
969      if (ui.NewProps)
970        FromUpdateItemToFileItem(ui, file, file2);
971      else
972        GetFile(*db, ui.IndexInArchive, file, file2);
973    }
974    file.Size = 0;
975    file.HasStream = false;
976    file.IsDir = true;
977    file.Parent = treeFolder.Parent;
978
979    treeFolderToArcIndex[i] = newDatabase.Files.Size();
980    newDatabase.AddFile(file, file2, treeFolder.Name);
981
982    if (totalSecureDataSize != 0)
983      newDatabase.SecureIDs.Add(secureID);
984  }
985  */
986
987  {
988    /* ---------- Write non-AUX dirs and Empty files ---------- */
989    CRecordVector<int> emptyRefs;
990    for (i = 0; i < updateItems.Size(); i++)
991    {
992      const CUpdateItem &ui = updateItems[i];
993      if (ui.NewData)
994      {
995        if (ui.HasStream())
996          continue;
997      }
998      else if (ui.IndexInArchive != -1 && db->Files[ui.IndexInArchive].HasStream)
999        continue;
1000      /*
1001      if (ui.TreeFolderIndex >= 0)
1002        continue;
1003      */
1004      emptyRefs.Add(i);
1005    }
1006    emptyRefs.Sort(CompareEmptyItems, (void *)&updateItems);
1007    for (i = 0; i < emptyRefs.Size(); i++)
1008    {
1009      const CUpdateItem &ui = updateItems[emptyRefs[i]];
1010      CFileItem file;
1011      CFileItem2 file2;
1012      UString name;
1013      if (ui.NewProps)
1014      {
1015        FromUpdateItemToFileItem(ui, file, file2);
1016        name = ui.Name;
1017      }
1018      else
1019      {
1020        GetFile(*db, ui.IndexInArchive, file, file2);
1021        db->GetPath(ui.IndexInArchive, name);
1022      }
1023
1024      /*
1025      if (totalSecureDataSize != 0)
1026        newDatabase.SecureIDs.Add(ui.SecureIndex);
1027      file.Parent = ui.ParentFolderIndex;
1028      */
1029      newDatabase.AddFile(file, file2, name);
1030    }
1031  }
1032
1033  unsigned folderRefIndex = 0;
1034  lps->ProgressOffset = 0;
1035
1036  for (int groupIndex = 0; groupIndex < kNumGroupsMax; groupIndex++)
1037  {
1038    const CSolidGroup &group = groups[groupIndex];
1039
1040    CCompressionMethodMode method = *options.Method;
1041    MakeExeMethod(method, options.UseFilters, Is86Group(groupIndex), options.MaxFilter);
1042
1043    if (IsEncryptedGroup(groupIndex))
1044    {
1045      if (!method.PasswordIsDefined)
1046      {
1047        #ifndef _NO_CRYPTO
1048        if (getPasswordSpec)
1049          method.Password = getPasswordSpec->Password;
1050        #endif
1051        method.PasswordIsDefined = true;
1052      }
1053    }
1054    else
1055    {
1056      method.PasswordIsDefined = false;
1057      method.Password.Empty();
1058    }
1059
1060    CEncoder encoder(method);
1061
1062    for (; folderRefIndex < folderRefs.Size(); folderRefIndex++)
1063    {
1064      const CFolderRepack &rep = folderRefs[folderRefIndex];
1065      if (rep.Group != groupIndex)
1066        break;
1067      int folderIndex = rep.FolderIndex;
1068
1069      if (rep.NumCopyFiles == db->NumUnpackStreamsVector[folderIndex])
1070      {
1071        UInt64 packSize = db->GetFolderFullPackSize(folderIndex);
1072        RINOK(WriteRange(inStream, archive.SeqStream,
1073          db->GetFolderStreamPos(folderIndex, 0), packSize, progress));
1074        lps->ProgressOffset += packSize;
1075
1076        CFolder &folder = newDatabase.Folders.AddNew();
1077        db->ParseFolderInfo(folderIndex, folder);
1078        CNum startIndex = db->FoStartPackStreamIndex[folderIndex];
1079        for (unsigned j = 0; j < folder.PackStreams.Size(); j++)
1080        {
1081          newDatabase.PackSizes.Add(db->GetStreamPackSize(startIndex + j));
1082          // newDatabase.PackCRCsDefined.Add(db.PackCRCsDefined[startIndex + j]);
1083          // newDatabase.PackCRCs.Add(db.PackCRCs[startIndex + j]);
1084        }
1085
1086        UInt32 indexStart = db->FoToCoderUnpackSizes[folderIndex];
1087        UInt32 indexEnd = db->FoToCoderUnpackSizes[folderIndex + 1];
1088        for (; indexStart < indexEnd; indexStart++)
1089          newDatabase.CoderUnpackSizes.Add(db->CoderUnpackSizes[indexStart]);
1090      }
1091      else
1092      {
1093        CBoolVector extractStatuses;
1094
1095        CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
1096        CNum indexInFolder = 0;
1097
1098        for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
1099        {
1100          bool needExtract = false;
1101          if (db->Files[fi].HasStream)
1102          {
1103            indexInFolder++;
1104            int updateIndex = fileIndexToUpdateIndexMap[fi];
1105            if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
1106              needExtract = true;
1107          }
1108          extractStatuses.Add(needExtract);
1109        }
1110
1111        unsigned startPackIndex = newDatabase.PackSizes.Size();
1112        UInt64 curUnpackSize;
1113        {
1114          CMyComPtr<ISequentialInStream> sbInStream;
1115          {
1116            CMyComPtr<ISequentialOutStream> sbOutStream;
1117            sb.CreateStreams(&sbInStream, &sbOutStream);
1118            sb.ReInit();
1119            RINOK(threadDecoder.FosSpec->Init(db, db->FolderStartFileIndex[folderIndex], &extractStatuses, sbOutStream));
1120          }
1121
1122          threadDecoder.InStream = inStream;
1123          threadDecoder.Folders = (const CFolders *)db;
1124          threadDecoder.FolderIndex = folderIndex;
1125          threadDecoder.StartPos = db->ArcInfo.DataStartPosition; // db->GetFolderStreamPos(folderIndex, 0);
1126
1127          threadDecoder.Start();
1128
1129          RINOK(encoder.Encode(
1130              EXTERNAL_CODECS_LOC_VARS
1131              sbInStream, NULL, &inSizeForReduce,
1132              newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curUnpackSize,
1133              archive.SeqStream, newDatabase.PackSizes, progress));
1134
1135          threadDecoder.WaitExecuteFinish();
1136        }
1137
1138        RINOK(threadDecoder.Result);
1139
1140        for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
1141          lps->OutSize += newDatabase.PackSizes[startPackIndex];
1142        lps->InSize += curUnpackSize;
1143      }
1144
1145      newDatabase.NumUnpackStreamsVector.Add(rep.NumCopyFiles);
1146
1147      CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
1148
1149      CNum indexInFolder = 0;
1150      for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
1151      {
1152        CFileItem file;
1153        CFileItem2 file2;
1154        GetFile(*db, fi, file, file2);
1155        UString name;
1156        db->GetPath(fi, name);
1157        if (file.HasStream)
1158        {
1159          indexInFolder++;
1160          int updateIndex = fileIndexToUpdateIndexMap[fi];
1161          if (updateIndex >= 0)
1162          {
1163            const CUpdateItem &ui = updateItems[updateIndex];
1164            if (ui.NewData)
1165              continue;
1166            if (ui.NewProps)
1167            {
1168              CFileItem uf;
1169              FromUpdateItemToFileItem(ui, uf, file2);
1170              uf.Size = file.Size;
1171              uf.Crc = file.Crc;
1172              uf.CrcDefined = file.CrcDefined;
1173              uf.HasStream = file.HasStream;
1174              file = uf;
1175              name = ui.Name;
1176            }
1177            /*
1178            file.Parent = ui.ParentFolderIndex;
1179            if (ui.TreeFolderIndex >= 0)
1180              treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
1181            if (totalSecureDataSize != 0)
1182              newDatabase.SecureIDs.Add(ui.SecureIndex);
1183            */
1184            newDatabase.AddFile(file, file2, name);
1185          }
1186        }
1187      }
1188    }
1189
1190    unsigned numFiles = group.Indices.Size();
1191    if (numFiles == 0)
1192      continue;
1193    CRecordVector<CRefItem> refItems;
1194    refItems.ClearAndSetSize(numFiles);
1195    bool sortByType = (numSolidFiles > 1);
1196    for (i = 0; i < numFiles; i++)
1197      refItems[i] = CRefItem(group.Indices[i], updateItems[group.Indices[i]], sortByType);
1198    CSortParam sortParam;
1199    // sortParam.TreeFolders = &treeFolders;
1200    sortParam.SortByType = sortByType;
1201    refItems.Sort(CompareUpdateItems, (void *)&sortParam);
1202
1203    CObjArray<UInt32> indices(numFiles);
1204
1205    for (i = 0; i < numFiles; i++)
1206    {
1207      UInt32 index = refItems[i].Index;
1208      indices[i] = index;
1209      /*
1210      const CUpdateItem &ui = updateItems[index];
1211      CFileItem file;
1212      if (ui.NewProps)
1213        FromUpdateItemToFileItem(ui, file);
1214      else
1215        file = db.Files[ui.IndexInArchive];
1216      if (file.IsAnti || file.IsDir)
1217        return E_FAIL;
1218      newDatabase.Files.Add(file);
1219      */
1220    }
1221
1222    for (i = 0; i < numFiles;)
1223    {
1224      UInt64 totalSize = 0;
1225      int numSubFiles;
1226      UString prevExtension;
1227      for (numSubFiles = 0; i + numSubFiles < numFiles &&
1228          numSubFiles < numSolidFiles; numSubFiles++)
1229      {
1230        const CUpdateItem &ui = updateItems[indices[i + numSubFiles]];
1231        totalSize += ui.Size;
1232        if (totalSize > options.NumSolidBytes)
1233          break;
1234        if (options.SolidExtension)
1235        {
1236          UString ext = ui.GetExtension();
1237          if (numSubFiles == 0)
1238            prevExtension = ext;
1239          else
1240            if (!ext.IsEqualToNoCase(prevExtension))
1241              break;
1242        }
1243      }
1244      if (numSubFiles < 1)
1245        numSubFiles = 1;
1246
1247      CFolderInStream *inStreamSpec = new CFolderInStream;
1248      CMyComPtr<ISequentialInStream> solidInStream(inStreamSpec);
1249      inStreamSpec->Init(updateCallback, &indices[i], numSubFiles);
1250
1251      unsigned startPackIndex = newDatabase.PackSizes.Size();
1252      UInt64 curFolderUnpackSize;
1253      RINOK(encoder.Encode(
1254          EXTERNAL_CODECS_LOC_VARS
1255          solidInStream, NULL, &inSizeForReduce,
1256          newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curFolderUnpackSize,
1257          archive.SeqStream, newDatabase.PackSizes, progress));
1258
1259      for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
1260        lps->OutSize += newDatabase.PackSizes[startPackIndex];
1261
1262      lps->InSize += curFolderUnpackSize;
1263      // for ()
1264      // newDatabase.PackCRCsDefined.Add(false);
1265      // newDatabase.PackCRCs.Add(0);
1266
1267      CNum numUnpackStreams = 0;
1268      for (int subIndex = 0; subIndex < numSubFiles; subIndex++)
1269      {
1270        const CUpdateItem &ui = updateItems[indices[i + subIndex]];
1271        CFileItem file;
1272        CFileItem2 file2;
1273        UString name;
1274        if (ui.NewProps)
1275        {
1276          FromUpdateItemToFileItem(ui, file, file2);
1277          name = ui.Name;
1278        }
1279        else
1280        {
1281          GetFile(*db, ui.IndexInArchive, file, file2);
1282          db->GetPath(ui.IndexInArchive, name);
1283        }
1284        if (file2.IsAnti || file.IsDir)
1285          return E_FAIL;
1286
1287        /*
1288        CFileItem &file = newDatabase.Files[
1289              startFileIndexInDatabase + i + subIndex];
1290        */
1291        if (!inStreamSpec->Processed[subIndex])
1292        {
1293          continue;
1294          // file.Name += L".locked";
1295        }
1296
1297        file.Crc = inStreamSpec->CRCs[subIndex];
1298        file.Size = inStreamSpec->Sizes[subIndex];
1299        if (file.Size != 0)
1300        {
1301          file.CrcDefined = true;
1302          file.HasStream = true;
1303          numUnpackStreams++;
1304        }
1305        else
1306        {
1307          file.CrcDefined = false;
1308          file.HasStream = false;
1309        }
1310        /*
1311        file.Parent = ui.ParentFolderIndex;
1312        if (ui.TreeFolderIndex >= 0)
1313          treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
1314        if (totalSecureDataSize != 0)
1315          newDatabase.SecureIDs.Add(ui.SecureIndex);
1316        */
1317        newDatabase.AddFile(file, file2, name);
1318      }
1319      // numUnpackStreams = 0 is very bad case for locked files
1320      // v3.13 doesn't understand it.
1321      newDatabase.NumUnpackStreamsVector.Add(numUnpackStreams);
1322      i += numSubFiles;
1323    }
1324  }
1325
1326  if (folderRefIndex != folderRefs.Size())
1327    return E_FAIL;
1328
1329  RINOK(lps->SetCur());
1330
1331  /*
1332  folderRefs.ClearAndFree();
1333  fileIndexToUpdateIndexMap.ClearAndFree();
1334  groups.ClearAndFree();
1335  */
1336
1337  /*
1338  for (i = 0; i < newDatabase.Files.Size(); i++)
1339  {
1340    CFileItem &file = newDatabase.Files[i];
1341    file.Parent = treeFolderToArcIndex[file.Parent];
1342  }
1343
1344  if (totalSecureDataSize != 0)
1345  {
1346    newDatabase.SecureBuf.SetCapacity(totalSecureDataSize);
1347    size_t pos = 0;
1348    newDatabase.SecureSizes.Reserve(secureBlocks.Sorted.Size());
1349    for (i = 0; i < secureBlocks.Sorted.Size(); i++)
1350    {
1351      const CByteBuffer &buf = secureBlocks.Bufs[secureBlocks.Sorted[i]];
1352      size_t size = buf.GetCapacity();
1353      memcpy(newDatabase.SecureBuf + pos, buf, size);
1354      newDatabase.SecureSizes.Add((UInt32)size);
1355      pos += size;
1356    }
1357  }
1358  */
1359  newDatabase.ReserveDown();
1360  return S_OK;
1361}
1362
1363}}
1364