1 // 7zUpdate.cpp
2 
3 #include "StdAfx.h"
4 
5 #include "../../../../C/CpuArch.h"
6 
7 #include "../../Common/LimitedStreams.h"
8 #include "../../Common/ProgressUtils.h"
9 
10 #include "../../Common/CreateCoder.h"
11 
12 #include "../../Compress/CopyCoder.h"
13 
14 #include "../Common/ItemNameUtils.h"
15 #include "../Common/OutStreamWithCRC.h"
16 
17 #include "7zDecode.h"
18 #include "7zEncode.h"
19 #include "7zFolderInStream.h"
20 #include "7zHandler.h"
21 #include "7zOut.h"
22 #include "7zUpdate.h"
23 
24 namespace NArchive {
25 namespace N7z {
26 
27 static const UInt64 k_LZMA = 0x030101;
28 static const UInt64 k_BCJ  = 0x03030103;
29 static const UInt64 k_BCJ2 = 0x0303011B;
30 
31 static const wchar_t *kMatchFinderForBCJ2_LZMA = L"BT2";
32 static const UInt32 kDictionaryForBCJ2_LZMA = 1 << 20;
33 static const UInt32 kAlgorithmForBCJ2_LZMA = 1;
34 static const UInt32 kNumFastBytesForBCJ2_LZMA = 64;
35 
36 #ifdef MY_CPU_X86_OR_AMD64
37 #define USE_86_FILTER
38 #endif
39 
WriteRange(IInStream * inStream,ISequentialOutStream * outStream,UInt64 position,UInt64 size,ICompressProgressInfo * progress)40 static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream,
41     UInt64 position, UInt64 size, ICompressProgressInfo *progress)
42 {
43   RINOK(inStream->Seek(position, STREAM_SEEK_SET, 0));
44   CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream;
45   CMyComPtr<CLimitedSequentialInStream> inStreamLimited(streamSpec);
46   streamSpec->SetStream(inStream);
47   streamSpec->Init(size);
48 
49   NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder;
50   CMyComPtr<ICompressCoder> copyCoder = copyCoderSpec;
51   RINOK(copyCoder->Code(inStreamLimited, outStream, NULL, NULL, progress));
52   return (copyCoderSpec->TotalSize == size ? S_OK : E_FAIL);
53 }
54 
GetReverseSlashPos(const UString & name)55 static int GetReverseSlashPos(const UString &name)
56 {
57   int slashPos = name.ReverseFind(L'/');
58   #ifdef _WIN32
59   int slash1Pos = name.ReverseFind(L'\\');
60   slashPos = MyMax(slashPos, slash1Pos);
61   #endif
62   return slashPos;
63 }
64 
GetExtensionPos() const65 int CUpdateItem::GetExtensionPos() const
66 {
67   int slashPos = GetReverseSlashPos(Name);
68   int dotPos = Name.ReverseFind(L'.');
69   if (dotPos < 0 || (dotPos < slashPos && slashPos >= 0))
70     return Name.Length();
71   return dotPos + 1;
72 }
73 
GetExtension() const74 UString CUpdateItem::GetExtension() const
75 {
76   return Name.Mid(GetExtensionPos());
77 }
78 
79 #define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; }
80 
81 #define RINOZ_COMP(a, b) RINOZ(MyCompare(a, b))
82 
CompareBuffers(const CByteBuffer & a1,const CByteBuffer & a2)83 static int CompareBuffers(const CByteBuffer &a1, const CByteBuffer &a2)
84 {
85   size_t c1 = a1.GetCapacity();
86   size_t c2 = a2.GetCapacity();
87   RINOZ_COMP(c1, c2);
88   for (size_t i = 0; i < c1; i++)
89     RINOZ_COMP(a1[i], a2[i]);
90   return 0;
91 }
92 
CompareCoders(const CCoderInfo & c1,const CCoderInfo & c2)93 static int CompareCoders(const CCoderInfo &c1, const CCoderInfo &c2)
94 {
95   RINOZ_COMP(c1.NumInStreams, c2.NumInStreams);
96   RINOZ_COMP(c1.NumOutStreams, c2.NumOutStreams);
97   RINOZ_COMP(c1.MethodID, c2.MethodID);
98   return CompareBuffers(c1.Props, c2.Props);
99 }
100 
CompareBindPairs(const CBindPair & b1,const CBindPair & b2)101 static int CompareBindPairs(const CBindPair &b1, const CBindPair &b2)
102 {
103   RINOZ_COMP(b1.InIndex, b2.InIndex);
104   return MyCompare(b1.OutIndex, b2.OutIndex);
105 }
106 
CompareFolders(const CFolder & f1,const CFolder & f2)107 static int CompareFolders(const CFolder &f1, const CFolder &f2)
108 {
109   int s1 = f1.Coders.Size();
110   int s2 = f2.Coders.Size();
111   RINOZ_COMP(s1, s2);
112   int i;
113   for (i = 0; i < s1; i++)
114     RINOZ(CompareCoders(f1.Coders[i], f2.Coders[i]));
115   s1 = f1.BindPairs.Size();
116   s2 = f2.BindPairs.Size();
117   RINOZ_COMP(s1, s2);
118   for (i = 0; i < s1; i++)
119     RINOZ(CompareBindPairs(f1.BindPairs[i], f2.BindPairs[i]));
120   return 0;
121 }
122 
123 /*
124 static int CompareFiles(const CFileItem &f1, const CFileItem &f2)
125 {
126   return MyStringCompareNoCase(f1.Name, f2.Name);
127 }
128 */
129 
130 struct CFolderRepack
131 {
132   int FolderIndex;
133   int Group;
134   CNum NumCopyFiles;
135 };
136 
CompareFolderRepacks(const CFolderRepack * p1,const CFolderRepack * p2,void * param)137 static int CompareFolderRepacks(const CFolderRepack *p1, const CFolderRepack *p2, void *param)
138 {
139   RINOZ_COMP(p1->Group, p2->Group);
140   int i1 = p1->FolderIndex;
141   int i2 = p2->FolderIndex;
142   const CArchiveDatabaseEx &db = *(const CArchiveDatabaseEx *)param;
143   RINOZ(CompareFolders(
144       db.Folders[i1],
145       db.Folders[i2]));
146   return MyCompare(i1, i2);
147   /*
148   RINOZ_COMP(
149       db.NumUnpackStreamsVector[i1],
150       db.NumUnpackStreamsVector[i2]);
151   if (db.NumUnpackStreamsVector[i1] == 0)
152     return 0;
153   return CompareFiles(
154       db.Files[db.FolderStartFileIndex[i1]],
155       db.Files[db.FolderStartFileIndex[i2]]);
156   */
157 }
158 
159 ////////////////////////////////////////////////////////////
160 
CompareEmptyItems(const int * p1,const int * p2,void * param)161 static int CompareEmptyItems(const int *p1, const int *p2, void *param)
162 {
163   const CObjectVector<CUpdateItem> &updateItems = *(const CObjectVector<CUpdateItem> *)param;
164   const CUpdateItem &u1 = updateItems[*p1];
165   const CUpdateItem &u2 = updateItems[*p2];
166   if (u1.IsDir != u2.IsDir)
167     return (u1.IsDir) ? 1 : -1;
168   if (u1.IsDir)
169   {
170     if (u1.IsAnti != u2.IsAnti)
171       return (u1.IsAnti ? 1 : -1);
172     int n = MyStringCompareNoCase(u1.Name, u2.Name);
173     return -n;
174   }
175   if (u1.IsAnti != u2.IsAnti)
176     return (u1.IsAnti ? 1 : -1);
177   return MyStringCompareNoCase(u1.Name, u2.Name);
178 }
179 
180 static const char *g_Exts =
181   " lzma 7z ace arc arj bz bz2 deb lzo lzx gz pak rpm sit tgz tbz tbz2 tgz cab ha lha lzh rar zoo"
182   " zip jar ear war msi"
183   " 3gp avi mov mpeg mpg mpe wmv"
184   " aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav"
185   " swf "
186   " chm hxi hxs"
187   " gif jpeg jpg jp2 png tiff  bmp ico psd psp"
188   " awg ps eps cgm dxf svg vrml wmf emf ai md"
189   " cad dwg pps key sxi"
190   " max 3ds"
191   " iso bin nrg mdf img pdi tar cpio xpi"
192   " vfd vhd vud vmc vsv"
193   " vmdk dsk nvram vmem vmsd vmsn vmss vmtm"
194   " inl inc idl acf asa h hpp hxx c cpp cxx rc java cs pas bas vb cls ctl frm dlg def"
195   " f77 f f90 f95"
196   " asm sql manifest dep "
197   " mak clw csproj vcproj sln dsp dsw "
198   " class "
199   " bat cmd"
200   " xml xsd xsl xslt hxk hxc htm html xhtml xht mht mhtml htw asp aspx css cgi jsp shtml"
201   " awk sed hta js php php3 php4 php5 phptml pl pm py pyo rb sh tcl vbs"
202   " text txt tex ans asc srt reg ini doc docx mcw dot rtf hlp xls xlr xlt xlw ppt pdf"
203   " sxc sxd sxi sxg sxw stc sti stw stm odt ott odg otg odp otp ods ots odf"
204   " abw afp cwk lwp wpd wps wpt wrf wri"
205   " abf afm bdf fon mgf otf pcf pfa snf ttf"
206   " dbf mdb nsf ntf wdb db fdb gdb"
207   " exe dll ocx vbx sfx sys tlb awx com obj lib out o so "
208   " pdb pch idb ncb opt";
209 
GetExtIndex(const char * ext)210 int GetExtIndex(const char *ext)
211 {
212   int extIndex = 1;
213   const char *p = g_Exts;
214   for (;;)
215   {
216     char c = *p++;
217     if (c == 0)
218       return extIndex;
219     if (c == ' ')
220       continue;
221     int pos = 0;
222     for (;;)
223     {
224       char c2 = ext[pos++];
225       if (c2 == 0 && (c == 0 || c == ' '))
226         return extIndex;
227       if (c != c2)
228         break;
229       c = *p++;
230     }
231     extIndex++;
232     for (;;)
233     {
234       if (c == 0)
235         return extIndex;
236       if (c == ' ')
237         break;
238       c = *p++;
239     }
240   }
241 }
242 
243 struct CRefItem
244 {
245   const CUpdateItem *UpdateItem;
246   UInt32 Index;
247   UInt32 ExtensionPos;
248   UInt32 NamePos;
249   int ExtensionIndex;
CRefItemNArchive::N7z::CRefItem250   CRefItem(UInt32 index, const CUpdateItem &ui, bool sortByType):
251     UpdateItem(&ui),
252     Index(index),
253     ExtensionPos(0),
254     NamePos(0),
255     ExtensionIndex(0)
256   {
257     if (sortByType)
258     {
259       int slashPos = GetReverseSlashPos(ui.Name);
260       NamePos = ((slashPos >= 0) ? (slashPos + 1) : 0);
261       int dotPos = ui.Name.ReverseFind(L'.');
262       if (dotPos < 0 || (dotPos < slashPos && slashPos >= 0))
263         ExtensionPos = ui.Name.Length();
264       else
265       {
266         ExtensionPos = dotPos + 1;
267         UString us = ui.Name.Mid(ExtensionPos);
268         if (!us.IsEmpty())
269         {
270           us.MakeLower();
271           int i;
272           AString s;
273           for (i = 0; i < us.Length(); i++)
274           {
275             wchar_t c = us[i];
276             if (c >= 0x80)
277               break;
278             s += (char)c;
279           }
280           if (i == us.Length())
281             ExtensionIndex = GetExtIndex(s);
282           else
283             ExtensionIndex = 0;
284         }
285       }
286     }
287   }
288 };
289 
CompareUpdateItems(const CRefItem * p1,const CRefItem * p2,void * param)290 static int CompareUpdateItems(const CRefItem *p1, const CRefItem *p2, void *param)
291 {
292   const CRefItem &a1 = *p1;
293   const CRefItem &a2 = *p2;
294   const CUpdateItem &u1 = *a1.UpdateItem;
295   const CUpdateItem &u2 = *a2.UpdateItem;
296   int n;
297   if (u1.IsDir != u2.IsDir)
298     return (u1.IsDir) ? 1 : -1;
299   if (u1.IsDir)
300   {
301     if (u1.IsAnti != u2.IsAnti)
302       return (u1.IsAnti ? 1 : -1);
303     n = MyStringCompareNoCase(u1.Name, u2.Name);
304     return -n;
305   }
306   bool sortByType = *(bool *)param;
307   if (sortByType)
308   {
309     RINOZ_COMP(a1.ExtensionIndex, a2.ExtensionIndex);
310     RINOZ(MyStringCompareNoCase(u1.Name + a1.ExtensionPos, u2.Name + a2.ExtensionPos));
311     RINOZ(MyStringCompareNoCase(u1.Name + a1.NamePos, u2.Name + a2.NamePos));
312     if (!u1.MTimeDefined && u2.MTimeDefined) return 1;
313     if (u1.MTimeDefined && !u2.MTimeDefined) return -1;
314     if (u1.MTimeDefined && u2.MTimeDefined) RINOZ_COMP(u1.MTime, u2.MTime);
315     RINOZ_COMP(u1.Size, u2.Size);
316   }
317   return MyStringCompareNoCase(u1.Name, u2.Name);
318 }
319 
320 struct CSolidGroup
321 {
322   CRecordVector<UInt32> Indices;
323 };
324 
325 static wchar_t *g_ExeExts[] =
326 {
327   L"dll",
328   L"exe",
329   L"ocx",
330   L"sfx",
331   L"sys"
332 };
333 
IsExeExt(const UString & ext)334 static bool IsExeExt(const UString &ext)
335 {
336   for (int i = 0; i < sizeof(g_ExeExts) / sizeof(g_ExeExts[0]); i++)
337     if (ext.CompareNoCase(g_ExeExts[i]) == 0)
338       return true;
339   return false;
340 }
341 
342 #ifdef USE_86_FILTER
343 
GetMethodFull(UInt64 methodID,UInt32 numInStreams,CMethodFull & methodResult)344 static inline void GetMethodFull(UInt64 methodID, UInt32 numInStreams, CMethodFull &methodResult)
345 {
346   methodResult.Id = methodID;
347   methodResult.NumInStreams = numInStreams;
348   methodResult.NumOutStreams = 1;
349 }
350 
MakeExeMethod(const CCompressionMethodMode & method,bool bcj2Filter,CCompressionMethodMode & exeMethod)351 static void MakeExeMethod(const CCompressionMethodMode &method,
352     bool bcj2Filter, CCompressionMethodMode &exeMethod)
353 {
354   exeMethod = method;
355   if (bcj2Filter)
356   {
357     CMethodFull methodFull;
358     GetMethodFull(k_BCJ2, 4, methodFull);
359     exeMethod.Methods.Insert(0, methodFull);
360     GetMethodFull(k_LZMA, 1, methodFull);
361     {
362       CProp prop;
363       prop.Id = NCoderPropID::kAlgorithm;
364       prop.Value = kAlgorithmForBCJ2_LZMA;
365       methodFull.Props.Add(prop);
366     }
367     {
368       CProp prop;
369       prop.Id = NCoderPropID::kMatchFinder;
370       prop.Value = kMatchFinderForBCJ2_LZMA;
371       methodFull.Props.Add(prop);
372     }
373     {
374       CProp prop;
375       prop.Id = NCoderPropID::kDictionarySize;
376       prop.Value = kDictionaryForBCJ2_LZMA;
377       methodFull.Props.Add(prop);
378     }
379     {
380       CProp prop;
381       prop.Id = NCoderPropID::kNumFastBytes;
382       prop.Value = kNumFastBytesForBCJ2_LZMA;
383       methodFull.Props.Add(prop);
384     }
385     {
386       CProp prop;
387       prop.Id = NCoderPropID::kNumThreads;
388       prop.Value = (UInt32)1;
389       methodFull.Props.Add(prop);
390     }
391 
392     exeMethod.Methods.Add(methodFull);
393     exeMethod.Methods.Add(methodFull);
394     CBind bind;
395 
396     bind.OutCoder = 0;
397     bind.InStream = 0;
398 
399     bind.InCoder = 1;
400     bind.OutStream = 0;
401     exeMethod.Binds.Add(bind);
402 
403     bind.InCoder = 2;
404     bind.OutStream = 1;
405     exeMethod.Binds.Add(bind);
406 
407     bind.InCoder = 3;
408     bind.OutStream = 2;
409     exeMethod.Binds.Add(bind);
410   }
411   else
412   {
413     CMethodFull methodFull;
414     GetMethodFull(k_BCJ, 1, methodFull);
415     exeMethod.Methods.Insert(0, methodFull);
416     CBind bind;
417     bind.OutCoder = 0;
418     bind.InStream = 0;
419     bind.InCoder = 1;
420     bind.OutStream = 0;
421     exeMethod.Binds.Add(bind);
422   }
423 }
424 
425 #endif
426 
FromUpdateItemToFileItem(const CUpdateItem & ui,CFileItem & file,CFileItem2 & file2)427 static void FromUpdateItemToFileItem(const CUpdateItem &ui,
428     CFileItem &file, CFileItem2 &file2)
429 {
430   file.Name = NItemName::MakeLegalName(ui.Name);
431   if (ui.AttribDefined)
432     file.SetAttrib(ui.Attrib);
433 
434   file2.CTime = ui.CTime;  file2.CTimeDefined = ui.CTimeDefined;
435   file2.ATime = ui.ATime;  file2.ATimeDefined = ui.ATimeDefined;
436   file2.MTime = ui.MTime;  file2.MTimeDefined = ui.MTimeDefined;
437   file2.IsAnti = ui.IsAnti;
438   file2.StartPosDefined = false;
439 
440   file.Size = ui.Size;
441   file.IsDir = ui.IsDir;
442   file.HasStream = ui.HasStream();
443 }
444 
445 class CFolderOutStream2:
446   public ISequentialOutStream,
447   public CMyUnknownImp
448 {
449   COutStreamWithCRC *_crcStreamSpec;
450   CMyComPtr<ISequentialOutStream> _crcStream;
451   const CArchiveDatabaseEx *_db;
452   const CBoolVector *_extractStatuses;
453   CMyComPtr<ISequentialOutStream> _outStream;
454   UInt32 _startIndex;
455   int _currentIndex;
456   bool _fileIsOpen;
457   UInt64 _rem;
458 
459   void OpenFile();
460   void CloseFile();
461   HRESULT CloseFileAndSetResult();
462   HRESULT ProcessEmptyFiles();
463 public:
464   MY_UNKNOWN_IMP
465 
CFolderOutStream2()466   CFolderOutStream2()
467   {
468     _crcStreamSpec = new COutStreamWithCRC;
469     _crcStream = _crcStreamSpec;
470   }
471 
472   HRESULT Init(const CArchiveDatabaseEx *db, UInt32 startIndex,
473       const CBoolVector *extractStatuses, ISequentialOutStream *outStream);
474   void ReleaseOutStream();
CheckFinishedState() const475   HRESULT CheckFinishedState() const { return (_currentIndex == _extractStatuses->Size()) ? S_OK: E_FAIL; }
476 
477   STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize);
478 };
479 
Init(const CArchiveDatabaseEx * db,UInt32 startIndex,const CBoolVector * extractStatuses,ISequentialOutStream * outStream)480 HRESULT CFolderOutStream2::Init(const CArchiveDatabaseEx *db, UInt32 startIndex,
481     const CBoolVector *extractStatuses, ISequentialOutStream *outStream)
482 {
483   _db = db;
484   _startIndex = startIndex;
485   _extractStatuses = extractStatuses;
486   _outStream = outStream;
487 
488   _currentIndex = 0;
489   _fileIsOpen = false;
490   return ProcessEmptyFiles();
491 }
492 
ReleaseOutStream()493 void CFolderOutStream2::ReleaseOutStream()
494 {
495   _outStream.Release();
496   _crcStreamSpec->ReleaseStream();
497 }
498 
OpenFile()499 void CFolderOutStream2::OpenFile()
500 {
501   _crcStreamSpec->SetStream((*_extractStatuses)[_currentIndex] ? _outStream : NULL);
502   _crcStreamSpec->Init(true);
503   _fileIsOpen = true;
504   _rem = _db->Files[_startIndex + _currentIndex].Size;
505 }
506 
CloseFile()507 void CFolderOutStream2::CloseFile()
508 {
509   _crcStreamSpec->ReleaseStream();
510   _fileIsOpen = false;
511   _currentIndex++;
512 }
513 
CloseFileAndSetResult()514 HRESULT CFolderOutStream2::CloseFileAndSetResult()
515 {
516   const CFileItem &file = _db->Files[_startIndex + _currentIndex];
517   CloseFile();
518   return (file.IsDir || !file.CrcDefined || file.Crc == _crcStreamSpec->GetCRC()) ? S_OK: S_FALSE;
519 }
520 
ProcessEmptyFiles()521 HRESULT CFolderOutStream2::ProcessEmptyFiles()
522 {
523   while (_currentIndex < _extractStatuses->Size() && _db->Files[_startIndex + _currentIndex].Size == 0)
524   {
525     OpenFile();
526     RINOK(CloseFileAndSetResult());
527   }
528   return S_OK;
529 }
530 
Write(const void * data,UInt32 size,UInt32 * processedSize)531 STDMETHODIMP CFolderOutStream2::Write(const void *data, UInt32 size, UInt32 *processedSize)
532 {
533   if (processedSize != NULL)
534     *processedSize = 0;
535   while (size != 0)
536   {
537     if (_fileIsOpen)
538     {
539       UInt32 cur = size < _rem ? size : (UInt32)_rem;
540       RINOK(_crcStream->Write(data, cur, &cur));
541       if (cur == 0)
542         break;
543       data = (const Byte *)data + cur;
544       size -= cur;
545       _rem -= cur;
546       if (processedSize != NULL)
547         *processedSize += cur;
548       if (_rem == 0)
549       {
550         RINOK(CloseFileAndSetResult());
551         RINOK(ProcessEmptyFiles());
552         continue;
553       }
554     }
555     else
556     {
557       RINOK(ProcessEmptyFiles());
558       if (_currentIndex == _extractStatuses->Size())
559       {
560         // we don't support partial extracting
561         return E_FAIL;
562       }
563       OpenFile();
564     }
565   }
566   return S_OK;
567 }
568 
569 class CThreadDecoder: public CVirtThread
570 {
571 public:
572   HRESULT Result;
573   CMyComPtr<IInStream> InStream;
574 
575   CFolderOutStream2 *FosSpec;
576   CMyComPtr<ISequentialOutStream> Fos;
577 
578   UInt64 StartPos;
579   const UInt64 *PackSizes;
580   const CFolder *Folder;
581   #ifndef _NO_CRYPTO
582   CMyComPtr<ICryptoGetTextPassword> GetTextPassword;
583   #endif
584 
585   DECL_EXTERNAL_CODECS_VARS
586   CDecoder Decoder;
587 
588   #ifndef _7ZIP_ST
589   bool MtMode;
590   UInt32 NumThreads;
591   #endif
592 
CThreadDecoder()593   CThreadDecoder():
594     Decoder(true)
595   {
596     #ifndef _7ZIP_ST
597     MtMode = false;
598     NumThreads = 1;
599     #endif
600     FosSpec = new CFolderOutStream2;
601     Fos = FosSpec;
602     Result = E_FAIL;
603   }
604   virtual void Execute();
605 };
606 
Execute()607 void CThreadDecoder::Execute()
608 {
609   try
610   {
611     #ifndef _NO_CRYPTO
612     bool passwordIsDefined;
613     #endif
614     Result = Decoder.Decode(
615       EXTERNAL_CODECS_VARS
616       InStream,
617       StartPos,
618       PackSizes,
619       *Folder,
620       Fos,
621       NULL
622       #ifndef _NO_CRYPTO
623       , GetTextPassword, passwordIsDefined
624       #endif
625       #ifndef _7ZIP_ST
626       , MtMode, NumThreads
627       #endif
628       );
629   }
630   catch(...)
631   {
632     Result = E_FAIL;
633   }
634   if (Result == S_OK)
635     Result = FosSpec->CheckFinishedState();
636   FosSpec->ReleaseOutStream();
637 }
638 
Is86FilteredFolder(const CFolder & f)639 bool static Is86FilteredFolder(const CFolder &f)
640 {
641   for (int i = 0; i < f.Coders.Size(); i++)
642   {
643     CMethodId m = f.Coders[i].MethodID;
644     if (m == k_BCJ || m == k_BCJ2)
645       return true;
646   }
647   return false;
648 }
649 
650 #ifndef _NO_CRYPTO
651 
652 class CCryptoGetTextPassword:
653   public ICryptoGetTextPassword,
654   public CMyUnknownImp
655 {
656 public:
657   UString Password;
658 
659   MY_UNKNOWN_IMP
660   STDMETHOD(CryptoGetTextPassword)(BSTR *password);
661 };
662 
CryptoGetTextPassword(BSTR * password)663 STDMETHODIMP CCryptoGetTextPassword::CryptoGetTextPassword(BSTR *password)
664 {
665   return StringToBstr(Password, password);
666 }
667 
668 #endif
669 
670 static const int kNumGroupsMax = 4;
671 
672 #ifdef USE_86_FILTER
Is86Group(int group)673 static bool Is86Group(int group) { return (group & 1) != 0; }
674 #endif
IsEncryptedGroup(int group)675 static bool IsEncryptedGroup(int group) { return (group & 2) != 0; }
GetGroupIndex(bool encrypted,int bcjFiltered)676 static int GetGroupIndex(bool encrypted, int bcjFiltered)
677   { return (encrypted ? 2 : 0) + (bcjFiltered ? 1 : 0); }
678 
Update(DECL_EXTERNAL_CODECS_LOC_VARS IInStream * inStream,const CArchiveDatabaseEx * db,const CObjectVector<CUpdateItem> & updateItems,COutArchive & archive,CArchiveDatabase & newDatabase,ISequentialOutStream * seqOutStream,IArchiveUpdateCallback * updateCallback,const CUpdateOptions & options,ICryptoGetTextPassword * getDecoderPassword)679 HRESULT Update(
680     DECL_EXTERNAL_CODECS_LOC_VARS
681     IInStream *inStream,
682     const CArchiveDatabaseEx *db,
683     const CObjectVector<CUpdateItem> &updateItems,
684     COutArchive &archive,
685     CArchiveDatabase &newDatabase,
686     ISequentialOutStream *seqOutStream,
687     IArchiveUpdateCallback *updateCallback,
688     const CUpdateOptions &options
689     #ifndef _NO_CRYPTO
690     , ICryptoGetTextPassword *getDecoderPassword
691     #endif
692     )
693 {
694   UInt64 numSolidFiles = options.NumSolidFiles;
695   if (numSolidFiles == 0)
696     numSolidFiles = 1;
697   /*
698   CMyComPtr<IOutStream> outStream;
699   RINOK(seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStream));
700   if (!outStream)
701     return E_NOTIMPL;
702   */
703 
704   UInt64 startBlockSize = db != 0 ? db->ArchiveInfo.StartPosition: 0;
705   if (startBlockSize > 0 && !options.RemoveSfxBlock)
706   {
707     RINOK(WriteRange(inStream, seqOutStream, 0, startBlockSize, NULL));
708   }
709 
710   CRecordVector<int> fileIndexToUpdateIndexMap;
711   CRecordVector<CFolderRepack> folderRefs;
712   UInt64 complexity = 0;
713   UInt64 inSizeForReduce2 = 0;
714   bool needEncryptedRepack = false;
715   if (db != 0)
716   {
717     fileIndexToUpdateIndexMap.Reserve(db->Files.Size());
718     int i;
719     for (i = 0; i < db->Files.Size(); i++)
720       fileIndexToUpdateIndexMap.Add(-1);
721 
722     for (i = 0; i < updateItems.Size(); i++)
723     {
724       int index = updateItems[i].IndexInArchive;
725       if (index != -1)
726         fileIndexToUpdateIndexMap[index] = i;
727     }
728 
729     for (i = 0; i < db->Folders.Size(); i++)
730     {
731       CNum indexInFolder = 0;
732       CNum numCopyItems = 0;
733       CNum numUnpackStreams = db->NumUnpackStreamsVector[i];
734       UInt64 repackSize = 0;
735       for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++)
736       {
737         const CFileItem &file = db->Files[fi];
738         if (file.HasStream)
739         {
740           indexInFolder++;
741           int updateIndex = fileIndexToUpdateIndexMap[fi];
742           if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
743           {
744             numCopyItems++;
745             repackSize += file.Size;
746           }
747         }
748       }
749 
750       if (numCopyItems == 0)
751         continue;
752 
753       CFolderRepack rep;
754       rep.FolderIndex = i;
755       rep.NumCopyFiles = numCopyItems;
756       const CFolder &f = db->Folders[i];
757       bool isEncrypted = f.IsEncrypted();
758       rep.Group = GetGroupIndex(isEncrypted, Is86FilteredFolder(f));
759       folderRefs.Add(rep);
760       if (numCopyItems == numUnpackStreams)
761         complexity += db->GetFolderFullPackSize(i);
762       else
763       {
764         complexity += repackSize;
765         if (repackSize > inSizeForReduce2)
766           inSizeForReduce2 = repackSize;
767         if (isEncrypted)
768           needEncryptedRepack = true;
769       }
770     }
771     folderRefs.Sort(CompareFolderRepacks, (void *)db);
772   }
773 
774   UInt64 inSizeForReduce = 0;
775   int i;
776   for (i = 0; i < updateItems.Size(); i++)
777   {
778     const CUpdateItem &ui = updateItems[i];
779     if (ui.NewData)
780     {
781       complexity += ui.Size;
782       if (numSolidFiles != 1)
783         inSizeForReduce += ui.Size;
784       else if (ui.Size > inSizeForReduce)
785         inSizeForReduce = ui.Size;
786     }
787   }
788 
789   if (inSizeForReduce2 > inSizeForReduce)
790     inSizeForReduce = inSizeForReduce2;
791 
792   const UInt32 kMinReduceSize = (1 << 16);
793   if (inSizeForReduce < kMinReduceSize)
794     inSizeForReduce = kMinReduceSize;
795 
796   RINOK(updateCallback->SetTotal(complexity));
797 
798   CLocalProgress *lps = new CLocalProgress;
799   CMyComPtr<ICompressProgressInfo> progress = lps;
800   lps->Init(updateCallback, true);
801 
802   CThreadDecoder threadDecoder;
803   if (!folderRefs.IsEmpty())
804   {
805     #ifdef EXTERNAL_CODECS
806     threadDecoder._codecsInfo = codecsInfo;
807     threadDecoder._externalCodecs = *externalCodecs;
808     #endif
809     RINOK(threadDecoder.Create());
810   }
811 
812   CObjectVector<CSolidGroup> groups;
813   for (i = 0; i < kNumGroupsMax; i++)
814     groups.Add(CSolidGroup());
815 
816   {
817     // ---------- Split files to 2 groups ----------
818 
819     bool useFilters = options.UseFilters;
820     const CCompressionMethodMode &method = *options.Method;
821     if (method.Methods.Size() != 1 || method.Binds.Size() != 0)
822       useFilters = false;
823     for (i = 0; i < updateItems.Size(); i++)
824     {
825       const CUpdateItem &ui = updateItems[i];
826       if (!ui.NewData || !ui.HasStream())
827         continue;
828       bool filteredGroup = false;
829       if (useFilters)
830       {
831         int dotPos = ui.Name.ReverseFind(L'.');
832         if (dotPos >= 0)
833           filteredGroup = IsExeExt(ui.Name.Mid(dotPos + 1));
834       }
835       groups[GetGroupIndex(method.PasswordIsDefined, filteredGroup)].Indices.Add(i);
836     }
837   }
838 
839   #ifndef _NO_CRYPTO
840 
841   CCryptoGetTextPassword *getPasswordSpec = NULL;
842   if (needEncryptedRepack)
843   {
844     getPasswordSpec = new CCryptoGetTextPassword;
845     threadDecoder.GetTextPassword = getPasswordSpec;
846 
847     if (options.Method->PasswordIsDefined)
848       getPasswordSpec->Password = options.Method->Password;
849     else
850     {
851       if (!getDecoderPassword)
852         return E_NOTIMPL;
853       CMyComBSTR password;
854       RINOK(getDecoderPassword->CryptoGetTextPassword(&password));
855       getPasswordSpec->Password = password;
856     }
857   }
858 
859   #endif
860 
861   // ---------- Compress ----------
862 
863   RINOK(archive.Create(seqOutStream, false));
864   RINOK(archive.SkipPrefixArchiveHeader());
865 
866   int folderRefIndex = 0;
867   lps->ProgressOffset = 0;
868 
869   for (int groupIndex = 0; groupIndex < kNumGroupsMax; groupIndex++)
870   {
871     const CSolidGroup &group = groups[groupIndex];
872 
873     CCompressionMethodMode method;
874     #ifdef USE_86_FILTER
875     if (Is86Group(groupIndex))
876       MakeExeMethod(*options.Method, options.MaxFilter, method);
877     else
878     #endif
879       method = *options.Method;
880 
881     if (IsEncryptedGroup(groupIndex))
882     {
883       if (!method.PasswordIsDefined)
884       {
885         #ifndef _NO_CRYPTO
886         if (getPasswordSpec)
887           method.Password = getPasswordSpec->Password;
888         #endif
889         method.PasswordIsDefined = true;
890       }
891     }
892     else
893     {
894       method.PasswordIsDefined = false;
895       method.Password.Empty();
896     }
897 
898     CEncoder encoder(method);
899 
900     for (; folderRefIndex < folderRefs.Size(); folderRefIndex++)
901     {
902       const CFolderRepack &rep = folderRefs[folderRefIndex];
903       if (rep.Group != groupIndex)
904         break;
905       int folderIndex = rep.FolderIndex;
906 
907       if (rep.NumCopyFiles == db->NumUnpackStreamsVector[folderIndex])
908       {
909         UInt64 packSize = db->GetFolderFullPackSize(folderIndex);
910         RINOK(WriteRange(inStream, archive.SeqStream,
911           db->GetFolderStreamPos(folderIndex, 0), packSize, progress));
912         lps->ProgressOffset += packSize;
913 
914         const CFolder &folder = db->Folders[folderIndex];
915         CNum startIndex = db->FolderStartPackStreamIndex[folderIndex];
916         for (int j = 0; j < folder.PackStreams.Size(); j++)
917         {
918           newDatabase.PackSizes.Add(db->PackSizes[startIndex + j]);
919           // newDatabase.PackCRCsDefined.Add(db.PackCRCsDefined[startIndex + j]);
920           // newDatabase.PackCRCs.Add(db.PackCRCs[startIndex + j]);
921         }
922         newDatabase.Folders.Add(folder);
923       }
924       else
925       {
926         CStreamBinder sb;
927         RINOK(sb.CreateEvents());
928         CMyComPtr<ISequentialOutStream> sbOutStream;
929         CMyComPtr<ISequentialInStream> sbInStream;
930         sb.CreateStreams(&sbInStream, &sbOutStream);
931         CBoolVector extractStatuses;
932 
933         CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
934         CNum indexInFolder = 0;
935 
936         for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
937         {
938           bool needExtract = false;
939           if (db->Files[fi].HasStream)
940           {
941             indexInFolder++;
942             int updateIndex = fileIndexToUpdateIndexMap[fi];
943             if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
944               needExtract = true;
945           }
946           extractStatuses.Add(needExtract);
947         }
948 
949         RINOK(threadDecoder.FosSpec->Init(db, db->FolderStartFileIndex[folderIndex], &extractStatuses, sbOutStream));
950         sbOutStream.Release();
951 
952         threadDecoder.InStream = inStream;
953         threadDecoder.Folder = &db->Folders[folderIndex];
954         threadDecoder.StartPos = db->GetFolderStreamPos(folderIndex, 0);
955         threadDecoder.PackSizes = &db->PackSizes[db->FolderStartPackStreamIndex[folderIndex]];
956 
957         threadDecoder.Start();
958 
959         int startPackIndex = newDatabase.PackSizes.Size();
960         CFolder newFolder;
961         RINOK(encoder.Encode(
962           EXTERNAL_CODECS_LOC_VARS
963           sbInStream, NULL, &inSizeForReduce, newFolder,
964           archive.SeqStream, newDatabase.PackSizes, progress));
965 
966         threadDecoder.WaitFinish();
967 
968         RINOK(threadDecoder.Result);
969 
970         for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
971           lps->OutSize += newDatabase.PackSizes[startPackIndex];
972         lps->InSize += newFolder.GetUnpackSize();
973 
974         newDatabase.Folders.Add(newFolder);
975       }
976 
977       newDatabase.NumUnpackStreamsVector.Add(rep.NumCopyFiles);
978 
979       CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
980 
981       CNum indexInFolder = 0;
982       for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
983       {
984         CFileItem file;
985         CFileItem2 file2;
986         db->GetFile(fi, file, file2);
987         if (file.HasStream)
988         {
989           indexInFolder++;
990           int updateIndex = fileIndexToUpdateIndexMap[fi];
991           if (updateIndex >= 0)
992           {
993             const CUpdateItem &ui = updateItems[updateIndex];
994             if (ui.NewData)
995               continue;
996             if (ui.NewProps)
997             {
998               CFileItem uf;
999               FromUpdateItemToFileItem(ui, uf, file2);
1000               uf.Size = file.Size;
1001               uf.Crc = file.Crc;
1002               uf.CrcDefined = file.CrcDefined;
1003               uf.HasStream = file.HasStream;
1004               file = uf;
1005             }
1006             newDatabase.AddFile(file, file2);
1007           }
1008         }
1009       }
1010     }
1011 
1012     int numFiles = group.Indices.Size();
1013     if (numFiles == 0)
1014       continue;
1015     CRecordVector<CRefItem> refItems;
1016     refItems.Reserve(numFiles);
1017     bool sortByType = (numSolidFiles > 1);
1018     for (i = 0; i < numFiles; i++)
1019       refItems.Add(CRefItem(group.Indices[i], updateItems[group.Indices[i]], sortByType));
1020     refItems.Sort(CompareUpdateItems, (void *)&sortByType);
1021 
1022     CRecordVector<UInt32> indices;
1023     indices.Reserve(numFiles);
1024 
1025     for (i = 0; i < numFiles; i++)
1026     {
1027       UInt32 index = refItems[i].Index;
1028       indices.Add(index);
1029       /*
1030       const CUpdateItem &ui = updateItems[index];
1031       CFileItem file;
1032       if (ui.NewProps)
1033         FromUpdateItemToFileItem(ui, file);
1034       else
1035         file = db.Files[ui.IndexInArchive];
1036       if (file.IsAnti || file.IsDir)
1037         return E_FAIL;
1038       newDatabase.Files.Add(file);
1039       */
1040     }
1041 
1042     for (i = 0; i < numFiles;)
1043     {
1044       UInt64 totalSize = 0;
1045       int numSubFiles;
1046       UString prevExtension;
1047       for (numSubFiles = 0; i + numSubFiles < numFiles &&
1048           numSubFiles < numSolidFiles; numSubFiles++)
1049       {
1050         const CUpdateItem &ui = updateItems[indices[i + numSubFiles]];
1051         totalSize += ui.Size;
1052         if (totalSize > options.NumSolidBytes)
1053           break;
1054         if (options.SolidExtension)
1055         {
1056           UString ext = ui.GetExtension();
1057           if (numSubFiles == 0)
1058             prevExtension = ext;
1059           else
1060             if (ext.CompareNoCase(prevExtension) != 0)
1061               break;
1062         }
1063       }
1064       if (numSubFiles < 1)
1065         numSubFiles = 1;
1066 
1067       CFolderInStream *inStreamSpec = new CFolderInStream;
1068       CMyComPtr<ISequentialInStream> solidInStream(inStreamSpec);
1069       inStreamSpec->Init(updateCallback, &indices[i], numSubFiles);
1070 
1071       CFolder folderItem;
1072 
1073       int startPackIndex = newDatabase.PackSizes.Size();
1074       RINOK(encoder.Encode(
1075           EXTERNAL_CODECS_LOC_VARS
1076           solidInStream, NULL, &inSizeForReduce, folderItem,
1077           archive.SeqStream, newDatabase.PackSizes, progress));
1078 
1079       for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
1080         lps->OutSize += newDatabase.PackSizes[startPackIndex];
1081 
1082       lps->InSize += folderItem.GetUnpackSize();
1083       // for ()
1084       // newDatabase.PackCRCsDefined.Add(false);
1085       // newDatabase.PackCRCs.Add(0);
1086 
1087       newDatabase.Folders.Add(folderItem);
1088 
1089       CNum numUnpackStreams = 0;
1090       for (int subIndex = 0; subIndex < numSubFiles; subIndex++)
1091       {
1092         const CUpdateItem &ui = updateItems[indices[i + subIndex]];
1093         CFileItem file;
1094         CFileItem2 file2;
1095         if (ui.NewProps)
1096           FromUpdateItemToFileItem(ui, file, file2);
1097         else
1098           db->GetFile(ui.IndexInArchive, file, file2);
1099         if (file2.IsAnti || file.IsDir)
1100           return E_FAIL;
1101 
1102         /*
1103         CFileItem &file = newDatabase.Files[
1104               startFileIndexInDatabase + i + subIndex];
1105         */
1106         if (!inStreamSpec->Processed[subIndex])
1107         {
1108           continue;
1109           // file.Name += L".locked";
1110         }
1111 
1112         file.Crc = inStreamSpec->CRCs[subIndex];
1113         file.Size = inStreamSpec->Sizes[subIndex];
1114         if (file.Size != 0)
1115         {
1116           file.CrcDefined = true;
1117           file.HasStream = true;
1118           numUnpackStreams++;
1119         }
1120         else
1121         {
1122           file.CrcDefined = false;
1123           file.HasStream = false;
1124         }
1125         newDatabase.AddFile(file, file2);
1126       }
1127       // numUnpackStreams = 0 is very bad case for locked files
1128       // v3.13 doesn't understand it.
1129       newDatabase.NumUnpackStreamsVector.Add(numUnpackStreams);
1130       i += numSubFiles;
1131     }
1132   }
1133 
1134   if (folderRefIndex != folderRefs.Size())
1135     return E_FAIL;
1136 
1137   /*
1138   folderRefs.ClearAndFree();
1139   fileIndexToUpdateIndexMap.ClearAndFree();
1140   groups.ClearAndFree();
1141   */
1142 
1143   {
1144     // ---------- Write Folders & Empty Files ----------
1145 
1146     CRecordVector<int> emptyRefs;
1147     for (i = 0; i < updateItems.Size(); i++)
1148     {
1149       const CUpdateItem &ui = updateItems[i];
1150       if (ui.NewData)
1151       {
1152         if (ui.HasStream())
1153           continue;
1154       }
1155       else if (ui.IndexInArchive != -1 && db->Files[ui.IndexInArchive].HasStream)
1156         continue;
1157       emptyRefs.Add(i);
1158     }
1159     emptyRefs.Sort(CompareEmptyItems, (void *)&updateItems);
1160     for (i = 0; i < emptyRefs.Size(); i++)
1161     {
1162       const CUpdateItem &ui = updateItems[emptyRefs[i]];
1163       CFileItem file;
1164       CFileItem2 file2;
1165       if (ui.NewProps)
1166         FromUpdateItemToFileItem(ui, file, file2);
1167       else
1168         db->GetFile(ui.IndexInArchive, file, file2);
1169       newDatabase.AddFile(file, file2);
1170     }
1171   }
1172 
1173   newDatabase.ReserveDown();
1174   return S_OK;
1175 }
1176 
1177 }}
1178