1 /* Lzma2Enc.c -- LZMA2 Encoder
2 2010-09-24 : Igor Pavlov : Public domain */
3
4 /* #include <stdio.h> */
5 #include <string.h>
6
7 /* #define _7ZIP_ST */
8
9 #include "Lzma2Enc.h"
10
11 #ifndef _7ZIP_ST
12 #include "MtCoder.h"
13 #else
14 #define NUM_MT_CODER_THREADS_MAX 1
15 #endif
16
17 #define LZMA2_CONTROL_LZMA (1 << 7)
18 #define LZMA2_CONTROL_COPY_NO_RESET 2
19 #define LZMA2_CONTROL_COPY_RESET_DIC 1
20 #define LZMA2_CONTROL_EOF 0
21
22 #define LZMA2_LCLP_MAX 4
23
24 #define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))
25
26 #define LZMA2_PACK_SIZE_MAX (1 << 16)
27 #define LZMA2_COPY_CHUNK_SIZE LZMA2_PACK_SIZE_MAX
28 #define LZMA2_UNPACK_SIZE_MAX (1 << 21)
29 #define LZMA2_KEEP_WINDOW_SIZE LZMA2_UNPACK_SIZE_MAX
30
31 #define LZMA2_CHUNK_SIZE_COMPRESSED_MAX ((1 << 16) + 16)
32
33
34 #define PRF(x) /* x */
35
36 /* ---------- CLzma2EncInt ---------- */
37
38 typedef struct
39 {
40 CLzmaEncHandle enc;
41 UInt64 srcPos;
42 Byte props;
43 Bool needInitState;
44 Bool needInitProp;
45 } CLzma2EncInt;
46
Lzma2EncInt_Init(CLzma2EncInt * p,const CLzma2EncProps * props)47 static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props)
48 {
49 Byte propsEncoded[LZMA_PROPS_SIZE];
50 SizeT propsSize = LZMA_PROPS_SIZE;
51 RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps));
52 RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize));
53 p->srcPos = 0;
54 p->props = propsEncoded[0];
55 p->needInitState = True;
56 p->needInitProp = True;
57 return SZ_OK;
58 }
59
60 SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize,
61 ISzAlloc *alloc, ISzAlloc *allocBig);
62 SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen,
63 UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig);
64 SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit,
65 Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize);
66 const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp);
67 void LzmaEnc_Finish(CLzmaEncHandle pp);
68 void LzmaEnc_SaveState(CLzmaEncHandle pp);
69 void LzmaEnc_RestoreState(CLzmaEncHandle pp);
70
71
Lzma2EncInt_EncodeSubblock(CLzma2EncInt * p,Byte * outBuf,size_t * packSizeRes,ISeqOutStream * outStream)72 static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
73 size_t *packSizeRes, ISeqOutStream *outStream)
74 {
75 size_t packSizeLimit = *packSizeRes;
76 size_t packSize = packSizeLimit;
77 UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX;
78 unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0);
79 Bool useCopyBlock;
80 SRes res;
81
82 *packSizeRes = 0;
83 if (packSize < lzHeaderSize)
84 return SZ_ERROR_OUTPUT_EOF;
85 packSize -= lzHeaderSize;
86
87 LzmaEnc_SaveState(p->enc);
88 res = LzmaEnc_CodeOneMemBlock(p->enc, p->needInitState,
89 outBuf + lzHeaderSize, &packSize, LZMA2_PACK_SIZE_MAX, &unpackSize);
90
91 PRF(printf("\npackSize = %7d unpackSize = %7d ", packSize, unpackSize));
92
93 if (unpackSize == 0)
94 return res;
95
96 if (res == SZ_OK)
97 useCopyBlock = (packSize + 2 >= unpackSize || packSize > (1 << 16));
98 else
99 {
100 if (res != SZ_ERROR_OUTPUT_EOF)
101 return res;
102 res = SZ_OK;
103 useCopyBlock = True;
104 }
105
106 if (useCopyBlock)
107 {
108 size_t destPos = 0;
109 PRF(printf("################# COPY "));
110 while (unpackSize > 0)
111 {
112 UInt32 u = (unpackSize < LZMA2_COPY_CHUNK_SIZE) ? unpackSize : LZMA2_COPY_CHUNK_SIZE;
113 if (packSizeLimit - destPos < u + 3)
114 return SZ_ERROR_OUTPUT_EOF;
115 outBuf[destPos++] = (Byte)(p->srcPos == 0 ? LZMA2_CONTROL_COPY_RESET_DIC : LZMA2_CONTROL_COPY_NO_RESET);
116 outBuf[destPos++] = (Byte)((u - 1) >> 8);
117 outBuf[destPos++] = (Byte)(u - 1);
118 memcpy(outBuf + destPos, LzmaEnc_GetCurBuf(p->enc) - unpackSize, u);
119 unpackSize -= u;
120 destPos += u;
121 p->srcPos += u;
122 if (outStream)
123 {
124 *packSizeRes += destPos;
125 if (outStream->Write(outStream, outBuf, destPos) != destPos)
126 return SZ_ERROR_WRITE;
127 destPos = 0;
128 }
129 else
130 *packSizeRes = destPos;
131 /* needInitState = True; */
132 }
133 LzmaEnc_RestoreState(p->enc);
134 return SZ_OK;
135 }
136 {
137 size_t destPos = 0;
138 UInt32 u = unpackSize - 1;
139 UInt32 pm = (UInt32)(packSize - 1);
140 unsigned mode = (p->srcPos == 0) ? 3 : (p->needInitState ? (p->needInitProp ? 2 : 1) : 0);
141
142 PRF(printf(" "));
143
144 outBuf[destPos++] = (Byte)(LZMA2_CONTROL_LZMA | (mode << 5) | ((u >> 16) & 0x1F));
145 outBuf[destPos++] = (Byte)(u >> 8);
146 outBuf[destPos++] = (Byte)u;
147 outBuf[destPos++] = (Byte)(pm >> 8);
148 outBuf[destPos++] = (Byte)pm;
149
150 if (p->needInitProp)
151 outBuf[destPos++] = p->props;
152
153 p->needInitProp = False;
154 p->needInitState = False;
155 destPos += packSize;
156 p->srcPos += unpackSize;
157
158 if (outStream)
159 if (outStream->Write(outStream, outBuf, destPos) != destPos)
160 return SZ_ERROR_WRITE;
161 *packSizeRes = destPos;
162 return SZ_OK;
163 }
164 }
165
166 /* ---------- Lzma2 Props ---------- */
167
Lzma2EncProps_Init(CLzma2EncProps * p)168 void Lzma2EncProps_Init(CLzma2EncProps *p)
169 {
170 LzmaEncProps_Init(&p->lzmaProps);
171 p->numTotalThreads = -1;
172 p->numBlockThreads = -1;
173 p->blockSize = 0;
174 }
175
Lzma2EncProps_Normalize(CLzma2EncProps * p)176 void Lzma2EncProps_Normalize(CLzma2EncProps *p)
177 {
178 int t1, t1n, t2, t3;
179 {
180 CLzmaEncProps lzmaProps = p->lzmaProps;
181 LzmaEncProps_Normalize(&lzmaProps);
182 t1n = lzmaProps.numThreads;
183 }
184
185 t1 = p->lzmaProps.numThreads;
186 t2 = p->numBlockThreads;
187 t3 = p->numTotalThreads;
188
189 if (t2 > NUM_MT_CODER_THREADS_MAX)
190 t2 = NUM_MT_CODER_THREADS_MAX;
191
192 if (t3 <= 0)
193 {
194 if (t2 <= 0)
195 t2 = 1;
196 t3 = t1n * t2;
197 }
198 else if (t2 <= 0)
199 {
200 t2 = t3 / t1n;
201 if (t2 == 0)
202 {
203 t1 = 1;
204 t2 = t3;
205 }
206 if (t2 > NUM_MT_CODER_THREADS_MAX)
207 t2 = NUM_MT_CODER_THREADS_MAX;
208 }
209 else if (t1 <= 0)
210 {
211 t1 = t3 / t2;
212 if (t1 == 0)
213 t1 = 1;
214 }
215 else
216 t3 = t1n * t2;
217
218 p->lzmaProps.numThreads = t1;
219 p->numBlockThreads = t2;
220 p->numTotalThreads = t3;
221 LzmaEncProps_Normalize(&p->lzmaProps);
222
223 if (p->blockSize == 0)
224 {
225 UInt32 dictSize = p->lzmaProps.dictSize;
226 UInt64 blockSize = (UInt64)dictSize << 2;
227 const UInt32 kMinSize = (UInt32)1 << 20;
228 const UInt32 kMaxSize = (UInt32)1 << 28;
229 if (blockSize < kMinSize) blockSize = kMinSize;
230 if (blockSize > kMaxSize) blockSize = kMaxSize;
231 if (blockSize < dictSize) blockSize = dictSize;
232 p->blockSize = (size_t)blockSize;
233 }
234 }
235
Progress(ICompressProgress * p,UInt64 inSize,UInt64 outSize)236 static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize)
237 {
238 return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK;
239 }
240
241 /* ---------- Lzma2 ---------- */
242
243 typedef struct
244 {
245 Byte propEncoded;
246 CLzma2EncProps props;
247
248 Byte *outBuf;
249
250 ISzAlloc *alloc;
251 ISzAlloc *allocBig;
252
253 CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX];
254
255 #ifndef _7ZIP_ST
256 CMtCoder mtCoder;
257 #endif
258
259 } CLzma2Enc;
260
261
262 /* ---------- Lzma2EncThread ---------- */
263
Lzma2Enc_EncodeMt1(CLzma2EncInt * p,CLzma2Enc * mainEncoder,ISeqOutStream * outStream,ISeqInStream * inStream,ICompressProgress * progress)264 static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder,
265 ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
266 {
267 UInt64 packTotal = 0;
268 SRes res = SZ_OK;
269
270 if (mainEncoder->outBuf == 0)
271 {
272 mainEncoder->outBuf = (Byte *)IAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX);
273 if (mainEncoder->outBuf == 0)
274 return SZ_ERROR_MEM;
275 }
276 RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
277 RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE,
278 mainEncoder->alloc, mainEncoder->allocBig));
279 for (;;)
280 {
281 size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX;
282 res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream);
283 if (res != SZ_OK)
284 break;
285 packTotal += packSize;
286 res = Progress(progress, p->srcPos, packTotal);
287 if (res != SZ_OK)
288 break;
289 if (packSize == 0)
290 break;
291 }
292 LzmaEnc_Finish(p->enc);
293 if (res == SZ_OK)
294 {
295 Byte b = 0;
296 if (outStream->Write(outStream, &b, 1) != 1)
297 return SZ_ERROR_WRITE;
298 }
299 return res;
300 }
301
302 #ifndef _7ZIP_ST
303
304 typedef struct
305 {
306 IMtCoderCallback funcTable;
307 CLzma2Enc *lzma2Enc;
308 } CMtCallbackImp;
309
MtCallbackImp_Code(void * pp,unsigned index,Byte * dest,size_t * destSize,const Byte * src,size_t srcSize,int finished)310 static SRes MtCallbackImp_Code(void *pp, unsigned index, Byte *dest, size_t *destSize,
311 const Byte *src, size_t srcSize, int finished)
312 {
313 CMtCallbackImp *imp = (CMtCallbackImp *)pp;
314 CLzma2Enc *mainEncoder = imp->lzma2Enc;
315 CLzma2EncInt *p = &mainEncoder->coders[index];
316
317 SRes res = SZ_OK;
318 {
319 size_t destLim = *destSize;
320 *destSize = 0;
321
322 if (srcSize != 0)
323 {
324 RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
325
326 RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE,
327 mainEncoder->alloc, mainEncoder->allocBig));
328
329 while (p->srcPos < srcSize)
330 {
331 size_t packSize = destLim - *destSize;
332 res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL);
333 if (res != SZ_OK)
334 break;
335 *destSize += packSize;
336
337 if (packSize == 0)
338 {
339 res = SZ_ERROR_FAIL;
340 break;
341 }
342
343 if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK)
344 {
345 res = SZ_ERROR_PROGRESS;
346 break;
347 }
348 }
349 LzmaEnc_Finish(p->enc);
350 if (res != SZ_OK)
351 return res;
352 }
353 if (finished)
354 {
355 if (*destSize == destLim)
356 return SZ_ERROR_OUTPUT_EOF;
357 dest[(*destSize)++] = 0;
358 }
359 }
360 return res;
361 }
362
363 #endif
364
365 /* ---------- Lzma2Enc ---------- */
366
Lzma2Enc_Create(ISzAlloc * alloc,ISzAlloc * allocBig)367 CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig)
368 {
369 CLzma2Enc *p = (CLzma2Enc *)alloc->Alloc(alloc, sizeof(CLzma2Enc));
370 if (p == 0)
371 return NULL;
372 Lzma2EncProps_Init(&p->props);
373 Lzma2EncProps_Normalize(&p->props);
374 p->outBuf = 0;
375 p->alloc = alloc;
376 p->allocBig = allocBig;
377 {
378 unsigned i;
379 for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
380 p->coders[i].enc = 0;
381 }
382 #ifndef _7ZIP_ST
383 MtCoder_Construct(&p->mtCoder);
384 #endif
385
386 return p;
387 }
388
Lzma2Enc_Destroy(CLzma2EncHandle pp)389 void Lzma2Enc_Destroy(CLzma2EncHandle pp)
390 {
391 CLzma2Enc *p = (CLzma2Enc *)pp;
392 unsigned i;
393 for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
394 {
395 CLzma2EncInt *t = &p->coders[i];
396 if (t->enc)
397 {
398 LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig);
399 t->enc = 0;
400 }
401 }
402
403 #ifndef _7ZIP_ST
404 MtCoder_Destruct(&p->mtCoder);
405 #endif
406
407 IAlloc_Free(p->alloc, p->outBuf);
408 IAlloc_Free(p->alloc, pp);
409 }
410
Lzma2Enc_SetProps(CLzma2EncHandle pp,const CLzma2EncProps * props)411 SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props)
412 {
413 CLzma2Enc *p = (CLzma2Enc *)pp;
414 CLzmaEncProps lzmaProps = props->lzmaProps;
415 LzmaEncProps_Normalize(&lzmaProps);
416 if (lzmaProps.lc + lzmaProps.lp > LZMA2_LCLP_MAX)
417 return SZ_ERROR_PARAM;
418 p->props = *props;
419 Lzma2EncProps_Normalize(&p->props);
420 return SZ_OK;
421 }
422
Lzma2Enc_WriteProperties(CLzma2EncHandle pp)423 Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp)
424 {
425 CLzma2Enc *p = (CLzma2Enc *)pp;
426 unsigned i;
427 UInt32 dicSize = LzmaEncProps_GetDictSize(&p->props.lzmaProps);
428 for (i = 0; i < 40; i++)
429 if (dicSize <= LZMA2_DIC_SIZE_FROM_PROP(i))
430 break;
431 return (Byte)i;
432 }
433
Lzma2Enc_Encode(CLzma2EncHandle pp,ISeqOutStream * outStream,ISeqInStream * inStream,ICompressProgress * progress)434 SRes Lzma2Enc_Encode(CLzma2EncHandle pp,
435 ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
436 {
437 CLzma2Enc *p = (CLzma2Enc *)pp;
438 int i;
439
440 for (i = 0; i < p->props.numBlockThreads; i++)
441 {
442 CLzma2EncInt *t = &p->coders[i];
443 if (t->enc == NULL)
444 {
445 t->enc = LzmaEnc_Create(p->alloc);
446 if (t->enc == NULL)
447 return SZ_ERROR_MEM;
448 }
449 }
450
451 #ifndef _7ZIP_ST
452 if (p->props.numBlockThreads <= 1)
453 #endif
454 return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress);
455
456 #ifndef _7ZIP_ST
457
458 {
459 CMtCallbackImp mtCallback;
460
461 mtCallback.funcTable.Code = MtCallbackImp_Code;
462 mtCallback.lzma2Enc = p;
463
464 p->mtCoder.progress = progress;
465 p->mtCoder.inStream = inStream;
466 p->mtCoder.outStream = outStream;
467 p->mtCoder.alloc = p->alloc;
468 p->mtCoder.mtCallback = &mtCallback.funcTable;
469
470 p->mtCoder.blockSize = p->props.blockSize;
471 p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16;
472 p->mtCoder.numThreads = p->props.numBlockThreads;
473
474 return MtCoder_Code(&p->mtCoder);
475 }
476 #endif
477 }
478