1 /* XzEnc.c -- Xz Encode
2 2009-06-04 : Igor Pavlov : Public domain */
3
4 #include <stdlib.h>
5 #include <string.h>
6
7 #include "7zCrc.h"
8 #include "Alloc.h"
9 #include "Bra.h"
10 #include "CpuArch.h"
11 #ifdef USE_SUBBLOCK
12 #include "SbEnc.h"
13 #endif
14
15 #include "XzEnc.h"
16
SzBigAlloc(void * p,size_t size)17 static void *SzBigAlloc(void *p, size_t size) { p = p; return BigAlloc(size); }
SzBigFree(void * p,void * address)18 static void SzBigFree(void *p, void *address) { p = p; BigFree(address); }
19 static ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree };
20
SzAlloc(void * p,size_t size)21 static void *SzAlloc(void *p, size_t size) { p = p; return MyAlloc(size); }
SzFree(void * p,void * address)22 static void SzFree(void *p, void *address) { p = p; MyFree(address); }
23 static ISzAlloc g_Alloc = { SzAlloc, SzFree };
24
25 #define XzBlock_ClearFlags(p) (p)->flags = 0;
26 #define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1);
27 #define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE;
28 #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE;
29
WriteBytes(ISeqOutStream * s,const void * buf,UInt32 size)30 static SRes WriteBytes(ISeqOutStream *s, const void *buf, UInt32 size)
31 {
32 return (s->Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE;
33 }
34
WriteBytesAndCrc(ISeqOutStream * s,const void * buf,UInt32 size,UInt32 * crc)35 static SRes WriteBytesAndCrc(ISeqOutStream *s, const void *buf, UInt32 size, UInt32 *crc)
36 {
37 *crc = CrcUpdate(*crc, buf, size);
38 return WriteBytes(s, buf, size);
39 }
40
Xz_WriteHeader(CXzStreamFlags f,ISeqOutStream * s)41 SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s)
42 {
43 UInt32 crc;
44 Byte header[XZ_STREAM_HEADER_SIZE];
45 memcpy(header, XZ_SIG, XZ_SIG_SIZE);
46 header[XZ_SIG_SIZE] = (Byte)(f >> 8);
47 header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF);
48 crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE);
49 SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc);
50 return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE);
51 }
52
XzBlock_WriteHeader(const CXzBlock * p,ISeqOutStream * s)53 SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s)
54 {
55 Byte header[XZ_BLOCK_HEADER_SIZE_MAX];
56
57 unsigned pos = 1;
58 int numFilters, i;
59 header[pos++] = p->flags;
60
61 if (XzBlock_HasPackSize(p)) pos += Xz_WriteVarInt(header + pos, p->packSize);
62 if (XzBlock_HasUnpackSize(p)) pos += Xz_WriteVarInt(header + pos, p->unpackSize);
63 numFilters = XzBlock_GetNumFilters(p);
64 for (i = 0; i < numFilters; i++)
65 {
66 const CXzFilter *f = &p->filters[i];
67 pos += Xz_WriteVarInt(header + pos, f->id);
68 pos += Xz_WriteVarInt(header + pos, f->propsSize);
69 memcpy(header + pos, f->props, f->propsSize);
70 pos += f->propsSize;
71 }
72 while((pos & 3) != 0)
73 header[pos++] = 0;
74 header[0] = (Byte)(pos >> 2);
75 SetUi32(header + pos, CrcCalc(header, pos));
76 return WriteBytes(s, header, pos + 4);
77 }
78
Xz_WriteFooter(CXzStream * p,ISeqOutStream * s)79 SRes Xz_WriteFooter(CXzStream *p, ISeqOutStream *s)
80 {
81 Byte buf[32];
82 UInt64 globalPos;
83 {
84 UInt32 crc = CRC_INIT_VAL;
85 unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks);
86 size_t i;
87
88 globalPos = pos;
89 buf[0] = 0;
90 RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
91 for (i = 0; i < p->numBlocks; i++)
92 {
93 const CXzBlockSizes *block = &p->blocks[i];
94 pos = Xz_WriteVarInt(buf, block->totalSize);
95 pos += Xz_WriteVarInt(buf + pos, block->unpackSize);
96 globalPos += pos;
97 RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
98 }
99 pos = ((unsigned)globalPos & 3);
100 if (pos != 0)
101 {
102 buf[0] = buf[1] = buf[2] = 0;
103 RINOK(WriteBytesAndCrc(s, buf, 4 - pos, &crc));
104 globalPos += 4 - pos;
105 }
106 {
107 SetUi32(buf, CRC_GET_DIGEST(crc));
108 RINOK(WriteBytes(s, buf, 4));
109 globalPos += 4;
110 }
111 }
112
113 {
114 UInt32 indexSize = (UInt32)((globalPos >> 2) - 1);
115 SetUi32(buf + 4, indexSize);
116 buf[8] = (Byte)(p->flags >> 8);
117 buf[9] = (Byte)(p->flags & 0xFF);
118 SetUi32(buf, CrcCalc(buf + 4, 6));
119 memcpy(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE);
120 return WriteBytes(s, buf, 12);
121 }
122 }
123
Xz_AddIndexRecord(CXzStream * p,UInt64 unpackSize,UInt64 totalSize,ISzAlloc * alloc)124 SRes Xz_AddIndexRecord(CXzStream *p, UInt64 unpackSize, UInt64 totalSize, ISzAlloc *alloc)
125 {
126 if (p->blocks == 0 || p->numBlocksAllocated == p->numBlocks)
127 {
128 size_t num = (p->numBlocks + 1) * 2;
129 size_t newSize = sizeof(CXzBlockSizes) * num;
130 CXzBlockSizes *blocks;
131 if (newSize / sizeof(CXzBlockSizes) != num)
132 return SZ_ERROR_MEM;
133 blocks = alloc->Alloc(alloc, newSize);
134 if (blocks == 0)
135 return SZ_ERROR_MEM;
136 if (p->numBlocks != 0)
137 {
138 memcpy(blocks, p->blocks, p->numBlocks * sizeof(CXzBlockSizes));
139 Xz_Free(p, alloc);
140 }
141 p->blocks = blocks;
142 p->numBlocksAllocated = num;
143 }
144 {
145 CXzBlockSizes *block = &p->blocks[p->numBlocks++];
146 block->totalSize = totalSize;
147 block->unpackSize = unpackSize;
148 }
149 return SZ_OK;
150 }
151
152 /* ---------- CSeqCheckInStream ---------- */
153
154 typedef struct
155 {
156 ISeqInStream p;
157 ISeqInStream *realStream;
158 UInt64 processed;
159 CXzCheck check;
160 } CSeqCheckInStream;
161
SeqCheckInStream_Init(CSeqCheckInStream * p,int mode)162 void SeqCheckInStream_Init(CSeqCheckInStream *p, int mode)
163 {
164 p->processed = 0;
165 XzCheck_Init(&p->check, mode);
166 }
167
SeqCheckInStream_GetDigest(CSeqCheckInStream * p,Byte * digest)168 void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest)
169 {
170 XzCheck_Final(&p->check, digest);
171 }
172
SeqCheckInStream_Read(void * pp,void * data,size_t * size)173 static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size)
174 {
175 CSeqCheckInStream *p = (CSeqCheckInStream *)pp;
176 SRes res = p->realStream->Read(p->realStream, data, size);
177 XzCheck_Update(&p->check, data, *size);
178 p->processed += *size;
179 return res;
180 }
181
182 /* ---------- CSeqSizeOutStream ---------- */
183
184 typedef struct
185 {
186 ISeqOutStream p;
187 ISeqOutStream *realStream;
188 UInt64 processed;
189 } CSeqSizeOutStream;
190
MyWrite(void * pp,const void * data,size_t size)191 static size_t MyWrite(void *pp, const void *data, size_t size)
192 {
193 CSeqSizeOutStream *p = (CSeqSizeOutStream *)pp;
194 size = p->realStream->Write(p->realStream, data, size);
195 p->processed += size;
196 return size;
197 }
198
199 /* ---------- CSeqInFilter ---------- */
200
201 /*
202 typedef struct _IFilter
203 {
204 void *p;
205 void (*Free)(void *p, ISzAlloc *alloc);
206 SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAlloc *alloc);
207 void (*Init)(void *p);
208 size_t (*Filter)(void *p, Byte *data, SizeT destLen);
209 } IFilter;
210
211 #define FILT_BUF_SIZE (1 << 19)
212
213 typedef struct
214 {
215 ISeqInStream p;
216 ISeqInStream *realStream;
217 UInt32 x86State;
218 UInt32 ip;
219 UInt64 processed;
220 CXzCheck check;
221 Byte buf[FILT_BUF_SIZE];
222 UInt32 bufferPos;
223 UInt32 convertedPosBegin;
224 UInt32 convertedPosEnd;
225 IFilter *filter;
226 } CSeqInFilter;
227
228 static SRes SeqInFilter_Read(void *pp, void *data, size_t *size)
229 {
230 CSeqInFilter *p = (CSeqInFilter *)pp;
231 size_t remSize = *size;
232 *size = 0;
233
234 while (remSize > 0)
235 {
236 int i;
237 if (p->convertedPosBegin != p->convertedPosEnd)
238 {
239 UInt32 sizeTemp = p->convertedPosEnd - p->convertedPosBegin;
240 if (remSize < sizeTemp)
241 sizeTemp = (UInt32)remSize;
242 memmove(data, p->buf + p->convertedPosBegin, sizeTemp);
243 p->convertedPosBegin += sizeTemp;
244 data = (void *)((Byte *)data + sizeTemp);
245 remSize -= sizeTemp;
246 *size += sizeTemp;
247 break;
248 }
249 for (i = 0; p->convertedPosEnd + i < p->bufferPos; i++)
250 p->buf[i] = p->buf[i + p->convertedPosEnd];
251 p->bufferPos = i;
252 p->convertedPosBegin = p->convertedPosEnd = 0;
253 {
254 size_t processedSizeTemp = FILT_BUF_SIZE - p->bufferPos;
255 RINOK(p->realStream->Read(p->realStream, p->buf + p->bufferPos, &processedSizeTemp));
256 p->bufferPos = p->bufferPos + (UInt32)processedSizeTemp;
257 }
258 p->convertedPosEnd = (UInt32)p->filter->Filter(p->filter->p, p->buf, p->bufferPos);
259 if (p->convertedPosEnd == 0)
260 {
261 if (p->bufferPos == 0)
262 break;
263 else
264 {
265 p->convertedPosEnd = p->bufferPos;
266 continue;
267 }
268 }
269 if (p->convertedPosEnd > p->bufferPos)
270 {
271 for (; p->bufferPos < p->convertedPosEnd; p->bufferPos++)
272 p->buf[p->bufferPos] = 0;
273 p->convertedPosEnd = (UInt32)p->filter->Filter(p->filter->p, p->buf, p->bufferPos);
274 }
275 }
276 return SZ_OK;
277 }
278 */
279
280 /*
281 typedef struct
282 {
283 ISeqInStream p;
284 ISeqInStream *realStream;
285 CMixCoder mixCoder;
286 Byte buf[FILT_BUF_SIZE];
287 UInt32 bufPos;
288 UInt32 bufSize;
289 } CMixCoderSeqInStream;
290
291 static SRes CMixCoderSeqInStream_Read(void *pp, void *data, size_t *size)
292 {
293 CMixCoderSeqInStream *p = (CMixCoderSeqInStream *)pp;
294 SRes res = SZ_OK;
295 size_t remSize = *size;
296 *size = 0;
297 while (remSize > 0)
298 {
299 if (p->bufPos == p->bufSize)
300 {
301 size_t curSize;
302 p->bufPos = p->bufSize = 0;
303 if (*size != 0)
304 break;
305 curSize = FILT_BUF_SIZE;
306 RINOK(p->realStream->Read(p->realStream, p->buf, &curSize));
307 p->bufSize = (UInt32)curSize;
308 }
309 {
310 SizeT destLen = remSize;
311 SizeT srcLen = p->bufSize - p->bufPos;
312 res = MixCoder_Code(&p->mixCoder, data, &destLen, p->buf + p->bufPos, &srcLen, 0);
313 data = (void *)((Byte *)data + destLen);
314 remSize -= destLen;
315 *size += destLen;
316 p->bufPos += srcLen;
317 }
318 }
319 return res;
320 }
321 */
322
323 #ifdef USE_SUBBLOCK
324 typedef struct
325 {
326 ISeqInStream p;
327 CSubblockEnc sb;
328 UInt64 processed;
329 } CSbEncInStream;
330
SbEncInStream_Init(CSbEncInStream * p)331 void SbEncInStream_Init(CSbEncInStream *p)
332 {
333 p->processed = 0;
334 SubblockEnc_Init(&p->sb);
335 }
336
SbEncInStream_Read(void * pp,void * data,size_t * size)337 static SRes SbEncInStream_Read(void *pp, void *data, size_t *size)
338 {
339 CSbEncInStream *p = (CSbEncInStream *)pp;
340 SRes res = SubblockEnc_Read(&p->sb, data, size);
341 p->processed += *size;
342 return res;
343 }
344 #endif
345
346 typedef struct
347 {
348 /* CMixCoderSeqInStream inStream; */
349 CLzma2EncHandle lzma2;
350 #ifdef USE_SUBBLOCK
351 CSbEncInStream sb;
352 #endif
353 ISzAlloc *alloc;
354 ISzAlloc *bigAlloc;
355 } CLzma2WithFilters;
356
357
Lzma2WithFilters_Construct(CLzma2WithFilters * p,ISzAlloc * alloc,ISzAlloc * bigAlloc)358 static void Lzma2WithFilters_Construct(CLzma2WithFilters *p, ISzAlloc *alloc, ISzAlloc *bigAlloc)
359 {
360 p->alloc = alloc;
361 p->bigAlloc = bigAlloc;
362 p->lzma2 = NULL;
363 #ifdef USE_SUBBLOCK
364 p->sb.p.Read = SbEncInStream_Read;
365 SubblockEnc_Construct(&p->sb.sb, p->alloc);
366 #endif
367 }
368
Lzma2WithFilters_Create(CLzma2WithFilters * p)369 static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p)
370 {
371 p->lzma2 = Lzma2Enc_Create(p->alloc, p->bigAlloc);
372 if (p->lzma2 == 0)
373 return SZ_ERROR_MEM;
374 return SZ_OK;
375 }
376
Lzma2WithFilters_Free(CLzma2WithFilters * p)377 static void Lzma2WithFilters_Free(CLzma2WithFilters *p)
378 {
379 #ifdef USE_SUBBLOCK
380 SubblockEnc_Free(&p->sb.sb);
381 #endif
382 if (p->lzma2)
383 {
384 Lzma2Enc_Destroy(p->lzma2);
385 p->lzma2 = NULL;
386 }
387 }
388
Xz_Compress(CXzStream * xz,CLzma2WithFilters * lzmaf,ISeqOutStream * outStream,ISeqInStream * inStream,const CLzma2EncProps * lzma2Props,Bool useSubblock,ICompressProgress * progress)389 static SRes Xz_Compress(CXzStream *xz,
390 CLzma2WithFilters *lzmaf,
391 ISeqOutStream *outStream,
392 ISeqInStream *inStream,
393 const CLzma2EncProps *lzma2Props,
394 Bool useSubblock,
395 ICompressProgress *progress)
396 {
397 xz->flags = XZ_CHECK_CRC32;
398
399 RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, lzma2Props));
400 RINOK(Xz_WriteHeader(xz->flags, outStream));
401
402 {
403 CSeqCheckInStream checkInStream;
404 CSeqSizeOutStream seqSizeOutStream;
405 CXzBlock block;
406 int filterIndex = 0;
407
408 XzBlock_ClearFlags(&block);
409 XzBlock_SetNumFilters(&block, 1 + (useSubblock ? 1 : 0));
410
411 if (useSubblock)
412 {
413 CXzFilter *f = &block.filters[filterIndex++];
414 f->id = XZ_ID_Subblock;
415 f->propsSize = 0;
416 }
417
418 {
419 CXzFilter *f = &block.filters[filterIndex++];
420 f->id = XZ_ID_LZMA2;
421 f->propsSize = 1;
422 f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2);
423 }
424
425 seqSizeOutStream.p.Write = MyWrite;
426 seqSizeOutStream.realStream = outStream;
427 seqSizeOutStream.processed = 0;
428
429 RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.p));
430
431 checkInStream.p.Read = SeqCheckInStream_Read;
432 checkInStream.realStream = inStream;
433 SeqCheckInStream_Init(&checkInStream, XzFlags_GetCheckType(xz->flags));
434
435 #ifdef USE_SUBBLOCK
436 if (useSubblock)
437 {
438 lzmaf->sb.sb.inStream = &checkInStream.p;
439 SubblockEnc_Init(&lzmaf->sb.sb);
440 }
441 #endif
442
443 {
444 UInt64 packPos = seqSizeOutStream.processed;
445 SRes res = Lzma2Enc_Encode(lzmaf->lzma2, &seqSizeOutStream.p,
446 #ifdef USE_SUBBLOCK
447 useSubblock ? &lzmaf->sb.p:
448 #endif
449 &checkInStream.p,
450 progress);
451 RINOK(res);
452 block.unpackSize = checkInStream.processed;
453 block.packSize = seqSizeOutStream.processed - packPos;
454 }
455
456 {
457 unsigned padSize = 0;
458 Byte buf[128];
459 while((((unsigned)block.packSize + padSize) & 3) != 0)
460 buf[padSize++] = 0;
461 SeqCheckInStream_GetDigest(&checkInStream, buf + padSize);
462 RINOK(WriteBytes(&seqSizeOutStream.p, buf, padSize + XzFlags_GetCheckSize(xz->flags)));
463 RINOK(Xz_AddIndexRecord(xz, block.unpackSize, seqSizeOutStream.processed - padSize, &g_Alloc));
464 }
465 }
466 return Xz_WriteFooter(xz, outStream);
467 }
468
Xz_Encode(ISeqOutStream * outStream,ISeqInStream * inStream,const CLzma2EncProps * lzma2Props,Bool useSubblock,ICompressProgress * progress)469 SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream,
470 const CLzma2EncProps *lzma2Props, Bool useSubblock,
471 ICompressProgress *progress)
472 {
473 SRes res;
474 CXzStream xz;
475 CLzma2WithFilters lzmaf;
476 Xz_Construct(&xz);
477 Lzma2WithFilters_Construct(&lzmaf, &g_Alloc, &g_BigAlloc);
478 res = Lzma2WithFilters_Create(&lzmaf);
479 if (res == SZ_OK)
480 res = Xz_Compress(&xz, &lzmaf, outStream, inStream,
481 lzma2Props, useSubblock, progress);
482 Lzma2WithFilters_Free(&lzmaf);
483 Xz_Free(&xz, &g_Alloc);
484 return res;
485 }
486
Xz_EncodeEmpty(ISeqOutStream * outStream)487 SRes Xz_EncodeEmpty(ISeqOutStream *outStream)
488 {
489 SRes res;
490 CXzStream xz;
491 Xz_Construct(&xz);
492 res = Xz_WriteHeader(xz.flags, outStream);
493 if (res == SZ_OK)
494 res = Xz_WriteFooter(&xz, outStream);
495 Xz_Free(&xz, &g_Alloc);
496 return res;
497 }
498