1 /* LZ4frame API example : compress a file
2  * Modified from an example code by Zbigniew Jędrzejewski-Szmek
3  *
4  * This example streams an input file into an output file
5  * using a bounded memory budget.
6  * Input is read in chunks of IN_CHUNK_SIZE */
7 
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <string.h>
11 #include <errno.h>
12 #include <assert.h>
13 
14 #include <lz4frame.h>
15 
16 
17 #define IN_CHUNK_SIZE  (16*1024)
18 
19 static const LZ4F_preferences_t kPrefs = {
20     { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame,
21       0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
22     0,   /* compression level; 0 == default */
23     0,   /* autoflush */
24     0,   /* favor decompression speed */
25     { 0, 0, 0 },  /* reserved, must be set to 0 */
26 };
27 
28 
29 /* safe_fwrite() :
30  * performs fwrite(), ensure operation success, or immediately exit() */
safe_fwrite(void * buf,size_t eltSize,size_t nbElt,FILE * f)31 static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f)
32 {
33     size_t const writtenSize = fwrite(buf, eltSize, nbElt, f);
34     size_t const expectedSize = eltSize * nbElt;
35     assert(expectedSize / nbElt == eltSize);   /* check overflow */
36     if (writtenSize < expectedSize) {
37         if (ferror(f))  /* note : ferror() must follow fwrite */
38             fprintf(stderr, "Write failed \n");
39         else
40             fprintf(stderr, "Short write \n");
41         exit(1);
42     }
43 }
44 
45 
46 /* ================================================= */
47 /*     Streaming Compression example               */
48 /* ================================================= */
49 
50 typedef struct {
51     int error;
52     unsigned long long size_in;
53     unsigned long long size_out;
54 } compressResult_t;
55 
56 static compressResult_t
compress_file_internal(FILE * f_in,FILE * f_out,LZ4F_compressionContext_t ctx,void * inBuff,size_t inChunkSize,void * outBuff,size_t outCapacity)57 compress_file_internal(FILE* f_in, FILE* f_out,
58                        LZ4F_compressionContext_t ctx,
59                        void* inBuff,  size_t inChunkSize,
60                        void* outBuff, size_t outCapacity)
61 {
62     compressResult_t result = { 1, 0, 0 };  /* result for an error */
63     unsigned long long count_in = 0, count_out;
64 
65     assert(f_in != NULL); assert(f_out != NULL);
66     assert(ctx != NULL);
67     assert(outCapacity >= LZ4F_HEADER_SIZE_MAX);
68     assert(outCapacity >= LZ4F_compressBound(inChunkSize, &kPrefs));
69 
70     /* write frame header */
71     {   size_t const headerSize = LZ4F_compressBegin(ctx, outBuff, outCapacity, &kPrefs);
72         if (LZ4F_isError(headerSize)) {
73             printf("Failed to start compression: error %zu\n", headerSize);
74             return result;
75         }
76         count_out = headerSize;
77         printf("Buffer size is %zu bytes, header size %zu bytes\n", outCapacity, headerSize);
78         safe_fwrite(outBuff, 1, headerSize, f_out);
79     }
80 
81     /* stream file */
82     for (;;) {
83         size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, f_in);
84         if (readSize == 0) break; /* nothing left to read from input file */
85         count_in += readSize;
86 
87         size_t const compressedSize = LZ4F_compressUpdate(ctx,
88                                                 outBuff, outCapacity,
89                                                 inBuff, readSize,
90                                                 NULL);
91         if (LZ4F_isError(compressedSize)) {
92             printf("Compression failed: error %zu\n", compressedSize);
93             return result;
94         }
95 
96         printf("Writing %zu bytes\n", compressedSize);
97         safe_fwrite(outBuff, 1, compressedSize, f_out);
98         count_out += compressedSize;
99     }
100 
101     /* flush whatever remains within internal buffers */
102     {   size_t const compressedSize = LZ4F_compressEnd(ctx,
103                                                 outBuff, outCapacity,
104                                                 NULL);
105         if (LZ4F_isError(compressedSize)) {
106             printf("Failed to end compression: error %zu\n", compressedSize);
107             return result;
108         }
109 
110         printf("Writing %zu bytes\n", compressedSize);
111         safe_fwrite(outBuff, 1, compressedSize, f_out);
112         count_out += compressedSize;
113     }
114 
115     result.size_in = count_in;
116     result.size_out = count_out;
117     result.error = 0;
118     return result;
119 }
120 
121 static compressResult_t
compress_file(FILE * f_in,FILE * f_out)122 compress_file(FILE* f_in, FILE* f_out)
123 {
124     assert(f_in != NULL);
125     assert(f_out != NULL);
126 
127     /* ressource allocation */
128     LZ4F_compressionContext_t ctx;
129     size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
130     void* const src = malloc(IN_CHUNK_SIZE);
131     size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &kPrefs);   /* large enough for any input <= IN_CHUNK_SIZE */
132     void* const outbuff = malloc(outbufCapacity);
133 
134     compressResult_t result = { 1, 0, 0 };  /* == error (default) */
135     if (!LZ4F_isError(ctxCreation) && src && outbuff) {
136         result = compress_file_internal(f_in, f_out,
137                                         ctx,
138                                         src, IN_CHUNK_SIZE,
139                                         outbuff, outbufCapacity);
140     } else {
141         printf("error : ressource allocation failed \n");
142     }
143 
144     LZ4F_freeCompressionContext(ctx);   /* supports free on NULL */
145     free(src);
146     free(outbuff);
147     return result;
148 }
149 
150 
151 /* ================================================= */
152 /*     Streaming decompression example               */
153 /* ================================================= */
154 
get_block_size(const LZ4F_frameInfo_t * info)155 static size_t get_block_size(const LZ4F_frameInfo_t* info) {
156     switch (info->blockSizeID) {
157         case LZ4F_default:
158         case LZ4F_max64KB:  return 1 << 16;
159         case LZ4F_max256KB: return 1 << 18;
160         case LZ4F_max1MB:   return 1 << 20;
161         case LZ4F_max4MB:   return 1 << 22;
162         default:
163             printf("Impossible with expected frame specification (<=v1.6.1)\n");
164             exit(1);
165     }
166 }
167 
168 /* @return : 1==error, 0==success */
169 static int
decompress_file_internal(FILE * f_in,FILE * f_out,LZ4F_dctx * dctx,void * src,size_t srcCapacity,size_t filled,size_t alreadyConsumed,void * dst,size_t dstCapacity)170 decompress_file_internal(FILE* f_in, FILE* f_out,
171                          LZ4F_dctx* dctx,
172                          void* src, size_t srcCapacity, size_t filled, size_t alreadyConsumed,
173                          void* dst, size_t dstCapacity)
174 {
175     int firstChunk = 1;
176     size_t ret = 1;
177 
178     assert(f_in != NULL); assert(f_out != NULL);
179     assert(dctx != NULL);
180     assert(src != NULL); assert(srcCapacity > 0); assert(filled <= srcCapacity); assert(alreadyConsumed <= filled);
181     assert(dst != NULL); assert(dstCapacity > 0);
182 
183     /* Decompression */
184     while (ret != 0) {
185         /* Load more input */
186         size_t readSize = firstChunk ? filled : fread(src, 1, srcCapacity, f_in); firstChunk=0;
187         const void* srcPtr = src + alreadyConsumed; alreadyConsumed=0;
188         const void* const srcEnd = srcPtr + readSize;
189         if (readSize == 0 || ferror(f_in)) {
190             printf("Decompress: not enough input or error reading file\n");
191             return 1;
192         }
193 
194         /* Decompress:
195          * Continue while there is more input to read (srcPtr != srcEnd)
196          * and the frame isn't over (ret != 0)
197          */
198         while (srcPtr < srcEnd && ret != 0) {
199             /* Any data within dst has been flushed at this stage */
200             size_t dstSize = dstCapacity;
201             size_t srcSize = srcEnd - srcPtr;
202             ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL);
203             if (LZ4F_isError(ret)) {
204                 printf("Decompression error: %s\n", LZ4F_getErrorName(ret));
205                 return 1;
206             }
207             /* Flush output */
208             if (dstSize != 0) safe_fwrite(dst, 1, dstSize, f_out);
209             /* Update input */
210             srcPtr += srcSize;
211         }
212 
213         assert(srcPtr <= srcEnd);
214 
215         /* Ensure all input data has been consumed.
216          * It is valid to have multiple frames in the same file,
217          * but this example only supports one frame.
218          */
219         if (srcPtr < srcEnd) {
220             printf("Decompress: Trailing data left in file after frame\n");
221             return 1;
222         }
223     }
224 
225     /* Check that there isn't trailing data in the file after the frame.
226      * It is valid to have multiple frames in the same file,
227      * but this example only supports one frame.
228      */
229     {   size_t const readSize = fread(src, 1, 1, f_in);
230         if (readSize != 0 || !feof(f_in)) {
231             printf("Decompress: Trailing data left in file after frame\n");
232             return 1;
233     }   }
234 
235     return 0;
236 }
237 
238 
239 /* @return : 1==error, 0==completed */
240 static int
decompress_file_allocDst(FILE * f_in,FILE * f_out,LZ4F_dctx * dctx,void * src,size_t srcCapacity)241 decompress_file_allocDst(FILE* f_in, FILE* f_out,
242                         LZ4F_dctx* dctx,
243                         void* src, size_t srcCapacity)
244 {
245     assert(f_in != NULL); assert(f_out != NULL);
246     assert(dctx != NULL);
247     assert(src != NULL);
248     assert(srcCapacity >= LZ4F_HEADER_SIZE_MAX);  /* ensure LZ4F_getFrameInfo() can read enough data */
249 
250     /* Read Frame header */
251     size_t const readSize = fread(src, 1, srcCapacity, f_in);
252     if (readSize == 0 || ferror(f_in)) {
253         printf("Decompress: not enough input or error reading file\n");
254         return 1;
255     }
256 
257     LZ4F_frameInfo_t info;
258     size_t consumedSize = readSize;
259     {   size_t const fires = LZ4F_getFrameInfo(dctx, &info, src, &consumedSize);
260         if (LZ4F_isError(fires)) {
261             printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(fires));
262             return 1;
263     }   }
264 
265     /* Allocating enough space for an entire block isn't necessary for
266      * correctness, but it allows some memcpy's to be elided.
267      */
268     size_t const dstCapacity = get_block_size(&info);
269     void* const dst = malloc(dstCapacity);
270     if (!dst) { perror("decompress_file(dst)"); return 1; }
271 
272     int const decompressionResult = decompress_file_internal(
273                         f_in, f_out,
274                         dctx,
275                         src, srcCapacity, readSize-consumedSize, consumedSize,
276                         dst, dstCapacity);
277 
278     free(dst);
279     return decompressionResult;
280 }
281 
282 
283 /* @result : 1==error, 0==success */
decompress_file(FILE * f_in,FILE * f_out)284 static int decompress_file(FILE* f_in, FILE* f_out)
285 {
286     assert(f_in != NULL); assert(f_out != NULL);
287 
288     /* Ressource allocation */
289     void* const src = malloc(IN_CHUNK_SIZE);
290     if (!src) { perror("decompress_file(src)"); return 1; }
291 
292     LZ4F_dctx* dctx;
293     {   size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
294         if (LZ4F_isError(dctxStatus)) {
295             printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus));
296     }   }
297 
298     int const result = !dctx ? 1 /* error */ :
299                        decompress_file_allocDst(f_in, f_out, dctx, src, IN_CHUNK_SIZE);
300 
301     free(src);
302     LZ4F_freeDecompressionContext(dctx);   /* note : free works on NULL */
303     return result;
304 }
305 
306 
compareFiles(FILE * fp0,FILE * fp1)307 int compareFiles(FILE* fp0, FILE* fp1)
308 {
309     int result = 0;
310 
311     while (result==0) {
312         char b0[1024];
313         char b1[1024];
314         size_t const r0 = fread(b0, 1, sizeof(b0), fp0);
315         size_t const r1 = fread(b1, 1, sizeof(b1), fp1);
316 
317         result = (r0 != r1);
318         if (!r0 || !r1) break;
319         if (!result) result = memcmp(b0, b1, r0);
320     }
321 
322     return result;
323 }
324 
325 
main(int argc,const char ** argv)326 int main(int argc, const char **argv) {
327     char inpFilename[256] = { 0 };
328     char lz4Filename[256] = { 0 };
329     char decFilename[256] = { 0 };
330 
331     if (argc < 2) {
332         printf("Please specify input filename\n");
333         return 0;
334     }
335 
336     snprintf(inpFilename, 256, "%s", argv[1]);
337     snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
338     snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
339 
340     printf("inp = [%s]\n", inpFilename);
341     printf("lz4 = [%s]\n", lz4Filename);
342     printf("dec = [%s]\n", decFilename);
343 
344     /* compress */
345     {   FILE* const inpFp = fopen(inpFilename, "rb");
346         FILE* const outFp = fopen(lz4Filename, "wb");
347 
348         printf("compress : %s -> %s\n", inpFilename, lz4Filename);
349         compressResult_t const ret = compress_file(inpFp, outFp);
350 
351         fclose(outFp);
352         fclose(inpFp);
353 
354         if (ret.error) {
355             printf("compress : failed with code %i\n", ret.error);
356             return ret.error;
357         }
358         printf("%s: %zu → %zu bytes, %.1f%%\n",
359             inpFilename,
360             (size_t)ret.size_in, (size_t)ret.size_out,  /* might overflow is size_t is 32 bits and size_{in,out} > 4 GB */
361             (double)ret.size_out / ret.size_in * 100);
362         printf("compress : done\n");
363     }
364 
365     /* decompress */
366     {   FILE* const inpFp = fopen(lz4Filename, "rb");
367         FILE* const outFp = fopen(decFilename, "wb");
368 
369         printf("decompress : %s -> %s\n", lz4Filename, decFilename);
370         int const ret = decompress_file(inpFp, outFp);
371 
372         fclose(outFp);
373         fclose(inpFp);
374 
375         if (ret) {
376             printf("decompress : failed with code %i\n", ret);
377             return ret;
378         }
379         printf("decompress : done\n");
380     }
381 
382     /* verify */
383     {   FILE* const inpFp = fopen(inpFilename, "rb");
384         FILE* const decFp = fopen(decFilename, "rb");
385 
386         printf("verify : %s <-> %s\n", inpFilename, decFilename);
387         int const cmp = compareFiles(inpFp, decFp);
388 
389         fclose(decFp);
390         fclose(inpFp);
391 
392         if (cmp) {
393             printf("corruption detected : decompressed file differs from original\n");
394             return cmp;
395         }
396         printf("verify : OK\n");
397     }
398 
399     return 0;
400 }
401