1 /*
2 * Copyright (c) 2010, 2011, 2012, 2013
3 * Phillip Lougher <phillip@squashfs.org.uk>
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version 2,
8 * or (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 *
19 * xz_wrapper.c
20 *
21 * Support for XZ (LZMA2) compression using XZ Utils liblzma
22 * http://tukaani.org/xz/
23 */
24
25 #include <stdio.h>
26 #include <string.h>
27 #include <stdlib.h>
28 #include <lzma.h>
29
30 #include "squashfs_fs.h"
31 #include "xz_wrapper.h"
32 #include "compressor.h"
33
34 static struct bcj bcj[] = {
35 { "x86", LZMA_FILTER_X86, 0 },
36 { "powerpc", LZMA_FILTER_POWERPC, 0 },
37 { "ia64", LZMA_FILTER_IA64, 0 },
38 { "arm", LZMA_FILTER_ARM, 0 },
39 { "armthumb", LZMA_FILTER_ARMTHUMB, 0 },
40 { "sparc", LZMA_FILTER_SPARC, 0 },
41 { NULL, LZMA_VLI_UNKNOWN, 0 }
42 };
43
44 static int filter_count = 1;
45 static int dictionary_size = 0;
46 static float dictionary_percent = 0;
47
48
49 /*
50 * This function is called by the options parsing code in mksquashfs.c
51 * to parse any -X compressor option.
52 *
53 * Two specific options are supported:
54 * -Xbcj
55 * -Xdict-size
56 *
57 * This function returns:
58 * >=0 (number of additional args parsed) on success
59 * -1 if the option was unrecognised, or
60 * -2 if the option was recognised, but otherwise bad in
61 * some way (e.g. invalid parameter)
62 *
63 * Note: this function sets internal compressor state, but does not
64 * pass back the results of the parsing other than success/failure.
65 * The xz_dump_options() function is called later to get the options in
66 * a format suitable for writing to the filesystem.
67 */
xz_options(char * argv[],int argc)68 static int xz_options(char *argv[], int argc)
69 {
70 int i;
71 char *name;
72
73 if(strcmp(argv[0], "-Xbcj") == 0) {
74 if(argc < 2) {
75 fprintf(stderr, "xz: -Xbcj missing filter\n");
76 goto failed;
77 }
78
79 name = argv[1];
80 while(name[0] != '\0') {
81 for(i = 0; bcj[i].name; i++) {
82 int n = strlen(bcj[i].name);
83 if((strncmp(name, bcj[i].name, n) == 0) &&
84 (name[n] == '\0' ||
85 name[n] == ',')) {
86 if(bcj[i].selected == 0) {
87 bcj[i].selected = 1;
88 filter_count++;
89 }
90 name += name[n] == ',' ? n + 1 : n;
91 break;
92 }
93 }
94 if(bcj[i].name == NULL) {
95 fprintf(stderr, "xz: -Xbcj unrecognised "
96 "filter\n");
97 goto failed;
98 }
99 }
100
101 return 1;
102 } else if(strcmp(argv[0], "-Xdict-size") == 0) {
103 char *b;
104 float size;
105
106 if(argc < 2) {
107 fprintf(stderr, "xz: -Xdict-size missing dict-size\n");
108 goto failed;
109 }
110
111 size = strtof(argv[1], &b);
112 if(*b == '%') {
113 if(size <= 0 || size > 100) {
114 fprintf(stderr, "xz: -Xdict-size percentage "
115 "should be 0 < dict-size <= 100\n");
116 goto failed;
117 }
118
119 dictionary_percent = size;
120 dictionary_size = 0;
121 } else {
122 if((float) ((int) size) != size) {
123 fprintf(stderr, "xz: -Xdict-size can't be "
124 "fractional unless a percentage of the"
125 " block size\n");
126 goto failed;
127 }
128
129 dictionary_percent = 0;
130 dictionary_size = (int) size;
131
132 if(*b == 'k' || *b == 'K')
133 dictionary_size *= 1024;
134 else if(*b == 'm' || *b == 'M')
135 dictionary_size *= 1024 * 1024;
136 else if(*b != '\0') {
137 fprintf(stderr, "xz: -Xdict-size invalid "
138 "dict-size\n");
139 goto failed;
140 }
141 }
142
143 return 1;
144 }
145
146 return -1;
147
148 failed:
149 return -2;
150 }
151
152
153 /*
154 * This function is called after all options have been parsed.
155 * It is used to do post-processing on the compressor options using
156 * values that were not expected to be known at option parse time.
157 *
158 * In this case block_size may not be known until after -Xdict-size has
159 * been processed (in the case where -b is specified after -Xdict-size)
160 *
161 * This function returns 0 on successful post processing, or
162 * -1 on error
163 */
xz_options_post(int block_size)164 static int xz_options_post(int block_size)
165 {
166 /*
167 * if -Xdict-size has been specified use this to compute the datablock
168 * dictionary size
169 */
170 if(dictionary_size || dictionary_percent) {
171 int n;
172
173 if(dictionary_size) {
174 if(dictionary_size > block_size) {
175 fprintf(stderr, "xz: -Xdict-size is larger than"
176 " block_size\n");
177 goto failed;
178 }
179 } else
180 dictionary_size = block_size * dictionary_percent / 100;
181
182 if(dictionary_size < 8192) {
183 fprintf(stderr, "xz: -Xdict-size should be 8192 bytes "
184 "or larger\n");
185 goto failed;
186 }
187
188 /*
189 * dictionary_size must be storable in xz header as either
190 * 2^n or as 2^n+2^(n+1)
191 */
192 n = ffs(dictionary_size) - 1;
193 if(dictionary_size != (1 << n) &&
194 dictionary_size != ((1 << n) + (1 << (n + 1)))) {
195 fprintf(stderr, "xz: -Xdict-size is an unsupported "
196 "value, dict-size must be storable in xz "
197 "header\n");
198 fprintf(stderr, "as either 2^n or as 2^n+2^(n+1). "
199 "Example dict-sizes are 75%%, 50%%, 37.5%%, "
200 "25%%,\n");
201 fprintf(stderr, "or 32K, 16K, 8K etc.\n");
202 goto failed;
203 }
204
205 } else
206 /* No -Xdict-size specified, use defaults */
207 dictionary_size = block_size;
208
209 return 0;
210
211 failed:
212 return -1;
213 }
214
215
216 /*
217 * This function is called by mksquashfs to dump the parsed
218 * compressor options in a format suitable for writing to the
219 * compressor options field in the filesystem (stored immediately
220 * after the superblock).
221 *
222 * This function returns a pointer to the compression options structure
223 * to be stored (and the size), or NULL if there are no compression
224 * options
225 */
xz_dump_options(int block_size,int * size)226 static void *xz_dump_options(int block_size, int *size)
227 {
228 static struct comp_opts comp_opts;
229 int flags = 0, i;
230
231 /*
232 * don't store compressor specific options in file system if the
233 * default options are being used - no compressor options in the
234 * file system means the default options are always assumed
235 *
236 * Defaults are:
237 * metadata dictionary size: SQUASHFS_METADATA_SIZE
238 * datablock dictionary size: block_size
239 * 1 filter
240 */
241 if(dictionary_size == block_size && filter_count == 1)
242 return NULL;
243
244 for(i = 0; bcj[i].name; i++)
245 flags |= bcj[i].selected << i;
246
247 comp_opts.dictionary_size = dictionary_size;
248 comp_opts.flags = flags;
249
250 SQUASHFS_INSWAP_COMP_OPTS(&comp_opts);
251
252 *size = sizeof(comp_opts);
253 return &comp_opts;
254 }
255
256
257 /*
258 * This function is a helper specifically for the append mode of
259 * mksquashfs. Its purpose is to set the internal compressor state
260 * to the stored compressor options in the passed compressor options
261 * structure.
262 *
263 * In effect this function sets up the compressor options
264 * to the same state they were when the filesystem was originally
265 * generated, this is to ensure on appending, the compressor uses
266 * the same compression options that were used to generate the
267 * original filesystem.
268 *
269 * Note, even if there are no compressor options, this function is still
270 * called with an empty compressor structure (size == 0), to explicitly
271 * set the default options, this is to ensure any user supplied
272 * -X options on the appending mksquashfs command line are over-ridden
273 *
274 * This function returns 0 on sucessful extraction of options, and
275 * -1 on error
276 */
xz_extract_options(int block_size,void * buffer,int size)277 static int xz_extract_options(int block_size, void *buffer, int size)
278 {
279 struct comp_opts *comp_opts = buffer;
280 int flags, i, n;
281
282 if(size == 0) {
283 /* set defaults */
284 dictionary_size = block_size;
285 flags = 0;
286 } else {
287 /* check passed comp opts struct is of the correct length */
288 if(size != sizeof(struct comp_opts))
289 goto failed;
290
291 SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
292
293 dictionary_size = comp_opts->dictionary_size;
294 flags = comp_opts->flags;
295
296 /*
297 * check that the dictionary size seems correct - the dictionary
298 * size should 2^n or 2^n+2^(n+1)
299 */
300 n = ffs(dictionary_size) - 1;
301 if(dictionary_size != (1 << n) &&
302 dictionary_size != ((1 << n) + (1 << (n + 1))))
303 goto failed;
304 }
305
306 filter_count = 1;
307 for(i = 0; bcj[i].name; i++) {
308 if((flags >> i) & 1) {
309 bcj[i].selected = 1;
310 filter_count ++;
311 } else
312 bcj[i].selected = 0;
313 }
314
315 return 0;
316
317 failed:
318 fprintf(stderr, "xz: error reading stored compressor options from "
319 "filesystem!\n");
320
321 return -1;
322 }
323
324
xz_display_options(void * buffer,int size)325 void xz_display_options(void *buffer, int size)
326 {
327 struct comp_opts *comp_opts = buffer;
328 int dictionary_size, flags, printed;
329 int i, n;
330
331 /* check passed comp opts struct is of the correct length */
332 if(size != sizeof(struct comp_opts))
333 goto failed;
334
335 SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
336
337 dictionary_size = comp_opts->dictionary_size;
338 flags = comp_opts->flags;
339
340 /*
341 * check that the dictionary size seems correct - the dictionary
342 * size should 2^n or 2^n+2^(n+1)
343 */
344 n = ffs(dictionary_size) - 1;
345 if(dictionary_size != (1 << n) &&
346 dictionary_size != ((1 << n) + (1 << (n + 1))))
347 goto failed;
348
349 printf("\tDictionary size %d\n", dictionary_size);
350
351 printed = 0;
352 for(i = 0; bcj[i].name; i++) {
353 if((flags >> i) & 1) {
354 if(printed)
355 printf(", ");
356 else
357 printf("\tFilters selected: ");
358 printf("%s", bcj[i].name);
359 printed = 1;
360 }
361 }
362
363 if(!printed)
364 printf("\tNo filters specified\n");
365 else
366 printf("\n");
367
368 return;
369
370 failed:
371 fprintf(stderr, "xz: error reading stored compressor options from "
372 "filesystem!\n");
373 }
374
375
376 /*
377 * This function is called by mksquashfs to initialise the
378 * compressor, before compress() is called.
379 *
380 * This function returns 0 on success, and
381 * -1 on error
382 */
xz_init(void ** strm,int block_size,int datablock)383 static int xz_init(void **strm, int block_size, int datablock)
384 {
385 int i, j, filters = datablock ? filter_count : 1;
386 struct filter *filter = malloc(filters * sizeof(struct filter));
387 struct xz_stream *stream;
388
389 if(filter == NULL)
390 goto failed;
391
392 stream = *strm = malloc(sizeof(struct xz_stream));
393 if(stream == NULL)
394 goto failed2;
395
396 stream->filter = filter;
397 stream->filters = filters;
398
399 memset(filter, 0, filters * sizeof(struct filter));
400
401 stream->dictionary_size = datablock ? dictionary_size :
402 SQUASHFS_METADATA_SIZE;
403
404 filter[0].filter[0].id = LZMA_FILTER_LZMA2;
405 filter[0].filter[0].options = &stream->opt;
406 filter[0].filter[1].id = LZMA_VLI_UNKNOWN;
407
408 for(i = 0, j = 1; datablock && bcj[i].name; i++) {
409 if(bcj[i].selected) {
410 filter[j].buffer = malloc(block_size);
411 if(filter[j].buffer == NULL)
412 goto failed3;
413 filter[j].filter[0].id = bcj[i].id;
414 filter[j].filter[1].id = LZMA_FILTER_LZMA2;
415 filter[j].filter[1].options = &stream->opt;
416 filter[j].filter[2].id = LZMA_VLI_UNKNOWN;
417 j++;
418 }
419 }
420
421 return 0;
422
423 failed3:
424 for(i = 1; i < filters; i++)
425 free(filter[i].buffer);
426 free(stream);
427
428 failed2:
429 free(filter);
430
431 failed:
432 return -1;
433 }
434
435
xz_compress(void * strm,void * dest,void * src,int size,int block_size,int * error)436 static int xz_compress(void *strm, void *dest, void *src, int size,
437 int block_size, int *error)
438 {
439 int i;
440 lzma_ret res = 0;
441 struct xz_stream *stream = strm;
442 struct filter *selected = NULL;
443
444 stream->filter[0].buffer = dest;
445
446 for(i = 0; i < stream->filters; i++) {
447 struct filter *filter = &stream->filter[i];
448
449 if(lzma_lzma_preset(&stream->opt, LZMA_PRESET_DEFAULT))
450 goto failed;
451
452 stream->opt.dict_size = stream->dictionary_size;
453
454 filter->length = 0;
455 res = lzma_stream_buffer_encode(filter->filter,
456 LZMA_CHECK_CRC32, NULL, src, size, filter->buffer,
457 &filter->length, block_size);
458
459 if(res == LZMA_OK) {
460 if(!selected || selected->length > filter->length)
461 selected = filter;
462 } else if(res != LZMA_BUF_ERROR)
463 goto failed;
464 }
465
466 if(!selected)
467 /*
468 * Output buffer overflow. Return out of buffer space
469 */
470 return 0;
471
472 if(selected->buffer != dest)
473 memcpy(dest, selected->buffer, selected->length);
474
475 return (int) selected->length;
476
477 failed:
478 /*
479 * All other errors return failure, with the compressor
480 * specific error code in *error
481 */
482 *error = res;
483 return -1;
484 }
485
486
xz_uncompress(void * dest,void * src,int size,int outsize,int * error)487 static int xz_uncompress(void *dest, void *src, int size, int outsize,
488 int *error)
489 {
490 size_t src_pos = 0;
491 size_t dest_pos = 0;
492 uint64_t memlimit = MEMLIMIT;
493
494 lzma_ret res = lzma_stream_buffer_decode(&memlimit, 0, NULL,
495 src, &src_pos, size, dest, &dest_pos, outsize);
496
497 if(res == LZMA_OK && size == (int) src_pos)
498 return (int) dest_pos;
499 else {
500 *error = res;
501 return -1;
502 }
503 }
504
505
xz_usage()506 void xz_usage()
507 {
508 fprintf(stderr, "\t -Xbcj filter1,filter2,...,filterN\n");
509 fprintf(stderr, "\t\tCompress using filter1,filter2,...,filterN in");
510 fprintf(stderr, " turn\n\t\t(in addition to no filter), and choose");
511 fprintf(stderr, " the best compression.\n");
512 fprintf(stderr, "\t\tAvailable filters: x86, arm, armthumb,");
513 fprintf(stderr, " powerpc, sparc, ia64\n");
514 fprintf(stderr, "\t -Xdict-size <dict-size>\n");
515 fprintf(stderr, "\t\tUse <dict-size> as the XZ dictionary size. The");
516 fprintf(stderr, " dictionary size\n\t\tcan be specified as a");
517 fprintf(stderr, " percentage of the block size, or as an\n\t\t");
518 fprintf(stderr, "absolute value. The dictionary size must be less");
519 fprintf(stderr, " than or equal\n\t\tto the block size and 8192 bytes");
520 fprintf(stderr, " or larger. It must also be\n\t\tstorable in the xz");
521 fprintf(stderr, " header as either 2^n or as 2^n+2^(n+1).\n\t\t");
522 fprintf(stderr, "Example dict-sizes are 75%%, 50%%, 37.5%%, 25%%, or");
523 fprintf(stderr, " 32K, 16K, 8K\n\t\tetc.\n");
524 }
525
526
527 struct compressor xz_comp_ops = {
528 .init = xz_init,
529 .compress = xz_compress,
530 .uncompress = xz_uncompress,
531 .options = xz_options,
532 .options_post = xz_options_post,
533 .dump_options = xz_dump_options,
534 .extract_options = xz_extract_options,
535 .display_options = xz_display_options,
536 .usage = xz_usage,
537 .id = XZ_COMPRESSION,
538 .name = "xz",
539 .supported = 1
540 };
541