1#!/usr/bin/env python3
2"""Test zstd interoperability between versions"""
3
4# ################################################################
5# Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
6# All rights reserved.
7#
8# This source code is licensed under both the BSD-style license (found in the
9# LICENSE file in the root directory of this source tree) and the GPLv2 (found
10# in the COPYING file in the root directory of this source tree).
11# You may select, at your option, one of the above-listed licenses.
12# ################################################################
13
14import filecmp
15import glob
16import hashlib
17import os
18import shutil
19import sys
20import subprocess
21from subprocess import Popen, PIPE
22
23repo_url = 'https://github.com/facebook/zstd.git'
24tmp_dir_name = 'tests/versionsTest'
25make_cmd = 'make'
26git_cmd = 'git'
27test_dat_src = 'README.md'
28test_dat = 'test_dat'
29head = 'vdevel'
30dict_source = 'dict_source'
31dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c '
32dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h'
33
34
35def execute(command, print_output=False, print_error=True, param_shell=False):
36    popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=param_shell)
37    stdout_lines, stderr_lines = popen.communicate()
38    stderr_lines = stderr_lines.decode("utf-8")
39    stdout_lines = stdout_lines.decode("utf-8")
40    if print_output:
41        print(stdout_lines)
42        print(stderr_lines)
43    if popen.returncode is not None and popen.returncode != 0:
44        if not print_output and print_error:
45            print(stderr_lines)
46    return popen.returncode
47
48
49def proc(cmd_args, pipe=True, dummy=False):
50    if dummy:
51        return
52    if pipe:
53        subproc = Popen(cmd_args, stdout=PIPE, stderr=PIPE)
54    else:
55        subproc = Popen(cmd_args)
56    return subproc.communicate()
57
58
59def make(args, pipe=True):
60    return proc([make_cmd] + args, pipe)
61
62
63def git(args, pipe=True):
64    return proc([git_cmd] + args, pipe)
65
66
67def get_git_tags():
68    stdout, stderr = git(['tag', '-l', 'v[0-9].[0-9].[0-9]'])
69    tags = stdout.decode('utf-8').split()
70    return tags
71
72
73def create_dict(tag, dict_source_path):
74    dict_name = 'dict.' + tag
75    if not os.path.isfile(dict_name):
76        cFiles = glob.glob(dict_source_path + "/*.c")
77        hFiles = glob.glob(dict_source_path + "/*.h")
78        if tag == 'v0.5.0':
79            result = execute('./dictBuilder.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
80        else:
81            result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
82        if result == 0:
83            print(dict_name + ' created')
84        else:
85            print('ERROR: creating of ' + dict_name + ' failed')
86    else:
87        print(dict_name + ' already exists')
88
89
90def dict_compress_sample(tag, sample):
91    dict_name = 'dict.' + tag
92    DEVNULL = open(os.devnull, 'wb')
93    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f',   sample], stderr=DEVNULL) == 0:
94        os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst')
95    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f',  sample], stderr=DEVNULL) == 0:
96        os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst')
97    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f',  sample], stderr=DEVNULL) == 0:
98        os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst')
99    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0:
100        os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst')
101    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0:
102        os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst')
103    # zstdFiles = glob.glob("*.zst*")
104    # print(zstdFiles)
105    print(tag + " : dict compression completed")
106
107
108def compress_sample(tag, sample):
109    DEVNULL = open(os.devnull, 'wb')
110    if subprocess.call(['./zstd.' + tag, '-f',   sample], stderr=DEVNULL) == 0:
111        os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst')
112    if subprocess.call(['./zstd.' + tag, '-5f',  sample], stderr=DEVNULL) == 0:
113        os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodict.zst')
114    if subprocess.call(['./zstd.' + tag, '-9f',  sample], stderr=DEVNULL) == 0:
115        os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodict.zst')
116    if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0:
117        os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodict.zst')
118    if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0:
119        os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodict.zst')
120    # zstdFiles = glob.glob("*.zst*")
121    # print(zstdFiles)
122    print(tag + " : compression completed")
123
124
125# http://stackoverflow.com/a/19711609/2132223
126def sha1_of_file(filepath):
127    with open(filepath, 'rb') as f:
128        return hashlib.sha1(f.read()).hexdigest()
129
130
131def remove_duplicates():
132    list_of_zst = sorted(glob.glob('*.zst'))
133    for i, ref_zst in enumerate(list_of_zst):
134        if not os.path.isfile(ref_zst):
135            continue
136        for j in range(i + 1, len(list_of_zst)):
137            compared_zst = list_of_zst[j]
138            if not os.path.isfile(compared_zst):
139                continue
140            if filecmp.cmp(ref_zst, compared_zst):
141                os.remove(compared_zst)
142                print('duplicated : {} == {}'.format(ref_zst, compared_zst))
143
144
145def decompress_zst(tag):
146    dec_error = 0
147    list_zst = sorted(glob.glob('*_nodict.zst'))
148    for file_zst in list_zst:
149        print(file_zst, end=' ')
150        print(tag, end=' ')
151        file_dec = file_zst + '_d64_' + tag + '.dec'
152        if tag <= 'v0.5.0':
153            params = ['./zstd.' + tag, '-df', file_zst, file_dec]
154        else:
155            params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec]
156        if execute(params) == 0:
157            if not filecmp.cmp(file_dec, test_dat):
158                print('ERR !! ')
159                dec_error = 1
160            else:
161                print('OK     ')
162        else:
163            print('command does not work')
164            dec_error = 1
165    return dec_error
166
167
168def decompress_dict(tag):
169    dec_error = 0
170    list_zst = sorted(glob.glob('*_dictio.zst'))
171    for file_zst in list_zst:
172        dict_tag = file_zst[0:len(file_zst)-11]  # remove "_dictio.zst"
173        if head in dict_tag: # find vdevel
174            dict_tag = head
175        else:
176            dict_tag = dict_tag[dict_tag.rfind('v'):]
177        if tag == 'v0.6.0' and dict_tag < 'v0.6.0':
178            continue
179        dict_name = 'dict.' + dict_tag
180        print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ')
181        file_dec = file_zst + '_d64_' + tag + '.dec'
182        if tag <= 'v0.5.0':
183            params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec]
184        else:
185            params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec]
186        if execute(params) == 0:
187            if not filecmp.cmp(file_dec, test_dat):
188                print('ERR !! ')
189                dec_error = 1
190            else:
191                print('OK     ')
192        else:
193            print('command does not work')
194            dec_error = 1
195    return dec_error
196
197
198if __name__ == '__main__':
199    error_code = 0
200    base_dir = os.getcwd() + '/..'                  # /path/to/zstd
201    tmp_dir = base_dir + '/' + tmp_dir_name         # /path/to/zstd/tests/versionsTest
202    clone_dir = tmp_dir + '/' + 'zstd'              # /path/to/zstd/tests/versionsTest/zstd
203    dict_source_path = tmp_dir + '/' + dict_source  # /path/to/zstd/tests/versionsTest/dict_source
204    programs_dir = base_dir + '/programs'           # /path/to/zstd/programs
205    os.makedirs(tmp_dir, exist_ok=True)
206
207    # since Travis clones limited depth, we should clone full repository
208    if not os.path.isdir(clone_dir):
209        git(['clone', repo_url, clone_dir])
210
211    shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat)
212
213    # Retrieve all release tags
214    print('Retrieve all release tags :')
215    os.chdir(clone_dir)
216    alltags = get_git_tags() + [head]
217    tags = [t for t in alltags if t >= 'v0.5.0']
218    print(tags)
219
220    # Build all release zstd
221    for tag in tags:
222        os.chdir(base_dir)
223        dst_zstd = '{}/zstd.{}'.format(tmp_dir, tag)  # /path/to/zstd/tests/versionsTest/zstd.<TAG>
224        if not os.path.isfile(dst_zstd) or tag == head:
225            if tag != head:
226                r_dir = '{}/{}'.format(tmp_dir, tag)  # /path/to/zstd/tests/versionsTest/<TAG>
227                os.makedirs(r_dir, exist_ok=True)
228                os.chdir(clone_dir)
229                git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
230                if tag == 'v0.5.0':
231                    os.chdir(r_dir + '/dictBuilder')  # /path/to/zstd/tests/versionsTest/v0.5.0/dictBuilder
232                    make(['clean', 'dictBuilder'], False)
233                    shutil.copy2('dictBuilder', '{}/dictBuilder.{}'.format(tmp_dir, tag))
234                os.chdir(r_dir + '/programs')  # /path/to/zstd/tests/versionsTest/<TAG>/programs
235                make(['clean', 'zstd'], False)
236            else:
237                os.chdir(programs_dir)
238                make(['zstd'], False)
239            shutil.copy2('zstd',   dst_zstd)
240
241    # remove any remaining *.zst and *.dec from previous test
242    os.chdir(tmp_dir)
243    for compressed in glob.glob("*.zst"):
244        os.remove(compressed)
245    for dec in glob.glob("*.dec"):
246        os.remove(dec)
247
248    # copy *.c and *.h to a temporary directory ("dict_source")
249    if not os.path.isdir(dict_source_path):
250        os.mkdir(dict_source_path)
251        print('cp ' + dict_files + ' ' + dict_source_path)
252        execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True)
253
254    print('Compress test.dat by all released zstd')
255
256    error_code = 0
257    for tag in tags:
258        print(tag)
259        if tag >= 'v0.5.0':
260            create_dict(tag, dict_source_path)
261            dict_compress_sample(tag, test_dat)
262            remove_duplicates()
263            error_code += decompress_dict(tag)
264        compress_sample(tag, test_dat)
265        remove_duplicates()
266        error_code += decompress_zst(tag)
267
268    print('')
269    print('Enumerate different compressed files')
270    zstds = sorted(glob.glob('*.zst'))
271    for zstd in zstds:
272        print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd))
273
274    if error_code != 0:
275        print('======  ERROR !!!  =======')
276
277    sys.exit(error_code)
278