1# Copyright 2007 Baptiste Lepilleur and The JsonCpp Authors
2# Distributed under MIT license, or public domain if desired and
3# recognized in your jurisdiction.
4# See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
5
6from __future__ import print_function
7from __future__ import unicode_literals
8from io import open
9from glob import glob
10import sys
11import os
12import os.path
13import optparse
14
15VALGRIND_CMD = 'valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes '
16
17def getStatusOutput(cmd):
18    """
19    Return int, unicode (for both Python 2 and 3).
20    Note: os.popen().close() would return None for 0.
21    """
22    print(cmd, file=sys.stderr)
23    pipe = os.popen(cmd)
24    process_output = pipe.read()
25    try:
26        # We have been using os.popen(). When we read() the result
27        # we get 'str' (bytes) in py2, and 'str' (unicode) in py3.
28        # Ugh! There must be a better way to handle this.
29        process_output = process_output.decode('utf-8')
30    except AttributeError:
31        pass  # python3
32    status = pipe.close()
33    return status, process_output
34def compareOutputs(expected, actual, message):
35    expected = expected.strip().replace('\r','').split('\n')
36    actual = actual.strip().replace('\r','').split('\n')
37    diff_line = 0
38    max_line_to_compare = min(len(expected), len(actual))
39    for index in range(0,max_line_to_compare):
40        if expected[index].strip() != actual[index].strip():
41            diff_line = index + 1
42            break
43    if diff_line == 0 and len(expected) != len(actual):
44        diff_line = max_line_to_compare+1
45    if diff_line == 0:
46        return None
47    def safeGetLine(lines, index):
48        index += -1
49        if index >= len(lines):
50            return ''
51        return lines[index].strip()
52    return """  Difference in %s at line %d:
53  Expected: '%s'
54  Actual:   '%s'
55""" % (message, diff_line,
56       safeGetLine(expected,diff_line),
57       safeGetLine(actual,diff_line))
58
59def safeReadFile(path):
60    try:
61        return open(path, 'rt', encoding = 'utf-8').read()
62    except IOError as e:
63        return '<File "%s" is missing: %s>' % (path,e)
64
65class FailError(Exception):
66    def __init__(self, msg):
67        super(Exception, self).__init__(msg)
68
69def runAllTests(jsontest_executable_path, input_dir = None,
70                 use_valgrind=False, with_json_checker=False,
71                 writerClass='StyledWriter'):
72    if not input_dir:
73        input_dir = os.path.join(os.getcwd(), 'data')
74    tests = glob(os.path.join(input_dir, '*.json'))
75    if with_json_checker:
76        all_tests = glob(os.path.join(input_dir, '../jsonchecker', '*.json'))
77        # These tests fail with strict json support, but pass with JsonCPP's
78        # extra leniency features. When adding a new exclusion to this list,
79        # remember to add the test's number and reasoning here:
80        known = ["fail{}.json".format(n) for n in [
81            4, 9, # fail because we allow trailing commas
82            7,    # fails because we allow commas after close
83            8,    # fails because we allow extra close
84            10,   # fails because we allow extra values after close
85            13,   # fails because we allow leading zeroes in numbers
86            18,   # fails because we allow deeply nested values
87            25,   # fails because we allow tab characters in strings
88            27,   # fails because we allow string line breaks
89        ]]
90        test_jsonchecker = [ test for test in all_tests
91                             if os.path.basename(test) not in known]
92
93    else:
94        test_jsonchecker = []
95
96    failed_tests = []
97    valgrind_path = use_valgrind and VALGRIND_CMD or ''
98    for input_path in tests + test_jsonchecker:
99        expect_failure = os.path.basename(input_path).startswith('fail')
100        is_json_checker_test = (input_path in test_jsonchecker) or expect_failure
101        print('TESTING:', input_path, end=' ')
102        options = is_json_checker_test and '--json-checker' or ''
103        options += ' --json-writer %s'%writerClass
104        cmd = '%s%s %s "%s"' % (            valgrind_path, jsontest_executable_path, options,
105            input_path)
106        status, process_output = getStatusOutput(cmd)
107        if is_json_checker_test:
108            if expect_failure:
109                if not status:
110                    print('FAILED')
111                    failed_tests.append((input_path, 'Parsing should have failed:\n%s' %
112                                          safeReadFile(input_path)))
113                else:
114                    print('OK')
115            else:
116                if status:
117                    print('FAILED')
118                    failed_tests.append((input_path, 'Parsing failed:\n' + process_output))
119                else:
120                    print('OK')
121        else:
122            base_path = os.path.splitext(input_path)[0]
123            actual_output = safeReadFile(base_path + '.actual')
124            actual_rewrite_output = safeReadFile(base_path + '.actual-rewrite')
125            open(base_path + '.process-output', 'wt', encoding = 'utf-8').write(process_output)
126            if status:
127                print('parsing failed')
128                failed_tests.append((input_path, 'Parsing failed:\n' + process_output))
129            else:
130                expected_output_path = os.path.splitext(input_path)[0] + '.expected'
131                expected_output = open(expected_output_path, 'rt', encoding = 'utf-8').read()
132                detail = (compareOutputs(expected_output, actual_output, 'input')
133                            or compareOutputs(expected_output, actual_rewrite_output, 'rewrite'))
134                if detail:
135                    print('FAILED')
136                    failed_tests.append((input_path, detail))
137                else:
138                    print('OK')
139
140    if failed_tests:
141        print()
142        print('Failure details:')
143        for failed_test in failed_tests:
144            print('* Test', failed_test[0])
145            print(failed_test[1])
146            print()
147        print('Test results: %d passed, %d failed.' % (len(tests)-len(failed_tests),
148                                                       len(failed_tests)))
149        raise FailError(repr(failed_tests))
150    else:
151        print('All %d tests passed.' % len(tests))
152
153def main():
154    from optparse import OptionParser
155    parser = OptionParser(usage="%prog [options] <path to jsontestrunner.exe> [test case directory]")
156    parser.add_option("--valgrind",
157                  action="store_true", dest="valgrind", default=False,
158                  help="run all the tests using valgrind to detect memory leaks")
159    parser.add_option("-c", "--with-json-checker",
160                  action="store_true", dest="with_json_checker", default=False,
161                  help="run all the tests from the official JSONChecker test suite of json.org")
162    parser.enable_interspersed_args()
163    options, args = parser.parse_args()
164
165    if len(args) < 1 or len(args) > 2:
166        parser.error('Must provides at least path to jsontestrunner executable.')
167        sys.exit(1)
168
169    jsontest_executable_path = os.path.normpath(os.path.abspath(args[0]))
170    if len(args) > 1:
171        input_path = os.path.normpath(os.path.abspath(args[1]))
172    else:
173        input_path = None
174    runAllTests(jsontest_executable_path, input_path,
175                         use_valgrind=options.valgrind,
176                         with_json_checker=options.with_json_checker,
177                         writerClass='StyledWriter')
178    runAllTests(jsontest_executable_path, input_path,
179                         use_valgrind=options.valgrind,
180                         with_json_checker=options.with_json_checker,
181                         writerClass='StyledStreamWriter')
182    runAllTests(jsontest_executable_path, input_path,
183                         use_valgrind=options.valgrind,
184                         with_json_checker=options.with_json_checker,
185                         writerClass='BuiltStyledStreamWriter')
186
187if __name__ == '__main__':
188    try:
189        main()
190    except FailError:
191        sys.exit(1)
192