1import sys
2import os
3import os.path
4from glob import glob
5import optparse
6
7VALGRIND_CMD = 'valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes '
8
9def compareOutputs( expected, actual, message ):
10    expected = expected.strip().replace('\r','').split('\n')
11    actual = actual.strip().replace('\r','').split('\n')
12    diff_line = 0
13    max_line_to_compare = min( len(expected), len(actual) )
14    for index in xrange(0,max_line_to_compare):
15        if expected[index].strip() != actual[index].strip():
16            diff_line = index + 1
17            break
18    if diff_line == 0 and len(expected) != len(actual):
19        diff_line = max_line_to_compare+1
20    if diff_line == 0:
21        return None
22    def safeGetLine( lines, index ):
23        index += -1
24        if index >= len(lines):
25            return ''
26        return lines[index].strip()
27    return """  Difference in %s at line %d:
28  Expected: '%s'
29  Actual:   '%s'
30""" % (message, diff_line,
31       safeGetLine(expected,diff_line),
32       safeGetLine(actual,diff_line) )
33
34def safeReadFile( path ):
35    try:
36        return file( path, 'rt' ).read()
37    except IOError, e:
38        return '<File "%s" is missing: %s>' % (path,e)
39
40def runAllTests( jsontest_executable_path, input_dir = None,
41                 use_valgrind=False, with_json_checker=False ):
42    if not input_dir:
43        input_dir = os.path.join( os.getcwd(), 'data' )
44    tests = glob( os.path.join( input_dir, '*.json' ) )
45    if with_json_checker:
46        test_jsonchecker = glob( os.path.join( input_dir, '../jsonchecker', '*.json' ) )
47    else:
48        test_jsonchecker = []
49    failed_tests = []
50    valgrind_path = use_valgrind and VALGRIND_CMD or ''
51    for input_path in tests + test_jsonchecker:
52        expect_failure = os.path.basename( input_path ).startswith( 'fail' )
53        is_json_checker_test = (input_path in test_jsonchecker) or expect_failure
54        print 'TESTING:', input_path,
55        options = is_json_checker_test and '--json-checker' or ''
56        pipe = os.popen( "%s%s %s %s" % (
57            valgrind_path, jsontest_executable_path, options,
58            input_path) )
59        process_output = pipe.read()
60        status = pipe.close()
61        if is_json_checker_test:
62            if expect_failure:
63                if status is None:
64                    print 'FAILED'
65                    failed_tests.append( (input_path, 'Parsing should have failed:\n%s' %
66                                          safeReadFile(input_path)) )
67                else:
68                    print 'OK'
69            else:
70                if status is not None:
71                    print 'FAILED'
72                    failed_tests.append( (input_path, 'Parsing failed:\n' + process_output) )
73                else:
74                    print 'OK'
75        else:
76            base_path = os.path.splitext(input_path)[0]
77            actual_output = safeReadFile( base_path + '.actual' )
78            actual_rewrite_output = safeReadFile( base_path + '.actual-rewrite' )
79            file(base_path + '.process-output','wt').write( process_output )
80            if status:
81                print 'parsing failed'
82                failed_tests.append( (input_path, 'Parsing failed:\n' + process_output) )
83            else:
84                expected_output_path = os.path.splitext(input_path)[0] + '.expected'
85                expected_output = file( expected_output_path, 'rt' ).read()
86                detail = ( compareOutputs( expected_output, actual_output, 'input' )
87                            or compareOutputs( expected_output, actual_rewrite_output, 'rewrite' ) )
88                if detail:
89                    print 'FAILED'
90                    failed_tests.append( (input_path, detail) )
91                else:
92                    print 'OK'
93
94    if failed_tests:
95        print
96        print 'Failure details:'
97        for failed_test in failed_tests:
98            print '* Test', failed_test[0]
99            print failed_test[1]
100            print
101        print 'Test results: %d passed, %d failed.' % (len(tests)-len(failed_tests),
102                                                       len(failed_tests) )
103        return 1
104    else:
105        print 'All %d tests passed.' % len(tests)
106        return 0
107
108def main():
109    from optparse import OptionParser
110    parser = OptionParser( usage="%prog [options] <path to jsontestrunner.exe> [test case directory]" )
111    parser.add_option("--valgrind",
112                  action="store_true", dest="valgrind", default=False,
113                  help="run all the tests using valgrind to detect memory leaks")
114    parser.add_option("-c", "--with-json-checker",
115                  action="store_true", dest="with_json_checker", default=False,
116                  help="run all the tests from the official JSONChecker test suite of json.org")
117    parser.enable_interspersed_args()
118    options, args = parser.parse_args()
119
120    if len(args) < 1 or len(args) > 2:
121        parser.error( 'Must provides at least path to jsontestrunner executable.' )
122        sys.exit( 1 )
123
124    jsontest_executable_path = os.path.normpath( os.path.abspath( args[0] ) )
125    if len(args) > 1:
126        input_path = os.path.normpath( os.path.abspath( args[1] ) )
127    else:
128        input_path = None
129    status = runAllTests( jsontest_executable_path, input_path,
130                          use_valgrind=options.valgrind, with_json_checker=options.with_json_checker )
131    sys.exit( status )
132
133if __name__ == '__main__':
134    main()
135