1#! /usr/bin/env python
2#
3# Copyright 2018 - The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18# Parses the output of parse_ltp_{make,make_install} and generates a
19# corresponding Android.mk.
20#
21# This process is split into two steps so this second step can later be replaced
22# with an Android.bp generator.
23
24from __future__ import print_function
25
26"""Tool for comparing 2 LTP projects to find added / removed tests & testsuites"""
27
28import os
29import argparse
30import os.path
31import sys
32
33def scan_tests(ltp_root, suite):
34    ''' Find all tests that are run as part of given test suite in LTP.
35
36    Args:
37        ltp_root: Path ot the LTP project.
38        suite: Name of testsuite.
39
40    Returns:
41        List of tests names that are run as part of the given test suite.
42    '''
43
44    tests = []
45    if not suite:
46        return tests
47
48    runtest_dir = ltp_root + os.path.sep + 'runtest'
49    test_suiteFile = runtest_dir + os.path.sep + suite
50    if not os.path.isfile(test_suiteFile):
51        print ('No tests defined for suite {}',format(suite))
52        return tests
53
54    with open(test_suiteFile) as f:
55        for line in f:
56            line = line.strip()
57            if line and not line.startswith('#'):
58                tests.append(line.split()[0])
59    f.close()
60    return tests
61
62def scan_test_suites(ltp_root, scenario):
63    ''' Find all testuites and tests run as part of given LTP scenario
64
65    Args:
66        ltp_root: Path to the LTP project.
67        scenario: name of the scenario (found in ltp_root/scenario_groups). E.g. "vts"
68
69    Returns:
70        List of testsuite names that are run as part of given scenario (e.g. vts).
71        If scenario is not specified, return all testsuites.
72    '''
73
74    runtest_dir = ltp_root + os.path.sep + 'runtest'
75    if not os.path.isdir(runtest_dir):
76        print ('Invalid ltp_root {}, runtest directory doesnt exist'.format(ltp_root))
77        sys.exit(2)
78
79    test_suites = []
80    if scenario:
81        scenarioFile = ltp_root + os.path.sep + 'scenario_groups' + os.path.sep + scenario
82        if not os.path.isfile(scenarioFile):
83            return test_suites
84        with open(scenarioFile) as f:
85            for line in f:
86                line = line.strip()
87                if not line.startswith('#'):
88                    test_suites.append(line)
89        return test_suites
90
91    runtest_dir = ltp_root + os.path.sep + 'runtest'
92    test_suites = [f for f in os.listdir(runtest_dir) if os.path.isfile(runtest_dir + os.path.sep + f)]
93
94    return test_suites
95
96def scan_ltp(ltp_root, scenario):
97    ''' scan LTP project and return all tests and testuites present.
98
99    Args:
100        ltp_root: Path to the LTP project.
101        scenario: specific scenario we want to scan. e.g. vts
102
103    Returns:
104        Dictionary of all LTP test names keyed by testsuite they are run as part of.
105        E.g.
106             {
107                 'mem': ['oom01', 'mem02',...],
108                 'syscalls': ['open01', 'read02',...],
109                 ...
110             }
111    '''
112
113    if not os.path.isdir(ltp_root):
114        print ('ltp_root {} does not exist'.format(ltp_root))
115        sys.exit(1)
116
117    test_suites = scan_test_suites(ltp_root, scenario)
118    if not test_suites:
119        print ('No Testsuites found for scenario {}'.format(scenario))
120        sys.exit(3)
121
122    ltp_tests = {}
123    for suite in test_suites:
124        ltp_tests[suite] = scan_tests(ltp_root, suite)
125    return ltp_tests
126
127def show_diff(ltp_tests_1, ltp_tests_2):
128    ''' Find and print diff between testcases in 2 LTP project checkouts.
129
130    Args:
131        ltp_tests_1: dictionary of tests keyed by test suite names
132        ltp_tests_2: dictionary of tests keyed by test suite names
133    '''
134    DEFAULT_WIDTH = 8
135    test_suites1 = set(sorted(ltp_tests_1.keys()))
136    test_suites2 = set(sorted(ltp_tests_2.keys()))
137
138    # Generate lists of deleted, added and common test suites between
139    # LTP1 & LTP2
140    deleted_test_suites = sorted(test_suites1.difference(test_suites2))
141    added_test_suites = sorted(test_suites2.difference(test_suites1))
142    common_test_suites = sorted(test_suites1.intersection(test_suites2))
143
144    deleted_tests = []
145    added_tests = []
146    for suite in common_test_suites:
147        tests1 = set(sorted(ltp_tests_1[suite]))
148        tests2 = set(sorted(ltp_tests_2[suite]))
149
150        exclusive_test1 = tests1.difference(tests2)
151        exclusive_test2 = tests2.difference(tests1)
152        for e in exclusive_test1:
153            deleted_tests.append(suite + '.' + e)
154        for e in exclusive_test2:
155            added_tests.append(suite + '.' + e)
156
157    # find the maximum width of a test suite name or test name
158    # we have to print to decide the alignment.
159    if not deleted_test_suites:
160        width_suites = DEFAULT_WIDTH
161    else:
162        width_suites = max([len(x) for x in deleted_test_suites])
163
164    if not deleted_tests:
165        width_tests = DEFAULT_WIDTH
166    else:
167        width_tests = max([len(x) for x in deleted_tests])
168    width = max(width_suites, width_tests);
169
170    # total rows we have to print
171    total_rows = max(len(deleted_test_suites), len(added_test_suites))
172    if total_rows > 0:
173        print ('{:*^{len}}'.format(' Tests Suites ', len=width*2+2))
174        print ('{:>{len}} {:>{len}}'.format('Deleted', 'Added', len=width))
175        for i in range(total_rows):
176            print ('{:>{len}} {:>{len}}'.format('' if i >= len(deleted_test_suites) else str(deleted_test_suites[i]),
177                                 '' if i >= len(added_test_suites) else str(added_test_suites[i]), len=width))
178
179    print('')
180    # total rows we have to print
181    total_rows = max(len(deleted_tests), len(added_tests))
182    if total_rows:
183        print ('{:*^{len}}'.format(' Tests ', len=width*2+2))
184        print ('{:>{len}} {:>{len}}'.format('Deleted', 'Added', len=width))
185        for i in range(total_rows):
186            print ('{:>{len}} {:>{len}}'.format('' if i >= len(deleted_tests) else str(deleted_tests[i]),
187                                 '' if i >= len(added_tests) else str(added_tests[i]), len=width))
188def main():
189    arg_parser = argparse.ArgumentParser(
190        description='Diff 2 LTP projects for supported test cases')
191    arg_parser.add_argument('--ltp-root1',
192                            dest='ltp_root1',
193                            required=True,
194                            help="LTP Root Directory before merge")
195    arg_parser.add_argument('--ltp-root2',
196                            dest='ltp_root2',
197                            required=True,
198                            help="LTP Root Directory after merge")
199    arg_parser.add_argument('--scenario', default=None,
200                            dest='scenario',
201                            help="LTP scenario to list tests for")
202    args = arg_parser.parse_args()
203
204    ltp_tests1 = scan_ltp(args.ltp_root1, args.scenario)
205    ltp_tests2 = scan_ltp(args.ltp_root2, args.scenario)
206    show_diff(ltp_tests1, ltp_tests2)
207
208if __name__ == '__main__':
209    main()
210