1#
2# Copyright (C) 2017 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16
17import importlib
18import json
19import logging
20import os
21import sys
22import time
23import yaml
24
25from vts.runners.host import asserts
26from vts.runners.host import base_test
27from vts.runners.host import config_parser
28from vts.runners.host import keys
29from vts.runners.host import records
30from vts.runners.host import test_runner
31from vts.utils.python.io import capture_printout
32from vts.utils.python.io import file_util
33
34from mobly import test_runner as mobly_test_runner
35
36
37LIST_TEST_OUTPUT_START = '==========> '
38LIST_TEST_OUTPUT_END = ' <=========='
39# Temp directory inside python log path. The name is required to be
40# the set value for tradefed to skip reading contents as logs.
41TEMP_DIR_NAME = 'temp'
42CONFIG_FILE_NAME = 'test_config.yaml'
43MOBLY_RESULT_JSON_FILE_NAME = 'test_run_summary.json'
44MOBLY_RESULT_YAML_FILE_NAME = 'test_summary.yaml'
45
46
47MOBLY_CONFIG_TEXT = '''TestBeds:
48  - Name: {module_name}
49    Controllers:
50        AndroidDevice:
51          - serial: {serial1}
52          - serial: {serial2}
53
54MoblyParams:
55    LogPath: {log_path}
56'''
57
58#TODO(yuexima):
59# 1. make DEVICES_REQUIRED configurable
60# 2. add include filter function
61DEVICES_REQUIRED = 2
62
63RESULT_KEY_TYPE = 'Type'
64RESULT_TYPE_SUMMARY = 'Summary'
65RESULT_TYPE_RECORD = 'Record'
66RESULT_TYPE_TEST_NAME_LIST = 'TestNameList'
67RESULT_TYPE_CONTROLLER_INFO = 'ControllerInfo'
68
69
70class MoblyTest(base_test.BaseTestClass):
71    '''Template class for running mobly test cases.
72
73    Attributes:
74        mobly_dir: string, mobly test temp directory for mobly runner
75        mobly_config_file_path: string, mobly test config file path
76        result_handlers: dict, map of result type and handler functions
77    '''
78    def setUpClass(self):
79        asserts.assertEqual(
80            len(self.android_devices), DEVICES_REQUIRED,
81            'Exactly %s devices are required for this test.' % DEVICES_REQUIRED
82        )
83
84        for ad in self.android_devices:
85            logging.debug('Android device serial: %s' % ad.serial)
86
87        logging.debug('Test cases: %s' % self.ListTestCases())
88
89        self.mobly_dir = os.path.join(logging.log_path, TEMP_DIR_NAME,
90                                      'mobly', str(time.time()))
91
92        file_util.Makedirs(self.mobly_dir)
93
94        logging.debug('mobly log path: %s' % self.mobly_dir)
95
96        self.result_handlers = {
97            RESULT_TYPE_SUMMARY: self.HandleSimplePrint,
98            RESULT_TYPE_RECORD: self.HandleRecord,
99            RESULT_TYPE_TEST_NAME_LIST: self.HandleSimplePrint,
100            RESULT_TYPE_CONTROLLER_INFO: self.HandleSimplePrint,
101        }
102
103    def tearDownClass(self):
104        ''' Clear the mobly directory.'''
105        file_util.Rmdirs(self.mobly_dir, ignore_errors=True)
106
107    def PrepareConfigFile(self):
108        '''Prepare mobly config file for running test.'''
109        self.mobly_config_file_path = os.path.join(self.mobly_dir,
110                                                   CONFIG_FILE_NAME)
111        config_text = MOBLY_CONFIG_TEXT.format(
112              module_name=self.test_module_name,
113              serial1=self.android_devices[0].serial,
114              serial2=self.android_devices[1].serial,
115              log_path=self.mobly_dir
116        )
117        with open(self.mobly_config_file_path, 'w') as f:
118            f.write(config_text)
119
120    def ListTestCases(self):
121        '''List test cases.
122
123        Returns:
124            List of string, test names.
125        '''
126        classes = mobly_test_runner._find_test_class()
127
128        with capture_printout.CaptureStdout() as output:
129            mobly_test_runner._print_test_names(classes)
130
131        test_names = []
132
133        for line in output:
134            if (not line.startswith(LIST_TEST_OUTPUT_START)
135                and line.endswith(LIST_TEST_OUTPUT_END)):
136                test_names.append(line)
137                tr_record = records.TestResultRecord(line, self.test_module_name)
138                self.results.requested.append(tr_record)
139
140        return test_names
141
142    def RunMoblyModule(self):
143        '''Execute mobly test module.'''
144        # Because mobly and vts uses a similar runner, both will modify
145        # log_path from python logging. The following step is to preserve
146        # log path after mobly test finishes.
147
148        # An alternative way is to start a new python process through shell
149        # command. In that case, test print out needs to be piped.
150        # This will also help avoid log overlapping
151
152        logger = logging.getLogger()
153        logger_path = logger.log_path
154        logging_path = logging.log_path
155
156        try:
157            mobly_test_runner.main(argv=['-c', self.mobly_config_file_path])
158        finally:
159            logger.log_path = logger_path
160            logging.log_path = logging_path
161
162    def GetMoblyResults(self):
163        '''Get mobly module run results and put in vts results.'''
164        file_handlers = (
165            (MOBLY_RESULT_YAML_FILE_NAME, self.ParseYamlResults),
166            (MOBLY_RESULT_JSON_FILE_NAME, self.ParseJsonResults),
167        )
168
169        for pair in file_handlers:
170            file_path = file_util.FindFile(self.mobly_dir, pair[0])
171
172            if file_path:
173                logging.debug('Mobly test yaml result path: %s', file_path)
174                pair[1](file_path)
175                return
176
177        asserts.fail('Mobly test result file not found.')
178
179    def generateAllTests(self):
180        '''Run the mobly test module and parse results.'''
181        #TODO(yuexima): report test names
182
183        self.PrepareConfigFile()
184        self.RunMoblyModule()
185        #TODO(yuexima): check whether DEBUG logs from mobly run are included
186        self.GetMoblyResults()
187
188    def ParseJsonResults(self, result_path):
189        '''Parse mobly test json result.
190
191        Args:
192            result_path: string, result json file path.
193        '''
194        with open(path, 'r') as f:
195            mobly_summary = json.load(f)
196
197        mobly_results = mobly_summary['Results']
198        for result in mobly_results:
199            logging.debug('Adding result for %s' % result[records.TestResultEnums.RECORD_NAME])
200            record = records.TestResultRecord(result[records.TestResultEnums.RECORD_NAME])
201            record.test_class = result[records.TestResultEnums.RECORD_CLASS]
202            record.begin_time = result[records.TestResultEnums.RECORD_BEGIN_TIME]
203            record.end_time = result[records.TestResultEnums.RECORD_END_TIME]
204            record.result = result[records.TestResultEnums.RECORD_RESULT]
205            record.uid = result[records.TestResultEnums.RECORD_UID]
206            record.extras = result[records.TestResultEnums.RECORD_EXTRAS]
207            record.details = result[records.TestResultEnums.RECORD_DETAILS]
208            record.extra_errors = result[records.TestResultEnums.RECORD_EXTRA_ERRORS]
209
210            self.results.addRecord(record)
211
212    def ParseYamlResults(self, result_path):
213        '''Parse mobly test yaml result.
214
215        Args:
216            result_path: string, result yaml file path.
217        '''
218        with open(result_path, 'r') as stream:
219            try:
220                docs = yaml.load_all(stream)
221                for doc in docs:
222                    type = doc.get(RESULT_KEY_TYPE)
223                    if type is None:
224                        logging.warn(
225                            'Mobly result document type unrecognized: %s', doc)
226                        continue
227
228                    logging.debug('Parsing result type: %s', type)
229
230                    handler = self.result_handlers.get(type)
231                    if handler is None:
232                        logging.debug('Unknown result type: %s', type)
233                        handler = self.HandleSimplePrint
234
235                    handler(doc)
236            except yaml.YAMLError as exc:
237                print(exc)
238
239    def HandleRecord(self, doc):
240        '''Handle record result document type.
241
242        Args:
243            doc: dict, result document item
244        '''
245        logging.debug('Adding result for %s' % doc.get(records.TestResultEnums.RECORD_NAME))
246        record = records.TestResultRecord(doc.get(records.TestResultEnums.RECORD_NAME))
247        record.test_class = doc.get(records.TestResultEnums.RECORD_CLASS)
248        record.begin_time = doc.get(records.TestResultEnums.RECORD_BEGIN_TIME)
249        record.end_time = doc.get(records.TestResultEnums.RECORD_END_TIME)
250        record.result = doc.get(records.TestResultEnums.RECORD_RESULT)
251        record.uid = doc.get(records.TestResultEnums.RECORD_UID)
252        record.extras = doc.get(records.TestResultEnums.RECORD_EXTRAS)
253        record.details = doc.get(records.TestResultEnums.RECORD_DETAILS)
254        record.extra_errors = doc.get(records.TestResultEnums.RECORD_EXTRA_ERRORS)
255
256        # 'Stacktrace' in yaml result is ignored. 'Stacktrace' is a more
257        # detailed version of record.details when exception is emitted.
258
259        self.results.addRecord(record)
260
261    def HandleSimplePrint(self, doc):
262        '''Simply print result document to log.
263
264        Args:
265            doc: dict, result document item
266        '''
267        for k, v in doc.items():
268            logging.debug(str(k) + ": " + str(v))
269
270def GetTestModuleNames():
271    '''Returns a list of mobly test module specified in test configuration.'''
272    configs = config_parser.load_test_config_file(sys.argv[1])
273    reduce_func = lambda x, y: x + y.get(keys.ConfigKeys.MOBLY_TEST_MODULE, [])
274    return reduce(reduce_func, configs, [])
275
276def ImportTestModules():
277    '''Dynamically import mobly test modules.'''
278    for module_name in GetTestModuleNames():
279        module, cls = module_name.rsplit('.', 1)
280        sys.modules['__main__'].__dict__[cls] = getattr(
281            importlib.import_module(module), cls)
282
283if __name__ == "__main__":
284    ImportTestModules()
285    test_runner.main()
286