1# 2# Copyright (C) 2020 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16 17import os 18import logging 19 20import ltp_configs 21import ltp_enums 22import test_case 23from configs import stable_tests 24from configs import disabled_tests 25from common import filter_utils 26 27ltp_test_template = ' <option name="test-command-line" key="%s" value="&env_setup_cmd; ;' \ 28 ' cd <p_bin_dir; ; %s" />' 29 30class LtpTestCases(object): 31 """Load a ltp vts testcase definition file and parse it into a generator. 32 33 Attributes: 34 _data_path: string, the vts data path on host side 35 _filter_func: function, a filter method that will emit exception if a test is filtered 36 _ltp_tests_filter: list of string, filter for tests that are stable and disabled 37 _ltp_binaries: list of string, All ltp binaries that generate in build time 38 _ltp_config_lines: list of string: the context of the generated config 39 """ 40 41 def __init__(self, android_build_top, filter_func): 42 self._android_build_top = android_build_top 43 self._filter_func = filter_func 44 self._ltp_tests_filter = filter_utils.Filter( 45 [ i[0] for i in stable_tests.STABLE_TESTS ], 46 disabled_tests.DISABLED_TESTS, 47 enable_regex=True) 48 self._ltp_tests_filter.ExpandBitness() 49 self._ltp_binaries = [] 50 self._ltp_config_lines = [] 51 52 def ValidateDefinition(self, line): 53 """Validate a tab delimited test case definition. 54 55 Will check whether the given line of definition has three parts 56 separated by tabs. 57 It will also trim leading and ending white spaces for each part 58 in returned tuple (if valid). 59 60 Returns: 61 A tuple in format (test suite, test name, test command) if 62 definition is valid. None otherwise. 63 """ 64 items = [ 65 item.strip() 66 for item in line.split(ltp_enums.Delimiters.TESTCASE_DEFINITION) 67 ] 68 if not len(items) == 3 or not items: 69 return None 70 else: 71 return items 72 73 def ReadConfigTemplateFile(self): 74 """Read the template of the config file and return the context. 75 76 Returns: 77 String. 78 """ 79 file_name = ltp_configs.LTP_CONFIG_TEMPLATE_FILE_NAME 80 file_path = os.path.join(self._android_build_top, ltp_configs.LTP_CONFIG_TEMPLATE_DIR, file_name) 81 with open(file_path, 'r') as f: 82 return f.read() 83 84 def GetKernelModuleControllerOption(self, arch, n_bit, is_low_mem=False, is_hwasan=False): 85 """Get the Option of KernelModuleController. 86 87 Args: 88 arch: String, arch 89 n_bit: int, bitness 90 run_staging: bool, whether to use staging configuration 91 is_low_mem: bool, whether to use low memory device configuration 92 93 Returns: 94 String. 95 """ 96 arch_template = ' <option name="arch" value="{}"/>\n' 97 is_low_mem_template = ' <option name="is-low-mem" value="{}"/>\n' 98 is_hwasan_template = ' <option name="is-hwasan" value="{}"/>' 99 option_lines = arch_template + is_low_mem_template + is_hwasan_template 100 if n_bit == '64': 101 n_bit_string = str(n_bit) if arch == 'arm' else ('_'+str(n_bit)) 102 else: 103 n_bit_string = '' 104 arch_name = arch + n_bit_string 105 is_low_mem = 'true' if is_low_mem else 'false' 106 is_hwasan = 'true' if is_hwasan else 'false' 107 option_lines = option_lines.format(arch_name, 108 str(is_low_mem).lower(), 109 str(is_hwasan).lower()) 110 return option_lines 111 112 def GetLtpBinaries(self): 113 """Check the binary exist in the command. 114 115 Args: 116 command: String, the test command 117 118 Returns: 119 bool: True if the binary in the gen.bp 120 """ 121 gen_bp_path = os.path.join(self._android_build_top, ltp_configs.LTP_GEN_BINARY_BP) 122 for line in open(gen_bp_path, 'r'): 123 line = line.strip() 124 if not line or line.startswith('#'): 125 continue 126 if line.startswith("stem:") or line.startswith('filename:'): 127 ltp_binary = line.split('"')[1] 128 self._ltp_binaries.append(ltp_binary) 129 130 def IsLtpBinaryExist(self, commands): 131 """Check the binary exist in the command. 132 133 Args: 134 command: String, the test command 135 136 Returns: 137 bool: True if the binary in the gen.bp 138 """ 139 all_commands = commands.split(';') 140 for cmd in all_commands: 141 cmd = cmd.strip() 142 binary_name = cmd.split(' ')[0] 143 if binary_name in self._ltp_binaries: 144 return True 145 logging.info("Ltp binary not exist in cmd of '%s'", commands) 146 return False 147 148 def GenConfig(self, 149 arch, 150 n_bit, 151 test_filter, 152 output_file, 153 run_staging=False, 154 is_low_mem=False, 155 is_hwasan=False): 156 """Read the definition file and generate the test config. 157 158 Args: 159 arch: String, arch 160 n_bit: int, bitness 161 test_filter: Filter object, test name filter from base_test 162 output_file: String, the file path of the generating config 163 run_staging: bool, whether to use staging configuration 164 is_low_mem: bool, whether to use low memory device configuration 165 """ 166 self.GetLtpBinaries() 167 scenario_groups = (ltp_configs.TEST_SUITES_LOW_MEM 168 if is_low_mem else ltp_configs.TEST_SUITES) 169 logging.info('LTP scenario groups: %s', scenario_groups) 170 start_append_test_keyword = 'option name="per-binary-timeout"' 171 config_lines = self.ReadConfigTemplateFile() 172 module_controller_option = self.GetKernelModuleControllerOption(arch, n_bit, 173 is_low_mem, 174 is_hwasan) 175 test_case_string = '' 176 run_scritp = self.GenerateLtpRunScript(scenario_groups, is_hwasan=is_hwasan) 177 for line in run_scritp: 178 items = self.ValidateDefinition(line) 179 if not items: 180 continue 181 182 testsuite, testname, command = items 183 if is_low_mem and testsuite.endswith( 184 ltp_configs.LOW_MEMORY_SCENARIO_GROUP_SUFFIX): 185 testsuite = testsuite[:-len( 186 ltp_configs.LOW_MEMORY_SCENARIO_GROUP_SUFFIX)] 187 188 # Tests failed to build will have prefix "DISABLED_" 189 if testname.startswith("DISABLED_"): 190 logging.info("[Parser] Skipping test case {}-{}. Reason: " 191 "not built".format(testsuite, testname)) 192 continue 193 194 # Some test cases have hardcoded "/tmp" in the command 195 # we replace that with ltp_configs.TMPDIR 196 command = command.replace('/tmp', ltp_configs.TMPDIR) 197 198 testcase = test_case.TestCase( 199 testsuite=testsuite, testname=testname, command=command) 200 test_display_name = "{}_{}bit".format(str(testcase), n_bit) 201 202 # Check runner's base_test filtering method 203 try: 204 self._filter_func(test_display_name) 205 except: 206 logging.info("[Parser] Skipping test case %s. Reason: " 207 "filtered" % testcase.fullname) 208 testcase.is_filtered = True 209 testcase.note = "filtered" 210 211 logging.info('ltp_test_cases Load(): test_display_name = %s\n' 212 'cmd = %s', test_display_name, command) 213 214 # For skipping tests that are not designed or ready for Android, 215 # check for bit specific test in disabled list as well as non-bit specific 216 if ((self._ltp_tests_filter.IsInExcludeFilter(str(testcase)) or 217 self._ltp_tests_filter.IsInExcludeFilter(test_display_name)) and 218 not test_filter.IsInIncludeFilter(test_display_name)): 219 logging.info("[Parser] Skipping test case %s. Reason: " 220 "disabled" % testcase.fullname) 221 continue 222 223 # For separating staging tests from stable tests 224 if not self._ltp_tests_filter.IsInIncludeFilter(test_display_name): 225 if not run_staging and not test_filter.IsInIncludeFilter( 226 test_display_name): 227 # Skip staging tests in stable run 228 continue 229 else: 230 testcase.is_staging = True 231 testcase.note = "staging" 232 else: 233 if run_staging: 234 # Skip stable tests in staging run 235 continue 236 237 if not testcase.is_staging: 238 for x in stable_tests.STABLE_TESTS: 239 if x[0] == test_display_name and x[1]: 240 testcase.is_mandatory = True 241 break 242 if self.IsLtpBinaryExist(command): 243 logging.info("[Parser] Adding test case %s." % testcase.fullname) 244 # Some test cases contain semicolons in their commands, 245 # and we replace them with && 246 command = command.replace(';', '&&') 247 # Replace the original command with '/data/local/tmp/ltp' 248 # e.g. mm.mmapstress07 249 command = command.replace(ltp_configs.LTPDIR, '<p_dir;') 250 ltp_test_line = ltp_test_template % (test_display_name, command) 251 test_case_string += (ltp_test_line + '\n') 252 nativetest_bit_path = '64' if n_bit == '64' else '' 253 config_lines = config_lines.format(nativetest_bit_path, module_controller_option, 254 test_case_string) 255 with open(output_file, 'w') as f: 256 f.write(config_lines) 257 258 def ReadCommentedTxt(self, filepath): 259 '''Read a lines of a file that are not commented by #. 260 261 Args: 262 filepath: string, path of file to read 263 264 Returns: 265 A set of string representing non-commented lines in given file 266 ''' 267 if not filepath: 268 logging.error('Invalid file path') 269 return None 270 271 with open(filepath, 'r') as f: 272 lines_gen = (line.strip() for line in f) 273 return set( 274 line for line in lines_gen 275 if line and not line.startswith('#')) 276 277 def GenerateLtpTestCases(self, testsuite, disabled_tests_list): 278 '''Generate test cases for each ltp test suite. 279 280 Args: 281 testsuite: string, test suite name 282 283 Returns: 284 A list of string 285 ''' 286 testsuite_script = os.path.join(self._android_build_top, 287 ltp_configs.LTP_RUNTEST_DIR, testsuite) 288 289 result = [] 290 for line in open(testsuite_script, 'r'): 291 line = line.strip() 292 if not line or line.startswith('#'): 293 continue 294 295 testname = line.split()[0] 296 testname_prefix = ('DISABLED_' 297 if testname in disabled_tests_list else '') 298 testname_modified = testname_prefix + testname 299 300 result.append("\t".join( 301 [testsuite, testname_modified, line[len(testname):].strip()])) 302 return result 303 304 def GenerateLtpRunScript(self, scenario_groups, is_hwasan=False): 305 '''Given a scenario group generate test case script. 306 307 Args: 308 scenario_groups: list of string, name of test scenario groups to use 309 310 Returns: 311 A list of string 312 ''' 313 disabled_tests_path = os.path.join( 314 self._android_build_top, ltp_configs.LTP_DISABLED_BUILD_TESTS_CONFIG_PATH) 315 disabled_tests_list = self.ReadCommentedTxt(disabled_tests_path) 316 if is_hwasan: 317 disabled_tests_list = disabled_tests_list.union(disabled_tests.DISABLED_TESTS_HWASAN) 318 319 result = [] 320 for testsuite in scenario_groups: 321 result.extend( 322 self.GenerateLtpTestCases(testsuite, disabled_tests_list)) 323 return result 324