1# Copyright 2018, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16Utils for finder classes. 17""" 18 19# pylint: disable=line-too-long 20 21from __future__ import print_function 22 23import logging 24import multiprocessing 25import os 26import pickle 27import re 28import subprocess 29import time 30import xml.etree.ElementTree as ET 31 32import atest_decorator 33import atest_error 34import atest_enum 35import constants 36 37from metrics import metrics_utils 38 39# Helps find apk files listed in a test config (AndroidTest.xml) file. 40# Matches "filename.apk" in <option name="foo", value="filename.apk" /> 41# We want to make sure we don't grab apks with paths in their name since we 42# assume the apk name is the build target. 43_APK_RE = re.compile(r'^[^/]+\.apk$', re.I) 44# RE for checking if TEST or TEST_F is in a cc file or not. 45_CC_CLASS_RE = re.compile(r'^[ ]*TEST(_F|_P)?[ ]*\(', re.I) 46# RE for checking if there exists one of the methods in java file. 47_JAVA_METHODS_PATTERN = r'.*[ ]+({0})\(.*' 48# RE for checking if there exists one of the methods in cc file. 49_CC_METHODS_PATTERN = r'^[ ]*TEST(_F|_P)?[ ]*\(.*,[ ]*({0})\).*' 50# Parse package name from the package declaration line of a java or 51# a kotlin file. 52# Group matches "foo.bar" of line "package foo.bar;" or "package foo.bar" 53_PACKAGE_RE = re.compile(r'\s*package\s+(?P<package>[^(;|\s)]+)\s*', re.I) 54# Matches install paths in module_info to install location(host or device). 55_HOST_PATH_RE = re.compile(r'.*\/host\/.*', re.I) 56_DEVICE_PATH_RE = re.compile(r'.*\/target\/.*', re.I) 57 58# Explanation of FIND_REFERENCE_TYPEs: 59# ---------------------------------- 60# 0. CLASS: Name of a java/kotlin class, usually file is named the same 61# (HostTest lives in HostTest.java or HostTest.kt) 62# 1. QUALIFIED_CLASS: Like CLASS but also contains the package in front like 63# com.android.tradefed.testtype.HostTest. 64# 2. PACKAGE: Name of a java package. 65# 3. INTEGRATION: XML file name in one of the 4 integration config directories. 66# 4. CC_CLASS: Name of a cc class. 67 68FIND_REFERENCE_TYPE = atest_enum.AtestEnum(['CLASS', 69 'QUALIFIED_CLASS', 70 'PACKAGE', 71 'INTEGRATION', 72 'CC_CLASS']) 73# Get cpu count. 74_CPU_COUNT = 0 if os.uname()[0] == 'Linux' else multiprocessing.cpu_count() 75 76# Unix find commands for searching for test files based on test type input. 77# Note: Find (unlike grep) exits with status 0 if nothing found. 78FIND_CMDS = { 79 FIND_REFERENCE_TYPE.CLASS: r"find {0} {1} -type f" 80 r"| egrep '.*/{2}\.(kt|java)$' || true", 81 FIND_REFERENCE_TYPE.QUALIFIED_CLASS: r"find {0} {1} -type f" 82 r"| egrep '.*{2}\.(kt|java)$' || true", 83 FIND_REFERENCE_TYPE.PACKAGE: r"find {0} {1} -wholename " 84 r"'*{2}' -type d -print", 85 FIND_REFERENCE_TYPE.INTEGRATION: r"find {0} {1} -wholename " 86 r"'*{2}.xml' -print", 87 # Searching a test among files where the absolute paths contain *test*. 88 # If users complain atest couldn't find a CC_CLASS, ask them to follow the 89 # convention that the filename or dirname must contain *test*, where *test* 90 # is case-insensitive. 91 FIND_REFERENCE_TYPE.CC_CLASS: r"find {0} {1} -type f -print" 92 r"| egrep -i '/*test.*\.(cc|cpp)$'" 93 r"| xargs -P" + str(_CPU_COUNT) + 94 r" egrep -sH '^[ ]*TEST(_F|_P)?[ ]*\({2}' " 95 " || true" 96} 97 98# Map ref_type with its index file. 99FIND_INDEXES = { 100 FIND_REFERENCE_TYPE.CLASS: constants.CLASS_INDEX, 101 FIND_REFERENCE_TYPE.QUALIFIED_CLASS: constants.QCLASS_INDEX, 102 FIND_REFERENCE_TYPE.PACKAGE: constants.PACKAGE_INDEX, 103 FIND_REFERENCE_TYPE.INTEGRATION: constants.INT_INDEX, 104 FIND_REFERENCE_TYPE.CC_CLASS: constants.CC_CLASS_INDEX 105} 106 107# XML parsing related constants. 108_COMPATIBILITY_PACKAGE_PREFIX = "com.android.compatibility" 109_CTS_JAR = "cts-tradefed" 110_XML_PUSH_DELIM = '->' 111_APK_SUFFIX = '.apk' 112# Setup script for device perf tests. 113_PERF_SETUP_LABEL = 'perf-setup.sh' 114 115# XML tags. 116_XML_NAME = 'name' 117_XML_VALUE = 'value' 118 119# VTS xml parsing constants. 120_VTS_TEST_MODULE = 'test-module-name' 121_VTS_MODULE = 'module-name' 122_VTS_BINARY_SRC = 'binary-test-source' 123_VTS_PUSH_GROUP = 'push-group' 124_VTS_PUSH = 'push' 125_VTS_BINARY_SRC_DELIM = '::' 126_VTS_PUSH_DIR = os.path.join(os.environ.get(constants.ANDROID_BUILD_TOP, ''), 127 'test', 'vts', 'tools', 'vts-tradefed', 'res', 128 'push_groups') 129_VTS_PUSH_SUFFIX = '.push' 130_VTS_BITNESS = 'append-bitness' 131_VTS_BITNESS_TRUE = 'true' 132_VTS_BITNESS_32 = '32' 133_VTS_BITNESS_64 = '64' 134_VTS_TEST_FILE = 'test-file-name' 135_VTS_APK = 'apk' 136# Matches 'DATA/target' in '_32bit::DATA/target' 137_VTS_BINARY_SRC_DELIM_RE = re.compile(r'.*::(?P<target>.*)$') 138_VTS_OUT_DATA_APP_PATH = 'DATA/app' 139 140# pylint: disable=inconsistent-return-statements 141def split_methods(user_input): 142 """Split user input string into test reference and list of methods. 143 144 Args: 145 user_input: A string of the user's input. 146 Examples: 147 class_name 148 class_name#method1,method2 149 path 150 path#method1,method2 151 Returns: 152 A tuple. First element is String of test ref and second element is 153 a set of method name strings or empty list if no methods included. 154 Exception: 155 atest_error.TooManyMethodsError raised when input string is trying to 156 specify too many methods in a single positional argument. 157 158 Examples of unsupported input strings: 159 module:class#method,class#method 160 class1#method,class2#method 161 path1#method,path2#method 162 """ 163 parts = user_input.split('#') 164 if len(parts) == 1: 165 return parts[0], frozenset() 166 if len(parts) == 2: 167 return parts[0], frozenset(parts[1].split(',')) 168 raise atest_error.TooManyMethodsError( 169 'Too many methods specified with # character in user input: %s.' 170 '\n\nOnly one class#method combination supported per positional' 171 ' argument. Multiple classes should be separated by spaces: ' 172 'class#method class#method') 173 174 175# pylint: disable=inconsistent-return-statements 176def get_fully_qualified_class_name(test_path): 177 """Parse the fully qualified name from the class java file. 178 179 Args: 180 test_path: A string of absolute path to the java class file. 181 182 Returns: 183 A string of the fully qualified class name. 184 185 Raises: 186 atest_error.MissingPackageName if no class name can be found. 187 """ 188 with open(test_path) as class_file: 189 for line in class_file: 190 match = _PACKAGE_RE.match(line) 191 if match: 192 package = match.group('package') 193 cls = os.path.splitext(os.path.split(test_path)[1])[0] 194 return '%s.%s' % (package, cls) 195 raise atest_error.MissingPackageNameError('%s: Test class java file' 196 'does not contain a package' 197 'name.'% test_path) 198 199 200def has_cc_class(test_path): 201 """Find out if there is any test case in the cc file. 202 203 Args: 204 test_path: A string of absolute path to the cc file. 205 206 Returns: 207 Boolean: has cc class in test_path or not. 208 """ 209 with open(test_path) as class_file: 210 for line in class_file: 211 match = _CC_CLASS_RE.match(line) 212 if match: 213 return True 214 return False 215 216 217def get_package_name(file_name): 218 """Parse the package name from a java file. 219 220 Args: 221 file_name: A string of the absolute path to the java file. 222 223 Returns: 224 A string of the package name or None 225 """ 226 with open(file_name) as data: 227 for line in data: 228 match = _PACKAGE_RE.match(line) 229 if match: 230 return match.group('package') 231 232 233def has_method_in_file(test_path, methods): 234 """Find out if there is at least one method in the file. 235 236 Note: This method doesn't handle if method is in comment sections or not. 237 If the file has any method(even in comment sections), it will return True. 238 239 Args: 240 test_path: A string of absolute path to the test file. 241 methods: A set of method names. 242 243 Returns: 244 Boolean: there is at least one method in test_path. 245 """ 246 if not os.path.isfile(test_path): 247 return False 248 methods_re = None 249 if constants.JAVA_EXT_RE.match(test_path): 250 methods_re = re.compile(_JAVA_METHODS_PATTERN.format( 251 '|'.join([r'%s' % x for x in methods]))) 252 elif constants.CC_EXT_RE.match(test_path): 253 methods_re = re.compile(_CC_METHODS_PATTERN.format( 254 '|'.join([r'%s' % x for x in methods]))) 255 if methods_re: 256 with open(test_path) as test_file: 257 for line in test_file: 258 match = re.match(methods_re, line) 259 if match: 260 return True 261 return False 262 263 264def extract_test_path(output, methods=None): 265 """Extract the test path from the output of a unix 'find' command. 266 267 Example of find output for CLASS find cmd: 268 /<some_root>/cts/tests/jank/src/android/jank/cts/ui/CtsDeviceJankUi.java 269 270 Args: 271 output: A string or list output of a unix 'find' command. 272 methods: A set of method names. 273 274 Returns: 275 A list of the test paths or None if output is '' or None. 276 """ 277 if not output: 278 return None 279 verified_tests = set() 280 if isinstance(output, str): 281 output = output.splitlines() 282 for test in output: 283 # compare CC_OUTPUT_RE with output 284 match_obj = constants.CC_OUTPUT_RE.match(test) 285 if match_obj: 286 # cc/cpp 287 fpath = match_obj.group('file_path') 288 if not methods or match_obj.group('method_name') in methods: 289 verified_tests.add(fpath) 290 else: 291 # TODO (b/138997521) - Atest checks has_method_in_file of a class 292 # without traversing its parent classes. A workaround for this is 293 # do not check has_method_in_file. Uncomment below when a solution 294 # to it is applied. 295 # java/kt 296 #if not methods or has_method_in_file(test, methods): 297 verified_tests.add(test) 298 return extract_test_from_tests(sorted(list(verified_tests))) 299 300 301def extract_test_from_tests(tests): 302 """Extract the test path from the tests. 303 304 Return the test to run from tests. If more than one option, prompt the user 305 to select multiple ones. Supporting formats: 306 - An integer. E.g. 0 307 - Comma-separated integers. E.g. 1,3,5 308 - A range of integers denoted by the starting integer separated from 309 the end integer by a dash, '-'. E.g. 1-3 310 311 Args: 312 tests: A string list which contains multiple test paths. 313 314 Returns: 315 A string list of paths. 316 """ 317 count = len(tests) 318 if count <= 1: 319 return tests if count else None 320 mtests = set() 321 try: 322 numbered_list = ['%s: %s' % (i, t) for i, t in enumerate(tests)] 323 numbered_list.append('%s: All' % count) 324 print('Multiple tests found:\n{0}'.format('\n'.join(numbered_list))) 325 test_indices = input("Please enter numbers of test to use. If none of " 326 "above option matched, keep searching for other " 327 "possible tests.\n(multiple selection is supported, " 328 "e.g. '1' or '0,1' or '0-2'): ") 329 for idx in re.sub(r'(\s)', '', test_indices).split(','): 330 indices = idx.split('-') 331 len_indices = len(indices) 332 if len_indices > 0: 333 start_index = min(int(indices[0]), int(indices[len_indices-1])) 334 end_index = max(int(indices[0]), int(indices[len_indices-1])) 335 # One of input is 'All', return all options. 336 if count in (start_index, end_index): 337 return tests 338 mtests.update(tests[start_index:(end_index+1)]) 339 except (ValueError, IndexError, AttributeError, TypeError) as err: 340 logging.debug('%s', err) 341 print('None of above option matched, keep searching for other' 342 ' possible tests...') 343 return list(mtests) 344 345 346@atest_decorator.static_var("cached_ignore_dirs", []) 347def _get_ignored_dirs(): 348 """Get ignore dirs in find command. 349 350 Since we can't construct a single find cmd to find the target and 351 filter-out the dir with .out-dir, .find-ignore and $OUT-DIR. We have 352 to run the 1st find cmd to find these dirs. Then, we can use these 353 results to generate the real find cmd. 354 355 Return: 356 A list of the ignore dirs. 357 """ 358 out_dirs = _get_ignored_dirs.cached_ignore_dirs 359 if not out_dirs: 360 build_top = os.environ.get(constants.ANDROID_BUILD_TOP) 361 find_out_dir_cmd = (r'find %s -maxdepth 2 ' 362 r'-type f \( -name ".out-dir" -o -name ' 363 r'".find-ignore" \)') % build_top 364 out_files = subprocess.check_output(find_out_dir_cmd, shell=True) 365 if isinstance(out_files, bytes): 366 out_files = out_files.decode() 367 # Get all dirs with .out-dir or .find-ignore 368 if out_files: 369 out_files = out_files.splitlines() 370 for out_file in out_files: 371 if out_file: 372 out_dirs.append(os.path.dirname(out_file.strip())) 373 # Get the out folder if user specified $OUT_DIR 374 custom_out_dir = os.environ.get(constants.ANDROID_OUT_DIR) 375 if custom_out_dir: 376 user_out_dir = None 377 if os.path.isabs(custom_out_dir): 378 user_out_dir = custom_out_dir 379 else: 380 user_out_dir = os.path.join(build_top, custom_out_dir) 381 # only ignore the out_dir when it under $ANDROID_BUILD_TOP 382 if build_top in user_out_dir: 383 if user_out_dir not in out_dirs: 384 out_dirs.append(user_out_dir) 385 _get_ignored_dirs.cached_ignore_dirs = out_dirs 386 return out_dirs 387 388 389def _get_prune_cond_of_ignored_dirs(): 390 """Get the prune condition of ignore dirs. 391 392 Generation a string of the prune condition in the find command. 393 It will filter-out the dir with .out-dir, .find-ignore and $OUT-DIR. 394 Because they are the out dirs, we don't have to find them. 395 396 Return: 397 A string of the prune condition of the ignore dirs. 398 """ 399 out_dirs = _get_ignored_dirs() 400 prune_cond = r'-type d \( -name ".*"' 401 for out_dir in out_dirs: 402 prune_cond += r' -o -path %s' % out_dir 403 prune_cond += r' \) -prune -o' 404 return prune_cond 405 406 407def run_find_cmd(ref_type, search_dir, target, methods=None): 408 """Find a path to a target given a search dir and a target name. 409 410 Args: 411 ref_type: An AtestEnum of the reference type. 412 search_dir: A string of the dirpath to search in. 413 target: A string of what you're trying to find. 414 methods: A set of method names. 415 416 Return: 417 A list of the path to the target. 418 If the search_dir is inexistent, None will be returned. 419 """ 420 # If module_info.json is outdated, finding in the search_dir can result in 421 # raising exception. Return null immediately can guild users to run 422 # --rebuild-module-info to resolve the problem. 423 if not os.path.isdir(search_dir): 424 logging.debug('\'%s\' does not exist!', search_dir) 425 return None 426 ref_name = FIND_REFERENCE_TYPE[ref_type] 427 start = time.time() 428 if os.path.isfile(FIND_INDEXES[ref_type]): 429 _dict, out = {}, None 430 with open(FIND_INDEXES[ref_type], 'rb') as index: 431 try: 432 _dict = pickle.load(index, encoding='utf-8') 433 except (TypeError, IOError, EOFError, pickle.UnpicklingError) as err: 434 logging.debug('Exception raised: %s', err) 435 metrics_utils.handle_exc_and_send_exit_event( 436 constants.ACCESS_CACHE_FAILURE) 437 os.remove(FIND_INDEXES[ref_type]) 438 if _dict.get(target): 439 logging.debug('Found %s in %s', target, FIND_INDEXES[ref_type]) 440 out = [path for path in _dict.get(target) if search_dir in path] 441 else: 442 prune_cond = _get_prune_cond_of_ignored_dirs() 443 if '.' in target: 444 target = target.replace('.', '/') 445 find_cmd = FIND_CMDS[ref_type].format(search_dir, prune_cond, target) 446 logging.debug('Executing %s find cmd: %s', ref_name, find_cmd) 447 out = subprocess.check_output(find_cmd, shell=True) 448 if isinstance(out, bytes): 449 out = out.decode() 450 logging.debug('%s find cmd out: %s', ref_name, out) 451 logging.debug('%s find completed in %ss', ref_name, time.time() - start) 452 return extract_test_path(out, methods) 453 454 455def find_class_file(search_dir, class_name, is_native_test=False, methods=None): 456 """Find a path to a class file given a search dir and a class name. 457 458 Args: 459 search_dir: A string of the dirpath to search in. 460 class_name: A string of the class to search for. 461 is_native_test: A boolean variable of whether to search for a native 462 test or not. 463 methods: A set of method names. 464 465 Return: 466 A list of the path to the java/cc file. 467 """ 468 if is_native_test: 469 ref_type = FIND_REFERENCE_TYPE.CC_CLASS 470 elif '.' in class_name: 471 ref_type = FIND_REFERENCE_TYPE.QUALIFIED_CLASS 472 else: 473 ref_type = FIND_REFERENCE_TYPE.CLASS 474 return run_find_cmd(ref_type, search_dir, class_name, methods) 475 476 477def is_equal_or_sub_dir(sub_dir, parent_dir): 478 """Return True sub_dir is sub dir or equal to parent_dir. 479 480 Args: 481 sub_dir: A string of the sub directory path. 482 parent_dir: A string of the parent directory path. 483 484 Returns: 485 A boolean of whether both are dirs and sub_dir is sub of parent_dir 486 or is equal to parent_dir. 487 """ 488 # avoid symlink issues with real path 489 parent_dir = os.path.realpath(parent_dir) 490 sub_dir = os.path.realpath(sub_dir) 491 if not os.path.isdir(sub_dir) or not os.path.isdir(parent_dir): 492 return False 493 return os.path.commonprefix([sub_dir, parent_dir]) == parent_dir 494 495 496def find_parent_module_dir(root_dir, start_dir, module_info): 497 """From current dir search up file tree until root dir for module dir. 498 499 Args: 500 root_dir: A string of the dir that is the parent of the start dir. 501 start_dir: A string of the dir to start searching up from. 502 module_info: ModuleInfo object containing module information from the 503 build system. 504 505 Returns: 506 A string of the module dir relative to root, None if no Module Dir 507 found. There may be multiple testable modules at this level. 508 509 Exceptions: 510 ValueError: Raised if cur_dir not dir or not subdir of root dir. 511 """ 512 if not is_equal_or_sub_dir(start_dir, root_dir): 513 raise ValueError('%s not in repo %s' % (start_dir, root_dir)) 514 auto_gen_dir = None 515 current_dir = start_dir 516 while current_dir != root_dir: 517 # TODO (b/112904944) - migrate module_finder functions to here and 518 # reuse them. 519 rel_dir = os.path.relpath(current_dir, root_dir) 520 # Check if actual config file here 521 if os.path.isfile(os.path.join(current_dir, constants.MODULE_CONFIG)): 522 return rel_dir 523 # Check module_info if auto_gen config or robo (non-config) here 524 for mod in module_info.path_to_module_info.get(rel_dir, []): 525 if module_info.is_robolectric_module(mod): 526 return rel_dir 527 for test_config in mod.get(constants.MODULE_TEST_CONFIG, []): 528 if os.path.isfile(os.path.join(root_dir, test_config)): 529 return rel_dir 530 if mod.get('auto_test_config'): 531 auto_gen_dir = rel_dir 532 # Don't return for auto_gen, keep checking for real config, 533 # because common in cts for class in apk that's in hostside 534 # test setup. 535 current_dir = os.path.dirname(current_dir) 536 return auto_gen_dir 537 538 539def get_targets_from_xml(xml_file, module_info): 540 """Retrieve build targets from the given xml. 541 542 Just a helper func on top of get_targets_from_xml_root. 543 544 Args: 545 xml_file: abs path to xml file. 546 module_info: ModuleInfo class used to verify targets are valid modules. 547 548 Returns: 549 A set of build targets based on the signals found in the xml file. 550 """ 551 xml_root = ET.parse(xml_file).getroot() 552 return get_targets_from_xml_root(xml_root, module_info) 553 554 555def _get_apk_target(apk_target): 556 """Return the sanitized apk_target string from the xml. 557 558 The apk_target string can be of 2 forms: 559 - apk_target.apk 560 - apk_target.apk->/path/to/install/apk_target.apk 561 562 We want to return apk_target in both cases. 563 564 Args: 565 apk_target: String of target name to clean. 566 567 Returns: 568 String of apk_target to build. 569 """ 570 apk = apk_target.split(_XML_PUSH_DELIM, 1)[0].strip() 571 return apk[:-len(_APK_SUFFIX)] 572 573 574def _is_apk_target(name, value): 575 """Return True if XML option is an apk target. 576 577 We have some scenarios where an XML option can be an apk target: 578 - value is an apk file. 579 - name is a 'push' option where value holds the apk_file + other stuff. 580 581 Args: 582 name: String name of XML option. 583 value: String value of the XML option. 584 585 Returns: 586 True if it's an apk target we should build, False otherwise. 587 """ 588 if _APK_RE.match(value): 589 return True 590 if name == 'push' and value.endswith(_APK_SUFFIX): 591 return True 592 return False 593 594 595def get_targets_from_xml_root(xml_root, module_info): 596 """Retrieve build targets from the given xml root. 597 598 We're going to pull the following bits of info: 599 - Parse any .apk files listed in the config file. 600 - Parse option value for "test-module-name" (for vts10 tests). 601 - Look for the perf script. 602 603 Args: 604 module_info: ModuleInfo class used to verify targets are valid modules. 605 xml_root: ElementTree xml_root for us to look through. 606 607 Returns: 608 A set of build targets based on the signals found in the xml file. 609 """ 610 targets = set() 611 option_tags = xml_root.findall('.//option') 612 for tag in option_tags: 613 target_to_add = None 614 name = tag.attrib[_XML_NAME].strip() 615 value = tag.attrib[_XML_VALUE].strip() 616 if _is_apk_target(name, value): 617 target_to_add = _get_apk_target(value) 618 elif _PERF_SETUP_LABEL in value: 619 targets.add(_PERF_SETUP_LABEL) 620 continue 621 622 # Let's make sure we can actually build the target. 623 if target_to_add and module_info.is_module(target_to_add): 624 targets.add(target_to_add) 625 elif target_to_add: 626 logging.warning('Build target (%s) not present in module info, ' 627 'skipping build', target_to_add) 628 629 # TODO (b/70813166): Remove this lookup once all runtime dependencies 630 # can be listed as a build dependencies or are in the base test harness. 631 nodes_with_class = xml_root.findall(".//*[@class]") 632 for class_attr in nodes_with_class: 633 fqcn = class_attr.attrib['class'].strip() 634 if fqcn.startswith(_COMPATIBILITY_PACKAGE_PREFIX): 635 targets.add(_CTS_JAR) 636 logging.debug('Targets found in config file: %s', targets) 637 return targets 638 639 640def _get_vts_push_group_targets(push_file, rel_out_dir): 641 """Retrieve vts10 push group build targets. 642 643 A push group file is a file that list out test dependencies and other push 644 group files. Go through the push file and gather all the test deps we need. 645 646 Args: 647 push_file: Name of the push file in the VTS 648 rel_out_dir: Abs path to the out dir to help create vts10 build targets. 649 650 Returns: 651 Set of string which represent build targets. 652 """ 653 targets = set() 654 full_push_file_path = os.path.join(_VTS_PUSH_DIR, push_file) 655 # pylint: disable=invalid-name 656 with open(full_push_file_path) as f: 657 for line in f: 658 target = line.strip() 659 # Skip empty lines. 660 if not target: 661 continue 662 663 # This is a push file, get the targets from it. 664 if target.endswith(_VTS_PUSH_SUFFIX): 665 targets |= _get_vts_push_group_targets(line.strip(), 666 rel_out_dir) 667 continue 668 sanitized_target = target.split(_XML_PUSH_DELIM, 1)[0].strip() 669 targets.add(os.path.join(rel_out_dir, sanitized_target)) 670 return targets 671 672 673def _specified_bitness(xml_root): 674 """Check if the xml file contains the option append-bitness. 675 676 Args: 677 xml_root: abs path to xml file. 678 679 Returns: 680 True if xml specifies to append-bitness, False otherwise. 681 """ 682 option_tags = xml_root.findall('.//option') 683 for tag in option_tags: 684 value = tag.attrib[_XML_VALUE].strip() 685 name = tag.attrib[_XML_NAME].strip() 686 if name == _VTS_BITNESS and value == _VTS_BITNESS_TRUE: 687 return True 688 return False 689 690 691def _get_vts_binary_src_target(value, rel_out_dir): 692 """Parse out the vts10 binary src target. 693 694 The value can be in the following pattern: 695 - {_32bit,_64bit,_IPC32_32bit}::DATA/target (DATA/target) 696 - DATA/target->/data/target (DATA/target) 697 - out/host/linx-x86/bin/VtsSecuritySelinuxPolicyHostTest (the string as 698 is) 699 700 Args: 701 value: String of the XML option value to parse. 702 rel_out_dir: String path of out dir to prepend to target when required. 703 704 Returns: 705 String of the target to build. 706 """ 707 # We'll assume right off the bat we can use the value as is and modify it if 708 # necessary, e.g. out/host/linux-x86/bin... 709 target = value 710 # _32bit::DATA/target 711 match = _VTS_BINARY_SRC_DELIM_RE.match(value) 712 if match: 713 target = os.path.join(rel_out_dir, match.group('target')) 714 # DATA/target->/data/target 715 elif _XML_PUSH_DELIM in value: 716 target = value.split(_XML_PUSH_DELIM, 1)[0].strip() 717 target = os.path.join(rel_out_dir, target) 718 return target 719 720 721def get_plans_from_vts_xml(xml_file): 722 """Get configs which are included by xml_file. 723 724 We're looking for option(include) to get all dependency plan configs. 725 726 Args: 727 xml_file: Absolute path to xml file. 728 729 Returns: 730 A set of plan config paths which are depended by xml_file. 731 """ 732 if not os.path.exists(xml_file): 733 raise atest_error.XmlNotExistError('%s: The xml file does' 734 'not exist' % xml_file) 735 plans = set() 736 xml_root = ET.parse(xml_file).getroot() 737 plans.add(xml_file) 738 option_tags = xml_root.findall('.//include') 739 if not option_tags: 740 return plans 741 # Currently, all vts10 xmls live in the same dir : 742 # https://android.googlesource.com/platform/test/vts/+/master/tools/vts-tradefed/res/config/ 743 # If the vts10 plans start using folders to organize the plans, the logic here 744 # should be changed. 745 xml_dir = os.path.dirname(xml_file) 746 for tag in option_tags: 747 name = tag.attrib[_XML_NAME].strip() 748 plans |= get_plans_from_vts_xml(os.path.join(xml_dir, name + ".xml")) 749 return plans 750 751 752def get_targets_from_vts_xml(xml_file, rel_out_dir, module_info): 753 """Parse a vts10 xml for test dependencies we need to build. 754 755 We have a separate vts10 parsing function because we make a big assumption 756 on the targets (the way they're formatted and what they represent) and we 757 also create these build targets in a very special manner as well. 758 The 6 options we're looking for are: 759 - binary-test-source 760 - push-group 761 - push 762 - test-module-name 763 - test-file-name 764 - apk 765 766 Args: 767 module_info: ModuleInfo class used to verify targets are valid modules. 768 rel_out_dir: Abs path to the out dir to help create vts10 build targets. 769 xml_file: abs path to xml file. 770 771 Returns: 772 A set of build targets based on the signals found in the xml file. 773 """ 774 xml_root = ET.parse(xml_file).getroot() 775 targets = set() 776 option_tags = xml_root.findall('.//option') 777 for tag in option_tags: 778 value = tag.attrib[_XML_VALUE].strip() 779 name = tag.attrib[_XML_NAME].strip() 780 if name in [_VTS_TEST_MODULE, _VTS_MODULE]: 781 if module_info.is_module(value): 782 targets.add(value) 783 else: 784 logging.warning('vts10 test module (%s) not present in module ' 785 'info, skipping build', value) 786 elif name == _VTS_BINARY_SRC: 787 targets.add(_get_vts_binary_src_target(value, rel_out_dir)) 788 elif name == _VTS_PUSH_GROUP: 789 # Look up the push file and parse out build artifacts (as well as 790 # other push group files to parse). 791 targets |= _get_vts_push_group_targets(value, rel_out_dir) 792 elif name == _VTS_PUSH: 793 # Parse out the build artifact directly. 794 push_target = value.split(_XML_PUSH_DELIM, 1)[0].strip() 795 # If the config specified append-bitness, append the bits suffixes 796 # to the target. 797 if _specified_bitness(xml_root): 798 targets.add(os.path.join( 799 rel_out_dir, push_target + _VTS_BITNESS_32)) 800 targets.add(os.path.join( 801 rel_out_dir, push_target + _VTS_BITNESS_64)) 802 else: 803 targets.add(os.path.join(rel_out_dir, push_target)) 804 elif name == _VTS_TEST_FILE: 805 # The _VTS_TEST_FILE values can be set in 2 possible ways: 806 # 1. test_file.apk 807 # 2. DATA/app/test_file/test_file.apk 808 # We'll assume that test_file.apk (#1) is in an expected path (but 809 # that is not true, see b/76158619) and create the full path for it 810 # and then append the _VTS_TEST_FILE value to targets to build. 811 target = os.path.join(rel_out_dir, value) 812 # If value is just an APK, specify the path that we expect it to be in 813 # e.g. out/host/linux-x86/vts10/android-vts10/testcases/DATA/app/test_file/test_file.apk 814 head, _ = os.path.split(value) 815 if not head: 816 target = os.path.join(rel_out_dir, _VTS_OUT_DATA_APP_PATH, 817 _get_apk_target(value), value) 818 targets.add(target) 819 elif name == _VTS_APK: 820 targets.add(os.path.join(rel_out_dir, value)) 821 logging.debug('Targets found in config file: %s', targets) 822 return targets 823 824 825def get_dir_path_and_filename(path): 826 """Return tuple of dir and file name from given path. 827 828 Args: 829 path: String of path to break up. 830 831 Returns: 832 Tuple of (dir, file) paths. 833 """ 834 if os.path.isfile(path): 835 dir_path, file_path = os.path.split(path) 836 else: 837 dir_path, file_path = path, None 838 return dir_path, file_path 839 840 841def get_cc_filter(class_name, methods): 842 """Get the cc filter. 843 844 Args: 845 class_name: class name of the cc test. 846 methods: a list of method names. 847 848 Returns: 849 A formatted string for cc filter. 850 Ex: "class1.method1:class1.method2" or "class1.*" 851 """ 852 if methods: 853 sorted_methods = sorted(list(methods)) 854 return ":".join(["%s.%s" % (class_name, x) for x in sorted_methods]) 855 return "%s.*" % class_name 856 857 858def search_integration_dirs(name, int_dirs): 859 """Search integration dirs for name and return full path. 860 861 Args: 862 name: A string of plan name needed to be found. 863 int_dirs: A list of path needed to be searched. 864 865 Returns: 866 A list of the test path. 867 Ask user to select if multiple tests are found. 868 None if no matched test found. 869 """ 870 root_dir = os.environ.get(constants.ANDROID_BUILD_TOP) 871 test_files = [] 872 for integration_dir in int_dirs: 873 abs_path = os.path.join(root_dir, integration_dir) 874 test_paths = run_find_cmd(FIND_REFERENCE_TYPE.INTEGRATION, abs_path, 875 name) 876 if test_paths: 877 test_files.extend(test_paths) 878 return extract_test_from_tests(test_files) 879 880 881def get_int_dir_from_path(path, int_dirs): 882 """Search integration dirs for the given path and return path of dir. 883 884 Args: 885 path: A string of path needed to be found. 886 int_dirs: A list of path needed to be searched. 887 888 Returns: 889 A string of the test dir. None if no matched path found. 890 """ 891 root_dir = os.environ.get(constants.ANDROID_BUILD_TOP) 892 if not os.path.exists(path): 893 return None 894 dir_path, file_name = get_dir_path_and_filename(path) 895 int_dir = None 896 for possible_dir in int_dirs: 897 abs_int_dir = os.path.join(root_dir, possible_dir) 898 if is_equal_or_sub_dir(dir_path, abs_int_dir): 899 int_dir = abs_int_dir 900 break 901 if not file_name: 902 logging.warning('Found dir (%s) matching input (%s).' 903 ' Referencing an entire Integration/Suite dir' 904 ' is not supported. If you are trying to reference' 905 ' a test by its path, please input the path to' 906 ' the integration/suite config file itself.', 907 int_dir, path) 908 return None 909 return int_dir 910 911 912def get_install_locations(installed_paths): 913 """Get install locations from installed paths. 914 915 Args: 916 installed_paths: List of installed_paths from module_info. 917 918 Returns: 919 Set of install locations from module_info installed_paths. e.g. 920 set(['host', 'device']) 921 """ 922 install_locations = set() 923 for path in installed_paths: 924 if _HOST_PATH_RE.match(path): 925 install_locations.add(constants.DEVICELESS_TEST) 926 elif _DEVICE_PATH_RE.match(path): 927 install_locations.add(constants.DEVICE_TEST) 928 return install_locations 929 930 931def get_levenshtein_distance(test_name, module_name, 932 dir_costs=constants.COST_TYPO): 933 """Return an edit distance between test_name and module_name. 934 935 Levenshtein Distance has 3 actions: delete, insert and replace. 936 dis_costs makes each action weigh differently. 937 938 Args: 939 test_name: A keyword from the users. 940 module_name: A testable module name. 941 dir_costs: A tuple which contains 3 integer, where dir represents 942 Deletion, Insertion and Replacement respectively. 943 For guessing typos: (1, 1, 1) gives the best result. 944 For searching keywords, (8, 1, 5) gives the best result. 945 946 Returns: 947 An edit distance integer between test_name and module_name. 948 """ 949 rows = len(test_name) + 1 950 cols = len(module_name) + 1 951 deletion, insertion, replacement = dir_costs 952 953 # Creating a Dynamic Programming Matrix and weighting accordingly. 954 dp_matrix = [[0 for _ in range(cols)] for _ in range(rows)] 955 # Weigh rows/deletion 956 for row in range(1, rows): 957 dp_matrix[row][0] = row * deletion 958 # Weigh cols/insertion 959 for col in range(1, cols): 960 dp_matrix[0][col] = col * insertion 961 # The core logic of LD 962 for col in range(1, cols): 963 for row in range(1, rows): 964 if test_name[row-1] == module_name[col-1]: 965 cost = 0 966 else: 967 cost = replacement 968 dp_matrix[row][col] = min(dp_matrix[row-1][col] + deletion, 969 dp_matrix[row][col-1] + insertion, 970 dp_matrix[row-1][col-1] + cost) 971 972 return dp_matrix[row][col] 973 974 975def is_test_from_kernel_xml(xml_file, test_name): 976 """Check if test defined in xml_file. 977 978 A kernel test can be defined like: 979 <option name="test-command-line" key="test_class_1" value="command 1" /> 980 where key is the name of test class and method of the runner. This method 981 returns True if the test_name was defined in the given xml_file. 982 983 Args: 984 xml_file: Absolute path to xml file. 985 test_name: test_name want to find. 986 987 Returns: 988 True if test_name in xml_file, False otherwise. 989 """ 990 if not os.path.exists(xml_file): 991 raise atest_error.XmlNotExistError('%s: The xml file does' 992 'not exist' % xml_file) 993 xml_root = ET.parse(xml_file).getroot() 994 option_tags = xml_root.findall('.//option') 995 for option_tag in option_tags: 996 if option_tag.attrib['name'] == 'test-command-line': 997 if option_tag.attrib['key'] == test_name: 998 return True 999 return False 1000