1# Copyright 2018, The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""
16Utils for finder classes.
17"""
18
19import logging
20import os
21import re
22import subprocess
23import time
24import xml.etree.ElementTree as ET
25
26# pylint: disable=import-error
27import atest_error
28import atest_enum
29import constants
30
31# Helps find apk files listed in a test config (AndroidTest.xml) file.
32# Matches "filename.apk" in <option name="foo", value="filename.apk" />
33# We want to make sure we don't grab apks with paths in their name since we
34# assume the apk name is the build target.
35_APK_RE = re.compile(r'^[^/]+\.apk$', re.I)
36# Parse package name from the package declaration line of a java file.
37# Group matches "foo.bar" of line "package foo.bar;"
38_PACKAGE_RE = re.compile(r'\s*package\s+(?P<package>[^;]+)\s*;\s*', re.I)
39
40# Explanation of FIND_REFERENCE_TYPEs:
41# ----------------------------------
42# 0. CLASS: Name of a java class, usually file is named the same (HostTest lives
43#           in HostTest.java)
44# 1. QUALIFIED_CLASS: Like CLASS but also contains the package in front like
45#.                    com.android.tradefed.testtype.HostTest.
46# 2. PACKAGE: Name of a java package.
47# 3. INTEGRATION: XML file name in one of the 4 integration config directories.
48
49FIND_REFERENCE_TYPE = atest_enum.AtestEnum(['CLASS', 'QUALIFIED_CLASS',
50                                            'PACKAGE', 'INTEGRATION', ])
51
52# Unix find commands for searching for test files based on test type input.
53# Note: Find (unlike grep) exits with status 0 if nothing found.
54FIND_CMDS = {
55    FIND_REFERENCE_TYPE.CLASS : r"find %s -type d %s -prune -o -type f -name "
56                                r"'%s.java' -print",
57    FIND_REFERENCE_TYPE.QUALIFIED_CLASS: r"find %s -type d %s -prune -o "
58                                         r"-wholename '*%s.java' -print",
59    FIND_REFERENCE_TYPE.PACKAGE: r"find %s -type d %s -prune -o -wholename "
60                                 r"'*%s' -type d -print",
61    FIND_REFERENCE_TYPE.INTEGRATION: r"find %s -type d %s -prune -o -wholename "
62                                     r"'*%s.xml' -print"
63}
64
65# XML parsing related constants.
66_COMPATIBILITY_PACKAGE_PREFIX = "com.android.compatibility"
67_CTS_JAR = "cts-tradefed"
68_XML_PUSH_DELIM = '->'
69_APK_SUFFIX = '.apk'
70# Setup script for device perf tests.
71_PERF_SETUP_LABEL = 'perf-setup.sh'
72
73# XML tags.
74_XML_NAME = 'name'
75_XML_VALUE = 'value'
76
77# VTS xml parsing constants.
78_VTS_TEST_MODULE = 'test-module-name'
79_VTS_MODULE = 'module-name'
80_VTS_BINARY_SRC = 'binary-test-source'
81_VTS_PUSH_GROUP = 'push-group'
82_VTS_PUSH = 'push'
83_VTS_BINARY_SRC_DELIM = '::'
84_VTS_PUSH_DIR = os.path.join(os.environ.get(constants.ANDROID_BUILD_TOP, ''),
85                             'test', 'vts', 'tools', 'vts-tradefed', 'res',
86                             'push_groups')
87_VTS_PUSH_SUFFIX = '.push'
88_VTS_BITNESS = 'append-bitness'
89_VTS_BITNESS_TRUE = 'true'
90_VTS_BITNESS_32 = '32'
91_VTS_BITNESS_64 = '64'
92# Matches 'DATA/target' in '_32bit::DATA/target'
93_VTS_BINARY_SRC_DELIM_RE = re.compile(r'.*::(?P<target>.*)$')
94
95# pylint: disable=inconsistent-return-statements
96def split_methods(user_input):
97    """Split user input string into test reference and list of methods.
98
99    Args:
100        user_input: A string of the user's input.
101                    Examples:
102                        class_name
103                        class_name#method1,method2
104                        path
105                        path#method1,method2
106    Returns:
107        A tuple. First element is String of test ref and second element is
108        a set of method name strings or empty list if no methods included.
109    Exception:
110        atest_error.TooManyMethodsError raised when input string is trying to
111        specify too many methods in a single positional argument.
112
113        Examples of unsupported input strings:
114            module:class#method,class#method
115            class1#method,class2#method
116            path1#method,path2#method
117    """
118    parts = user_input.split('#')
119    if len(parts) == 1:
120        return parts[0], frozenset()
121    elif len(parts) == 2:
122        return parts[0], frozenset(parts[1].split(','))
123    raise atest_error.TooManyMethodsError(
124        'Too many methods specified with # character in user input: %s.'
125        '\n\nOnly one class#method combination supported per positional'
126        ' argument. Multiple classes should be separated by spaces: '
127        'class#method class#method')
128
129
130# pylint: disable=inconsistent-return-statements
131def get_fully_qualified_class_name(test_path):
132    """Parse the fully qualified name from the class java file.
133
134    Args:
135        test_path: A string of absolute path to the java class file.
136
137    Returns:
138        A string of the fully qualified class name.
139
140    Raises:
141        atest_error.MissingPackageName if no class name can be found.
142    """
143    with open(test_path) as class_file:
144        for line in class_file:
145            match = _PACKAGE_RE.match(line)
146            if match:
147                package = match.group('package')
148                cls = os.path.splitext(os.path.split(test_path)[1])[0]
149                return '%s.%s' % (package, cls)
150    raise atest_error.MissingPackageNameError(test_path)
151
152
153def get_package_name(file_name):
154    """Parse the package name from a java file.
155
156    Args:
157        file_name: A string of the absolute path to the java file.
158
159    Returns:
160        A string of the package name or None
161      """
162    with open(file_name) as data:
163        for line in data:
164            match = _PACKAGE_RE.match(line)
165            if match:
166                return match.group('package')
167
168
169def extract_test_path(output):
170    """Extract the test path from the output of a unix 'find' command.
171
172    Example of find output for CLASS find cmd:
173    /<some_root>/cts/tests/jank/src/android/jank/cts/ui/CtsDeviceJankUi.java
174
175    Args:
176        output: A string output of a unix 'find' command.
177
178    Returns:
179        A string of the test path or None if output is '' or None.
180    """
181    if not output:
182        return None
183    tests = output.strip('\n').split('\n')
184    count = len(tests)
185    test_index = 0
186    if count > 1:
187        numbered_list = ['%s: %s' % (i, t) for i, t in enumerate(tests)]
188        print 'Multiple tests found:\n%s' % '\n'.join(numbered_list)
189        test_index = int(raw_input('Please enter number of test to use:'))
190    return tests[test_index]
191
192
193def static_var(varname, value):
194    """Decorator to cache static variable."""
195
196    def fun_var_decorate(func):
197        """Set the static variable in a function."""
198        setattr(func, varname, value)
199        return func
200    return fun_var_decorate
201
202
203@static_var("cached_ignore_dirs", [])
204def _get_ignored_dirs():
205    """Get ignore dirs in find command.
206
207    Since we can't construct a single find cmd to find the target and
208    filter-out the dir with .out-dir, .find-ignore and $OUT-DIR. We have
209    to run the 1st find cmd to find these dirs. Then, we can use these
210    results to generate the real find cmd.
211
212    Return:
213        A list of the ignore dirs.
214    """
215    out_dirs = _get_ignored_dirs.cached_ignore_dirs
216    if not out_dirs:
217        build_top = os.environ.get(constants.ANDROID_BUILD_TOP)
218        find_out_dir_cmd = (r'find %s -maxdepth 2 '
219                            r'-type f \( -name ".out-dir" -o -name '
220                            r'".find-ignore" \)') % build_top
221        out_files = subprocess.check_output(find_out_dir_cmd, shell=True)
222        # Get all dirs with .out-dir or .find-ignore
223        if out_files:
224            out_files = out_files.splitlines()
225            for out_file in out_files:
226                if out_file:
227                    out_dirs.append(os.path.dirname(out_file.strip()))
228        # Get the out folder if user specified $OUT_DIR
229        custom_out_dir = os.environ.get(constants.ANDROID_OUT_DIR)
230        if custom_out_dir:
231            user_out_dir = None
232            if os.path.isabs(custom_out_dir):
233                user_out_dir = custom_out_dir
234            else:
235                user_out_dir = os.path.join(build_top, custom_out_dir)
236            # only ignore the out_dir when it under $ANDROID_BUILD_TOP
237            if build_top in user_out_dir:
238                if user_out_dir not in out_dirs:
239                    out_dirs.append(user_out_dir)
240        _get_ignored_dirs.cached_ignore_dirs = out_dirs
241    return out_dirs
242
243
244def _get_prune_cond_of_ignored_dirs():
245    """Get the prune condition of ignore dirs.
246
247    Generation a string of the prune condition in the find command.
248    It will filter-out the dir with .out-dir, .find-ignore and $OUT-DIR.
249    Because they are the out dirs, we don't have to find them.
250
251    Return:
252        A string of the prune condition of the ignore dirs.
253    """
254    out_dirs = _get_ignored_dirs()
255    prune_cond = r'\( -name ".*"'
256    for out_dir in out_dirs:
257        prune_cond += r' -o -path %s' % out_dir
258    prune_cond += r' \)'
259    return prune_cond
260
261
262def run_find_cmd(ref_type, search_dir, target):
263    """Find a path to a target given a search dir and a target name.
264
265    Args:
266        ref_type: An AtestEnum of the reference type.
267        search_dir: A string of the dirpath to search in.
268        target: A string of what you're trying to find.
269
270    Return:
271        A string of the path to the target.
272    """
273    prune_cond = _get_prune_cond_of_ignored_dirs()
274    find_cmd = FIND_CMDS[ref_type] % (search_dir, prune_cond, target)
275    start = time.time()
276    ref_name = FIND_REFERENCE_TYPE[ref_type]
277    logging.debug('Executing %s find cmd: %s', ref_name, find_cmd)
278    out = subprocess.check_output(find_cmd, shell=True)
279    logging.debug('%s find completed in %ss', ref_name, time.time() - start)
280    logging.debug('%s find cmd out: %s', ref_name, out)
281    return extract_test_path(out)
282
283
284def find_class_file(search_dir, class_name):
285    """Find a path to a class file given a search dir and a class name.
286
287    Args:
288        search_dir: A string of the dirpath to search in.
289        class_name: A string of the class to search for.
290
291    Return:
292        A string of the path to the java file.
293    """
294    if '.' in class_name:
295        find_target = class_name.replace('.', '/')
296        ref_type = FIND_REFERENCE_TYPE.QUALIFIED_CLASS
297    else:
298        find_target = class_name
299        ref_type = FIND_REFERENCE_TYPE.CLASS
300    return run_find_cmd(ref_type, search_dir, find_target)
301
302
303def is_equal_or_sub_dir(sub_dir, parent_dir):
304    """Return True sub_dir is sub dir or equal to parent_dir.
305
306    Args:
307      sub_dir: A string of the sub directory path.
308      parent_dir: A string of the parent directory path.
309
310    Returns:
311        A boolean of whether both are dirs and sub_dir is sub of parent_dir
312        or is equal to parent_dir.
313    """
314    # avoid symlink issues with real path
315    parent_dir = os.path.realpath(parent_dir)
316    sub_dir = os.path.realpath(sub_dir)
317    if not os.path.isdir(sub_dir) or not os.path.isdir(parent_dir):
318        return False
319    return os.path.commonprefix([sub_dir, parent_dir]) == parent_dir
320
321
322def is_robolectric_module(mod_info):
323    """Check if a module is a robolectric module.
324
325    Args:
326        mod_info: ModuleInfo to check.
327
328    Returns:
329        True if module is a robolectric module, False otherwise.
330    """
331    if mod_info:
332        return (mod_info.get(constants.MODULE_CLASS, [None])[0] ==
333                constants.MODULE_CLASS_ROBOLECTRIC)
334    return False
335
336def is_2nd_arch_module(module_info):
337    """Check if a codule is 2nd architecture module
338
339    Args:
340        module_info: ModuleInfo to check.
341
342    Returns:
343        True is the module is 2nd architecture module, False otherwise.
344
345    """
346    for_2nd_arch = module_info.get(constants.MODULE_FOR_2ND_ARCH, [])
347    return for_2nd_arch and for_2nd_arch[0]
348
349def find_parent_module_dir(root_dir, start_dir, module_info):
350    """From current dir search up file tree until root dir for module dir.
351
352    Args:
353      start_dir: A string of the dir to start searching up from.
354      root_dir: A string  of the dir that is the parent of the start dir.
355      module_info: ModuleInfo object containing module information from the
356                   build system.
357
358    Returns:
359        A string of the module dir relative to root.
360
361    Exceptions:
362        ValueError: Raised if cur_dir not dir or not subdir of root dir.
363        atest_error.TestWithNoModuleError: Raised if no Module Dir found.
364    """
365    if not is_equal_or_sub_dir(start_dir, root_dir):
366        raise ValueError('%s not in repo %s' % (start_dir, root_dir))
367    module_dir = None
368    current_dir = start_dir
369    while current_dir != root_dir:
370        # If we find an AndroidTest.xml, we know we found the right directory.
371        if os.path.isfile(os.path.join(current_dir, constants.MODULE_CONFIG)):
372            module_dir = os.path.relpath(current_dir, root_dir)
373            break
374        # If we haven't found a possible auto-generated config location, check
375        # now.
376        if not module_dir:
377            rel_dir = os.path.relpath(current_dir, root_dir)
378            module_list = module_info.path_to_module_info.get(rel_dir, [])
379            # Verify only one module at this level has an auto_test_config.
380            if len([x for x in module_list
381                    if x.get('auto_test_config') and not is_2nd_arch_module(x)]) == 1:
382                # We found a single test module!
383                module_dir = rel_dir
384                # But keep searching in case there's an AndroidTest.xml in a
385                # parent folder. Example: a class belongs to an test apk that's
386                # part of a hostside test setup (common in cts).
387            # Check if a robolectric module lives here.
388            for mod in module_list:
389                if is_robolectric_module(mod):
390                    module_dir = rel_dir
391                    break
392        current_dir = os.path.dirname(current_dir)
393    if not module_dir:
394        raise atest_error.TestWithNoModuleError('No Parent Module Dir for: %s'
395                                                % start_dir)
396    return module_dir
397
398
399def get_targets_from_xml(xml_file, module_info):
400    """Retrieve build targets from the given xml.
401
402    Just a helper func on top of get_targets_from_xml_root.
403
404    Args:
405        xml_file: abs path to xml file.
406        module_info: ModuleInfo class used to verify targets are valid modules.
407
408    Returns:
409        A set of build targets based on the signals found in the xml file.
410    """
411    xml_root = ET.parse(xml_file).getroot()
412    return get_targets_from_xml_root(xml_root, module_info)
413
414
415def _get_apk_target(apk_target):
416    """Return the sanitized apk_target string from the xml.
417
418    The apk_target string can be of 2 forms:
419      - apk_target.apk
420      - apk_target.apk->/path/to/install/apk_target.apk
421
422    We want to return apk_target in both cases.
423
424    Args:
425        apk_target: String of target name to clean.
426
427    Returns:
428        String of apk_target to build.
429    """
430    apk = apk_target.split(_XML_PUSH_DELIM, 1)[0].strip()
431    return apk[:-len(_APK_SUFFIX)]
432
433
434def _is_apk_target(name, value):
435    """Return True if XML option is an apk target.
436
437    We have some scenarios where an XML option can be an apk target:
438      - value is an apk file.
439      - name is a 'push' option where value holds the apk_file + other stuff.
440
441    Args:
442        name: String name of XML option.
443        value: String value of the XML option.
444
445    Returns:
446        True if it's an apk target we should build, False otherwise.
447    """
448    if _APK_RE.match(value):
449        return True
450    if name == 'push' and value.endswith(_APK_SUFFIX):
451        return True
452    return False
453
454
455def get_targets_from_xml_root(xml_root, module_info):
456    """Retrieve build targets from the given xml root.
457
458    We're going to pull the following bits of info:
459      - Parse any .apk files listed in the config file.
460      - Parse option value for "test-module-name" (for vts tests).
461      - Look for the perf script.
462
463    Args:
464        module_info: ModuleInfo class used to verify targets are valid modules.
465        xml_root: ElementTree xml_root for us to look through.
466
467    Returns:
468        A set of build targets based on the signals found in the xml file.
469    """
470    targets = set()
471    option_tags = xml_root.findall('.//option')
472    for tag in option_tags:
473        target_to_add = None
474        name = tag.attrib[_XML_NAME].strip()
475        value = tag.attrib[_XML_VALUE].strip()
476        if _is_apk_target(name, value):
477            target_to_add = _get_apk_target(value)
478        elif _PERF_SETUP_LABEL in value:
479            targets.add(_PERF_SETUP_LABEL)
480            continue
481
482        # Let's make sure we can actually build the target.
483        if target_to_add and module_info.is_module(target_to_add):
484            targets.add(target_to_add)
485        elif target_to_add:
486            logging.warning('Build target (%s) not present in module info, '
487                            'skipping build', target_to_add)
488
489    # TODO (b/70813166): Remove this lookup once all runtime dependencies
490    # can be listed as a build dependencies or are in the base test harness.
491    nodes_with_class = xml_root.findall(".//*[@class]")
492    for class_attr in nodes_with_class:
493        fqcn = class_attr.attrib['class'].strip()
494        if fqcn.startswith(_COMPATIBILITY_PACKAGE_PREFIX):
495            targets.add(_CTS_JAR)
496    logging.debug('Targets found in config file: %s', targets)
497    return targets
498
499
500def _get_vts_push_group_targets(push_file, rel_out_dir):
501    """Retrieve vts push group build targets.
502
503    A push group file is a file that list out test dependencies and other push
504    group files. Go through the push file and gather all the test deps we need.
505
506    Args:
507        push_file: Name of the push file in the VTS
508        rel_out_dir: Abs path to the out dir to help create vts build targets.
509
510    Returns:
511        Set of string which represent build targets.
512    """
513    targets = set()
514    full_push_file_path = os.path.join(_VTS_PUSH_DIR, push_file)
515    # pylint: disable=invalid-name
516    with open(full_push_file_path) as f:
517        for line in f:
518            target = line.strip()
519            # Skip empty lines.
520            if not target:
521                continue
522
523            # This is a push file, get the targets from it.
524            if target.endswith(_VTS_PUSH_SUFFIX):
525                targets |= _get_vts_push_group_targets(line.strip(),
526                                                       rel_out_dir)
527                continue
528            sanitized_target = target.split(_XML_PUSH_DELIM, 1)[0].strip()
529            targets.add(os.path.join(rel_out_dir, sanitized_target))
530    return targets
531
532
533def _specified_bitness(xml_root):
534    """Check if the xml file contains the option append-bitness.
535
536    Args:
537        xml_root: abs path to xml file.
538
539    Returns:
540        True if xml specifies to append-bitness, False otherwise.
541    """
542    option_tags = xml_root.findall('.//option')
543    for tag in option_tags:
544        value = tag.attrib[_XML_VALUE].strip()
545        name = tag.attrib[_XML_NAME].strip()
546        if name == _VTS_BITNESS and value == _VTS_BITNESS_TRUE:
547            return True
548    return False
549
550
551def _get_vts_binary_src_target(value, rel_out_dir):
552    """Parse out the vts binary src target.
553
554    The value can be in the following pattern:
555      - {_32bit,_64bit,_IPC32_32bit}::DATA/target (DATA/target)
556      - DATA/target->/data/target (DATA/target)
557      - out/host/linx-x86/bin/VtsSecuritySelinuxPolicyHostTest (the string as
558        is)
559
560    Args:
561        value: String of the XML option value to parse.
562        rel_out_dir: String path of out dir to prepend to target when required.
563
564    Returns:
565        String of the target to build.
566    """
567    # We'll assume right off the bat we can use the value as is and modify it if
568    # necessary, e.g. out/host/linux-x86/bin...
569    target = value
570    # _32bit::DATA/target
571    match = _VTS_BINARY_SRC_DELIM_RE.match(value)
572    if match:
573        target = os.path.join(rel_out_dir, match.group('target'))
574    # DATA/target->/data/target
575    elif _XML_PUSH_DELIM in value:
576        target = value.split(_XML_PUSH_DELIM, 1)[0].strip()
577        target = os.path.join(rel_out_dir, target)
578    return target
579
580
581def get_targets_from_vts_xml(xml_file, rel_out_dir, module_info):
582    """Parse a vts xml for test dependencies we need to build.
583
584    We have a separate vts parsing function because we make a big assumption
585    on the targets (the way they're formatted and what they represent) and we
586    also create these build targets in a very special manner as well.
587    The 4 options we're looking for are:
588      - binary-test-source
589      - push-group
590      - push
591      - test-module-name
592
593    Args:
594        module_info: ModuleInfo class used to verify targets are valid modules.
595        rel_out_dir: Abs path to the out dir to help create vts build targets.
596        xml_file: abs path to xml file.
597
598    Returns:
599        A set of build targets based on the signals found in the xml file.
600    """
601    xml_root = ET.parse(xml_file).getroot()
602    targets = set()
603    option_tags = xml_root.findall('.//option')
604    for tag in option_tags:
605        value = tag.attrib[_XML_VALUE].strip()
606        name = tag.attrib[_XML_NAME].strip()
607        if name in [_VTS_TEST_MODULE, _VTS_MODULE]:
608            if module_info.is_module(value):
609                targets.add(value)
610            else:
611                logging.warning('vts test module (%s) not present in module '
612                                'info, skipping build', value)
613        elif name == _VTS_BINARY_SRC:
614            targets.add(_get_vts_binary_src_target(value, rel_out_dir))
615        elif name == _VTS_PUSH_GROUP:
616            # Look up the push file and parse out build artifacts (as well as
617            # other push group files to parse).
618            targets |= _get_vts_push_group_targets(value, rel_out_dir)
619        elif name == _VTS_PUSH:
620            # Parse out the build artifact directly.
621            push_target = value.split(_XML_PUSH_DELIM, 1)[0].strip()
622            # If the config specified append-bitness, append the bits suffixes
623            # to the target.
624            if _specified_bitness(xml_root):
625                targets.add(os.path.join(rel_out_dir, push_target + _VTS_BITNESS_32))
626                targets.add(os.path.join(rel_out_dir, push_target + _VTS_BITNESS_64))
627            else:
628                targets.add(os.path.join(rel_out_dir, push_target))
629    logging.debug('Targets found in config file: %s', targets)
630    return targets
631
632
633def get_dir_path_and_filename(path):
634    """Return tuple of dir and file name from given path.
635
636    Args:
637        path: String of path to break up.
638
639    Returns:
640        Tuple of (dir, file) paths.
641    """
642    if os.path.isfile(path):
643        dir_path, file_path = os.path.split(path)
644    else:
645        dir_path, file_path = path, None
646    return dir_path, file_path
647