1# Copyright 2017, The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""
16Utility functions for atest.
17"""
18
19import itertools
20import logging
21import os
22import re
23import subprocess
24import sys
25import urllib2
26
27import constants
28
29_MAKE_CMD = '%s/build/soong/soong_ui.bash' % os.environ.get(
30    constants.ANDROID_BUILD_TOP)
31_BUILD_CMD = [_MAKE_CMD, '--make-mode']
32_BASH_RESET_CODE = '\033[0m\n'
33# Arbitrary number to limit stdout for failed runs in _run_limited_output.
34# Reason for its use is that the make command itself has its own carriage
35# return output mechanism that when collected line by line causes the streaming
36# full_output list to be extremely large.
37_FAILED_OUTPUT_LINE_LIMIT = 100
38# Regular expression to match the start of a ninja compile:
39# ex: [ 99% 39710/39711]
40_BUILD_COMPILE_STATUS = re.compile(r'\[\s*(\d{1,3}%\s+)?\d+/\d+\]')
41_BUILD_FAILURE = 'FAILED: '
42
43
44def _capture_fail_section(full_log):
45    """Return the error message from the build output.
46
47    Args:
48        full_log: List of strings representing full output of build.
49
50    Returns:
51        capture_output: List of strings that are build errors.
52    """
53    am_capturing = False
54    capture_output = []
55    for line in full_log:
56        if am_capturing and _BUILD_COMPILE_STATUS.match(line):
57            break
58        if am_capturing or line.startswith(_BUILD_FAILURE):
59            capture_output.append(line)
60            am_capturing = True
61            continue
62    return capture_output
63
64
65def _run_limited_output(cmd, env_vars=None):
66    """Runs a given command and streams the output on a single line in stdout.
67
68    Args:
69        cmd: A list of strings representing the command to run.
70        env_vars: Optional arg. Dict of env vars to set during build.
71
72    Raises:
73        subprocess.CalledProcessError: When the command exits with a non-0
74            exitcode.
75    """
76    # Send stderr to stdout so we only have to deal with a single pipe.
77    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
78                            stderr=subprocess.STDOUT, env=env_vars)
79    sys.stdout.write('\n')
80    # Determine the width of the terminal. We'll need to clear this many
81    # characters when carriage returning.
82    _, term_width = os.popen('stty size', 'r').read().split()
83    term_width = int(term_width)
84    white_space = " " * int(term_width)
85    full_output = []
86    while proc.poll() is None:
87        line = proc.stdout.readline()
88        # Readline will often return empty strings.
89        if not line:
90            continue
91        full_output.append(line)
92        # Trim the line to the width of the terminal.
93        # Note: Does not handle terminal resizing, which is probably not worth
94        #       checking the width every loop.
95        if len(line) >= term_width:
96            line = line[:term_width - 1]
97        # Clear the last line we outputted.
98        sys.stdout.write('\r%s\r' % white_space)
99        sys.stdout.write('%s' % line.strip())
100        sys.stdout.flush()
101    # Reset stdout (on bash) to remove any custom formatting and newline.
102    sys.stdout.write(_BASH_RESET_CODE)
103    sys.stdout.flush()
104    # Wait for the Popen to finish completely before checking the returncode.
105    proc.wait()
106    if proc.returncode != 0:
107        # Parse out the build error to output.
108        output = _capture_fail_section(full_output)
109        if not output:
110            output = full_output
111        if len(output) >= _FAILED_OUTPUT_LINE_LIMIT:
112            output = output[-_FAILED_OUTPUT_LINE_LIMIT:]
113        output = 'Output (may be trimmed):\n%s' % ''.join(output)
114        raise subprocess.CalledProcessError(proc.returncode, cmd, output)
115
116
117def build(build_targets, verbose=False, env_vars=None):
118    """Shell out and make build_targets.
119
120    Args:
121        build_targets: A set of strings of build targets to make.
122        verbose: Optional arg. If True output is streamed to the console.
123                 If False, only the last line of the build output is outputted.
124        env_vars: Optional arg. Dict of env vars to set during build.
125
126    Returns:
127        Boolean of whether build command was successful, True if nothing to
128        build.
129    """
130    if not build_targets:
131        logging.debug('No build targets, skipping build.')
132        return True
133    full_env_vars = os.environ.copy()
134    if env_vars:
135        full_env_vars.update(env_vars)
136    logging.info('Building targets: %s', ' '.join(build_targets))
137    cmd = _BUILD_CMD + list(build_targets)
138    logging.debug('Executing command: %s', cmd)
139    try:
140        if verbose:
141            subprocess.check_call(cmd, stderr=subprocess.STDOUT,
142                                  env=full_env_vars)
143        else:
144            # TODO: Save output to a log file.
145            _run_limited_output(cmd, env_vars=full_env_vars)
146        logging.info('Build successful')
147        return True
148    except subprocess.CalledProcessError as err:
149        logging.error('Error building: %s', build_targets)
150        if err.output:
151            logging.error(err.output)
152        return False
153
154
155def _can_upload_to_result_server():
156    """Return Boolean if we can talk to result server."""
157    # TODO: Also check if we have a slow connection to result server.
158    if constants.RESULT_SERVER:
159        try:
160            urllib2.urlopen(constants.RESULT_SERVER,
161                            timeout=constants.RESULT_SERVER_TIMEOUT).close()
162            return True
163        # pylint: disable=broad-except
164        except Exception as err:
165            logging.debug('Talking to result server raised exception: %s', err)
166    return False
167
168
169def get_result_server_args():
170    """Return list of args for communication with result server."""
171    if _can_upload_to_result_server():
172        return constants.RESULT_SERVER_ARGS
173    return []
174
175
176def sort_and_group(iterable, key):
177    """Sort and group helper function."""
178    return itertools.groupby(sorted(iterable, key=key), key=key)
179