1#!/bin/sh
2#
3# Copyright (C) 2018 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17#
18# This test script to be used by the build server.
19# It is supposed to be executed from trusty root directory
20# and expects the following environment variables:
21#
22""":" # Shell script (in docstring to appease pylint)
23
24# Find and invoke hermetic python3 interpreter
25. "`dirname $0`/envsetup.sh"; exec "$PY3" "$0" "$@"
26# Shell script end
27Run tests for a project.
28"""
29
30import argparse
31from enum import Enum
32import importlib
33import os
34import re
35import subprocess
36import sys
37import time
38from typing import Optional
39
40from trusty_build_config import PortType, TrustyCompositeTest, TrustyTest
41from trusty_build_config import TrustyAndroidTest, TrustyBuildConfig
42from trusty_build_config import TrustyHostTest, TrustyRebootCommand
43
44
45TEST_STATUS = Enum("TEST_STATUS", ["PASSED", "FAILED", "SKIPPED"])
46
47class TestResult:
48    """Stores results for a single test.
49
50    Attributes:
51        test: Name of the test.
52        status: Test's integer return code, or None if this test was skipped.
53        retried: True if this test was retried.
54    """
55    test: str
56    status: Optional[int]
57    retried: bool
58
59    def __init__(self, test: str, status: Optional[int], retried: bool):
60        self.test = test
61        self.status = status
62        self.retried = retried
63
64    def test_status(self) -> TEST_STATUS:
65        if self.status is None:
66            return TEST_STATUS.SKIPPED
67        return TEST_STATUS.PASSED if self.status == 0 else TEST_STATUS.FAILED
68
69    def failed(self) -> bool:
70        return self.test_status() == TEST_STATUS.FAILED
71
72    def __format__(self, _format_spec: str) -> str:
73        return f"{self.test:s} returned {self.status:d}"
74
75
76class TestResults(object):
77    """Stores test results.
78
79    Attributes:
80        project: Name of project that tests were run on.
81        passed: True if all tests passed, False if one or more tests failed.
82        passed_count: Number of tests passed.
83        failed_count: Number of tests failed.
84        flaked_count: Number of tests that failed then passed on second try.
85        retried_count: Number of tests that were given a second try.
86        test_results: List of tuples storing test name an status.
87    """
88
89    def __init__(self, project):
90        """Inits TestResults with project name and empty test results."""
91        self.project = project
92        self.passed = True
93        self.passed_count = 0
94        self.failed_count = 0
95        self.skipped_count = 0
96        self.flaked_count = 0
97        self.retried_count = 0
98        self.test_results = []
99
100    def add_result(self, result: TestResult):
101        """Add a test result."""
102        self.test_results.append(result)
103        if result.test_status() == TEST_STATUS.PASSED:
104            self.passed_count += 1
105            if result.retried:
106                self.flaked_count += 1
107        elif result.test_status() == TEST_STATUS.FAILED:
108            self.failed_count += 1
109            self.passed = False
110        elif result.test_status() == TEST_STATUS.SKIPPED:
111            self.skipped_count += 1
112
113        if result.retried:
114            self.retried_count += 1
115
116    def print_results(self, print_failed_only=False):
117        """Print test results."""
118        if print_failed_only:
119            if self.passed:
120                return
121            sys.stdout.flush()
122            out = sys.stderr
123        else:
124            out = sys.stdout
125        test_count = self.passed_count + self.failed_count + self.skipped_count
126        test_attempted = self.passed_count + self.failed_count
127        out.write(
128            "\n"
129            f"There were {test_count} defined for project {self.project}.\n"
130            f"{test_attempted} ran and {self.skipped_count} were skipped."
131        )
132        if test_count:
133            for result in self.test_results:
134                match (result.test_status(), result.retried, print_failed_only):
135                    case (TEST_STATUS.FAILED, _, _):
136                        out.write(f"[  FAILED  ] {result.test}\n")
137                    case (TEST_STATUS.SKIPPED, _, False):
138                        out.write(f"[  SKIPPED ] {result.test}\n")
139                    case (TEST_STATUS.PASSED, retried, False):
140                        out.write(f"[       OK ] {result.test}\n")
141                        if retried:
142                            out.write(
143                                f"WARNING: {result.test} was re-run and "
144                                "passed on second try; it may be flaky\n"
145                            )
146
147            out.write(
148                f"[==========] {test_count} tests ran for project "
149                f"{self.project}.\n"
150            )
151            if self.passed_count and not print_failed_only:
152                out.write(f"[  PASSED  ] {self.passed_count} tests.\n")
153            if self.failed_count:
154                out.write(f"[  FAILED  ] {self.failed_count} tests.\n")
155            if self.skipped_count:
156                out.write(f"[  SKIPPED ] {self.skipped_count} tests.\n")
157            if self.flaked_count > 0:
158                out.write(
159                    f"WARNING: {self.flaked_count} tests passed when "
160                    "re-run which indicates that they may be flaky.\n"
161                )
162            if self.retried_count == MAX_RETRIES:
163                out.write(
164                    f"WARNING: hit MAX_RETRIES({MAX_RETRIES}) during "
165                    "testing after which point, no tests were retried.\n"
166                )
167
168
169class MultiProjectTestResults:
170    """Stores results from testing multiple projects.
171
172    Attributes:
173        test_results: List containing the results for each project.
174        failed_projects: List of projects with test failures.
175        tests_passed: Count of test passes across all projects.
176        tests_failed: Count of test failures across all projects.
177        had_passes: Count of all projects with any test passes.
178        had_failures: Count of all projects with any test failures.
179    """
180
181    def __init__(self, test_results: list[TestResults]):
182        self.test_results = test_results
183        self.failed_projects = []
184        self.tests_passed = 0
185        self.tests_failed = 0
186        self.tests_skipped = 0
187        self.had_passes = 0
188        self.had_failures = 0
189        self.had_skip = 0
190
191        for result in self.test_results:
192            if not result.passed:
193                self.failed_projects.append(result.project)
194            self.tests_passed += result.passed_count
195            self.tests_failed += result.failed_count
196            self.tests_skipped += result.skipped_count
197            if result.passed_count:
198                self.had_passes += 1
199            if result.failed_count:
200                self.had_failures += 1
201            if result.skipped_count:
202                self.had_skip += 1
203
204    def print_results(self):
205        """Prints the test results to stdout and stderr."""
206        for test_result in self.test_results:
207            test_result.print_results()
208
209        sys.stdout.write("\n")
210        if self.had_passes:
211            sys.stdout.write(
212                f"[  PASSED  ] {self.tests_passed} tests in "
213                f"{self.had_passes} projects.\n"
214            )
215        if self.had_failures:
216            sys.stdout.write(
217                f"[  FAILED  ] {self.tests_failed} tests in "
218                f"{self.had_failures} projects.\n"
219            )
220            sys.stdout.flush()
221        if self.had_skip:
222            sys.stdout.write(
223                f"[  SKIPPED ] {self.tests_skipped} tests in "
224                f"{self.had_skip} projects.\n"
225            )
226            sys.stdout.flush()
227
228            # Print the failed tests again to stderr as the build server will
229            # store this in a separate file with a direct link from the build
230            # status page. The full build long page on the build server, buffers
231            # stdout and stderr and interleaves them at random. By printing
232            # the summary to both stderr and stdout, we get at least one of them
233            # at the bottom of that file.
234            for test_result in self.test_results:
235                test_result.print_results(print_failed_only=True)
236            sys.stderr.write(
237                f"[  FAILED  ] {self.tests_failed,} tests in "
238                f"{self.had_failures} projects.\n"
239            )
240
241
242def test_should_run(testname: str, test_filters: Optional[list[re.Pattern]]):
243    """Check if test should run.
244
245    Args:
246        testname: Name of test to check.
247        test_filters: Regex list that limits the tests to run.
248
249    Returns:
250        True if test_filters list is empty or None, True if testname matches any
251        regex in test_filters, False otherwise.
252    """
253    if not test_filters:
254        return True
255    for r in test_filters:
256        if r.search(testname):
257            return True
258    return False
259
260
261def projects_to_test(
262    build_config: TrustyBuildConfig,
263    projects: list[str],
264    test_filters: list[re.Pattern],
265    run_disabled_tests: bool = False,
266) -> list[str]:
267    """Checks which projects have any of the specified tests.
268
269    Args:
270        build_config: TrustyBuildConfig object.
271        projects: Names of the projects to search for active tests.
272        test_filters: List that limits the tests to run. Projects without any
273          tests that match a filter will be skipped.
274        run_disabled_tests: Also run disabled tests from config file.
275
276    Returns:
277        A list of projects with tests that should be run
278    """
279
280    def has_test(name: str):
281        project = build_config.get_project(name)
282        for test in project.tests:
283            if not test.enabled and not run_disabled_tests:
284                continue
285            if test_should_run(test.name, test_filters):
286                return True
287        return False
288
289    return [project for project in projects if has_test(project)]
290
291
292# Put a global cap on the number of retries to detect flaky tests such that we
293# do not risk increasing the time to try all tests substantially. This should be
294# fine since *most* tests are not flaky.
295# TODO: would it be better to put a cap on the time spent retrying tests? We may
296#       not want to retry long running tests.
297MAX_RETRIES = 10
298
299
300def run_tests(
301    build_config: TrustyBuildConfig,
302    root: os.PathLike,
303    project: str,
304    run_disabled_tests: bool = False,
305    test_filters: Optional[list[re.Pattern]] = None,
306    verbose: bool = False,
307    debug_on_error: bool = False,
308    emulator: bool = True,
309) -> TestResults:
310    """Run tests for a project.
311
312    Args:
313        build_config: TrustyBuildConfig object.
314        root: Trusty build root output directory.
315        project: Project name.
316        run_disabled_tests: Also run disabled tests from config file.
317        test_filters: Optional list that limits the tests to run.
318        verbose: Enable debug output.
319        debug_on_error: Wait for debugger connection on errors.
320
321    Returns:
322        TestResults object listing overall and detailed test results.
323    """
324    project_config = build_config.get_project(project=project)
325    project_root = f"{root}/build-{project}"
326
327    test_results = TestResults(project)
328    test_env = None
329    test_runner = None
330
331    def load_test_environment():
332        sys.path.append(project_root)
333        try:
334            if run := sys.modules.get("run"):
335                if not run.__file__.startswith(project_root):
336                    # run module was imported for another project and needs
337                    # to be replaced with the one for the current project.
338                    run = importlib.reload(run)
339            else:
340                # first import in this interpreter instance, we use importlib
341                # rather than a regular import statement since it avoids
342                # linter warnings.
343                run = importlib.import_module("run")
344            sys.path.pop()
345        except ImportError:
346            return None
347
348        return run
349
350    def print_test_command(name, cmd: Optional[list[str]] = None):
351        print()
352        print("Running", name, "on", test_results.project)
353        if cmd:
354            print(
355                "Command line:", " ".join([s.replace(" ", "\\ ") for s in cmd])
356            )
357        sys.stdout.flush()
358
359    def run_test(
360        test, parent_test: Optional[TrustyCompositeTest] = None, retry=True
361    ) -> Optional[TestResult]:
362        """Execute a single test and print out helpful information
363
364        Returns:
365            The results of running this test, or None for non-tests, like
366            reboots or tests that don't work in this environment.
367        """
368        nonlocal test_env, test_runner
369        cmd = test.command[1:]
370        disable_rpmb = True if "--disable_rpmb" in cmd else None
371
372        test_start_time = time.time()
373
374        if not emulator and not isinstance(test, TrustyHostTest):
375            return None
376
377        match test:
378            case TrustyHostTest():
379                # append nice and expand path to command
380                cmd = ["nice", f"{project_root}/{test.command[0]}"] + cmd
381                print_test_command(test.name, cmd)
382                cmd_status = subprocess.call(cmd)
383                result = TestResult(test.name, cmd_status, False)
384            case TrustyCompositeTest():
385                status_code: Optional[int] = 0
386                for subtest in test.sequence:
387                    subtest_result = run_test(subtest, test, retry)
388                    if subtest_result and subtest_result.failed():
389                        status_code = subtest_result.status
390                        # fail the composite test with the same status code as
391                        # the first failing subtest
392                        break
393                result = TestResult(test.name, status_code, False)
394
395            case TrustyTest():
396                # Benchmark runs on QEMU are meaningless and take a lot of
397                # CI time. One can still run the bootport test manually
398                # if desired
399                if test.port_type == PortType.BENCHMARK:
400                    return TestResult(test.name, None, False)
401                else:
402                    if isinstance(test, TrustyAndroidTest):
403                        print_test_command(test.name, [test.shell_command])
404                    else:
405                        # port tests are identified by their port name,
406                        # no command
407                        print_test_command(test.name)
408
409                    if not test_env:
410                        test_env = load_test_environment()
411                    if test_env:
412                        if not test_runner:
413                            test_runner = test_env.init(
414                                android=build_config.android,
415                                disable_rpmb=disable_rpmb,
416                                verbose=verbose,
417                                debug_on_error=debug_on_error,
418                            )
419                        cmd_status = test_env.run_test(test_runner, cmd)
420                        result = TestResult(test.name, cmd_status, False)
421                    else:
422                        return TestResult(test.name, None, False)
423            case TrustyRebootCommand() if parent_test:
424                assert isinstance(parent_test, TrustyCompositeTest)
425                if test_env:
426                    test_env.shutdown(test_runner)
427                    test_runner = None
428                    print("Shut down test environment on", test_results.project)
429                # return early so we do not report the time to reboot or try to
430                # add the reboot command to test results.
431                return None
432            case TrustyRebootCommand():
433                raise RuntimeError(
434                    "Reboot may only be used inside compositetest"
435                )
436            case _:
437                raise NotImplementedError(f"Don't know how to run {test.name}")
438
439        elapsed = time.time() - test_start_time
440        print( f"{result} after {elapsed:.3f} seconds")
441
442        can_retry = retry and test_results.retried_count < MAX_RETRIES
443        if result and result.failed() and can_retry:
444            print(
445                f"retrying potentially flaky test {test.name} on",
446                test_results.project,
447            )
448            # TODO: first retry the test without restarting the test
449            #       environment and if that fails, restart and then
450            #       retry if < MAX_RETRIES.
451            if test_env:
452                test_env.shutdown(test_runner)
453                test_runner = None
454            retried_result = run_test(test, parent_test, retry=False)
455            # Know this is the kind of test that returns a status b/c it failed
456            assert retried_result is not None
457            retried_result.retried = True
458            return retried_result
459        else:
460            # Test passed, was skipped, or we're not retrying it.
461            return result
462
463    # the retry mechanism is intended to allow a batch run of all tests to pass
464    # even if a small handful of tests exhibit flaky behavior. If a test filter
465    # was provided or debug on error is set, we are most likely not doing a
466    # batch run (as is the case for presubmit testing) meaning that it is
467    # not all that helpful to retry failing tests vs. finishing the run faster.
468    retry = test_filters is None and not debug_on_error
469    try:
470        for test in project_config.tests:
471            if not test.enabled and not run_disabled_tests:
472                continue
473            if not test_should_run(test.name, test_filters):
474                continue
475
476            if result := run_test(test, None, retry):
477                test_results.add_result(result)
478    finally:
479        # finally is used here to make sure that we attempt to shutdown the
480        # test environment no matter whether an exception was raised or not
481        # and no matter what kind of test caused an exception to be raised.
482        if test_env:
483            test_env.shutdown(test_runner)
484        # any saved exception from the try block will be re-raised here
485
486    return test_results
487
488
489def test_projects(
490    build_config: TrustyBuildConfig,
491    root: os.PathLike,
492    projects: list[str],
493    run_disabled_tests: bool = False,
494    test_filters: Optional[list[re.Pattern]] = None,
495    verbose: bool = False,
496    debug_on_error: bool = False,
497    emulator: bool = True,
498) -> MultiProjectTestResults:
499    """Run tests for multiple project.
500
501    Args:
502        build_config: TrustyBuildConfig object.
503        root: Trusty build root output directory.
504        projects: Names of the projects to run tests for.
505        run_disabled_tests: Also run disabled tests from config file.
506        test_filters: Optional list that limits the tests to run. Projects
507          without any tests that match a filter will be skipped.
508        verbose: Enable debug output.
509        debug_on_error: Wait for debugger connection on errors.
510
511    Returns:
512        MultiProjectTestResults listing overall and detailed test results.
513    """
514    if test_filters:
515        projects = projects_to_test(
516            build_config,
517            projects,
518            test_filters,
519            run_disabled_tests=run_disabled_tests,
520        )
521
522    results = []
523    for project in projects:
524        results.append(
525            run_tests(
526                build_config,
527                root,
528                project,
529                run_disabled_tests=run_disabled_tests,
530                test_filters=test_filters,
531                verbose=verbose,
532                debug_on_error=debug_on_error,
533                emulator=emulator,
534            )
535        )
536    return MultiProjectTestResults(results)
537
538
539def default_root() -> str:
540    script_dir = os.path.dirname(os.path.abspath(__file__))
541    top = os.path.abspath(os.path.join(script_dir, "../../../../.."))
542    return os.path.join(top, "build-root")
543
544
545def main():
546    parser = argparse.ArgumentParser()
547    parser.add_argument(
548        "project", type=str, nargs="+", help="Project(s) to test."
549    )
550    parser.add_argument(
551        "--build-root",
552        type=str,
553        default=default_root(),
554        help="Root of intermediate build directory.",
555    )
556    parser.add_argument(
557        "--run_disabled_tests",
558        help="Also run disabled tests from config file.",
559        action="store_true",
560    )
561    parser.add_argument(
562        "--test",
563        type=str,
564        action="append",
565        help="Only run tests that match the provided regexes.",
566    )
567    parser.add_argument(
568        "--verbose", help="Enable debug output.", action="store_true"
569    )
570    parser.add_argument(
571        "--debug_on_error",
572        help="Wait for debugger connection on errors.",
573        action="store_true",
574    )
575    args = parser.parse_args()
576
577    build_config = TrustyBuildConfig()
578
579    test_filters = (
580        [re.compile(test) for test in args.test] if args.test else None
581    )
582    test_results = test_projects(
583        build_config,
584        args.build_root,
585        args.project,
586        run_disabled_tests=args.run_disabled_tests,
587        test_filters=test_filters,
588        verbose=args.verbose,
589        debug_on_error=args.debug_on_error,
590    )
591    test_results.print_results()
592
593    if test_results.failed_projects:
594        sys.exit(1)
595
596
597if __name__ == "__main__":
598    main()
599