1import errno
2import os
3import time
4
5import lit.Test        # pylint: disable=import-error
6import lit.TestRunner  # pylint: disable=import-error
7import lit.util        # pylint: disable=import-error
8
9from libcxx.test.executor import LocalExecutor as LocalExecutor
10import libcxx.util
11
12
13class LibcxxTestFormat(object):
14    """
15    Custom test format handler for use with the test format use by libc++.
16
17    Tests fall into two categories:
18      FOO.pass.cpp - Executable test which should compile, run, and exit with
19                     code 0.
20      FOO.fail.cpp - Negative test case which is expected to fail compilation.
21      FOO.sh.cpp   - A test that uses LIT's ShTest format.
22    """
23
24    def __init__(self, cxx, use_verify_for_fail, execute_external,
25                 executor, exec_env):
26        self.cxx = cxx
27        self.use_verify_for_fail = use_verify_for_fail
28        self.execute_external = execute_external
29        self.executor = executor
30        self.exec_env = dict(exec_env)
31
32    # TODO: Move this into lit's FileBasedTest
33    def getTestsInDirectory(self, testSuite, path_in_suite,
34                            litConfig, localConfig):
35        source_path = testSuite.getSourcePath(path_in_suite)
36        for filename in os.listdir(source_path):
37            # Ignore dot files and excluded tests.
38            if filename.startswith('.') or filename in localConfig.excludes:
39                continue
40
41            filepath = os.path.join(source_path, filename)
42            if not os.path.isdir(filepath):
43                if any([filename.endswith(ext)
44                        for ext in localConfig.suffixes]):
45                    yield lit.Test.Test(testSuite, path_in_suite + (filename,),
46                                        localConfig)
47
48    def execute(self, test, lit_config):
49        while True:
50            try:
51                return self._execute(test, lit_config)
52            except OSError as oe:
53                if oe.errno != errno.ETXTBSY:
54                    raise
55                time.sleep(0.1)
56
57    def _execute(self, test, lit_config):
58        name = test.path_in_suite[-1]
59        is_sh_test = name.endswith('.sh.cpp')
60        is_pass_test = name.endswith('.pass.cpp')
61        is_fail_test = name.endswith('.fail.cpp')
62
63        if test.config.unsupported:
64            return (lit.Test.UNSUPPORTED,
65                    "A lit.local.cfg marked this unsupported")
66
67        res = lit.TestRunner.parseIntegratedTestScript(
68            test, require_script=is_sh_test)
69        # Check if a result for the test was returned. If so return that
70        # result.
71        if isinstance(res, lit.Test.Result):
72            return res
73        if lit_config.noExecute:
74            return lit.Test.Result(lit.Test.PASS)
75        # res is not an instance of lit.test.Result. Expand res into its parts.
76        script, tmpBase, execDir = res
77        # Check that we don't have run lines on tests that don't support them.
78        if not is_sh_test and len(script) != 0:
79            lit_config.fatal('Unsupported RUN line found in test %s' % name)
80
81        # Dispatch the test based on its suffix.
82        if is_sh_test:
83            if not isinstance(self.executor, LocalExecutor):
84                # We can't run ShTest tests with a executor yet.
85                # For now, bail on trying to run them
86                return lit.Test.UNSUPPORTED, 'ShTest format not yet supported'
87            return lit.TestRunner._runShTest(test, lit_config,
88                                             self.execute_external, script,
89                                             tmpBase, execDir)
90        elif is_fail_test:
91            return self._evaluate_fail_test(test)
92        elif is_pass_test:
93            return self._evaluate_pass_test(test, tmpBase, execDir, lit_config)
94        else:
95            # No other test type is supported
96            assert False
97
98    def _clean(self, exec_path):  # pylint: disable=no-self-use
99        libcxx.util.cleanFile(exec_path)
100
101    def _evaluate_pass_test(self, test, tmpBase, execDir, lit_config):
102        source_path = test.getSourcePath()
103        exec_path = tmpBase + '.exe'
104        object_path = tmpBase + '.o'
105        # Create the output directory if it does not already exist.
106        lit.util.mkdir_p(os.path.dirname(tmpBase))
107        try:
108            # Compile the test
109            cmd, out, err, rc = self.cxx.compileLinkTwoSteps(
110                source_path, out=exec_path, object_file=object_path,
111                cwd=execDir)
112            compile_cmd = cmd
113            if rc != 0:
114                report = libcxx.util.makeReport(cmd, out, err, rc)
115                report += "Compilation failed unexpectedly!"
116                return lit.Test.FAIL, report
117            # Run the test
118            local_cwd = os.path.dirname(source_path)
119            env = None
120            if self.exec_env:
121                env = self.exec_env
122            # TODO: Only list actually needed files in file_deps.
123            # Right now we just mark all of the .dat files in the same
124            # directory as dependencies, but it's likely less than that. We
125            # should add a `// FILE-DEP: foo.dat` to each test to track this.
126            data_files = [os.path.join(local_cwd, f)
127                          for f in os.listdir(local_cwd) if f.endswith('.dat')]
128            cmd, out, err, rc = self.executor.run(exec_path, [exec_path],
129                                                  local_cwd, data_files, env)
130            if rc != 0:
131                report = libcxx.util.makeReport(cmd, out, err, rc)
132                report = "Compiled With: %s\n%s" % (compile_cmd, report)
133                report += "Compiled test failed unexpectedly!"
134                return lit.Test.FAIL, report
135            return lit.Test.PASS, ''
136        finally:
137            # Note that cleanup of exec_file happens in `_clean()`. If you
138            # override this, cleanup is your reponsibility.
139            libcxx.util.cleanFile(object_path)
140            self._clean(exec_path)
141
142    def _evaluate_fail_test(self, test):
143        source_path = test.getSourcePath()
144        with open(source_path, 'r') as f:
145            contents = f.read()
146        verify_tags = ['expected-note', 'expected-remark', 'expected-warning',
147                       'expected-error', 'expected-no-diagnostics']
148        use_verify = self.use_verify_for_fail and \
149                     any([tag in contents for tag in verify_tags])
150        extra_flags = []
151        if use_verify:
152            extra_flags += ['-Xclang', '-verify',
153                            '-Xclang', '-verify-ignore-unexpected=note']
154        cmd, out, err, rc = self.cxx.compile(source_path, out=os.devnull,
155                                             flags=extra_flags)
156        expected_rc = 0 if use_verify else 1
157        if rc == expected_rc:
158            return lit.Test.PASS, ''
159        else:
160            report = libcxx.util.makeReport(cmd, out, err, rc)
161            report_msg = ('Expected compilation to fail!' if not use_verify else
162                          'Expected compilation using verify to pass!')
163            return lit.Test.FAIL, report + report_msg + '\n'
164