1#!/usr/bin/python
2
3# Script to compare testsuite failures against a list of known-to-fail
4# tests.
5
6# Contributed by Diego Novillo <dnovillo@google.com>
7# Overhaul by Krystian Baclawski <kbaclawski@google.com>
8#
9# Copyright (C) 2011 Free Software Foundation, Inc.
10#
11# This file is part of GCC.
12#
13# GCC is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 3, or (at your option)
16# any later version.
17#
18# GCC is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with GCC; see the file COPYING.  If not, write to
25# the Free Software Foundation, 51 Franklin Street, Fifth Floor,
26# Boston, MA 02110-1301, USA.
27"""This script provides a coarser XFAILing mechanism that requires no
28detailed DejaGNU markings.  This is useful in a variety of scenarios:
29
30- Development branches with many known failures waiting to be fixed.
31- Release branches with known failures that are not considered
32  important for the particular release criteria used in that branch.
33
34The script must be executed from the toplevel build directory.  When
35executed it will:
36
371) Determine the target built: TARGET
382) Determine the source directory: SRCDIR
393) Look for a failure manifest file in
40   <SRCDIR>/contrib/testsuite-management/<TARGET>.xfail
414) Collect all the <tool>.sum files from the build tree.
425) Produce a report stating:
43   a) Failures expected in the manifest but not present in the build.
44   b) Failures in the build not expected in the manifest.
456) If all the build failures are expected in the manifest, it exits
46   with exit code 0.  Otherwise, it exits with error code 1.
47"""
48
49import optparse
50import logging
51import os
52import sys
53
54sys.path.append(os.path.dirname(os.path.abspath(__file__)))
55
56from dejagnu.manifest import Manifest
57from dejagnu.summary import DejaGnuTestResult
58from dejagnu.summary import DejaGnuTestRun
59
60# Pattern for naming manifest files.  The first argument should be
61# the toplevel GCC source directory.  The second argument is the
62# target triple used during the build.
63_MANIFEST_PATH_PATTERN = '%s/contrib/testsuite-management/%s.xfail'
64
65
66def GetMakefileVars(makefile_path):
67  assert os.path.exists(makefile_path)
68
69  with open(makefile_path) as lines:
70    kvs = [line.split('=', 1) for line in lines if '=' in line]
71
72    return dict((k.strip(), v.strip()) for k, v in kvs)
73
74
75def GetSumFiles(build_dir):
76  summaries = []
77
78  for root, _, filenames in os.walk(build_dir):
79    summaries.extend([os.path.join(root, filename)
80                      for filename in filenames if filename.endswith('.sum')])
81
82  return map(os.path.normpath, summaries)
83
84
85def ValidBuildDirectory(build_dir, target):
86  mandatory_paths = [build_dir, os.path.join(build_dir, 'Makefile')]
87
88  extra_paths = [os.path.join(build_dir, target),
89                 os.path.join(build_dir, 'build-%s' % target)]
90
91  return (all(map(os.path.exists, mandatory_paths)) and
92          any(map(os.path.exists, extra_paths)))
93
94
95def GetManifestPath(build_dir):
96  makefile = GetMakefileVars(os.path.join(build_dir, 'Makefile'))
97  srcdir = makefile['srcdir']
98  target = makefile['target']
99
100  if not ValidBuildDirectory(build_dir, target):
101    target = makefile['target_alias']
102
103  if not ValidBuildDirectory(build_dir, target):
104    logging.error('%s is not a valid GCC top level build directory.', build_dir)
105    sys.exit(1)
106
107  logging.info('Discovered source directory: "%s"', srcdir)
108  logging.info('Discovered build target: "%s"', target)
109
110  return _MANIFEST_PATH_PATTERN % (srcdir, target)
111
112
113def CompareResults(manifest, actual):
114  """Compare sets of results and return two lists:
115     - List of results present in MANIFEST but missing from ACTUAL.
116     - List of results present in ACTUAL but missing from MANIFEST.
117  """
118  # Report all the actual results not present in the manifest.
119  actual_vs_manifest = actual - manifest
120
121  # Filter out tests marked flaky.
122  manifest_without_flaky_tests = set(filter(lambda result: not result.flaky,
123                                            manifest))
124
125  # Simlarly for all the tests in the manifest.
126  manifest_vs_actual = manifest_without_flaky_tests - actual
127
128  return actual_vs_manifest, manifest_vs_actual
129
130
131def LogResults(level, results):
132  log_fun = getattr(logging, level)
133
134  for num, result in enumerate(sorted(results), start=1):
135    log_fun('  %d) %s', num, result)
136
137
138def CheckExpectedResults(manifest_path, build_dir):
139  logging.info('Reading manifest file: "%s"', manifest_path)
140
141  manifest = set(Manifest.FromFile(manifest_path))
142
143  logging.info('Getting actual results from build directory: "%s"',
144               os.path.realpath(build_dir))
145
146  summaries = GetSumFiles(build_dir)
147
148  actual = set()
149
150  for summary in summaries:
151    test_run = DejaGnuTestRun.FromFile(summary)
152    failures = set(Manifest.FromDejaGnuTestRun(test_run))
153    actual.update(failures)
154
155  if manifest:
156    logging.debug('Tests expected to fail:')
157    LogResults('debug', manifest)
158
159  if actual:
160    logging.debug('Actual test failures:')
161    LogResults('debug', actual)
162
163  actual_vs_manifest, manifest_vs_actual = CompareResults(manifest, actual)
164
165  if actual_vs_manifest:
166    logging.info('Build results not in the manifest:')
167    LogResults('info', actual_vs_manifest)
168
169  if manifest_vs_actual:
170    logging.info('Manifest results not present in the build:')
171    LogResults('info', manifest_vs_actual)
172    logging.info('NOTE: This is not a failure!  ',
173                 'It just means that the manifest expected these tests to '
174                 'fail, but they worked in this configuration.')
175
176  if actual_vs_manifest or manifest_vs_actual:
177    sys.exit(1)
178
179  logging.info('No unexpected failures.')
180
181
182def ProduceManifest(manifest_path, build_dir, overwrite):
183  if os.path.exists(manifest_path) and not overwrite:
184    logging.error('Manifest file "%s" already exists.', manifest_path)
185    logging.error('Use --force to overwrite.')
186    sys.exit(1)
187
188  testruns = map(DejaGnuTestRun.FromFile, GetSumFiles(build_dir))
189  manifests = map(Manifest.FromDejaGnuTestRun, testruns)
190
191  with open(manifest_path, 'w') as manifest_file:
192    manifest_strings = [manifest.Generate() for manifest in manifests]
193    logging.info('Writing manifest to "%s".', manifest_path)
194    manifest_file.write('\n'.join(manifest_strings))
195
196
197def Main(argv):
198  parser = optparse.OptionParser(usage=__doc__)
199  parser.add_option(
200      '-b',
201      '--build_dir',
202      dest='build_dir',
203      action='store',
204      metavar='PATH',
205      default=os.getcwd(),
206      help='Build directory to check. (default: current directory)')
207  parser.add_option('-m',
208                    '--manifest',
209                    dest='manifest',
210                    action='store_true',
211                    help='Produce the manifest for the current build.')
212  parser.add_option(
213      '-f',
214      '--force',
215      dest='force',
216      action='store_true',
217      help=('Overwrite an existing manifest file, if user requested creating '
218            'new one. (default: False)'))
219  parser.add_option('-v',
220                    '--verbose',
221                    dest='verbose',
222                    action='store_true',
223                    help='Increase verbosity.')
224  options, _ = parser.parse_args(argv[1:])
225
226  if options.verbose:
227    logging.root.setLevel(logging.DEBUG)
228
229  manifest_path = GetManifestPath(options.build_dir)
230
231  if options.manifest:
232    ProduceManifest(manifest_path, options.build_dir, options.force)
233  else:
234    CheckExpectedResults(manifest_path, options.build_dir)
235
236
237if __name__ == '__main__':
238  logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
239  Main(sys.argv)
240