1# python3
2# Copyright (C) 2019 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16"""Grep warnings messages and output HTML tables or warning counts in CSV.
17
18Default is to output warnings in HTML tables grouped by warning severity.
19Use option --byproject to output tables grouped by source file projects.
20Use option --gencsv to output warning counts in CSV format.
21
22Default input file is build.log, which can be changed with the --log flag.
23"""
24
25# List of important data structures and functions in this script.
26#
27# To parse and keep warning message in the input file:
28#   severity:                classification of message severity
29#   warn_patterns:
30#   warn_patterns[w]['category']     tool that issued the warning, not used now
31#   warn_patterns[w]['description']  table heading
32#   warn_patterns[w]['members']      matched warnings from input
33#   warn_patterns[w]['patterns']     regular expressions to match warnings
34#   warn_patterns[w]['projects'][p]  number of warnings of pattern w in p
35#   warn_patterns[w]['severity']     severity tuple
36#   project_list[p][0]               project name
37#   project_list[p][1]               regular expression to match a project path
38#   project_patterns[p]              re.compile(project_list[p][1])
39#   project_names[p]                 project_list[p][0]
40#   warning_messages     array of each warning message, without source url
41#   warning_links        array of each warning code search link; for 'chrome'
42#   warning_records      array of [idx to warn_patterns,
43#                                  idx to project_names,
44#                                  idx to warning_messages,
45#                                  idx to warning_links]
46#   parse_input_file
47#
48import argparse
49import io
50import multiprocessing
51import os
52import re
53import sys
54
55# pylint:disable=relative-beyond-top-level,no-name-in-module
56# suppress false positive of no-name-in-module warnings
57from . import android_project_list
58from . import chrome_project_list
59from . import cpp_warn_patterns as cpp_patterns
60from . import html_writer
61from . import java_warn_patterns as java_patterns
62from . import make_warn_patterns as make_patterns
63from . import other_warn_patterns as other_patterns
64from . import tidy_warn_patterns as tidy_patterns
65
66
67def parse_args(use_google3):
68  """Define and parse the args. Return the parse_args() result."""
69  parser = argparse.ArgumentParser(
70      description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
71  parser.add_argument('--capacitor_path', default='',
72                      help='Save capacitor warning file to the passed absolute'
73                      ' path')
74  # csvpath has a different naming than the above path because historically the
75  # original Android script used csvpath, so other scripts rely on it
76  parser.add_argument('--csvpath', default='',
77                      help='Save CSV warning file to the passed path')
78  parser.add_argument('--gencsv', action='store_true',
79                      help='Generate CSV file with number of various warnings')
80  parser.add_argument('--csvwithdescription', default='',
81                      help="""Save CSV warning file to the passed path this csv
82                            will contain all the warning descriptions""")
83  parser.add_argument('--byproject', action='store_true',
84                      help='Separate warnings in HTML output by project names')
85  parser.add_argument('--url', default='',
86                      help='Root URL of an Android source code tree prefixed '
87                      'before files in warnings')
88  parser.add_argument('--separator', default='?l=',
89                      help='Separator between the end of a URL and the line '
90                      'number argument. e.g. #')
91  parser.add_argument('--processes', default=multiprocessing.cpu_count(),
92                      type=int,
93                      help='Number of parallel processes to process warnings')
94  # Old Android build scripts call warn.py without --platform,
95  # so the default platform is set to 'android'.
96  parser.add_argument('--platform', default='android',
97                      choices=['chrome', 'android'],
98                      help='Platform of the build log')
99  # Old Android build scripts call warn.py with only a build.log file path.
100  parser.add_argument('--log', help='Path to build log file')
101  parser.add_argument(dest='buildlog', metavar='build.log',
102                      default='build.log', nargs='?',
103                      help='Path to build.log file')
104  flags = parser.parse_args()
105  if not flags.log:
106    flags.log = flags.buildlog
107  if not use_google3 and not os.path.exists(flags.log):
108    sys.exit('Cannot find log file: ' + flags.log)
109  return flags
110
111
112def get_project_names(project_list):
113  """Get project_names from project_list."""
114  return [p[0] for p in project_list]
115
116
117def find_project_index(line, project_patterns):
118  """Return the index to the project pattern array."""
119  for idx, pattern in enumerate(project_patterns):
120    if pattern.match(line):
121      return idx
122  return -1
123
124
125def classify_one_warning(warning, link, results, project_patterns,
126                         warn_patterns):
127  """Classify one warning line."""
128  for idx, pattern in enumerate(warn_patterns):
129    for cpat in pattern['compiled_patterns']:
130      if cpat.match(warning):
131        project_idx = find_project_index(warning, project_patterns)
132        results.append([warning, link, idx, project_idx])
133        return
134  # If we end up here, there was a problem parsing the log
135  # probably caused by 'make -j' mixing the output from
136  # 2 or more concurrent compiles
137
138
139def remove_prefix(src, sub):
140  """Remove everything before last occurrence of substring sub in string src."""
141  if sub in src:
142    inc_sub = src.rfind(sub)
143    return src[inc_sub:]
144  return src
145
146
147# TODO(emmavukelj): Don't have any generate_*_cs_link functions call
148# normalize_path a second time (the first time being in parse_input_file)
149def generate_cs_link(warning_line, flags, android_root=None):
150  """Try to add code search HTTP URL prefix."""
151  if flags.platform == 'chrome':
152    return generate_chrome_cs_link(warning_line, flags)
153  if flags.platform == 'android':
154    return generate_android_cs_link(warning_line, flags, android_root)
155  return 'https://cs.corp.google.com/'
156
157
158def generate_android_cs_link(warning_line, flags, android_root):
159  """Generate the code search link for a warning line in Android."""
160  # max_splits=2 -> only 3 items
161  raw_path, line_number_str, _ = warning_line.split(':', 2)
162  normalized_path = normalize_path(raw_path, flags, android_root)
163  if not flags.url:
164    return normalized_path
165  link_path = flags.url + '/' + normalized_path
166  if line_number_str.isdigit():
167    link_path += flags.separator + line_number_str
168  return link_path
169
170
171def generate_chrome_cs_link(warning_line, flags):
172  """Generate the code search link for a warning line in Chrome."""
173  split_line = warning_line.split(':')
174  raw_path = split_line[0]
175  normalized_path = normalize_path(raw_path, flags)
176  link_base = 'https://cs.chromium.org/'
177  link_add = 'chromium'
178  link_path = None
179
180  # Basically just going through a few specific directory cases and specifying
181  # the proper behavior for that case. This list of cases was accumulated
182  # through trial and error manually going through the warnings.
183  #
184  # This code pattern of using case-specific "if"s instead of "elif"s looks
185  # possibly accidental and mistaken but it is intentional because some paths
186  # fall under several cases (e.g. third_party/lib/nghttp2_frame.c) and for
187  # those we want the most specific case to be applied. If there is reliable
188  # knowledge of exactly where these occur, this could be changed to "elif"s
189  # but there is no reliable set of paths falling under multiple cases at the
190  # moment.
191  if '/src/third_party' in raw_path:
192    link_path = remove_prefix(raw_path, '/src/third_party/')
193  if '/chrome_root/src_internal/' in raw_path:
194    link_path = remove_prefix(raw_path, '/chrome_root/src_internal/')
195    link_path = link_path[len('/chrome_root'):]  # remove chrome_root
196  if '/chrome_root/src/' in raw_path:
197    link_path = remove_prefix(raw_path, '/chrome_root/src/')
198    link_path = link_path[len('/chrome_root'):]  # remove chrome_root
199  if '/libassistant/' in raw_path:
200    link_add = 'eureka_internal/chromium/src'
201    link_base = 'https://cs.corp.google.com/'  # internal data
202    link_path = remove_prefix(normalized_path, '/libassistant/')
203  if raw_path.startswith('gen/'):
204    link_path = '/src/out/Debug/gen/' + normalized_path
205  if '/gen/' in raw_path:
206    return '%s?q=file:%s' % (link_base, remove_prefix(normalized_path, '/gen/'))
207
208  if not link_path and (raw_path.startswith('src/') or
209                        raw_path.startswith('src_internal/')):
210    link_path = '/%s' % raw_path
211
212  if not link_path:  # can't find specific link, send a query
213    return '%s?q=file:%s' % (link_base, normalized_path)
214
215  line_number = int(split_line[1])
216  link = '%s%s%s?l=%d' % (link_base, link_add, link_path, line_number)
217  return link
218
219
220def find_warn_py_and_android_root(path):
221  """Return android source root path if warn.py is found."""
222  parts = path.split('/')
223  for idx in reversed(range(2, len(parts))):
224    root_path = '/'.join(parts[:idx])
225    # Android root directory should contain this script.
226    if os.path.exists(root_path + '/build/make/tools/warn.py'):
227      return root_path
228  return ''
229
230
231def find_android_root(buildlog):
232  """Guess android source root from common prefix of file paths."""
233  # Use the longest common prefix of the absolute file paths
234  # of the first 10000 warning messages as the android_root.
235  warning_lines = []
236  warning_pattern = re.compile('^/[^ ]*/[^ ]*: warning: .*')
237  count = 0
238  for line in buildlog:
239    if warning_pattern.match(line):
240      warning_lines.append(line)
241      count += 1
242      if count > 9999:
243        break
244      # Try to find warn.py and use its location to find
245      # the source tree root.
246      if count < 100:
247        path = os.path.normpath(re.sub(':.*$', '', line))
248        android_root = find_warn_py_and_android_root(path)
249        if android_root:
250          return android_root
251  # Do not use common prefix of a small number of paths.
252  if count > 10:
253    # pytype: disable=wrong-arg-types
254    root_path = os.path.commonprefix(warning_lines)
255    # pytype: enable=wrong-arg-types
256    if len(root_path) > 2 and root_path[len(root_path) - 1] == '/':
257      return root_path[:-1]
258  return ''
259
260
261def remove_android_root_prefix(path, android_root):
262  """Remove android_root prefix from path if it is found."""
263  if path.startswith(android_root):
264    return path[1 + len(android_root):]
265  return path
266
267
268def normalize_path(path, flags, android_root=None):
269  """Normalize file path relative to src/ or src-internal/ directory."""
270  path = os.path.normpath(path)
271
272  if flags.platform == 'android':
273    if android_root:
274      return remove_android_root_prefix(path, android_root)
275    return path
276
277  # Remove known prefix of root path and normalize the suffix.
278  idx = path.find('chrome_root/')
279  if idx >= 0:
280    # remove chrome_root/, we want path relative to that
281    return path[idx + len('chrome_root/'):]
282  return path
283
284
285def normalize_warning_line(line, flags, android_root=None):
286  """Normalize file path relative to src directory in a warning line."""
287  line = re.sub(u'[\u2018\u2019]', '\'', line)
288  # replace non-ASCII chars to spaces
289  line = re.sub(u'[^\x00-\x7f]', ' ', line)
290  line = line.strip()
291  first_column = line.find(':')
292  return normalize_path(line[:first_column], flags,
293                        android_root) + line[first_column:]
294
295
296def parse_input_file_chrome(infile, flags):
297  """Parse Chrome input file, collect parameters and warning lines."""
298  platform_version = 'unknown'
299  board_name = 'unknown'
300  architecture = 'unknown'
301
302  # only handle warning lines of format 'file_path:line_no:col_no: warning: ...'
303  chrome_warning_pattern = r'^[^ ]*/[^ ]*:[0-9]+:[0-9]+: warning: .*'
304
305  warning_pattern = re.compile(chrome_warning_pattern)
306
307  # Collect all unique warning lines
308  # Remove the duplicated warnings save ~8% of time when parsing
309  # one typical build log than before
310  unique_warnings = dict()
311  for line in infile:
312    if warning_pattern.match(line):
313      normalized_line = normalize_warning_line(line, flags)
314      if normalized_line not in unique_warnings:
315        unique_warnings[normalized_line] = generate_cs_link(line, flags)
316    elif (platform_version == 'unknown' or board_name == 'unknown' or
317          architecture == 'unknown'):
318      result = re.match(r'.+Package:.+chromeos-base/chromeos-chrome-', line)
319      if result is not None:
320        platform_version = 'R' + line.split('chrome-')[1].split('_')[0]
321        continue
322      result = re.match(r'.+Source\sunpacked\sin\s(.+)', line)
323      if result is not None:
324        board_name = result.group(1).split('/')[2]
325        continue
326      result = re.match(r'.+USE:\s*([^\s]*).*', line)
327      if result is not None:
328        architecture = result.group(1)
329        continue
330
331  header_str = '%s - %s - %s' % (platform_version, board_name, architecture)
332  return unique_warnings, header_str
333
334
335def add_normalized_line_to_warnings(line, flags, android_root, unique_warnings):
336  """Parse/normalize path, updating warning line and add to warnings dict."""
337  normalized_line = normalize_warning_line(line, flags, android_root)
338  if normalized_line not in unique_warnings:
339    unique_warnings[normalized_line] = generate_cs_link(line, flags,
340                                                        android_root)
341  return unique_warnings
342
343
344def parse_input_file_android(infile, flags):
345  """Parse Android input file, collect parameters and warning lines."""
346  # pylint:disable=too-many-locals,too-many-branches
347  platform_version = 'unknown'
348  target_product = 'unknown'
349  target_variant = 'unknown'
350  android_root = find_android_root(infile)
351  infile.seek(0)
352
353  # rustc warning messages have two lines that should be combined:
354  #     warning: description
355  #        --> file_path:line_number:column_number
356  # Some warning messages have no file name:
357  #     warning: macro replacement list ... [bugprone-macro-parentheses]
358  # Some makefile warning messages have no line number:
359  #     some/path/file.mk: warning: description
360  # C/C++ compiler warning messages have line and column numbers:
361  #     some/path/file.c:line_number:column_number: warning: description
362  warning_pattern = re.compile('(^[^ ]*/[^ ]*: warning: .*)|(^warning: .*)')
363  warning_without_file = re.compile('^warning: .*')
364  rustc_file_position = re.compile('^[ ]+--> [^ ]*/[^ ]*:[0-9]+:[0-9]+')
365
366   # Collect all unique warning lines
367  # Remove the duplicated warnings save ~8% of time when parsing
368  # one typical build log than before
369  unique_warnings = dict()
370  line_counter = 0
371  prev_warning = ''
372  for line in infile:
373    if prev_warning:
374      if rustc_file_position.match(line):
375        # must be a rustc warning, combine 2 lines into one warning
376        line = line.strip().replace('--> ', '') + ': ' + prev_warning
377        unique_warnings = add_normalized_line_to_warnings(
378            line, flags, android_root, unique_warnings)
379        prev_warning = ''
380        continue
381      # add prev_warning, and then process the current line
382      prev_warning = 'unknown_source_file: ' + prev_warning
383      unique_warnings = add_normalized_line_to_warnings(
384          prev_warning, flags, android_root, unique_warnings)
385      prev_warning = ''
386
387    if warning_pattern.match(line):
388      if warning_without_file.match(line):
389        # save this line and combine it with the next line
390        prev_warning = line
391      else:
392        unique_warnings = add_normalized_line_to_warnings(
393            line, flags, android_root, unique_warnings)
394      continue
395
396    if line_counter < 100:
397      # save a little bit of time by only doing this for the first few lines
398      line_counter += 1
399      result = re.search('(?<=^PLATFORM_VERSION=).*', line)
400      if result is not None:
401        platform_version = result.group(0)
402      result = re.search('(?<=^TARGET_PRODUCT=).*', line)
403      if result is not None:
404        target_product = result.group(0)
405      result = re.search('(?<=^TARGET_BUILD_VARIANT=).*', line)
406      if result is not None:
407        target_variant = result.group(0)
408      result = re.search('(?<=^TOP=).*', line)
409      if result is not None:
410        android_root = result.group(1)
411
412  if android_root:
413    new_unique_warnings = dict()
414    for warning_line in unique_warnings:
415      normalized_line = normalize_warning_line(warning_line, flags,
416                                               android_root)
417      new_unique_warnings[normalized_line] = generate_android_cs_link(
418          warning_line, flags, android_root)
419    unique_warnings = new_unique_warnings
420
421  header_str = '%s - %s - %s' % (platform_version, target_product,
422                                 target_variant)
423  return unique_warnings, header_str
424
425
426def parse_input_file(infile, flags):
427  """Parse one input file for chrome or android."""
428  if flags.platform == 'chrome':
429    return parse_input_file_chrome(infile, flags)
430  if flags.platform == 'android':
431    return parse_input_file_android(infile, flags)
432  raise RuntimeError('parse_input_file not defined for platform %s' %
433                     flags.platform)
434
435
436def parse_compiler_output(compiler_output):
437  """Parse compiler output for relevant info."""
438  split_output = compiler_output.split(':', 3)  # 3 = max splits
439  file_path = split_output[0]
440  line_number = int(split_output[1])
441  col_number = int(split_output[2].split(' ')[0])
442  warning_message = split_output[3]
443  return file_path, line_number, col_number, warning_message
444
445
446def get_warn_patterns(platform):
447  """Get and initialize warn_patterns."""
448  warn_patterns = []
449  if platform == 'chrome':
450    warn_patterns = cpp_patterns.warn_patterns
451  elif platform == 'android':
452    warn_patterns = (make_patterns.warn_patterns + cpp_patterns.warn_patterns +
453                     java_patterns.warn_patterns + tidy_patterns.warn_patterns +
454                     other_patterns.warn_patterns)
455  else:
456    raise Exception('platform name %s is not valid' % platform)
457  for pattern in warn_patterns:
458    pattern['members'] = []
459    # Each warning pattern has a 'projects' dictionary, that
460    # maps a project name to number of warnings in that project.
461    pattern['projects'] = {}
462  return warn_patterns
463
464
465def get_project_list(platform):
466  """Return project list for appropriate platform."""
467  if platform == 'chrome':
468    return chrome_project_list.project_list
469  if platform == 'android':
470    return android_project_list.project_list
471  raise Exception('platform name %s is not valid' % platform)
472
473
474def parallel_classify_warnings(warning_data, args, project_names,
475                               project_patterns, warn_patterns,
476                               use_google3, create_launch_subprocs_fn,
477                               classify_warnings_fn):
478  """Classify all warning lines with num_cpu parallel processes."""
479  # pylint:disable=too-many-arguments,too-many-locals
480  num_cpu = args.processes
481  group_results = []
482
483  if num_cpu > 1:
484    # set up parallel processing for this...
485    warning_groups = [[] for _ in range(num_cpu)]
486    i = 0
487    for warning, link in warning_data.items():
488      warning_groups[i].append((warning, link))
489      i = (i + 1) % num_cpu
490    arg_groups = [[] for _ in range(num_cpu)]
491    for i, group in enumerate(warning_groups):
492      arg_groups[i] = [{
493          'group': group,
494          'project_patterns': project_patterns,
495          'warn_patterns': warn_patterns,
496          'num_processes': num_cpu
497      }]
498
499    group_results = create_launch_subprocs_fn(num_cpu,
500                                              classify_warnings_fn,
501                                              arg_groups,
502                                              group_results)
503  else:
504    group_results = []
505    for warning, link in warning_data.items():
506      classify_one_warning(warning, link, group_results,
507                           project_patterns, warn_patterns)
508    group_results = [group_results]
509
510  warning_messages = []
511  warning_links = []
512  warning_records = []
513  if use_google3:
514    group_results = [group_results]
515  for group_result in group_results:
516    for result in group_result:
517      for line, link, pattern_idx, project_idx in result:
518        pattern = warn_patterns[pattern_idx]
519        pattern['members'].append(line)
520        message_idx = len(warning_messages)
521        warning_messages.append(line)
522        link_idx = len(warning_links)
523        warning_links.append(link)
524        warning_records.append([pattern_idx, project_idx, message_idx,
525                                link_idx])
526        pname = '???' if project_idx < 0 else project_names[project_idx]
527        # Count warnings by project.
528        if pname in pattern['projects']:
529          pattern['projects'][pname] += 1
530        else:
531          pattern['projects'][pname] = 1
532  return warning_messages, warning_links, warning_records
533
534
535def process_log(logfile, flags, project_names, project_patterns, warn_patterns,
536                html_path, use_google3, create_launch_subprocs_fn,
537                classify_warnings_fn, logfile_object):
538  # pylint does not recognize g-doc-*
539  # pylint: disable=bad-option-value,g-doc-args
540  # pylint: disable=bad-option-value,g-doc-return-or-yield
541  # pylint: disable=too-many-arguments,too-many-locals
542  """Function that handles processing of a log.
543
544  This is isolated into its own function (rather than just taking place in main)
545  so that it can be used by both warn.py and the borg job process_gs_logs.py, to
546  avoid duplication of code.
547  Note that if the arguments to this function change, process_gs_logs.py must
548  be updated accordingly.
549  """
550  if logfile_object is None:
551    with io.open(logfile, encoding='utf-8') as log:
552      warning_lines_and_links, header_str = parse_input_file(log, flags)
553  else:
554    warning_lines_and_links, header_str = parse_input_file(
555        logfile_object, flags)
556  warning_messages, warning_links, warning_records = parallel_classify_warnings(
557      warning_lines_and_links, flags, project_names, project_patterns,
558      warn_patterns, use_google3, create_launch_subprocs_fn,
559      classify_warnings_fn)
560
561  html_writer.write_html(flags, project_names, warn_patterns, html_path,
562                         warning_messages, warning_links, warning_records,
563                         header_str)
564
565  return warning_messages, warning_links, warning_records, header_str
566
567
568def common_main(use_google3, create_launch_subprocs_fn, classify_warnings_fn,
569                logfile_object=None):
570  """Shared main function for Google3 and non-Google3 versions of warn.py."""
571  flags = parse_args(use_google3)
572  warn_patterns = get_warn_patterns(flags.platform)
573  project_list = get_project_list(flags.platform)
574
575  project_names = get_project_names(project_list)
576  project_patterns = [re.compile(p[1]) for p in project_list]
577
578  # html_path=None because we output html below if not outputting CSV
579  warning_messages, warning_links, warning_records, header_str = process_log(
580      logfile=flags.log, flags=flags, project_names=project_names,
581      project_patterns=project_patterns, warn_patterns=warn_patterns,
582      html_path=None, use_google3=use_google3,
583      create_launch_subprocs_fn=create_launch_subprocs_fn,
584      classify_warnings_fn=classify_warnings_fn,
585      logfile_object=logfile_object)
586
587  html_writer.write_out_csv(flags, warn_patterns, warning_messages,
588                            warning_links, warning_records, header_str,
589                            project_names)
590
591  # Return these values, so that caller can use them, if desired.
592  return flags, warning_messages, warning_records, warn_patterns
593