1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import datetime
6import glob
7import heapq
8import logging
9import os
10import os.path
11import random
12import re
13import shutil
14import subprocess as subprocess
15import sys
16import tempfile
17import time
18
19from catapult_base import cloud_storage  # pylint: disable=import-error
20import dependency_manager  # pylint: disable=import-error
21
22from telemetry.internal.util import binary_manager
23from telemetry.core import exceptions
24from telemetry.core import util
25from telemetry.internal.backends import browser_backend
26from telemetry.internal.backends.chrome import chrome_browser_backend
27from telemetry.internal.util import path
28
29
30def ParseCrashpadDateTime(date_time_str):
31  # Python strptime does not support time zone parsing, strip it.
32  date_time_parts = date_time_str.split()
33  if len(date_time_parts) >= 3:
34    date_time_str = ' '.join(date_time_parts[:2])
35  return datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S')
36
37
38def GetSymbolBinaries(minidump, arch_name, os_name):
39  # Returns binary file where symbols are located.
40  minidump_dump = binary_manager.FetchPath('minidump_dump', arch_name, os_name)
41  assert minidump_dump
42
43  symbol_binaries = []
44
45  minidump_cmd = [minidump_dump, minidump]
46  try:
47    with open(os.devnull, 'wb') as DEVNULL:
48      minidump_output = subprocess.check_output(minidump_cmd, stderr=DEVNULL)
49  except subprocess.CalledProcessError as e:
50    # For some reason minidump_dump always fails despite successful dumping.
51    minidump_output = e.output
52
53  minidump_binary_re = re.compile(r'\W+\(code_file\)\W+=\W\"(.*)\"')
54  for minidump_line in minidump_output.splitlines():
55    line_match = minidump_binary_re.match(minidump_line)
56    if line_match:
57      binary_path = line_match.group(1)
58      if not os.path.isfile(binary_path):
59        continue
60
61      # Filter out system binaries.
62      if (binary_path.startswith('/usr/lib/') or
63          binary_path.startswith('/System/Library/') or
64          binary_path.startswith('/lib/')):
65        continue
66
67      # Filter out other binary file types which have no symbols.
68      if (binary_path.endswith('.pak') or
69          binary_path.endswith('.bin') or
70          binary_path.endswith('.dat')):
71        continue
72
73      symbol_binaries.append(binary_path)
74  return symbol_binaries
75
76
77def GenerateBreakpadSymbols(minidump, arch, os_name, symbols_dir, browser_dir):
78  logging.info('Dumping breakpad symbols.')
79  generate_breakpad_symbols_command = binary_manager.FetchPath(
80      'generate_breakpad_symbols', arch, os_name)
81  if generate_breakpad_symbols_command is None:
82    return
83
84  for binary_path in GetSymbolBinaries(minidump, arch, os_name):
85    cmd = [
86        sys.executable,
87        generate_breakpad_symbols_command,
88        '--binary=%s' % binary_path,
89        '--symbols-dir=%s' % symbols_dir,
90        '--build-dir=%s' % browser_dir,
91        ]
92
93    try:
94      subprocess.check_output(cmd, stderr=open(os.devnull, 'w'))
95    except subprocess.CalledProcessError:
96      logging.warning('Failed to execute "%s"' % ' '.join(cmd))
97      return
98
99
100class DesktopBrowserBackend(chrome_browser_backend.ChromeBrowserBackend):
101  """The backend for controlling a locally-executed browser instance, on Linux,
102  Mac or Windows.
103  """
104  def __init__(self, desktop_platform_backend, browser_options, executable,
105               flash_path, is_content_shell, browser_directory,
106               output_profile_path, extensions_to_load):
107    super(DesktopBrowserBackend, self).__init__(
108        desktop_platform_backend,
109        supports_tab_control=not is_content_shell,
110        supports_extensions=not is_content_shell,
111        browser_options=browser_options,
112        output_profile_path=output_profile_path,
113        extensions_to_load=extensions_to_load)
114
115    # Initialize fields so that an explosion during init doesn't break in Close.
116    self._proc = None
117    self._tmp_profile_dir = None
118    self._tmp_output_file = None
119
120    self._executable = executable
121    if not self._executable:
122      raise Exception('Cannot create browser, no executable found!')
123
124    assert not flash_path or os.path.exists(flash_path)
125    self._flash_path = flash_path
126
127    self._is_content_shell = is_content_shell
128
129    if len(extensions_to_load) > 0 and is_content_shell:
130      raise browser_backend.ExtensionsNotSupportedException(
131          'Content shell does not support extensions.')
132
133    self._browser_directory = browser_directory
134    self._port = None
135    self._tmp_minidump_dir = tempfile.mkdtemp()
136    self._crash_service = None
137    if self.browser_options.enable_logging:
138      self._log_file_path = os.path.join(tempfile.mkdtemp(), 'chrome.log')
139    else:
140      self._log_file_path = None
141
142    self._SetupProfile()
143
144  @property
145  def log_file_path(self):
146    return self._log_file_path
147
148  @property
149  def supports_uploading_logs(self):
150    return (self.browser_options.logs_cloud_bucket and self.log_file_path and
151            os.path.isfile(self.log_file_path))
152
153  def _SetupProfile(self):
154    if not self.browser_options.dont_override_profile:
155      if self._output_profile_path:
156        self._tmp_profile_dir = self._output_profile_path
157      else:
158        self._tmp_profile_dir = tempfile.mkdtemp()
159
160      profile_dir = self.browser_options.profile_dir
161      if profile_dir:
162        assert self._tmp_profile_dir != profile_dir
163        if self._is_content_shell:
164          logging.critical('Profiles cannot be used with content shell')
165          sys.exit(1)
166        logging.info("Using profile directory:'%s'." % profile_dir)
167        shutil.rmtree(self._tmp_profile_dir)
168        shutil.copytree(profile_dir, self._tmp_profile_dir)
169    # No matter whether we're using an existing profile directory or
170    # creating a new one, always delete the well-known file containing
171    # the active DevTools port number.
172    port_file = self._GetDevToolsActivePortPath()
173    if os.path.isfile(port_file):
174      try:
175        os.remove(port_file)
176      except Exception as e:
177        logging.critical('Unable to remove DevToolsActivePort file: %s' % e)
178        sys.exit(1)
179
180  def _GetDevToolsActivePortPath(self):
181    return os.path.join(self.profile_directory, 'DevToolsActivePort')
182
183  def _GetCrashServicePipeName(self):
184    # Ensure a unique pipe name by using the name of the temp dir.
185    pipe = r'\\.\pipe\%s_service' % os.path.basename(self._tmp_minidump_dir)
186    return pipe
187
188  def _StartCrashService(self):
189    os_name = self.browser.platform.GetOSName()
190    if os_name != 'win':
191      return None
192    arch_name = self.browser.platform.GetArchName()
193    command = binary_manager.FetchPath('crash_service', arch_name, os_name)
194    if not command:
195      logging.warning('crash_service.exe not found for %s %s',
196                      arch_name, os_name)
197      return None
198    if not os.path.exists(command):
199      logging.warning('crash_service.exe not found for %s %s',
200                      arch_name, os_name)
201      return None
202
203    try:
204      crash_service = subprocess.Popen([
205          command,
206          '--no-window',
207          '--dumps-dir=%s' % self._tmp_minidump_dir,
208          '--pipe-name=%s' % self._GetCrashServicePipeName()])
209    except Exception:
210      logging.error(
211          'Failed to run %s --no-window --dump-dir=%s --pip-name=%s' % (
212            command, self._tmp_minidump_dir, self._GetCrashServicePipeName()))
213      logging.error('Running on platform: %s and arch: %s.', os_name, arch_name)
214      wmic_stdout, _ = subprocess.Popen(
215        ['wmic', 'process', 'get', 'CommandLine,Name,ProcessId,ParentProcessId',
216        '/format:csv'], stdout=subprocess.PIPE).communicate()
217      logging.error('Current running processes:\n%s' % wmic_stdout)
218      raise
219    return crash_service
220
221  def _GetCdbPath(self):
222    possible_paths = (
223        'Debugging Tools For Windows',
224        'Debugging Tools For Windows (x86)',
225        'Debugging Tools For Windows (x64)',
226        os.path.join('Windows Kits', '8.0', 'Debuggers', 'x86'),
227        os.path.join('Windows Kits', '8.0', 'Debuggers', 'x64'),
228        os.path.join('win_toolchain', 'vs2013_files', 'win8sdk', 'Debuggers',
229                     'x86'),
230        os.path.join('win_toolchain', 'vs2013_files', 'win8sdk', 'Debuggers',
231                     'x64'),
232    )
233    for possible_path in possible_paths:
234      app_path = os.path.join(possible_path, 'cdb.exe')
235      app_path = path.FindInstalledWindowsApplication(app_path)
236      if app_path:
237        return app_path
238    return None
239
240  def HasBrowserFinishedLaunching(self):
241    # In addition to the functional check performed by the base class, quickly
242    # check if the browser process is still alive.
243    if not self.IsBrowserRunning():
244      raise exceptions.ProcessGoneException(
245          "Return code: %d" % self._proc.returncode)
246    # Start DevTools on an ephemeral port and wait for the well-known file
247    # containing the port number to exist.
248    port_file = self._GetDevToolsActivePortPath()
249    if not os.path.isfile(port_file):
250      # File isn't ready yet. Return false. Will retry.
251      return False
252    # Attempt to avoid reading the file until it's populated.
253    got_port = False
254    try:
255      if os.stat(port_file).st_size > 0:
256        with open(port_file) as f:
257          port_string = f.read()
258          self._port = int(port_string)
259          logging.info('Discovered ephemeral port %s' % self._port)
260          got_port = True
261    except Exception:
262      # Both stat and open can throw exceptions.
263      pass
264    if not got_port:
265      # File isn't ready yet. Return false. Will retry.
266      return False
267    return super(DesktopBrowserBackend, self).HasBrowserFinishedLaunching()
268
269  def GetBrowserStartupArgs(self):
270    args = super(DesktopBrowserBackend, self).GetBrowserStartupArgs()
271    self._port = 0
272    logging.info('Requested remote debugging port: %d' % self._port)
273    args.append('--remote-debugging-port=%i' % self._port)
274    args.append('--enable-crash-reporter-for-testing')
275    if not self._is_content_shell:
276      args.append('--window-size=1280,1024')
277      if self._flash_path:
278        args.append('--ppapi-flash-path=%s' % self._flash_path)
279      if not self.browser_options.dont_override_profile:
280        args.append('--user-data-dir=%s' % self._tmp_profile_dir)
281    else:
282      args.append('--data-path=%s' % self._tmp_profile_dir)
283
284    trace_config_file = (self.platform_backend.tracing_controller_backend
285                         .GetChromeTraceConfigFile())
286    if trace_config_file:
287      args.append('--trace-config-file=%s' % trace_config_file)
288    return args
289
290  def Start(self):
291    assert not self._proc, 'Must call Close() before Start()'
292
293    args = [self._executable]
294    args.extend(self.GetBrowserStartupArgs())
295    if self.browser_options.startup_url:
296      args.append(self.browser_options.startup_url)
297    env = os.environ.copy()
298    env['CHROME_HEADLESS'] = '1'  # Don't upload minidumps.
299    env['BREAKPAD_DUMP_LOCATION'] = self._tmp_minidump_dir
300    env['CHROME_BREAKPAD_PIPE_NAME'] = self._GetCrashServicePipeName()
301    if self.browser_options.enable_logging:
302      sys.stderr.write(
303        'Chrome log file will be saved in %s\n' % self.log_file_path)
304      env['CHROME_LOG_FILE'] = self.log_file_path
305    self._crash_service = self._StartCrashService()
306    logging.info('Starting Chrome %s', args)
307    if not self.browser_options.show_stdout:
308      self._tmp_output_file = tempfile.NamedTemporaryFile('w', 0)
309      self._proc = subprocess.Popen(
310          args, stdout=self._tmp_output_file, stderr=subprocess.STDOUT, env=env)
311    else:
312      self._proc = subprocess.Popen(args, env=env)
313
314    try:
315      self._WaitForBrowserToComeUp()
316      # browser is foregrounded by default on Windows and Linux, but not Mac.
317      if self.browser.platform.GetOSName() == 'mac':
318        subprocess.Popen([
319          'osascript', '-e', ('tell application "%s" to activate' %
320                              self._executable)])
321      self._InitDevtoolsClientBackend()
322      if self._supports_extensions:
323        self._WaitForExtensionsToLoad()
324    except:
325      self.Close()
326      raise
327
328  @property
329  def pid(self):
330    if self._proc:
331      return self._proc.pid
332    return None
333
334  @property
335  def browser_directory(self):
336    return self._browser_directory
337
338  @property
339  def profile_directory(self):
340    return self._tmp_profile_dir
341
342  def IsBrowserRunning(self):
343    return self._proc and self._proc.poll() == None
344
345  def GetStandardOutput(self):
346    if not self._tmp_output_file:
347      if self.browser_options.show_stdout:
348        # This can happen in the case that loading the Chrome binary fails.
349        # We print rather than using logging here, because that makes a
350        # recursive call to this function.
351        print >> sys.stderr, "Can't get standard output with --show-stdout"
352      return ''
353    self._tmp_output_file.flush()
354    try:
355      with open(self._tmp_output_file.name) as f:
356        return f.read()
357    except IOError:
358      return ''
359
360  def _GetMostRecentCrashpadMinidump(self):
361    os_name = self.browser.platform.GetOSName()
362    arch_name = self.browser.platform.GetArchName()
363    try:
364      crashpad_database_util = binary_manager.FetchPath(
365          'crashpad_database_util', arch_name, os_name)
366      if not crashpad_database_util:
367        return None
368    except dependency_manager.NoPathFoundError:
369      return None
370
371    report_output = subprocess.check_output([
372        crashpad_database_util, '--database=' + self._tmp_minidump_dir,
373        '--show-pending-reports', '--show-completed-reports',
374        '--show-all-report-info'])
375
376    last_indentation = -1
377    reports_list = []
378    report_dict = {}
379    for report_line in report_output.splitlines():
380      # Report values are grouped together by the same indentation level.
381      current_indentation = 0
382      for report_char in report_line:
383        if not report_char.isspace():
384          break
385        current_indentation += 1
386
387      # Decrease in indentation level indicates a new report is being printed.
388      if current_indentation >= last_indentation:
389        report_key, report_value = report_line.split(':', 1)
390        if report_value:
391          report_dict[report_key.strip()] = report_value.strip()
392      elif report_dict:
393        try:
394          report_time = ParseCrashpadDateTime(report_dict['Creation time'])
395          report_path = report_dict['Path'].strip()
396          reports_list.append((report_time, report_path))
397        except (ValueError, KeyError) as e:
398          logging.warning('Crashpad report expected valid keys'
399                          ' "Path" and "Creation time": %s', e)
400        finally:
401          report_dict = {}
402
403      last_indentation = current_indentation
404
405    # Include the last report.
406    if report_dict:
407      try:
408        report_time = ParseCrashpadDateTime(report_dict['Creation time'])
409        report_path = report_dict['Path'].strip()
410        reports_list.append((report_time, report_path))
411      except (ValueError, KeyError) as e:
412        logging.warning('Crashpad report expected valid keys'
413                          ' "Path" and "Creation time": %s', e)
414
415    if reports_list:
416      _, most_recent_report_path = max(reports_list)
417      return most_recent_report_path
418
419    return None
420
421  def _GetMostRecentMinidump(self):
422    # Crashpad dump layout will be the standard eventually, check it first.
423    most_recent_dump = self._GetMostRecentCrashpadMinidump()
424
425    # Typical breakpad format is simply dump files in a folder.
426    if not most_recent_dump:
427      dumps = glob.glob(os.path.join(self._tmp_minidump_dir, '*.dmp'))
428      if dumps:
429        most_recent_dump = heapq.nlargest(1, dumps, os.path.getmtime)[0]
430
431    # As a sanity check, make sure the crash dump is recent.
432    if (most_recent_dump and
433        os.path.getmtime(most_recent_dump) < (time.time() - (5 * 60))):
434      logging.warning('Crash dump is older than 5 minutes. May not be correct.')
435
436    return most_recent_dump
437
438  def _IsExecutableStripped(self):
439    if self.browser.platform.GetOSName() == 'mac':
440      try:
441        symbols = subprocess.check_output(['/usr/bin/nm', self._executable])
442      except subprocess.CalledProcessError as err:
443        logging.warning('Error when checking whether executable is stripped: %s'
444                        % err.output)
445        # Just assume that binary is stripped to skip breakpad symbol generation
446        # if this check failed.
447        return True
448      num_symbols = len(symbols.splitlines())
449      # We assume that if there are more than 10 symbols the executable is not
450      # stripped.
451      return num_symbols < 10
452    else:
453      return False
454
455  def _GetStackFromMinidump(self, minidump):
456    os_name = self.browser.platform.GetOSName()
457    if os_name == 'win':
458      cdb = self._GetCdbPath()
459      if not cdb:
460        logging.warning('cdb.exe not found.')
461        return None
462      output = subprocess.check_output([cdb, '-y', self._browser_directory,
463                                        '-c', '.ecxr;k30;q', '-z', minidump])
464      # cdb output can start the stack with "ChildEBP", "Child-SP", and possibly
465      # other things we haven't seen yet. If we can't find the start of the
466      # stack, include output from the beginning.
467      stack_start = 0
468      stack_start_match = re.search("^Child(?:EBP|-SP)", output, re.MULTILINE)
469      if stack_start_match:
470        stack_start = stack_start_match.start()
471      stack_end = output.find('quit:')
472      return output[stack_start:stack_end]
473
474    arch_name = self.browser.platform.GetArchName()
475    stackwalk = binary_manager.FetchPath(
476        'minidump_stackwalk', arch_name, os_name)
477    if not stackwalk:
478      logging.warning('minidump_stackwalk binary not found.')
479      return None
480
481    with open(minidump, 'rb') as infile:
482      minidump += '.stripped'
483      with open(minidump, 'wb') as outfile:
484        outfile.write(''.join(infile.read().partition('MDMP')[1:]))
485
486    symbols_path = os.path.join(self._tmp_minidump_dir, 'symbols')
487
488    symbols = glob.glob(os.path.join(self._browser_directory, '*.breakpad*'))
489    if symbols:
490      for symbol in sorted(symbols, key=os.path.getmtime, reverse=True):
491        if not os.path.isfile(symbol):
492          continue
493        with open(symbol, 'r') as f:
494          fields = f.readline().split()
495          if not fields:
496            continue
497          sha = fields[3]
498          binary = ' '.join(fields[4:])
499        symbol_path = os.path.join(symbols_path, binary, sha)
500        if os.path.exists(symbol_path):
501          continue
502        os.makedirs(symbol_path)
503        shutil.copyfile(symbol, os.path.join(symbol_path, binary + '.sym'))
504    else:
505      # On some platforms generating the symbol table can be very time
506      # consuming, skip it if there's nothing to dump.
507      if self._IsExecutableStripped():
508        logging.info('%s appears to be stripped, skipping symbol dump.' % (
509            self._executable))
510        return
511
512      GenerateBreakpadSymbols(minidump, arch_name, os_name,
513                              symbols_path, self._browser_directory)
514
515    return subprocess.check_output([stackwalk, minidump, symbols_path],
516                                   stderr=open(os.devnull, 'w'))
517
518  def _UploadMinidumpToCloudStorage(self, minidump_path):
519    """ Upload minidump_path to cloud storage and return the cloud storage url.
520    """
521    remote_path = ('minidump-%s-%i.dmp' %
522                   (datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'),
523                    random.randint(0, 1000000)))
524    try:
525      return cloud_storage.Insert(cloud_storage.TELEMETRY_OUTPUT, remote_path,
526                                  minidump_path)
527    except cloud_storage.CloudStorageError as err:
528      logging.error('Cloud storage error while trying to upload dump: %s' %
529                    repr(err))
530      return '<Missing link>'
531
532  def GetStackTrace(self):
533    most_recent_dump = self._GetMostRecentMinidump()
534    if not most_recent_dump:
535      return 'No crash dump found.'
536    logging.info('Minidump found: %s' % most_recent_dump)
537    stack = self._GetStackFromMinidump(most_recent_dump)
538    if not stack:
539      cloud_storage_link = self._UploadMinidumpToCloudStorage(most_recent_dump)
540      return ('Failed to symbolize minidump. Raw stack is uploaded to cloud '
541              'storage: %s.' % cloud_storage_link)
542    return stack
543
544  def __del__(self):
545    self.Close()
546
547  def _TryCooperativeShutdown(self):
548    if self.browser.platform.IsCooperativeShutdownSupported():
549      # Ideally there would be a portable, cooperative shutdown
550      # mechanism for the browser. This seems difficult to do
551      # correctly for all embedders of the content API. The only known
552      # problem with unclean shutdown of the browser process is on
553      # Windows, where suspended child processes frequently leak. For
554      # now, just solve this particular problem. See Issue 424024.
555      if self.browser.platform.CooperativelyShutdown(self._proc, "chrome"):
556        try:
557          util.WaitFor(lambda: not self.IsBrowserRunning(), timeout=5)
558          logging.info('Successfully shut down browser cooperatively')
559        except exceptions.TimeoutException as e:
560          logging.warning('Failed to cooperatively shutdown. ' +
561                          'Proceeding to terminate: ' + str(e))
562
563  def Close(self):
564    super(DesktopBrowserBackend, self).Close()
565
566    # First, try to cooperatively shutdown.
567    if self.IsBrowserRunning():
568      self._TryCooperativeShutdown()
569
570    # Second, try to politely shutdown with SIGTERM.
571    if self.IsBrowserRunning():
572      self._proc.terminate()
573      try:
574        util.WaitFor(lambda: not self.IsBrowserRunning(), timeout=5)
575        self._proc = None
576      except exceptions.TimeoutException:
577        logging.warning('Failed to gracefully shutdown.')
578
579    # Shutdown aggressively if all above failed.
580    if self.IsBrowserRunning():
581      logging.warning('Proceed to kill the browser.')
582      self._proc.kill()
583    self._proc = None
584
585    if self._crash_service:
586      self._crash_service.kill()
587      self._crash_service = None
588
589    if self._output_profile_path:
590      # If we need the output then double check that it exists.
591      if not (self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir)):
592        raise Exception("No profile directory generated by Chrome: '%s'." %
593            self._tmp_profile_dir)
594    else:
595      # If we don't need the profile after the run then cleanup.
596      if self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir):
597        shutil.rmtree(self._tmp_profile_dir, ignore_errors=True)
598        self._tmp_profile_dir = None
599
600    if self._tmp_output_file:
601      self._tmp_output_file.close()
602      self._tmp_output_file = None
603
604    if self._tmp_minidump_dir:
605      shutil.rmtree(self._tmp_minidump_dir, ignore_errors=True)
606      self._tmp_minidump_dir = None
607