1#!/usr/bin/env python2
2
3# Copyright 2017 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""A script to parse apache error logs
8
9The script gets the contents of the log file through stdin, and emits a counter
10metric for the beginning of each error message it recognizes.
11"""
12from __future__ import print_function
13
14import argparse
15import re
16import sys
17
18import common
19
20from chromite.lib import metrics
21from chromite.lib import ts_mon_config
22# infra_libs comes from chromite's third_party modules.
23from infra_libs import ts_mon
24
25from autotest_lib.site_utils.stats import log_daemon_common
26
27
28LOOP_INTERVAL = 60
29ERROR_LOG_METRIC = '/chromeos/autotest/apache/error_log'
30ERROR_LOG_LINE_METRIC = '/chromeos/autotest/apache/error_log_line'
31SEGFAULT_METRIC = '/chromeos/autotest/apache/segfault_count'
32START_METRIC = '/chromeos/autotest/apache/start_count'
33STOP_METRIC = '/chromeos/autotest/apache/stop_count'
34
35ERROR_LOG_MATCHER = re.compile(
36    r'^\[[^]]+\] ' # The timestamp. We don't need this.
37    r'\[(mpm_event|core)?:(?P<log_level>\S+)\] '
38    r'\[pid \d+[^]]+\] ' # The PID, possibly followed by a task id.
39    # There may be other sections, such as [remote <ip>]
40    r'(?P<sections>\[[^]]+\] )*'
41    r'\S' # first character after pid must be non-space; otherwise it is
42          # indented, meaning it is just a continuation of a previous message.
43    r'(?P<mod_wsgi>od_wsgi)?' # Note: the 'm' of mod_wsgi was already matched.
44    r'(?P<rest>.*)'
45)
46
47def EmitSegfault(_m):
48    """Emits a Counter metric for segfaults.
49
50    @param _m: A regex match object
51    """
52    metrics.Counter(
53            SEGFAULT_METRIC,
54            description='A metric counting segfaults in apache',
55            field_spec=None,
56    ).increment()
57
58
59def EmitStart(_m):
60    """Emits a Counter metric for apache service starts.
61
62    @param _m: A regex match object
63    """
64
65    metrics.Counter(
66            START_METRIC,
67            description="A metric counting Apache service starts.",
68            field_spec=None,
69    ).increment()
70
71
72def EmitStop(_m, graceful):
73    """Emits a Counter metric for apache service stops
74
75    @param _m: A regex match object
76    @param graceful: Whether apache was stopped gracefully.
77    """
78    metrics.Counter(
79            STOP_METRIC,
80            description="A metric counting Apache service stops.",
81            field_spec=[ts_mon.BooleanField('graceful')]
82    ).increment(fields={
83        'graceful': graceful
84    })
85
86
87MESSAGE_PATTERNS = {
88        r'Segmentation fault': EmitSegfault,
89        r'configured -- resuming normal operations': EmitStart,
90        r'caught SIGTERM, shutting down': lambda m: EmitStop(m, graceful=True),
91        # TODO(phobbs) add log message for when Apache dies ungracefully
92}
93
94
95def EmitErrorLog(m):
96    """Emits a Counter metric for error log messages.
97
98    @param m: A regex match object
99    """
100    log_level = m.group('log_level') or ''
101    # It might be interesting to see whether the error/warning was emitted
102    # from python at the mod_wsgi process or not.
103    mod_wsgi_present = bool(m.group('mod_wsgi'))
104
105    metrics.Counter(ERROR_LOG_METRIC).increment(fields={
106        'log_level': log_level,
107        'mod_wsgi': mod_wsgi_present})
108
109    rest = m.group('rest')
110    for pattern, handler in MESSAGE_PATTERNS.iteritems():
111        if pattern in rest:
112            handler(m)
113
114
115def EmitErrorLogLine(_m):
116    """Emits a Counter metric for each error log line.
117
118    @param _m: A regex match object.
119    """
120    metrics.Counter(
121            ERROR_LOG_LINE_METRIC,
122            description="A count of lines emitted to the apache error log.",
123            field_spec=None,
124    ).increment()
125
126
127MATCHERS = [
128    (ERROR_LOG_MATCHER, EmitErrorLog),
129    (re.compile(r'.*'), EmitErrorLogLine),
130]
131
132
133def ParseArgs():
134    """Parses the command line arguments."""
135    p = argparse.ArgumentParser(
136        description='Parses apache logs and emits metrics to Monarch')
137    p.add_argument('--output-logfile')
138    p.add_argument('--debug-metrics-file',
139                   help='Output metrics to the given file instead of sending '
140                   'them to production.')
141    return p.parse_args()
142
143
144def Main():
145    """Sets up logging and runs matchers against stdin"""
146    args = ParseArgs()
147    log_daemon_common.SetupLogging(args)
148
149    # Set up metrics sending and go.
150    ts_mon_args = {}
151    if args.debug_metrics_file:
152        ts_mon_args['debug_file'] = args.debug_metrics_file
153
154    with ts_mon_config.SetupTsMonGlobalState('apache_error_log_metrics',
155                                             **ts_mon_args):
156      log_daemon_common.RunMatchers(sys.stdin, MATCHERS)
157      metrics.Flush()
158
159
160if __name__ == '__main__':
161    Main()
162