1#!/usr/bin/env python2.7
2#
3# Copyright 2017 gRPC authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Runs the entire bm_*.py pipeline, and possible comments on the PR """
17
18import bm_constants
19import bm_build
20import bm_run
21import bm_diff
22
23import sys
24import os
25import random
26import argparse
27import multiprocessing
28import subprocess
29
30sys.path.append(
31    os.path.join(
32        os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
33import comment_on_pr
34
35sys.path.append(
36    os.path.join(
37        os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests',
38        'python_utils'))
39import jobset
40
41
42def _args():
43    argp = argparse.ArgumentParser(
44        description='Perform diff on microbenchmarks')
45    argp.add_argument(
46        '-t',
47        '--track',
48        choices=sorted(bm_constants._INTERESTING),
49        nargs='+',
50        default=sorted(bm_constants._INTERESTING),
51        help='Which metrics to track')
52    argp.add_argument(
53        '-b',
54        '--benchmarks',
55        nargs='+',
56        choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
57        default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
58        help='Which benchmarks to run')
59    argp.add_argument(
60        '-d',
61        '--diff_base',
62        type=str,
63        help='Commit or branch to compare the current one to')
64    argp.add_argument(
65        '-o',
66        '--old',
67        default='old',
68        type=str,
69        help='Name of baseline run to compare to. Ususally just called "old"')
70    argp.add_argument(
71        '-r',
72        '--regex',
73        type=str,
74        default="",
75        help='Regex to filter benchmarks run')
76    argp.add_argument(
77        '-l',
78        '--loops',
79        type=int,
80        default=10,
81        help=
82        'Number of times to loops the benchmarks. More loops cuts down on noise'
83    )
84    argp.add_argument(
85        '-j',
86        '--jobs',
87        type=int,
88        default=multiprocessing.cpu_count(),
89        help='Number of CPUs to use')
90    argp.add_argument(
91        '--pr_comment_name',
92        type=str,
93        default="microbenchmarks",
94        help='Name that Jenkins will use to commen on the PR')
95    argp.add_argument('--counters', dest='counters', action='store_true')
96    argp.add_argument('--no-counters', dest='counters', action='store_false')
97    argp.set_defaults(counters=True)
98    args = argp.parse_args()
99    assert args.diff_base or args.old, "One of diff_base or old must be set!"
100    if args.loops < 3:
101        print "WARNING: This run will likely be noisy. Increase loops."
102    return args
103
104
105def eintr_be_gone(fn):
106    """Run fn until it doesn't stop because of EINTR"""
107
108    def inner(*args):
109        while True:
110            try:
111                return fn(*args)
112            except IOError, e:
113                if e.errno != errno.EINTR:
114                    raise
115
116    return inner
117
118
119def main(args):
120
121    bm_build.build('new', args.benchmarks, args.jobs, args.counters)
122
123    old = args.old
124    if args.diff_base:
125        old = 'old'
126        where_am_i = subprocess.check_output(
127            ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
128        subprocess.check_call(['git', 'checkout', args.diff_base])
129        try:
130            bm_build.build(old, args.benchmarks, args.jobs, args.counters)
131        finally:
132            subprocess.check_call(['git', 'checkout', where_am_i])
133            subprocess.check_call(['git', 'submodule', 'update'])
134
135    jobs_list = []
136    jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops,
137                                    args.regex, args.counters)
138    jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops,
139                                    args.regex, args.counters)
140
141    # shuffle all jobs to eliminate noise from GCE CPU drift
142    random.shuffle(jobs_list, random.SystemRandom().random)
143    jobset.run(jobs_list, maxjobs=args.jobs)
144
145    diff, note = bm_diff.diff(args.benchmarks, args.loops, args.regex,
146                              args.track, old, 'new', args.counters)
147    if diff:
148        text = '[%s] Performance differences noted:\n%s' % (
149            args.pr_comment_name, diff)
150    else:
151        text = '[%s] No significant performance differences' % args.pr_comment_name
152    if note:
153        text = note + '\n\n' + text
154    print('%s' % text)
155    comment_on_pr.comment_on_pr('```\n%s\n```' % text)
156
157
158if __name__ == '__main__':
159    args = _args()
160    main(args)
161