1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""
6Test to generate the AFDO profile for a set of ChromeOS benchmarks.
7
8This will run a pre-determined set of benchmarks on the DUT under
9the monitoring of the linux "perf" tool. The resulting perf.data
10file will then be copied to Google Storage (GS) where it can be
11used by the AFDO optimized build.
12
13Given that the telemetry benchmarks are quite unstable on ChromeOS at
14this point, this test also supports a mode where the benchmarks are
15executed outside of the telemetry framework. It is not the same as
16executing the benchmarks under telemetry because there is no telemetry
17measurement taken but, for the purposes of profiling Chrome, it should
18be pretty close.
19
20Example invocation:
21/usr/bin/test_that --debug --board=lumpy <DUT IP>
22  --args="ignore_failures=True local=True gs_test_location=True"
23  telemetry_AFDOGenerate
24"""
25
26import bz2
27import logging
28import os
29import time
30
31from autotest_lib.client.common_lib import error, utils
32from autotest_lib.server import autotest
33from autotest_lib.server import profilers
34from autotest_lib.server import test
35from autotest_lib.server import utils
36from autotest_lib.server.cros import telemetry_runner
37
38# List of benchmarks to run to capture profile information. This is
39# based on the "superhero" and "perf_v2" list and other telemetry
40# benchmarks. Goal is to have a short list that is as representative
41# as possible and takes a short time to execute. At this point the
42# list of benchmarks is in flux.
43TELEMETRY_AFDO_BENCHMARKS = (
44    ('page_cycler.typical_25', ('--pageset-repeat=2',)),
45    ('page_cycler.intl_ja_zh', ('--pageset-repeat=1',)),
46    ('page_cycler.intl_ar_fa_he', ('--pageset-repeat=1',)),
47    ('page_cycler.intl_es_fr_pt-BR', ('--pageset-repeat=1',)),
48    ('page_cycler.intl_ko_th_vi', ('--pageset-repeat=1',)),
49    ('page_cycler.intl_hi_ru', ('--pageset-repeat=1',)),
50    ('octane',),
51    ('kraken',),
52    ('speedometer',),
53    ('dromaeo.domcoreattr',),
54    ('dromaeo.domcoremodify',),
55    ('smoothness.tough_webgl_cases',)
56    )
57
58# Some benchmarks removed from the profile set:
59# 'page_cycler.morejs' -> uninteresting, seems to fail frequently,
60# 'page_cycler.moz' -> seems very old.
61# 'media.tough_video_cases' -> removed this because it does not bring
62#                              any benefit and takes more than 12 mins
63
64# List of boards where this test can be run.
65# Currently, this has only been tested on 'sandybridge' boards.
66VALID_BOARDS = ['butterfly', 'lumpy', 'parrot', 'stumpy']
67
68class telemetry_AFDOGenerate(test.test):
69    """
70    Run one or more telemetry benchmarks under the "perf" monitoring
71    tool, generate a "perf.data" file and upload to GS for comsumption
72    by the AFDO optimized build.
73    """
74    version = 1
75
76
77    def run_once(self, host, args):
78        """Run a set of telemetry benchmarks.
79
80        @param host: Host machine where test is run
81        @param args: A dictionary of the arguments that were passed
82                to this test.
83        @returns None.
84        """
85        self._host = host
86        host_board = host.get_board().split(':')[1]
87        if not host_board in VALID_BOARDS:
88            raise error.TestFail(
89                    'This test cannot be run on board %s' % host_board)
90
91        self._parse_args(args)
92
93        if self._minimal_telemetry:
94            self._run_tests_minimal_telemetry()
95        else:
96            self._telemetry_runner = telemetry_runner.TelemetryRunner(
97                    self._host, self._local)
98
99            for benchmark_info in TELEMETRY_AFDO_BENCHMARKS:
100                benchmark = benchmark_info[0]
101                args = () if len(benchmark_info) == 1 else benchmark_info[1]
102                try:
103                    self._run_test_with_retry(benchmark, *args)
104                except error.TestBaseException:
105                    if not self._ignore_failures:
106                        raise
107                    else:
108                        logging.info('Ignoring failure from benchmark %s.',
109                                     benchmark)
110
111
112    def after_run_once(self):
113        """After the profile information has been collected, compress it
114        and upload it to GS
115        """
116        PERF_FILE = 'perf.data'
117        COMP_PERF_FILE = 'chromeos-chrome-%s-%s.perf.data'
118        perf_data = os.path.join(self.profdir, PERF_FILE)
119        comp_data = os.path.join(self.profdir, COMP_PERF_FILE % (
120                self._arch, self._version))
121        compressed = self._compress_file(perf_data, comp_data)
122        self._gs_upload(compressed, os.path.basename(compressed))
123
124        # Also create copy of this file using "LATEST" as version so
125        # it can be found in case the builder is looking for a version
126        # number that does not match. It is ok to use a slighly old
127        # version of the this file for the optimized build
128        latest_data =  COMP_PERF_FILE % (self._arch, 'LATEST')
129        latest_compressed = self._get_compressed_name(latest_data)
130        self._gs_upload(compressed, latest_compressed)
131
132
133    def _parse_args(self, args):
134        """Parses input arguments to this autotest.
135
136        @param args: Options->values dictionary.
137        @raises error.TestFail if a bad option is passed.
138        """
139
140        # Set default values for the options.
141        # Architecture for which we are collecting afdo data.
142        self._arch = 'amd64'
143        # Use an alternate GS location where everyone can write.
144        # Set default depending on whether this is executing in
145        # the lab environment or not
146        self._gs_test_location = not utils.host_is_in_lab_zone(
147                self._host.hostname)
148        # Ignore individual test failures.
149        self._ignore_failures = False
150        # Use local copy of telemetry instead of using the dev server copy.
151        self._local = False
152        # Chrome version to which the AFDO data corresponds.
153        self._version, _ = self._host.get_chrome_version()
154        # Try to use the minimal support from Telemetry. The Telemetry
155        # benchmarks in ChromeOS are too flaky at this point. So, initially,
156        # this will be set to True by default.
157        self._minimal_telemetry = False
158
159        for option_name, value in args.iteritems():
160            if option_name == 'arch':
161                self._arch = value
162            elif option_name == 'gs_test_location':
163                self._gs_test_location = (value == 'True')
164            elif option_name == 'ignore_failures':
165                self._ignore_failures = (value == 'True')
166            elif option_name == 'local':
167                self._local = (value == 'True')
168            elif option_name == 'minimal_telemetry':
169                self._minimal_telemetry = (value == 'True')
170            elif option_name == 'version':
171                self._version = value
172            else:
173                raise error.TestFail('Unknown option passed: %s' % option_name)
174
175
176    def _run_test(self, benchmark, *args):
177        """Run the benchmark using Telemetry.
178
179        @param benchmark: Name of the benchmark to run.
180        @param args: Additional arguments to pass to the telemetry execution
181                     script.
182        @raises Raises error.TestFail if execution of test failed.
183                Also re-raise any exceptions thrown by run_telemetry benchmark.
184        """
185        try:
186            logging.info('Starting run for Telemetry benchmark %s', benchmark)
187            start_time = time.time()
188            result = self._telemetry_runner.run_telemetry_benchmark(
189                    benchmark, None, *args)
190            end_time = time.time()
191            logging.info('Completed Telemetry benchmark %s in %f seconds',
192                         benchmark, end_time - start_time)
193        except error.TestBaseException as e:
194            end_time = time.time()
195            logging.info('Got exception from Telemetry benchmark %s '
196                         'after %f seconds. Exception: %s',
197                         benchmark, end_time - start_time, str(e))
198            raise
199
200        # We dont generate any keyvals for this run. This is not
201        # an official run of the benchmark. We are just running it to get
202        # a profile from it.
203
204        if result.status is telemetry_runner.SUCCESS_STATUS:
205            logging.info('Benchmark %s succeeded', benchmark)
206        else:
207            raise error.TestFail('An error occurred while executing'
208                                 ' benchmark: %s' % benchmark)
209
210
211    def _run_test_with_retry(self, benchmark, *args):
212        """Run the benchmark using Telemetry. Retry in case of failure.
213
214        @param benchmark: Name of the benchmark to run.
215        @param args: Additional arguments to pass to the telemetry execution
216                     script.
217        @raises Re-raise any exceptions thrown by _run_test.
218        """
219
220        tried = False
221        while True:
222            try:
223                self._run_test(benchmark, *args)
224                logging.info('Benchmark %s succeeded on %s try',
225                             benchmark,
226                             'first' if not tried else 'second')
227                break
228            except error.TestBaseException:
229                if not tried:
230                   tried = True
231                   logging.info('Benchmark %s failed. Retrying ...',
232                                benchmark)
233                else:
234                    logging.info('Benchmark %s failed twice. Not retrying',
235                                  benchmark)
236                    raise
237
238
239    def _run_tests_minimal_telemetry(self):
240        """Run the benchmarks using the minimal support from Telemetry.
241
242        The benchmarks are run using a client side autotest test. This test
243        will control Chrome directly using the chrome.Chrome support and it
244        will ask Chrome to display the benchmark pages directly instead of
245        using the "page sets" and "measurements" support from Telemetry.
246        In this way we avoid using Telemetry benchmark support which is not
247        stable on ChromeOS yet.
248        """
249        AFDO_GENERATE_CLIENT_TEST = 'telemetry_AFDOGenerateClient'
250
251        # We dont want to "inherit" the profiler settings for this test
252        # to the client test. Doing so will end up in two instances of
253        # the profiler (perf) being executed at the same time.
254        # Filed a feature request about this. See crbug/342958.
255
256        # Save the current settings for profilers.
257        saved_profilers = self.job.profilers
258        saved_default_profile_only = self.job.default_profile_only
259
260        # Reset the state of the profilers.
261        self.job.default_profile_only = False
262        self.job.profilers = profilers.profilers(self.job)
263
264        # Execute the client side test.
265        client_at = autotest.Autotest(self._host)
266        client_at.run_test(AFDO_GENERATE_CLIENT_TEST, args='')
267
268        # Restore the settings for the profilers.
269        self.job.default_profile_only = saved_default_profile_only
270        self.job.profiler = saved_profilers
271
272
273    @staticmethod
274    def _get_compressed_name(name):
275        """Given a file name, return bz2 compressed name.
276        @param name: Name of uncompressed file.
277        @returns name of compressed file.
278        """
279        return name + '.bz2'
280
281    @staticmethod
282    def _compress_file(unc_file, com_file):
283        """Compresses specified file with bz2.
284
285        @param unc_file: name of file to compress.
286        @param com_file: prefix name of compressed file.
287        @raises error.TestFail if compression failed
288        @returns Name of compressed file.
289        """
290        dest = ''
291        with open(unc_file, 'r') as inp:
292            dest = telemetry_AFDOGenerate._get_compressed_name(com_file)
293            with bz2.BZ2File(dest, 'w') as out:
294                for data in inp:
295                    out.write(data)
296        if not dest or not os.path.isfile(dest):
297            raise error.TestFail('Could not compress %s' % unc_file)
298        return dest
299
300
301    def _gs_upload(self, local_file, remote_basename):
302        """Uploads file to google storage specific location.
303
304        @param local_file: name of file to upload.
305        @param remote_basename: basename of remote file.
306        @raises error.TestFail if upload failed.
307        @returns nothing.
308        """
309        GS_DEST = 'gs://chromeos-prebuilt/afdo-job/canonicals/%s'
310        GS_TEST_DEST = 'gs://chromeos-throw-away-bucket/afdo-job/canonicals/%s'
311        GS_ACL = 'project-private'
312
313        gs_dest = GS_TEST_DEST if self._gs_test_location else GS_DEST
314        remote_file = gs_dest % remote_basename
315
316        logging.info('About to upload to GS: %s', remote_file)
317        if not utils.gs_upload(local_file,
318                               remote_file,
319                               GS_ACL, result_dir=self.resultsdir):
320            logging.info('Failed upload to GS: %s', remote_file)
321            raise error.TestFail('Unable to gs upload %s to %s' %
322                                 (local_file, remote_file))
323
324        logging.info('Successfull upload to GS: %s', remote_file)
325