1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""This is a client side WebGL aquarium test.
6
7Description of some of the test result output:
8    - interframe time: The time elapsed between two frames. It is the elapsed
9            time between two consecutive calls to the render() function.
10    - render time: The time it takes in Javascript to construct a frame and
11            submit all the GL commands. It is the time it takes for a render()
12            function call to complete.
13"""
14
15import functools
16import logging
17import math
18import os
19import sampler
20import system_sampler
21import threading
22import time
23
24from autotest_lib.client.bin import fps_meter
25from autotest_lib.client.bin import utils
26from autotest_lib.client.common_lib import error
27from autotest_lib.client.common_lib.cros import chrome
28from autotest_lib.client.common_lib.cros import memory_eater
29from autotest_lib.client.cros.graphics import graphics_utils
30from autotest_lib.client.cros import perf
31from autotest_lib.client.cros import service_stopper
32from autotest_lib.client.cros.power import power_rapl, power_status, power_utils
33
34# Minimum battery charge percentage to run the test
35BATTERY_INITIAL_CHARGED_MIN = 10
36
37# Measurement duration in seconds.
38MEASUREMENT_DURATION = 30
39
40POWER_DESCRIPTION = 'avg_energy_rate_1000_fishes'
41
42# Time to exclude from calculation after playing a webgl demo [seconds].
43STABILIZATION_DURATION = 10
44
45
46class graphics_WebGLAquarium(graphics_utils.GraphicsTest):
47    """WebGL aquarium graphics test."""
48    version = 1
49
50    _backlight = None
51    _power_status = None
52    _service_stopper = None
53    _test_power = False
54    active_tab = None
55    flip_stats = {}
56    kernel_sampler = None
57    perf_keyval = {}
58    sampler_lock = None
59    test_duration_secs = 30
60    test_setting_num_fishes = 50
61    test_settings = {
62        50: ('setSetting2', 2),
63        1000: ('setSetting6', 6),
64    }
65
66    def setup(self):
67        """Testcase setup."""
68        tarball_path = os.path.join(self.bindir,
69                                    'webgl_aquarium_static.tar.bz2')
70        utils.extract_tarball_to_dir(tarball_path, self.srcdir)
71
72    def initialize(self):
73        """Testcase initialization."""
74        super(graphics_WebGLAquarium, self).initialize()
75        self.sampler_lock = threading.Lock()
76        # TODO: Create samplers for other platforms (e.g. x86).
77        if utils.get_board().lower() in ['daisy', 'daisy_spring']:
78            # Enable ExynosSampler on Exynos platforms.  The sampler looks for
79            # exynos-drm page flip states: 'wait_kds', 'rendered', 'prepared',
80            # and 'flipped' in kernel debugfs.
81
82            # Sample 3-second durtaion for every 5 seconds.
83            self.kernel_sampler = sampler.ExynosSampler(period=5, duration=3)
84            self.kernel_sampler.sampler_callback = self.exynos_sampler_callback
85            self.kernel_sampler.output_flip_stats = (
86                self.exynos_output_flip_stats)
87
88    def cleanup(self):
89        """Testcase cleanup."""
90        if self._backlight:
91            self._backlight.restore()
92        if self._service_stopper:
93            self._service_stopper.restore_services()
94        super(graphics_WebGLAquarium, self).cleanup()
95
96    def setup_webpage(self, browser, test_url, num_fishes):
97        """Open fish tank in a new tab.
98
99        @param browser: The Browser object to run the test with.
100        @param test_url: The URL to the aquarium test site.
101        @param num_fishes: The number of fishes to run the test with.
102        """
103        # Create tab and load page. Set the number of fishes when page is fully
104        # loaded.
105        tab = browser.tabs.New()
106        tab.Navigate(test_url)
107        tab.Activate()
108        self.active_tab = tab
109        tab.WaitForDocumentReadyStateToBeComplete()
110
111        # Set the number of fishes when document finishes loading.  Also reset
112        # our own FPS counter and start recording FPS and rendering time.
113        utils.wait_for_value(
114            lambda: tab.EvaluateJavaScript(
115                'if (document.readyState === "complete") {'
116                '  setSetting(document.getElementById("%s"), %d);'
117                '  g_crosFpsCounter.reset();'
118                '  true;'
119                '} else {'
120                '  false;'
121                '}' % self.test_settings[num_fishes]
122            ),
123            expected_value=True,
124            timeout_sec=30)
125
126        return tab
127
128    def tear_down_webpage(self):
129        """Close the tab containing testing webpage."""
130        # Do not close the tab when the sampler_callback is
131        # doing its work.
132        with self.sampler_lock:
133            self.active_tab.Close()
134            self.active_tab = None
135
136    def run_fish_test(self, browser, test_url, num_fishes, perf_log=True):
137        """Run the test with the given number of fishes.
138
139        @param browser: The Browser object to run the test with.
140        @param test_url: The URL to the aquarium test site.
141        @param num_fishes: The number of fishes to run the test with.
142        @param perf_log: Report perf data only if it's set to True.
143        """
144
145        tab = self.setup_webpage(browser, test_url, num_fishes)
146
147        if self.kernel_sampler:
148            self.kernel_sampler.start_sampling_thread()
149        time.sleep(self.test_duration_secs)
150        if self.kernel_sampler:
151            self.kernel_sampler.stop_sampling_thread()
152            self.kernel_sampler.output_flip_stats('flip_stats_%d' % num_fishes)
153            self.flip_stats = {}
154
155        # Get average FPS and rendering time, then close the tab.
156        avg_fps = tab.EvaluateJavaScript('g_crosFpsCounter.getAvgFps();')
157        if math.isnan(float(avg_fps)):
158            raise error.TestFail('Failed: Could not get FPS count.')
159
160        avg_interframe_time = tab.EvaluateJavaScript(
161            'g_crosFpsCounter.getAvgInterFrameTime();')
162        avg_render_time = tab.EvaluateJavaScript(
163            'g_crosFpsCounter.getAvgRenderTime();')
164        std_interframe_time = tab.EvaluateJavaScript(
165            'g_crosFpsCounter.getStdInterFrameTime();')
166        std_render_time = tab.EvaluateJavaScript(
167            'g_crosFpsCounter.getStdRenderTime();')
168        self.perf_keyval['avg_fps_%04d_fishes' % num_fishes] = avg_fps
169        self.perf_keyval['avg_interframe_time_%04d_fishes' % num_fishes] = (
170            avg_interframe_time)
171        self.perf_keyval['avg_render_time_%04d_fishes' % num_fishes] = (
172            avg_render_time)
173        self.perf_keyval['std_interframe_time_%04d_fishes' % num_fishes] = (
174            std_interframe_time)
175        self.perf_keyval['std_render_time_%04d_fishes' % num_fishes] = (
176            std_render_time)
177        logging.info('%d fish(es): Average FPS = %f, '
178                     'average render time = %f', num_fishes, avg_fps,
179                     avg_render_time)
180
181        if perf_log:
182            # Report frames per second to chromeperf/ dashboard.
183            self.output_perf_value(
184                description='avg_fps_%04d_fishes' % num_fishes,
185                value=avg_fps,
186                units='fps',
187                higher_is_better=True)
188
189            # Intel only: Record the power consumption for the next few seconds.
190            rapl_rate = power_rapl.get_rapl_measurement(
191                'rapl_%04d_fishes' % num_fishes)
192            # Remove entries that we don't care about.
193            rapl_rate = {key: rapl_rate[key]
194                         for key in rapl_rate.keys() if key.endswith('pwr')}
195            # Report to chromeperf/ dashboard.
196            for key, values in rapl_rate.iteritems():
197                self.output_perf_value(
198                    description=key,
199                    value=values,
200                    units='W',
201                    higher_is_better=False,
202                    graph='rapl_power_consumption'
203                )
204
205    def run_power_test(self, browser, test_url, ac_ok):
206        """Runs the webgl power consumption test and reports the perf results.
207
208        @param browser: The Browser object to run the test with.
209        @param test_url: The URL to the aquarium test site.
210        @param ac_ok: Boolean on whether its ok to have AC power supplied.
211        """
212
213        self._backlight = power_utils.Backlight()
214        self._backlight.set_default()
215
216        self._service_stopper = service_stopper.ServiceStopper(
217            service_stopper.ServiceStopper.POWER_DRAW_SERVICES)
218        self._service_stopper.stop_services()
219
220        if not ac_ok:
221            self._power_status = power_status.get_status()
222            # Verify that we are running on battery and the battery is
223            # sufficiently charged.
224            self._power_status.assert_battery_state(BATTERY_INITIAL_CHARGED_MIN)
225
226            measurements = [
227                power_status.SystemPower(self._power_status.battery_path)
228            ]
229
230        def get_power():
231            power_logger = power_status.PowerLogger(measurements)
232            power_logger.start()
233            time.sleep(STABILIZATION_DURATION)
234            start_time = time.time()
235            time.sleep(MEASUREMENT_DURATION)
236            power_logger.checkpoint('result', start_time)
237            keyval = power_logger.calc()
238            logging.info('Power output %s', keyval)
239            return keyval['result_' + measurements[0].domain + '_pwr']
240
241        self.run_fish_test(browser, test_url, 1000, perf_log=False)
242        if not ac_ok:
243            energy_rate = get_power()
244            # This is a power specific test so we are not capturing
245            # avg_fps and avg_render_time in this test.
246            self.perf_keyval[POWER_DESCRIPTION] = energy_rate
247            self.output_perf_value(
248                description=POWER_DESCRIPTION,
249                value=energy_rate,
250                units='W',
251                higher_is_better=False)
252
253    def exynos_sampler_callback(self, sampler_obj):
254        """Sampler callback function for ExynosSampler.
255
256        @param sampler_obj: The ExynosSampler object that invokes this callback
257                function.
258        """
259        if sampler_obj.stopped:
260            return
261
262        with self.sampler_lock:
263            now = time.time()
264            results = {}
265            info_str = ['\nfb_id wait_kds flipped']
266            for value in sampler_obj.frame_buffers.itervalues():
267                results[value.fb] = {}
268                for state, stats in value.states.iteritems():
269                    results[value.fb][state] = (stats.avg, stats.stdev)
270                info_str.append('%s: %s %s' % (value.fb,
271                                               results[value.fb]['wait_kds'][0],
272                                               results[value.fb]['flipped'][0]))
273            results['avg_fps'] = self.active_tab.EvaluateJavaScript(
274                'g_crosFpsCounter.getAvgFps();')
275            results['avg_render_time'] = self.active_tab.EvaluateJavaScript(
276                'g_crosFpsCounter.getAvgRenderTime();')
277            self.active_tab.ExecuteJavaScript('g_crosFpsCounter.reset();')
278            info_str.append('avg_fps: %s, avg_render_time: %s' %
279                            (results['avg_fps'], results['avg_render_time']))
280            self.flip_stats[now] = results
281            logging.info('\n'.join(info_str))
282
283    def exynos_output_flip_stats(self, file_name):
284        """Pageflip statistics output function for ExynosSampler.
285
286        @param file_name: The output file name.
287        """
288        # output format:
289        # time fb_id avg_rendered avg_prepared avg_wait_kds avg_flipped
290        # std_rendered std_prepared std_wait_kds std_flipped
291        with open(file_name, 'w') as f:
292            for t in sorted(self.flip_stats.keys()):
293                if ('avg_fps' in self.flip_stats[t] and
294                        'avg_render_time' in self.flip_stats[t]):
295                    f.write('%s %s %s\n' %
296                            (t, self.flip_stats[t]['avg_fps'],
297                             self.flip_stats[t]['avg_render_time']))
298                for fb, stats in self.flip_stats[t].iteritems():
299                    if not isinstance(fb, int):
300                        continue
301                    f.write('%s %s ' % (t, fb))
302                    f.write('%s %s %s %s ' % (stats['rendered'][0],
303                                              stats['prepared'][0],
304                                              stats['wait_kds'][0],
305                                              stats['flipped'][0]))
306                    f.write('%s %s %s %s\n' % (stats['rendered'][1],
307                                               stats['prepared'][1],
308                                               stats['wait_kds'][1],
309                                               stats['flipped'][1]))
310
311    def write_samples(self, samples, filename):
312        """Writes all samples to result dir with the file name "samples'.
313
314        @param samples: A list of all collected samples.
315        @param filename: The file name to save under result directory.
316        """
317        out_file = os.path.join(self.resultsdir, filename)
318        with open(out_file, 'w') as f:
319            for sample in samples:
320                print >> f, sample
321
322    def run_fish_test_with_memory_pressure(
323        self, browser, test_url, num_fishes, memory_pressure):
324        """Measure fps under memory pressure.
325
326        It measure FPS of WebGL aquarium while adding memory pressure. It runs
327        in 2 phases:
328          1. Allocate non-swappable memory until |memory_to_reserve_mb| is
329          remained. The memory is not accessed after allocated.
330          2. Run "active" memory consumer in the background. After allocated,
331          Its content is accessed sequentially by page and looped around
332          infinitely.
333        The second phase is opeared in two possible modes:
334          1. "single" mode, which means only one "active" memory consumer. After
335          running a single memory consumer with a given memory size, it waits
336          for a while to see if system can afford current memory pressure
337          (definition here is FPS > 5). If it does, kill current consumer and
338          launch another consumer with a larger memory size. The process keeps
339          going until system couldn't afford the load.
340          2. "multiple"mode. It simply launch memory consumers with a given size
341          one by one until system couldn't afford the load (e.g., FPS < 5).
342          In "single" mode, CPU load is lighter so we expect swap in/swap out
343          rate to be correlated to FPS better. In "multiple" mode, since there
344          are multiple busy loop processes, CPU pressure is another significant
345          cause of frame drop. Frame drop can happen easily due to busy CPU
346          instead of memory pressure.
347
348        @param browser: The Browser object to run the test with.
349        @param test_url: The URL to the aquarium test site.
350        @param num_fishes: The number of fishes to run the test with.
351        @param memory_pressure: Memory pressure parameters.
352        """
353        consumer_mode = memory_pressure.get('consumer_mode', 'single')
354        memory_to_reserve_mb = memory_pressure.get('memory_to_reserve_mb', 500)
355        # Empirical number to quickly produce memory pressure.
356        if consumer_mode == 'single':
357            default_consumer_size_mb = memory_to_reserve_mb + 100
358        else:
359            default_consumer_size_mb = memory_to_reserve_mb / 2
360        consumer_size_mb = memory_pressure.get(
361            'consumer_size_mb', default_consumer_size_mb)
362
363        # Setup fish tank.
364        self.setup_webpage(browser, test_url, num_fishes)
365
366        # Drop all file caches.
367        utils.drop_caches()
368
369        def fps_near_zero(fps_sampler):
370            """Returns whether recent fps goes down to near 0.
371
372            @param fps_sampler: A system_sampler.Sampler object.
373            """
374            last_fps = fps_sampler.get_last_avg_fps(6)
375            if last_fps:
376                logging.info('last fps %f', last_fps)
377                if last_fps <= 5:
378                    return True
379            return False
380
381        max_allocated_mb = 0
382        # Consume free memory and release them by the end.
383        with memory_eater.consume_free_memory(memory_to_reserve_mb):
384            fps_sampler = system_sampler.SystemSampler(
385                memory_eater.MemoryEater.get_active_consumer_pids)
386            end_condition = functools.partial(fps_near_zero, fps_sampler)
387            with fps_meter.FPSMeter(fps_sampler.sample):
388                # Collects some samples before running memory pressure.
389                time.sleep(5)
390                try:
391                    if consumer_mode == 'single':
392                        # A single run couldn't generate samples representative
393                        # enough.
394                        # First runs squeeze more inactive anonymous memory into
395                        # zram so in later runs we have a more stable memory
396                        # stat.
397                        max_allocated_mb = max(
398                            memory_eater.run_single_memory_pressure(
399                                consumer_size_mb, 100, end_condition, 10, 3,
400                                900),
401                            memory_eater.run_single_memory_pressure(
402                                consumer_size_mb, 20, end_condition, 10, 3,
403                                900),
404                            memory_eater.run_single_memory_pressure(
405                                consumer_size_mb, 10, end_condition, 10, 3,
406                                900))
407                    elif consumer_mode == 'multiple':
408                        max_allocated_mb = (
409                            memory_eater.run_multi_memory_pressure(
410                                consumer_size_mb, end_condition, 10, 900))
411                    else:
412                        raise error.TestFail(
413                            'Failed: Unsupported consumer mode.')
414                except memory_eater.TimeoutException as e:
415                    raise error.TestFail(e)
416
417        samples = fps_sampler.get_samples()
418        self.write_samples(samples, 'memory_pressure_fps_samples.txt')
419
420        self.perf_keyval['num_samples'] = len(samples)
421        self.perf_keyval['max_allocated_mb'] = max_allocated_mb
422
423        logging.info(self.perf_keyval)
424
425        self.output_perf_value(
426            description='max_allocated_mb_%d_fishes_reserved_%d_mb' % (
427                num_fishes, memory_to_reserve_mb),
428            value=max_allocated_mb,
429            units='MB',
430            higher_is_better=True)
431
432
433    @graphics_utils.GraphicsTest.failure_report_decorator('graphics_WebGLAquarium')
434    def run_once(self,
435                 test_duration_secs=30,
436                 test_setting_num_fishes=(50, 1000),
437                 power_test=False,
438                 ac_ok=False,
439                 memory_pressure=None):
440        """Find a browser with telemetry, and run the test.
441
442        @param test_duration_secs: The duration in seconds to run each scenario
443                for.
444        @param test_setting_num_fishes: A list of the numbers of fishes to
445                enable in the test.
446        @param power_test: Boolean on whether to run power_test
447        @param ac_ok: Boolean on whether its ok to have AC power supplied.
448        @param memory_pressure: A dictionay which specifies memory pressure
449                parameters:
450                'consumer_mode': 'single' or 'multiple' to have one or moultiple
451                concurrent memory consumers.
452                'consumer_size_mb': Amount of memory to allocate. In 'single'
453                mode, a single memory consumer would allocate memory by the
454                specific size. It then gradually allocates more memory until
455                FPS down to near 0. In 'multiple' mode, memory consumers of
456                this size would be spawn one by one until FPS down to near 0.
457                'memory_to_reserve_mb': Amount of memory to reserve before
458                running memory consumer. In practical we allocate mlocked
459                memory (i.e., not swappable) to consume free memory until this
460                amount of free memory remained.
461        """
462        self.test_duration_secs = test_duration_secs
463        self.test_setting_num_fishes = test_setting_num_fishes
464        pc_error_reason = None
465
466        with chrome.Chrome(logged_in=False, init_network_controller=True) as cr:
467            cr.browser.platform.SetHTTPServerDirectories(self.srcdir)
468            test_url = cr.browser.platform.http_server.UrlOf(
469                os.path.join(self.srcdir, 'aquarium.html'))
470
471            utils.report_temperature(self, 'temperature_1_start')
472            # Wrap the test run inside of a PerfControl instance to make machine
473            # behavior more consistent.
474            with perf.PerfControl() as pc:
475                if not pc.verify_is_valid():
476                    raise error.TestFail('Failed: %s' % pc.get_error_reason())
477                utils.report_temperature(self, 'temperature_2_before_test')
478
479                if memory_pressure:
480                    self.run_fish_test_with_memory_pressure(
481                        cr.browser, test_url, num_fishes=1000,
482                        memory_pressure=memory_pressure)
483                    self.tear_down_webpage()
484                elif power_test:
485                    self._test_power = True
486                    self.run_power_test(cr.browser, test_url, ac_ok)
487                    self.tear_down_webpage()
488                else:
489                    for n in self.test_setting_num_fishes:
490                        self.run_fish_test(cr.browser, test_url, n)
491                        self.tear_down_webpage()
492
493                if not pc.verify_is_valid():
494                    # Defer error handling until after perf report.
495                    pc_error_reason = pc.get_error_reason()
496
497        utils.report_temperature(self, 'temperature_3_after_test')
498        self.write_perf_keyval(self.perf_keyval)
499
500        if pc_error_reason:
501            raise error.TestWarn('Warning: %s' % pc_error_reason)
502