1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import collections
6import logging
7import os
8import re
9import time
10
11from math import sqrt
12
13from autotest_lib.client.bin import test, utils
14from autotest_lib.client.common_lib import error
15from autotest_lib.client.common_lib.cros import chrome
16from autotest_lib.client.cros.graphics import graphics_utils
17from autotest_lib.client.cros.video import helper_logger
18
19from telemetry.timeline import chrome_trace_config
20from telemetry.timeline import tracing_config
21from telemetry.timeline.model import TimelineModel
22
23TEST_PAGE = 'content.html'
24
25# The keys to access the content of memry stats.
26KEY_RENDERER = 'Renderer'
27KEY_BROWSER = 'Browser'
28KEY_GPU = 'GPU Process'
29
30# The number of iterations to be run before measuring the memory usage.
31# Just ensure we have fill up the caches/buffers so that we can get
32# a more stable/correct result.
33WARMUP_COUNT = 50
34
35# Number of iterations per measurement.
36EVALUATION_COUNT = 70
37
38# The minimal number of samples for memory-leak test.
39MEMORY_LEAK_CHECK_MIN_COUNT = 20
40
41# The approximate values of the student's t-distribution at 95% confidence.
42# See http://en.wikipedia.org/wiki/Student's_t-distribution
43T_095 = [None, # No value for degree of freedom 0
44    12.706205, 4.302653, 3.182446, 2.776445, 2.570582, 2.446912, 2.364624,
45     2.306004, 2.262157, 2.228139, 2.200985, 2.178813, 2.160369, 2.144787,
46     2.131450, 2.119905, 2.109816, 2.100922, 2.093024, 2.085963, 2.079614,
47     2.073873, 2.068658, 2.063899, 2.059539, 2.055529, 2.051831, 2.048407,
48     2.045230, 2.042272, 2.039513, 2.036933, 2.034515, 2.032245, 2.030108,
49     2.028094, 2.026192, 2.024394, 2.022691, 2.021075, 2.019541, 2.018082,
50     2.016692, 2.015368, 2.014103, 2.012896, 2.011741, 2.010635, 2.009575,
51     2.008559, 2.007584, 2.006647, 2.005746, 2.004879, 2.004045, 2.003241,
52     2.002465, 2.001717, 2.000995, 2.000298, 1.999624, 1.998972, 1.998341,
53     1.997730, 1.997138, 1.996564, 1.996008, 1.995469, 1.994945, 1.994437,
54     1.993943, 1.993464, 1.992997, 1.992543, 1.992102, 1.991673, 1.991254,
55     1.990847, 1.990450, 1.990063, 1.989686, 1.989319, 1.988960, 1.988610,
56     1.988268, 1.987934, 1.987608, 1.987290, 1.986979, 1.986675, 1.986377,
57     1.986086, 1.985802, 1.985523, 1.985251, 1.984984, 1.984723, 1.984467,
58     1.984217, 1.983972, 1.983731]
59
60# The memory leak (bytes/iteration) we can tolerate.
61MEMORY_LEAK_THRESHOLD = 1024 * 1024
62
63# Regular expression used to parse the content of '/proc/meminfo'
64# The content of the file looks like:
65# MemTotal:       65904768 kB
66# MemFree:        14248152 kB
67# Buffers:          508836 kB
68MEMINFO_RE = re.compile('^(\w+):\s+(\d+)', re.MULTILINE)
69MEMINFO_PATH = '/proc/meminfo'
70
71# We sum up the following values in '/proc/meminfo' to represent
72# the kernel memory usage.
73KERNEL_MEMORY_ENTRIES = ['Slab', 'Shmem', 'KernelStack', 'PageTables']
74
75MEM_TOTAL_ENTRY = 'MemTotal'
76
77# The default sleep time, in seconds.
78SLEEP_TIME = 1.5
79
80
81def _get_kernel_memory_usage():
82    with file(MEMINFO_PATH) as f:
83        mem_info = {x.group(1): int(x.group(2))
84                   for x in MEMINFO_RE.finditer(f.read())}
85    # Sum up the kernel memory usage (in KB) in mem_info
86    return sum(map(mem_info.get, KERNEL_MEMORY_ENTRIES))
87
88def _get_graphics_memory_usage():
89    """Get the memory usage (in KB) of the graphics module."""
90    key = 'gem_objects_bytes'
91    graphics_kernel_memory = graphics_utils.GraphicsKernelMemory()
92    usage = graphics_kernel_memory.get_memory_keyvals().get(key, 0)
93
94    if graphics_kernel_memory.num_errors:
95        logging.warning('graphics memory info is not available')
96        return 0
97
98    # The original value is in bytes
99    return usage / 1024
100
101def _get_linear_regression_slope(x, y):
102    """
103    Gets slope and the confidence interval of the linear regression based on
104    the given xs and ys.
105
106    This function returns a tuple (beta, delta), where the beta is the slope
107    of the linear regression and delta is the range of the confidence
108    interval, i.e., confidence interval = (beta + delta, beta - delta).
109    """
110    assert len(x) == len(y)
111    n = len(x)
112    sx, sy = sum(x), sum(y)
113    sxx = sum(v * v for v in x)
114    syy = sum(v * v for v in y)
115    sxy = sum(u * v for u, v in zip(x, y))
116    beta = float(n * sxy - sx * sy) / (n * sxx - sx * sx)
117    alpha = float(sy - beta * sx) / n
118    stderr2 = (n * syy - sy * sy -
119               beta * beta * (n * sxx - sx * sx)) / (n * (n - 2))
120    std_beta = sqrt((n * stderr2) / (n * sxx - sx * sx))
121    return (beta, T_095[n - 2] * std_beta)
122
123
124def _assert_no_memory_leak(name, mem_usage, threshold = MEMORY_LEAK_THRESHOLD):
125    """Helper function to check memory leak"""
126    index = range(len(mem_usage))
127    slope, delta = _get_linear_regression_slope(index, mem_usage)
128    logging.info('confidence interval: %s - %s, %s',
129                 name, slope - delta, slope + delta)
130    if (slope - delta > threshold):
131        logging.debug('memory usage for %s - %s', name, mem_usage)
132        raise error.TestError('leak detected: %s - %s' % (name, slope - delta))
133
134
135def _output_entries(out, entries):
136    out.write(' '.join(str(x) for x in entries) + '\n')
137    out.flush()
138
139
140class MemoryTest(object):
141    """The base class of all memory tests"""
142
143    def __init__(self, bindir):
144        self._bindir = bindir
145
146
147    def _open_new_tab(self, page_to_open):
148        tab = self.browser.tabs.New()
149        tab.Activate()
150        tab.Navigate(self.browser.platform.http_server.UrlOf(
151                os.path.join(self._bindir, page_to_open)))
152        tab.WaitForDocumentReadyStateToBeComplete()
153        return tab
154
155
156    def _get_memory_usage(self):
157        """Helper function to get the memory usage.
158
159        It returns a tuple of six elements:
160            (browser_usage, renderer_usage, gpu_usage, kernel_usage,
161             total_usage, graphics_usage)
162        All are expected in the unit of KB.
163
164        browser_usage: the RSS of the browser process
165        renderer_usage: the total RSS of all renderer processes
166        gpu_usage: the total RSS of all gpu processes
167        kernel_usage: the memory used in kernel
168        total_usage: the sum of the above memory usages. The graphics_usage is
169                     not included because the composition of the graphics
170                     memory is much more complicated (could be from video card,
171                     user space, or kenerl space). It doesn't make so much
172                     sense to sum it up with others.
173        graphics_usage: the memory usage reported by the graphics driver
174        """
175
176        config = tracing_config.TracingConfig()
177        config.chrome_trace_config.category_filter.AddExcludedCategory("*")
178        config.chrome_trace_config.category_filter.AddDisabledByDefault(
179                "disabled-by-default-memory-infra")
180        config.chrome_trace_config.SetMemoryDumpConfig(
181                chrome_trace_config.MemoryDumpConfig())
182        config.enable_chrome_trace = True
183        self.browser.platform.tracing_controller.StartTracing(config)
184
185        # Force to collect garbage before measuring memory
186        for t in self.browser.tabs:
187            t.CollectGarbage()
188
189        self.browser.DumpMemory()
190
191        trace_data = self.browser.platform.tracing_controller.StopTracing()
192        model = TimelineModel(trace_data)
193        memory_dump = model.IterGlobalMemoryDumps().next()
194        process_memory = collections.defaultdict(int)
195        for process_memory_dump in memory_dump.IterProcessMemoryDumps():
196            process_name = process_memory_dump.process_name
197            process_memory[process_name] += sum(
198                    process_memory_dump.GetMemoryUsage().values())
199
200        result = (process_memory[KEY_BROWSER] / 1024,
201                  process_memory[KEY_RENDERER] / 1024,
202                  process_memory[KEY_GPU] / 1024,
203                  _get_kernel_memory_usage())
204
205        # total = browser + renderer + gpu + kernal
206        result += (sum(result), _get_graphics_memory_usage())
207        return result
208
209
210    def initialize(self):
211        """A callback function. It is just called before the main loops."""
212        pass
213
214
215    def loop(self):
216        """A callback function. It is the main memory test function."""
217        pass
218
219
220    def cleanup(self):
221        """A callback function, executed after loop()."""
222        pass
223
224
225    def run(self, name, browser, videos, test,
226            warmup_count=WARMUP_COUNT,
227            eval_count=EVALUATION_COUNT):
228        """Runs this memory test case.
229
230        @param name: the name of the test.
231        @param browser: the telemetry entry of the browser under test.
232        @param videos: the videos to be used in the test.
233        @param test: the autotest itself, used to output performance values.
234        @param warmup_count: run loop() for warmup_count times to make sure the
235               memory usage has been stabalize.
236        @param eval_count: run loop() for eval_count times to measure the memory
237               usage.
238        """
239
240        self.browser = browser
241        self.videos = videos
242        self.name = name
243
244        names = ['browser', 'renderers', 'gpu', 'kernel', 'total', 'graphics']
245        result_log = open(os.path.join(test.resultsdir, '%s.log' % name), 'wt')
246        _output_entries(result_log, names)
247
248        self.initialize()
249        try:
250            for i in xrange(warmup_count):
251                self.loop()
252                _output_entries(result_log, self._get_memory_usage())
253
254            metrics = []
255            for i in xrange(eval_count):
256                self.loop()
257                results = self._get_memory_usage()
258                _output_entries(result_log, results)
259                metrics.append(results)
260
261                # Check memory leak when we have enough samples
262                if len(metrics) >= MEMORY_LEAK_CHECK_MIN_COUNT:
263                    # Assert no leak in the 'total' and 'graphics' usages
264                    for index in map(names.index, ('total', 'graphics')):
265                        _assert_no_memory_leak(
266                            self.name, [m[index] for m in metrics])
267
268            indices = range(len(metrics))
269
270            # Prefix the test name to each metric's name
271            fullnames = ['%s.%s' % (name, n) for n in names]
272
273            # Transpose metrics, and iterate each type of memory usage
274            for name, metric in zip(fullnames, zip(*metrics)):
275                memory_increase_per_run, _ = _get_linear_regression_slope(
276                    indices, metric)
277                logging.info('memory increment for %s - %s',
278                    name, memory_increase_per_run)
279                test.output_perf_value(description=name,
280                        value=memory_increase_per_run,
281                        units='KB', higher_is_better=False)
282        finally:
283            self.cleanup()
284
285
286def _change_source_and_play(tab, video):
287    tab.EvaluateJavaScript('changeSourceAndPlay("%s")' % video)
288
289
290def _assert_video_is_playing(tab):
291    if not tab.EvaluateJavaScript('isVideoPlaying()'):
292        raise error.TestError('video is stopped')
293
294    # The above check may fail. Be sure the video time is advancing.
295    startTime = tab.EvaluateJavaScript('getVideoCurrentTime()')
296
297    def _is_video_playing():
298        return startTime != tab.EvaluateJavaScript('getVideoCurrentTime()')
299
300    utils.poll_for_condition(
301            _is_video_playing, exception=error.TestError('video is stuck'))
302
303
304class OpenTabPlayVideo(MemoryTest):
305    """A memory test case:
306        Open a tab, play a video and close the tab.
307    """
308
309    def loop(self):
310        tab = self._open_new_tab(TEST_PAGE)
311        _change_source_and_play(tab, self.videos[0])
312        _assert_video_is_playing(tab)
313        time.sleep(SLEEP_TIME)
314        tab.Close()
315
316        # Wait a while for the closed tab to clean up all used resources
317        time.sleep(SLEEP_TIME)
318
319
320class PlayVideo(MemoryTest):
321    """A memory test case: keep playing a video."""
322
323    def initialize(self):
324        super(PlayVideo, self).initialize()
325        self.activeTab = self._open_new_tab(TEST_PAGE)
326        _change_source_and_play(self.activeTab, self.videos[0])
327
328
329    def loop(self):
330        time.sleep(SLEEP_TIME)
331        _assert_video_is_playing(self.activeTab)
332
333
334    def cleanup(self):
335        self.activeTab.Close()
336
337
338class ChangeVideoSource(MemoryTest):
339    """A memory test case: change the "src" property of <video> object to
340    load different video sources."""
341
342    def initialize(self):
343        super(ChangeVideoSource, self).initialize()
344        self.activeTab = self._open_new_tab(TEST_PAGE)
345
346
347    def loop(self):
348        for video in self.videos:
349            _change_source_and_play(self.activeTab, video)
350            time.sleep(SLEEP_TIME)
351            _assert_video_is_playing(self.activeTab)
352
353
354    def cleanup(self):
355        self.activeTab.Close()
356
357
358def _get_testcase_name(class_name, videos):
359    # Convert from Camel to underscrore.
360    s = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', class_name)
361    s = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s).lower()
362
363    # Get a shorter name from the first video's URL.
364    # For example, get 'tp101.mp4' from the URL:
365    # 'http://host/path/tpe101-1024x768-9123456780123456.mp4'
366    m = re.match('.*/(\w+)-.*\.(\w+)', videos[0])
367
368    return '%s.%s.%s' % (m.group(1), m.group(2), s)
369
370
371# Deprecate the logging messages at DEBUG level (and lower) in telemetry.
372# http://crbug.com/331992
373class TelemetryFilter(logging.Filter):
374    """Filter for telemetry logging."""
375
376    def filter(self, record):
377        return (record.levelno > logging.DEBUG or
378            'telemetry' not in record.pathname)
379
380
381class video_VideoDecodeMemoryUsage(test.test):
382    """This is a memory usage test for video playback."""
383    version = 1
384
385    @helper_logger.video_log_wrapper
386    def run_once(self, testcases):
387        last_error = None
388        logging.getLogger().addFilter(TelemetryFilter())
389
390        with chrome.Chrome(
391                extra_browser_args=helper_logger.chrome_vmodule_flag(),
392                init_network_controller=True) as cr:
393            cr.browser.platform.SetHTTPServerDirectories(self.bindir)
394            for class_name, videos in testcases:
395                name = _get_testcase_name(class_name, videos)
396                logging.info('run: %s - %s', name, videos)
397                try :
398                    test_case_class = globals()[class_name]
399                    test_case_class(self.bindir).run(
400                            name, cr.browser, videos, self)
401                except Exception as last_error:
402                    logging.exception('%s fail', name)
403                    # continue to next test case
404
405        if last_error:
406            raise  # the last_error
407