1# Copyright (c) 2017 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import collections
6import json
7import logging
8import numpy
9import operator
10import os
11import re
12import time
13import urllib
14import urllib2
15
16from autotest_lib.client.bin import utils
17from autotest_lib.client.common_lib import error
18from autotest_lib.client.common_lib import lsbrelease_utils
19from autotest_lib.client.common_lib.cros import retry
20from autotest_lib.client.cros.power import power_status
21from autotest_lib.client.cros.power import power_utils
22
23_HTML_CHART_STR = '''
24<!DOCTYPE html>
25<html>
26<head>
27<script type="text/javascript" src="https://www.gstatic.com/charts/loader.js">
28</script>
29<script type="text/javascript">
30    google.charts.load('current', {{'packages':['corechart']}});
31    google.charts.setOnLoadCallback(drawChart);
32    function drawChart() {{
33        var data = google.visualization.arrayToDataTable([
34{data}
35        ]);
36        var numDataCols = data.getNumberOfColumns() - 1;
37        var unit = '{unit}';
38        var options = {{
39            width: 1600,
40            height: 1200,
41            lineWidth: 1,
42            legend: {{ position: 'top', maxLines: 3 }},
43            vAxis: {{ viewWindow: {{min: 0}}, title: '{type} ({unit})' }},
44            hAxis: {{ viewWindow: {{min: 0}}, title: 'time (second)' }},
45        }};
46        var element = document.getElementById('{type}');
47        var chart;
48        if (unit == 'percent') {{
49            options['isStacked'] = true;
50            if (numDataCols == 2) {{
51                options['colors'] = ['#d32f2f', '#43a047']
52            }} else if (numDataCols <= 4) {{
53                options['colors'] = ['#d32f2f', '#f4c7c3', '#cddc39','#43a047'];
54            }} else if (numDataCols <= 9) {{
55                options['colors'] = ['#d32f2f', '#e57373', '#f4c7c3', '#ffccbc',
56                        '#f0f4c3', '#c8e6c9', '#cddc39', '#81c784', '#43a047'];
57            }}
58            chart = new google.visualization.SteppedAreaChart(element);
59        }} else {{
60            chart = new google.visualization.LineChart(element);
61        }}
62        chart.draw(data, options);
63    }}
64</script>
65</head>
66<body>
67<div id="{type}"></div>
68</body>
69</html>
70'''
71
72_HTML_LINK_STR = '''
73<!DOCTYPE html>
74<html>
75<body>
76<a href="http://chrome-power.appspot.com/dashboard?board={board}&test={test}&datetime={datetime}">
77  Link to power dashboard
78</a>
79</body>
80</html>
81'''
82
83
84class BaseDashboard(object):
85    """Base class that implements method for prepare and upload data to power
86    dashboard.
87    """
88
89    def __init__(self, logger, testname, start_ts=None, resultsdir=None,
90                 uploadurl=None):
91        """Create BaseDashboard objects.
92
93        Args:
94            logger: object that store the log. This will get convert to
95                    dictionary by self._convert()
96            testname: name of current test
97            start_ts: timestamp of when test started in seconds since epoch
98            resultsdir: directory to save the power json
99            uploadurl: url to upload power data
100        """
101        self._logger = logger
102        self._testname = testname
103        self._start_ts = start_ts if start_ts else time.time()
104        self._resultsdir = resultsdir
105        self._uploadurl = uploadurl
106
107    def _create_powerlog_dict(self, raw_measurement):
108        """Create powerlog dictionary from raw measurement data
109        Data format in go/power-dashboard-data.
110
111        Args:
112            raw_measurement: dictionary contains raw measurement data
113
114        Returns:
115            A dictionary of powerlog
116        """
117        powerlog_dict = {
118            'format_version': 5,
119            'timestamp': self._start_ts,
120            'test': self._testname,
121            'dut': self._create_dut_info_dict(raw_measurement['data'].keys()),
122            'power': raw_measurement,
123        }
124
125        return powerlog_dict
126
127    def _create_dut_info_dict(self, power_rails):
128        """Create a dictionary that contain information of the DUT.
129
130        MUST be implemented in subclass.
131
132        Args:
133            power_rails: list of measured power rails
134
135        Returns:
136            DUT info dictionary
137        """
138        raise NotImplementedError
139
140    def _save_json(self, powerlog_dict, resultsdir, filename='power_log.json'):
141        """Convert powerlog dict to human readable formatted JSON and
142        append to <resultsdir>/<filename>.
143
144        Args:
145            powerlog_dict: dictionary of power data
146            resultsdir: directory to save formatted JSON object
147            filename: filename to append to
148        """
149        if not os.path.exists(resultsdir):
150            raise error.TestError('resultsdir %s does not exist.' % resultsdir)
151        filename = os.path.join(resultsdir, filename)
152        json_str = json.dumps(powerlog_dict, indent=4, separators=(',', ': '),
153                              ensure_ascii=False)
154        json_str = utils.strip_non_printable(json_str)
155        with file(filename, 'a') as f:
156            f.write(json_str)
157
158    def _save_html(self, powerlog_dict, resultsdir, filename='power_log.html'):
159        """Convert powerlog dict to chart in HTML page and append to
160        <resultsdir>/<filename>.
161
162        Note that this results in multiple HTML objects in one file but Chrome
163        can render all of it in one page.
164
165        Args:
166            powerlog_dict: dictionary of power data
167            resultsdir: directory to save HTML page
168            filename: filename to append to
169        """
170        # Generate link to power dashboard,
171        board = powerlog_dict['dut']['board']
172        test = powerlog_dict['test']
173        datetime = time.strftime('%Y%m%d%H%M',
174                                 time.gmtime(powerlog_dict['timestamp']))
175
176        html_str = _HTML_LINK_STR.format(board=board,
177                                         test=test,
178                                         datetime=datetime)
179
180        # Create dict from type to sorted list of rail names.
181        rail_type = collections.defaultdict(list)
182        for r, t in powerlog_dict['power']['type'].iteritems():
183            rail_type[t].append(r)
184        for t in rail_type:
185            rail_type[t] = sorted(rail_type[t])
186
187        row_indent = ' ' * 12
188        for t in rail_type:
189            data_str_list = []
190
191            # Generate rail name data string.
192            header = ['time'] + rail_type[t]
193            header_str = row_indent + "['" + "', '".join(header) + "']"
194            data_str_list.append(header_str)
195
196            # Generate measurements data string.
197            for i in range(powerlog_dict['power']['sample_count']):
198                row = [str(i * powerlog_dict['power']['sample_duration'])]
199                for r in rail_type[t]:
200                    row.append(str(powerlog_dict['power']['data'][r][i]))
201                row_str = row_indent + '[' + ', '.join(row) + ']'
202                data_str_list.append(row_str)
203
204            data_str = ',\n'.join(data_str_list)
205            unit = powerlog_dict['power']['unit'][rail_type[t][0]]
206            html_str += _HTML_CHART_STR.format(data=data_str, unit=unit, type=t)
207
208        if not os.path.exists(resultsdir):
209            raise error.TestError('resultsdir %s does not exist.' % resultsdir)
210        filename = os.path.join(resultsdir, filename)
211        with file(filename, 'a') as f:
212            f.write(html_str)
213
214    def _upload(self, powerlog_dict, uploadurl):
215        """Convert powerlog dict to minimal size JSON and upload to dashboard.
216
217        Args:
218            powerlog_dict: dictionary of power data
219            uploadurl: url to upload the power data
220        """
221        json_str = json.dumps(powerlog_dict, ensure_ascii=False)
222        data_obj = {'data': utils.strip_non_printable(json_str)}
223        encoded = urllib.urlencode(data_obj)
224        req = urllib2.Request(uploadurl, encoded)
225
226        @retry.retry(urllib2.URLError, raiselist=[urllib2.HTTPError],
227                     timeout_min=5.0, delay_sec=1, backoff=2)
228        def _do_upload():
229            urllib2.urlopen(req)
230
231        _do_upload()
232
233    def _create_checkpoint_dict(self):
234        """Create dictionary for checkpoint.
235
236        @returns a dictionary of tags to their corresponding intervals in the
237                 following format:
238                 {
239                      tag1: [(start1, end1), (start2, end2), ...],
240                      tag2: [(start3, end3), (start4, end4), ...],
241                      ...
242                 }
243        """
244        raise NotImplementedError
245
246    def _tag_with_checkpoint(self, power_dict):
247        """Tag power_dict with checkpoint data.
248
249        This function translates the checkpoint intervals into a list of tags
250        for each data point.
251
252        @param power_dict: a dictionary with power data; assume this dictionary
253                           has attributes 'sample_count' and 'sample_duration'.
254        """
255        checkpoint_dict = self._create_checkpoint_dict()
256
257        # Create list of check point event tuple.
258        # Tuple format: (checkpoint_name:str, event_time:float, is_start:bool)
259        checkpoint_event_list = []
260        for name, intervals in checkpoint_dict.iteritems():
261            for start, finish in intervals:
262                checkpoint_event_list.append((name, start, True))
263                checkpoint_event_list.append((name, finish, False))
264
265        checkpoint_event_list = sorted(checkpoint_event_list,
266                                       key=operator.itemgetter(1))
267
268        # Add dummy check point at 1e9 seconds.
269        checkpoint_event_list.append(('dummy', 1e9, True))
270
271        interval_set = set()
272        event_index = 0
273        checkpoint_list = []
274        for i in range(power_dict['sample_count']):
275            curr_time = i * power_dict['sample_duration']
276
277            # Process every checkpoint event until current point of time
278            while checkpoint_event_list[event_index][1] <= curr_time:
279                name, _, is_start = checkpoint_event_list[event_index]
280                if is_start:
281                    interval_set.add(name)
282                else:
283                    interval_set.discard(name)
284                event_index += 1
285
286            checkpoint_list.append(list(interval_set))
287        power_dict['checkpoint'] = checkpoint_list
288
289    def _convert(self):
290        """Convert data from self._logger object to raw power measurement
291        dictionary.
292
293        MUST be implemented in subclass.
294
295        Return:
296            raw measurement dictionary
297        """
298        raise NotImplementedError
299
300    def upload(self):
301        """Upload powerlog to dashboard and save data to results directory.
302        """
303        raw_measurement = self._convert()
304        if raw_measurement is None:
305            return
306
307        powerlog_dict = self._create_powerlog_dict(raw_measurement)
308        if self._resultsdir is not None:
309            self._save_json(powerlog_dict, self._resultsdir)
310            self._save_html(powerlog_dict, self._resultsdir)
311        if self._uploadurl is not None:
312            self._upload(powerlog_dict, self._uploadurl)
313
314
315class ClientTestDashboard(BaseDashboard):
316    """Dashboard class for autotests that run on client side.
317    """
318
319    def __init__(self, logger, testname, start_ts, resultsdir, uploadurl, note):
320        """Create BaseDashboard objects.
321
322        Args:
323            logger: object that store the log. This will get convert to
324                    dictionary by self._convert()
325            testname: name of current test
326            start_ts: timestamp of when test started in seconds since epoch
327            resultsdir: directory to save the power json
328            uploadurl: url to upload power data
329            note: note for current test run
330        """
331        super(ClientTestDashboard, self).__init__(logger, testname, start_ts,
332                                                  resultsdir, uploadurl)
333        self._note = note
334
335
336    def _create_dut_info_dict(self, power_rails):
337        """Create a dictionary that contain information of the DUT.
338
339        Args:
340            power_rails: list of measured power rails
341
342        Returns:
343            DUT info dictionary
344        """
345        board = utils.get_board()
346        platform = utils.get_platform()
347
348        if not platform.startswith(board):
349            board += '_' + platform
350
351        if power_utils.has_hammer():
352            board += '_hammer'
353
354        dut_info_dict = {
355            'board': board,
356            'version': {
357                'hw': utils.get_hardware_revision(),
358                'milestone': lsbrelease_utils.get_chromeos_release_milestone(),
359                'os': lsbrelease_utils.get_chromeos_release_version(),
360                'channel': lsbrelease_utils.get_chromeos_channel(),
361                'firmware': utils.get_firmware_version(),
362                'ec': utils.get_ec_version(),
363                'kernel': utils.get_kernel_version(),
364            },
365            'sku': {
366                'cpu': utils.get_cpu_name(),
367                'memory_size': utils.get_mem_total_gb(),
368                'storage_size': utils.get_disk_size_gb(utils.get_root_device()),
369                'display_resolution': utils.get_screen_resolution(),
370            },
371            'ina': {
372                'version': 0,
373                'ina': power_rails,
374            },
375            'note': self._note,
376        }
377
378        if power_utils.has_battery():
379            status = power_status.get_status()
380            if status.battery:
381                # Round the battery size to nearest tenth because it is
382                # fluctuated for platform without battery nominal voltage data.
383                dut_info_dict['sku']['battery_size'] = round(
384                        status.battery.energy_full_design, 1)
385                dut_info_dict['sku']['battery_shutdown_percent'] = \
386                        power_utils.get_low_battery_shutdown_percent()
387        return dut_info_dict
388
389
390class MeasurementLoggerDashboard(ClientTestDashboard):
391    """Dashboard class for power_status.MeasurementLogger.
392    """
393
394    def __init__(self, logger, testname, resultsdir, uploadurl, note):
395        super(MeasurementLoggerDashboard, self).__init__(logger, testname, None,
396                                                         resultsdir, uploadurl,
397                                                         note)
398        self._unit = None
399        self._type = None
400        self._padded_domains = None
401
402    def _create_powerlog_dict(self, raw_measurement):
403        """Create powerlog dictionary from raw measurement data
404        Data format in go/power-dashboard-data.
405
406        Args:
407            raw_measurement: dictionary contains raw measurement data
408
409        Returns:
410            A dictionary of powerlog
411        """
412        powerlog_dict = \
413                super(MeasurementLoggerDashboard, self)._create_powerlog_dict(
414                        raw_measurement)
415
416        # Using start time of the logger as the timestamp of powerlog dict.
417        powerlog_dict['timestamp'] = self._logger.times[0]
418
419        return powerlog_dict
420
421    def _create_padded_domains(self):
422        """Pad the domains name for dashboard to make the domain name better
423        sorted in alphabetical order"""
424        pass
425
426    def _create_checkpoint_dict(self):
427        """Create dictionary for checkpoint.
428        """
429        start_time = self._logger.times[0]
430        return self._logger._checkpoint_logger.convert_relative(start_time)
431
432    def _convert(self):
433        """Convert data from power_status.MeasurementLogger object to raw
434        power measurement dictionary.
435
436        Return:
437            raw measurement dictionary or None if no readings
438        """
439        if len(self._logger.readings) == 0:
440            logging.warn('No readings in logger ... ignoring')
441            return None
442
443        power_dict = collections.defaultdict(dict, {
444            'sample_count': len(self._logger.readings),
445            'sample_duration': 0,
446            'average': dict(),
447            'data': dict(),
448        })
449        if power_dict['sample_count'] > 1:
450            total_duration = self._logger.times[-1] - self._logger.times[0]
451            power_dict['sample_duration'] = \
452                    1.0 * total_duration / (power_dict['sample_count'] - 1)
453
454        self._create_padded_domains()
455        for i, domain_readings in enumerate(zip(*self._logger.readings)):
456            if self._padded_domains:
457                domain = self._padded_domains[i]
458            else:
459                domain = self._logger.domains[i]
460            power_dict['data'][domain] = domain_readings
461            power_dict['average'][domain] = \
462                    numpy.average(power_dict['data'][domain])
463            if self._unit:
464                power_dict['unit'][domain] = self._unit
465            if self._type:
466                power_dict['type'][domain] = self._type
467
468        self._tag_with_checkpoint(power_dict)
469        return power_dict
470
471
472class PowerLoggerDashboard(MeasurementLoggerDashboard):
473    """Dashboard class for power_status.PowerLogger.
474    """
475
476    def __init__(self, logger, testname, resultsdir, uploadurl, note):
477        super(PowerLoggerDashboard, self).__init__(logger, testname, resultsdir,
478                                                   uploadurl, note)
479        self._unit = 'watt'
480        self._type = 'power'
481
482
483class TempLoggerDashboard(MeasurementLoggerDashboard):
484    """Dashboard class for power_status.TempLogger.
485    """
486
487    def __init__(self, logger, testname, resultsdir, uploadurl, note):
488        super(TempLoggerDashboard, self).__init__(logger, testname, resultsdir,
489                                                  uploadurl, note)
490        self._unit = 'celsius'
491        self._type = 'temperature'
492
493
494class KeyvalLogger(power_status.MeasurementLogger):
495    """Class for logging custom keyval data to power dashboard.
496
497    Each key should be unique and only map to one value.
498    See power_SpeedoMeter2 for implementation example.
499    """
500
501    def __init__(self, start_ts, end_ts):
502        # Do not call parent constructor to avoid making a new thread.
503        self.times = [start_ts]
504        self._duration_secs = end_ts - start_ts
505        self.keys = []
506        self.values = []
507        self.units = []
508        self.types = []
509
510    def is_unit_valid(self, unit):
511        """Make sure that unit of the data is supported unit."""
512        pattern = re.compile(r'^((kilo|mega|giga)hertz|'
513                             r'percent|celsius|fps|rpm|point|'
514                             r'(milli|micro)?(watt|volt|amp))$')
515        return pattern.match(unit) is not None
516
517    def add_item(self, key, value, unit, type_):
518        """Add a data point to the logger.
519
520        @param key: string, key of the data.
521        @param value: float, measurement value.
522        @param unit: string, unit for the data.
523        @param type: string, type of the data.
524        """
525        if not self.is_unit_valid(unit):
526            raise error.TestError(
527                    'Unit %s is not support in power dashboard.' % unit)
528        self.keys.append(key)
529        self.values.append(value)
530        self.units.append(unit)
531        self.types.append(type_)
532
533    def calc(self, mtype=None):
534        return {}
535
536    def save_results(self, resultsdir=None, fname_prefix=None):
537        pass
538
539
540class KeyvalLoggerDashboard(MeasurementLoggerDashboard):
541    """Dashboard class for custom keyval data in KeyvalLogger class."""
542
543    def _convert(self):
544        """Convert KeyvalLogger data to power dict."""
545        power_dict =  {
546            # 2 samples to show flat value spanning across duration of the test.
547            'sample_count': 2,
548            'sample_duration': self._logger._duration_secs,
549            'average': dict(zip(self._logger.keys, self._logger.values)),
550            'data': dict(zip(self._logger.keys,
551                             ([v, v] for v in self._logger.values))),
552            'unit': dict(zip(self._logger.keys, self._logger.units)),
553            'type': dict(zip(self._logger.keys, self._logger.types)),
554            'checkpoint': [[self._testname], [self._testname]],
555        }
556        return power_dict
557
558
559class CPUStatsLoggerDashboard(MeasurementLoggerDashboard):
560    """Dashboard class for power_status.CPUStatsLogger.
561    """
562    @staticmethod
563    def _split_domain(domain):
564        """Return domain_type and domain_name for given domain.
565
566        Example: Split ................... to ........... and .......
567                       cpuidle_C1E-SKL        cpuidle         C1E-SKL
568                       cpuidle_0_3_C0         cpuidle_0_3     C0
569                       cpupkg_C0_C1           cpupkg          C0_C1
570                       cpufreq_0_3_1512000    cpufreq_0_3     1512000
571
572        Args:
573            domain: cpu stat domain name to split
574
575        Return:
576            tuple of domain_type and domain_name
577        """
578        # Regex explanation
579        # .*?           matches type non-greedily                 (cpuidle)
580        # (?:_\d+)*     matches cpu part, ?: makes it not a group (_0_1_2_3)
581        # .*            matches name greedily                     (C0_C1)
582        return re.match(r'(.*?(?:_\d+)*)_(.*)', domain).groups()
583
584    def _convert(self):
585        power_dict = super(CPUStatsLoggerDashboard, self)._convert()
586        remove_rail = []
587        for rail in power_dict['data']:
588            if rail.startswith('wavg_cpu'):
589                power_dict['type'][rail] = 'cpufreq_wavg'
590                power_dict['unit'][rail] = 'kilohertz'
591            elif rail.startswith('wavg_gpu'):
592                power_dict['type'][rail] = 'gpufreq_wavg'
593                power_dict['unit'][rail] = 'megahertz'
594            else:
595                # Remove all aggregate stats, only 'non-c0' and 'non-C0_C1' now
596                if self._split_domain(rail)[1].startswith('non'):
597                    remove_rail.append(rail)
598                    continue
599                power_dict['type'][rail] = self._split_domain(rail)[0]
600                power_dict['unit'][rail] = 'percent'
601        for rail in remove_rail:
602            del power_dict['data'][rail]
603            del power_dict['average'][rail]
604        return power_dict
605
606    def _create_padded_domains(self):
607        """Padded number in the domain name with dot to make it sorted
608        alphabetically.
609
610        Example:
611        cpuidle_C1-SKL, cpuidle_C1E-SKL, cpuidle_C2-SKL, cpuidle_C10-SKL
612        will be changed to
613        cpuidle_C.1-SKL, cpuidle_C.1E-SKL, cpuidle_C.2-SKL, cpuidle_C10-SKL
614        which make it in alphabetically order.
615        """
616        longest = collections.defaultdict(int)
617        searcher = re.compile(r'\d+')
618        number_strs = []
619        splitted_domains = \
620                [self._split_domain(domain) for domain in self._logger.domains]
621        for domain_type, domain_name in splitted_domains:
622            result = searcher.search(domain_name)
623            if not result:
624                number_strs.append('')
625                continue
626            number_str = result.group(0)
627            number_strs.append(number_str)
628            longest[domain_type] = max(longest[domain_type], len(number_str))
629
630        self._padded_domains = []
631        for i in range(len(self._logger.domains)):
632            if not number_strs[i]:
633                self._padded_domains.append(self._logger.domains[i])
634                continue
635
636            domain_type, domain_name = splitted_domains[i]
637            formatter_component = '{:.>%ds}' % longest[domain_type]
638
639            # Change "cpuidle_C1E-SKL" to "cpuidle_C{:.>2s}E-SKL"
640            formatter_str = domain_type + '_' + \
641                    searcher.sub(formatter_component, domain_name, count=1)
642
643            # Run "cpuidle_C{:_>2s}E-SKL".format("1") to get "cpuidle_C.1E-SKL"
644            self._padded_domains.append(formatter_str.format(number_strs[i]))
645
646
647class VideoFpsLoggerDashboard(MeasurementLoggerDashboard):
648    """Dashboard class for power_status.VideoFpsLogger."""
649
650    def __init__(self, logger, testname, resultsdir, uploadurl, note):
651        super(VideoFpsLoggerDashboard, self).__init__(
652            logger, testname, resultsdir, uploadurl, note)
653        self._unit = 'fps'
654        self._type = 'fps'
655
656
657class FanRpmLoggerDashboard(MeasurementLoggerDashboard):
658    """Dashboard class for power_status.FanRpmLogger."""
659
660    def __init__(self, logger, testname, resultsdir, uploadurl, note):
661        super(FanRpmLoggerDashboard, self).__init__(
662            logger, testname, resultsdir, uploadurl, note)
663        self._unit = 'rpm'
664        self._type = 'fan'
665
666dashboard_factory = None
667def get_dashboard_factory():
668    global dashboard_factory
669    if not dashboard_factory:
670        dashboard_factory = LoggerDashboardFactory()
671    return dashboard_factory
672
673class LoggerDashboardFactory(object):
674    """Class to generate client test dashboard object from logger."""
675
676    loggerToDashboardDict = {
677        power_status.CPUStatsLogger: CPUStatsLoggerDashboard,
678        power_status.PowerLogger:    PowerLoggerDashboard,
679        power_status.TempLogger:     TempLoggerDashboard,
680        power_status.VideoFpsLogger: VideoFpsLoggerDashboard,
681        power_status.FanRpmLogger:   FanRpmLoggerDashboard,
682        KeyvalLogger:                KeyvalLoggerDashboard,
683    }
684
685    def registerDataType(self, logger_type, dashboard_type):
686        """Register new type of dashboard to the factory
687
688        @param logger_type: Type of logger to register
689        @param dashboard_type: Type of dashboard to register
690        """
691        self.loggerToDashboardDict[logger_type] = dashboard_type
692
693    def createDashboard(self, logger, testname, resultsdir=None,
694                        uploadurl=None, note=''):
695        """Create dashboard object"""
696        if uploadurl is None:
697            uploadurl = 'http://chrome-power.appspot.com/rapl'
698        dashboard = self.loggerToDashboardDict[type(logger)]
699        return dashboard(logger, testname, resultsdir, uploadurl, note)
700