1# Copyright (c) 2017 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import csv, datetime, glob, math, os, re, time
6
7from autotest_lib.client.bin import utils
8from autotest_lib.client.common_lib import error
9from autotest_lib.client.common_lib.cros import tpm_utils
10from autotest_lib.server import test
11from autotest_lib.server.cros import cfm_jmidata_log_collector
12from autotest_lib.server.cros.multimedia import remote_facade_factory
13
14_SHORT_TIMEOUT = 5
15_MEASUREMENT_DURATION_SECONDS = 10
16_TOTAL_TEST_DURATION_SECONDS = 900
17_PERF_RESULT_FILE = 'perf.csv'
18
19_BASE_DIR = '/home/chronos/user/Storage/ext/'
20_EXT_ID = 'ikfcpmgefdpheiiomgmhlmmkihchmdlj'
21_JMI_DIR = '/0*/File\ System/000/t/00/*'
22_JMI_SOURCE_DIR = _BASE_DIR + _EXT_ID + _JMI_DIR
23
24
25class enterprise_CFM_Perf(test.test):
26    """This is a server test which clears device TPM and runs
27    enterprise_RemoraRequisition client test to enroll the device in to hotrod
28    mode. After enrollment is successful, it collects and logs cpu, memory and
29    temperature data from the device under test."""
30    version = 1
31
32
33    def _cpu_usage(self):
34        """Returns cpu usage in %."""
35        cpu_usage_start = self.system_facade.get_cpu_usage()
36        time.sleep(_MEASUREMENT_DURATION_SECONDS)
37        cpu_usage_end = self.system_facade.get_cpu_usage()
38        return self.system_facade.compute_active_cpu_time(cpu_usage_start,
39                cpu_usage_end) * 100
40
41
42    def _memory_usage(self):
43        """Returns total used memory in %."""
44        total_memory = self.system_facade.get_mem_total()
45        return ((total_memory - self.system_facade.get_mem_free())
46                * 100 / total_memory)
47
48
49    def _temperature_data(self):
50        """Returns temperature sensor data in fahrenheit."""
51        ectool = self.client.run('ectool version', ignore_status=True)
52        if not ectool.exit_status:
53            ec_temp = self.system_facade.get_ec_temperatures()
54            return ec_temp[1]
55        else:
56            temp_sensor_name = 'temp0'
57            if not temp_sensor_name:
58                return 0
59            MOSYS_OUTPUT_RE = re.compile('(\w+)="(.*?)"')
60            values = {}
61            cmd = 'mosys -k sensor print thermal %s' % temp_sensor_name
62            for kv in MOSYS_OUTPUT_RE.finditer(self.client.run_output(cmd)):
63                key, value = kv.groups()
64                if key == 'reading':
65                    value = int(value)
66                values[key] = value
67            return values['reading']
68
69
70    def enroll_device_and_start_hangout(self):
71        """Enroll device into CFM and start hangout session."""
72        current_date = datetime.datetime.now().strftime("%Y-%m-%d")
73        hangout_name = current_date + '-cfm-perf'
74
75        self.cfm_facade.enroll_device()
76        self.cfm_facade.restart_chrome_for_cfm()
77        self.cfm_facade.wait_for_telemetry_commands()
78
79        if not self.cfm_facade.is_oobe_start_page():
80            self.cfm_facade.wait_for_oobe_start_page()
81
82        self.cfm_facade.skip_oobe_screen()
83        self.cfm_facade.start_new_hangout_session(hangout_name)
84
85
86    def collect_perf_data(self):
87        """Use system facade to collect performance data from the DUT using
88        xmlrpc and save it to csv file in results directory. Data collected
89        includes:
90                1. CPU usage
91                2. Memory usage
92                3. Thermal temperature
93                4. Timestamp
94                5. Board name
95                6. Build id
96        """
97        start_time = time.time()
98        perf_keyval = {}
99        cpu_usage_list = list()
100        memory_usage_list = list()
101        temperature_list = list()
102        board_name = self.system_facade.get_current_board()
103        build_id = self.system_facade.get_chromeos_release_version()
104        perf_file = open(os.path.join(self.resultsdir, _PERF_RESULT_FILE), 'w')
105        writer = csv.writer(perf_file)
106        writer.writerow(['cpu', 'memory', 'temperature', 'timestamp', 'board',
107                         'build'])
108        while (time.time() - start_time) < _TOTAL_TEST_DURATION_SECONDS:
109            perf_keyval['cpu_usage'] = self._cpu_usage()
110            perf_keyval['memory_usage'] = self._memory_usage()
111            perf_keyval['temperature'] = self._temperature_data()
112            writer.writerow([perf_keyval['cpu_usage'],
113                             perf_keyval['memory_usage'],
114                             perf_keyval['temperature'],
115                             time.strftime('%Y/%m/%d %H:%M:%S'),
116                             board_name,
117                             build_id])
118            self.write_perf_keyval(perf_keyval)
119            cpu_usage_list.append(perf_keyval['cpu_usage'])
120            memory_usage_list.append(perf_keyval['memory_usage'])
121            temperature_list.append(perf_keyval['temperature'])
122            time.sleep(_MEASUREMENT_DURATION_SECONDS)
123        perf_file.close()
124        utils.write_keyval(os.path.join(self.resultsdir, os.pardir),
125                           {'perf_csv_folder': self.resultsdir})
126        self.upload_perf_data(cpu_usage_list,
127                              memory_usage_list,
128                              temperature_list)
129
130
131    def upload_perf_data(self, cpu_usage, memory_usage, temperature):
132        """Write perf results to results-chart.json file for Perf Dashboard.
133
134        @param cpu_usage: list of cpu usage values
135        @param memory_usage: list of memory usage values
136        @param temperature: list of temperature values
137        """
138        avg_cpu_usage = sum(cpu_usage)/len(cpu_usage)
139        avg_memory_usage = sum(memory_usage)/len(memory_usage)
140        avg_temp = sum(temperature)/len(temperature)
141
142        peak_cpu_usage = max(cpu_usage)
143        peak_memory_usage = max(memory_usage)
144        peak_temp = max(temperature)
145
146        self.output_perf_value(description='average_cpu_usage',
147                value=avg_cpu_usage, units='percent', higher_is_better=False)
148        self.output_perf_value(description='average_memory_usage',
149                value=avg_memory_usage, units='percent', higher_is_better=False)
150        self.output_perf_value(description='average_temperature',
151                value=avg_temp, units='Celsius', higher_is_better=False)
152
153        self.output_perf_value(description='peak_cpu_usage',
154                value=peak_cpu_usage, units='percent', higher_is_better=False)
155        self.output_perf_value(description='peak_memory_usage',
156                value=peak_memory_usage, units='percent',
157                higher_is_better=False)
158        self.output_perf_value(description='peak_temperature',
159                value=peak_temp, units='Celsius', higher_is_better=False)
160
161
162    def _get_average(self, data_type, jmidata):
163        """Computes mean of a list of numbers.
164
165        @param data_type: Type of data to be retrieved from jmi data log.
166        @param jmidata: Raw jmi data log to parse.
167        @return Mean computed from the list of numbers.
168        """
169        data = self._get_data_from_jmifile(data_type, jmidata)
170        if not data:
171            return 0
172        return float(sum(data)) / len(data)
173
174
175    def _get_std_dev(self, data_type, jmidata):
176        """Computes standard deviation of a list of numbers.
177
178        @param data_type: Type of data to be retrieved from jmi data log.
179        @param jmidata: Raw jmi data log to parse.
180        @return Standard deviation computed from the list of numbers.
181        """
182        data = self._get_data_from_jmifile(data_type, jmidata)
183        n = len(data)
184        if not data or n == 1:
185            return 0
186        mean = float(sum(data)) / n
187        variance = sum([(elem - mean) ** 2 for elem in data]) / (n -1)
188        return math.sqrt(variance)
189
190
191    def _get_max_value(self, data_type, jmidata):
192        """Computes maximum value of a list of numbers.
193
194        @param data_type: Type of data to be retrieved from jmi data log.
195        @param jmidata: Raw jmi data log to parse.
196        @return Maxium value from the list of numbers.
197        """
198        data = self._get_data_from_jmifile(data_type, jmidata)
199        if not data:
200            return 0
201        return max(data)
202
203
204    def _get_sum(self, data_type, jmidata):
205        """Computes sum of a list of numbers.
206
207        @param data_type: Type of data to be retrieved from jmi data log.
208        @param jmidata: Raw jmi data log to parse.
209        @return Sum computed from the list of numbers.
210        """
211        data = self._get_data_from_jmifile(data_type, jmidata)
212        if not data:
213            return 0
214        return sum(data)
215
216
217    def _get_last_value(self, data_type, jmidata):
218        """Gets last value of a list of numbers.
219
220        @param data_type: Type of data to be retrieved from jmi data log.
221        @param jmidata: Raw jmi data log to parse.
222        @return Mean computed from the list of numbers.
223        """
224        data = self._get_data_from_jmifile(data_type, jmidata)
225        if not data:
226            return 0
227        return data[-1]
228
229
230    def _get_data_from_jmifile(self, data_type, jmidata):
231        """Gets data from jmidata log for given data type.
232
233        @param data_type: Type of data to be retrieved from jmi data log.
234        @param jmidata: Raw jmi data log to parse.
235        @return Data for given data type from jmidata log.
236        """
237        return cfm_jmidata_log_collector.GetDataFromLogs(
238                self, data_type, jmidata)
239
240
241    def _get_file_to_parse(self):
242        """Copy jmi logs from client to test's results directory.
243
244        @return The newest jmi log file.
245        """
246        self.client.get_file(_JMI_SOURCE_DIR, self.resultsdir)
247        source_jmi_files = self.resultsdir + '/0*'
248        if not source_jmi_files:
249            raise error.TestNAError('JMI data file not found.')
250        newest_file = max(glob.iglob(source_jmi_files), key=os.path.getctime)
251        return newest_file
252
253
254    def upload_jmidata(self):
255        """Write jmidata results to results-chart.json file for Perf Dashboard.
256        """
257        jmi_file = self._get_file_to_parse()
258        jmifile_to_parse = open(jmi_file, 'r')
259        jmidata = jmifile_to_parse.read()
260
261        self.output_perf_value(description='sum_vid_in_frames_decoded',
262                value=self._get_sum('frames_decoded', jmidata), units='frames',
263                higher_is_better=True)
264
265        self.output_perf_value(description='sum_vid_out_frames_encoded',
266                value=self._get_sum('frames_encoded', jmidata), units='frames',
267                higher_is_better=True)
268
269        self.output_perf_value(description='vid_out_adapt_changes',
270                value=self._get_last_value('adaptation_changes', jmidata),
271                units='count', higher_is_better=False)
272
273        self.output_perf_value(description='avg_video_out_encode_time',
274                value=self._get_average('average_encode_time', jmidata),
275                units='ms', higher_is_better=False)
276
277        self.output_perf_value(description='std_dev_video_out_encode_time',
278                value=self._get_std_dev('average_encode_time', jmidata),
279                units='ms', higher_is_better=False)
280
281        self.output_perf_value(description='max_video_out_encode_time',
282                value=self._get_max_value('average_encode_time', jmidata),
283                units='ms', higher_is_better=False)
284
285        self.output_perf_value(description='vid_out_bandwidth_adapt',
286                value=self._get_average('bandwidth_adaptation', jmidata),
287                units='bool', higher_is_better=False)
288
289        self.output_perf_value(description='vid_out_cpu_adapt',
290                value=self._get_average('cpu_adaptation', jmidata),
291                units='bool', higher_is_better=False)
292
293        self.output_perf_value(description='avg_video_in_res',
294                value=self._get_average('video_received_frame_height', jmidata),
295                units='resolution', higher_is_better=True)
296
297        self.output_perf_value(description='avg_video_out_res',
298                value=self._get_average('video_sent_frame_height', jmidata),
299                units='resolution', higher_is_better=True)
300
301        self.output_perf_value(description='std_dev_video_out_res',
302                value=self._get_std_dev('video_sent_frame_height', jmidata),
303                units='resolution', higher_is_better=True)
304
305        self.output_perf_value(description='avg_vid_in_framerate_decoded',
306                value=self._get_average('framerate_decoded', jmidata),
307                units='fps', higher_is_better=True)
308
309        self.output_perf_value(description='avg_vid_out_framerate_input',
310                value=self._get_average('framerate_outgoing', jmidata),
311                units='fps', higher_is_better=True)
312
313        self.output_perf_value(description='std_dev_vid_out_framerate_input',
314                value=self._get_std_dev('framerate_outgoing', jmidata),
315                units='fps', higher_is_better=True)
316
317        self.output_perf_value(description='avg_vid_in_framerate_to_renderer',
318                value=self._get_average('framerate_to_renderer', jmidata),
319                units='fps', higher_is_better=True)
320
321        self.output_perf_value(description='avg_vid_in_framerate_received',
322                value=self._get_average('framerate_received', jmidata),
323                units='fps', higher_is_better=True)
324
325        self.output_perf_value(description='avg_vid_out_framerate_sent',
326                value=self._get_average('framerate_sent', jmidata),
327                units='fps', higher_is_better=True)
328
329        self.output_perf_value(description='std_dev_vid_out_framerate_sent',
330                value=self._get_std_dev('framerate_sent', jmidata),
331                units='fps', higher_is_better=True)
332
333        self.output_perf_value(description='avg_vid_in_frame_width',
334                value=self._get_average('video_received_frame_width', jmidata),
335                units='fps', higher_is_better=True)
336
337        self.output_perf_value(description='avg_vid_out_frame_width',
338                value=self._get_average('video_sent_frame_width', jmidata),
339                units='fps', higher_is_better=True)
340
341        self.output_perf_value(description='avg_vid_out_encode_cpu_usage',
342                value=self._get_average('video_encode_cpu_usage', jmidata),
343                units='percent', higher_is_better=False)
344
345        total_vid_packets_sent = self._get_sum('video_packets_sent', jmidata)
346        total_vid_packets_lost = self._get_sum('video_packets_lost', jmidata)
347        lost_packet_percentage = float(total_vid_packets_lost)*100/ \
348                                 float(total_vid_packets_sent) if \
349                                 total_vid_packets_sent else 0
350
351        self.output_perf_value(description='lost_packet_percentage',
352                value=lost_packet_percentage, units='percent',
353                higher_is_better=False)
354
355        num_processors = self._get_data_from_jmifile('cpu_processors', jmidata)
356        total_cpu = self._get_average('cpu_percent', jmidata)
357        render_cpu = self._get_average('renderer_cpu_percent', jmidata)
358
359        cpu_percentage = total_cpu/num_processors if num_processors else 0
360        render_cpu_percent = render_cpu/num_processors if num_processors else 0
361
362        self.output_perf_value(description='avg_cpu_usage_jmi',
363                value=cpu_percentage,
364                units='percent', higher_is_better=False)
365
366        self.output_perf_value(description='avg_renderer_cpu_usage',
367                value=render_cpu_percent,
368                units='percent', higher_is_better=False)
369
370        self.output_perf_value(description='avg_browser_cpu_usage',
371                value=self._get_average('browser_cpu_percent', jmidata),
372                units='percent', higher_is_better=False)
373
374        self.output_perf_value(description='avg_gpu_cpu_usage',
375                value=self._get_average('gpu_cpu_percent', jmidata),
376                units='percent', higher_is_better=False)
377
378        self.output_perf_value(description='avg_active_streams',
379                value=self._get_average('num_active_vid_in_streams', jmidata),
380                units='count', higher_is_better=True)
381
382        self.output_perf_value(description='std_dev_active_streams',
383                value=self._get_std_dev('num_active_vid_in_streams', jmidata),
384                units='count', higher_is_better=True)
385
386
387    def run_once(self, host=None):
388        self.client = host
389
390        factory = remote_facade_factory.RemoteFacadeFactory(
391                host, no_chrome=True)
392        self.system_facade = factory.create_system_facade()
393        self.cfm_facade = factory.create_cfm_facade()
394
395        tpm_utils.ClearTPMOwnerRequest(self.client)
396
397        if self.client.servo:
398            self.client.servo.switch_usbkey('dut')
399            self.client.servo.set('usb_mux_sel3', 'dut_sees_usbkey')
400            time.sleep(_SHORT_TIMEOUT)
401            self.client.servo.set('dut_hub1_rst1', 'off')
402            time.sleep(_SHORT_TIMEOUT)
403
404        try:
405            self.enroll_device_and_start_hangout()
406            self.collect_perf_data()
407            self.cfm_facade.end_hangout_session()
408            self.upload_jmidata()
409        except Exception as e:
410            raise error.TestFail(str(e))
411
412        tpm_utils.ClearTPMOwnerRequest(self.client)
413