1#!/usr/bin/python
2
3"""Unit tests for the perf_uploader.py module.
4
5"""
6
7import json, unittest
8
9import common
10from autotest_lib.tko import models as tko_models
11from autotest_lib.tko.perf_upload import perf_uploader
12
13
14class test_aggregate_iterations(unittest.TestCase):
15    """Tests for the aggregate_iterations function."""
16
17    _PERF_ITERATION_DATA = {
18        '1': [
19            {
20                'description': 'metric1',
21                'value': 1,
22                'stddev': 0.0,
23                'units': 'units1',
24                'higher_is_better': True,
25                'graph': None
26            },
27            {
28                'description': 'metric2',
29                'value': 10,
30                'stddev': 0.0,
31                'units': 'units2',
32                'higher_is_better': True,
33                'graph': 'graph1',
34            },
35            {
36                'description': 'metric2',
37                'value': 100,
38                'stddev': 1.7,
39                'units': 'units3',
40                'higher_is_better': False,
41                'graph': 'graph2',
42            }
43        ],
44        '2': [
45            {
46                'description': 'metric1',
47                'value': 2,
48                'stddev': 0.0,
49                'units': 'units1',
50                'higher_is_better': True,
51                'graph': None,
52            },
53            {
54                'description': 'metric2',
55                'value': 20,
56                'stddev': 0.0,
57                'units': 'units2',
58                'higher_is_better': True,
59                'graph': 'graph1',
60            },
61            {
62                'description': 'metric2',
63                'value': 200,
64                'stddev': 21.2,
65                'units': 'units3',
66                'higher_is_better': False,
67                'graph': 'graph2',
68            }
69        ],
70    }
71
72
73    def setUp(self):
74        """Sets up for each test case."""
75        self._perf_values = []
76        for iter_num, iter_data in self._PERF_ITERATION_DATA.iteritems():
77            self._perf_values.append(
78                    tko_models.perf_value_iteration(iter_num, iter_data))
79
80
81
82class test_json_config_file_sanity(unittest.TestCase):
83    """Sanity tests for the JSON-formatted presentation config file."""
84
85    def test_parse_json(self):
86        """Verifies _parse_config_file function."""
87        perf_uploader._parse_config_file(
88                perf_uploader._PRESENTATION_CONFIG_FILE)
89
90
91    def test_proper_json(self):
92        """Verifies the file can be parsed as proper JSON."""
93        try:
94            with open(perf_uploader._PRESENTATION_CONFIG_FILE, 'r') as fp:
95                json.load(fp)
96        except:
97            self.fail('Presentation config file could not be parsed as JSON.')
98
99
100    def test_unique_test_names(self):
101        """Verifies that each test name appears only once in the JSON file."""
102        json_obj = []
103        try:
104            with open(perf_uploader._PRESENTATION_CONFIG_FILE, 'r') as fp:
105                json_obj = json.load(fp)
106        except:
107            self.fail('Presentation config file could not be parsed as JSON.')
108
109        name_set = set([x['autotest_name'] for x in json_obj])
110        self.assertEqual(len(name_set), len(json_obj),
111                         msg='Autotest names not unique in the JSON file.')
112
113
114    def test_required_master_name(self):
115        """Verifies that master name must be specified."""
116        json_obj = []
117        try:
118            with open(perf_uploader._PRESENTATION_CONFIG_FILE, 'r') as fp:
119                json_obj = json.load(fp)
120        except:
121            self.fail('Presentation config file could not be parsed as JSON.')
122
123        for entry in json_obj:
124            if not 'master_name' in entry:
125                self.fail('Missing master field for test %s.' %
126                          entry['autotest_name'])
127
128
129class test_gather_presentation_info(unittest.TestCase):
130    """Tests for the gather_presentation_info function."""
131
132    _PRESENT_INFO = {
133        'test_name': {
134            'master_name': 'new_master_name',
135            'dashboard_test_name': 'new_test_name',
136        }
137    }
138
139    _PRESENT_INFO_MISSING_MASTER = {
140        'test_name': {
141            'dashboard_test_name': 'new_test_name',
142        }
143    }
144
145    _PRESENT_INFO_REGEX = {
146        'test_name.*': {
147            'master_name': 'new_master_name',
148            'dashboard_test_name': 'new_test_name',
149        }
150    }
151
152    def test_test_name_regex_specified(self):
153        """Verifies gathers presentation info for regex search correctly"""
154        for test_name in ['test_name.arm.7.1', 'test_name.x86.7.1']:
155            result = perf_uploader._gather_presentation_info(
156                    self._PRESENT_INFO, 'test_name_P')
157            self.assertTrue(
158                    all([key in result for key in
159                         ['test_name', 'master_name']]),
160                    msg='Unexpected keys in resulting dictionary: %s' % result)
161            self.assertEqual(result['master_name'], 'new_master_name',
162                             msg='Unexpected "master_name" value: %s' %
163                                 result['master_name'])
164            self.assertEqual(result['test_name'], 'new_test_name',
165                             msg='Unexpected "test_name" value: %s' %
166                                 result['test_name'])
167
168    def test_test_name_specified(self):
169        """Verifies gathers presentation info correctly."""
170        result = perf_uploader._gather_presentation_info(
171                self._PRESENT_INFO, 'test_name')
172        self.assertTrue(
173                all([key in result for key in
174                     ['test_name', 'master_name']]),
175                msg='Unexpected keys in resulting dictionary: %s' % result)
176        self.assertEqual(result['master_name'], 'new_master_name',
177                         msg='Unexpected "master_name" value: %s' %
178                             result['master_name'])
179        self.assertEqual(result['test_name'], 'new_test_name',
180                         msg='Unexpected "test_name" value: %s' %
181                             result['test_name'])
182
183
184    def test_test_name_not_specified(self):
185        """Verifies exception raised if test is not there."""
186        self.assertRaises(
187                perf_uploader.PerfUploadingError,
188                perf_uploader._gather_presentation_info,
189                        self._PRESENT_INFO, 'other_test_name')
190
191
192    def test_master_not_specified(self):
193        """Verifies exception raised if master is not there."""
194        self.assertRaises(
195                perf_uploader.PerfUploadingError,
196                perf_uploader._gather_presentation_info,
197                    self._PRESENT_INFO_MISSING_MASTER, 'test_name')
198
199
200class test_get_id_from_version(unittest.TestCase):
201    """Tests for the _get_id_from_version function."""
202
203    def test_correctly_formatted_versions(self):
204        """Verifies that the expected ID is returned when input is OK."""
205        chrome_version = '27.0.1452.2'
206        cros_version = '27.3906.0.0'
207        # 1452.2 + 3906.0.0
208        # --> 01452 + 002 + 03906 + 000 + 00
209        # --> 14520020390600000
210        self.assertEqual(
211                14520020390600000,
212                perf_uploader._get_id_from_version(
213                        chrome_version, cros_version))
214
215        chrome_version = '25.10.1000.0'
216        cros_version = '25.1200.0.0'
217        # 1000.0 + 1200.0.0
218        # --> 01000 + 000 + 01200 + 000 + 00
219        # --> 10000000120000000
220        self.assertEqual(
221                10000000120000000,
222                perf_uploader._get_id_from_version(
223                        chrome_version, cros_version))
224
225    def test_returns_none_when_given_invalid_input(self):
226        """Checks the return value when invalid input is given."""
227        chrome_version = '27.0'
228        cros_version = '27.3906.0.0'
229        self.assertIsNone(perf_uploader._get_id_from_version(
230                chrome_version, cros_version))
231
232
233class test_get_version_numbers(unittest.TestCase):
234    """Tests for the _get_version_numbers function."""
235
236    def test_with_valid_versions(self):
237      """Checks the version numbers used when data is formatted as expected."""
238      self.assertEqual(
239              ('34.5678.9.0', '34.5.678.9'),
240              perf_uploader._get_version_numbers(
241                  {
242                      'CHROME_VERSION': '34.5.678.9',
243                      'CHROMEOS_RELEASE_VERSION': '5678.9.0',
244                  }))
245
246    def test_with_missing_version_raises_error(self):
247      """Checks that an error is raised when a version is missing."""
248      with self.assertRaises(perf_uploader.PerfUploadingError):
249          perf_uploader._get_version_numbers(
250              {
251                  'CHROMEOS_RELEASE_VERSION': '5678.9.0',
252              })
253
254    def test_with_unexpected_version_format_raises_error(self):
255      """Checks that an error is raised when there's a rN suffix."""
256      with self.assertRaises(perf_uploader.PerfUploadingError):
257          perf_uploader._get_version_numbers(
258              {
259                  'CHROME_VERSION': '34.5.678.9',
260                  'CHROMEOS_RELEASE_VERSION': '5678.9.0r1',
261              })
262
263    def test_with_valid_release_milestone(self):
264      """Checks the version numbers used when data is formatted as expected."""
265      self.assertEqual(
266              ('54.5678.9.0', '34.5.678.9'),
267              perf_uploader._get_version_numbers(
268                  {
269                      'CHROME_VERSION': '34.5.678.9',
270                      'CHROMEOS_RELEASE_VERSION': '5678.9.0',
271                      'CHROMEOS_RELEASE_CHROME_MILESTONE': '54',
272                  }))
273
274
275class test_format_for_upload(unittest.TestCase):
276    """Tests for the format_for_upload function."""
277
278    _PERF_DATA = {
279     "charts": {
280            "metric1": {
281                "summary": {
282                    "improvement_direction": "down",
283                    "type": "scalar",
284                    "units": "msec",
285                    "value": 2.7,
286                }
287            },
288            "metric2": {
289                "summary": {
290                    "improvement_direction": "up",
291                    "type": "scalar",
292                    "units": "frames_per_sec",
293                    "value": 101.35,
294                }
295            }
296        },
297    }
298    _PRESENT_INFO = {
299        'master_name': 'new_master_name',
300        'test_name': 'new_test_name',
301    }
302
303    def setUp(self):
304        self._perf_data = self._PERF_DATA
305
306    def _verify_result_string(self, actual_result, expected_result):
307        """Verifies a JSON string matches the expected result.
308
309        This function compares JSON objects rather than strings, because of
310        possible floating-point values that need to be compared using
311        assertAlmostEqual().
312
313        @param actual_result: The candidate JSON string.
314        @param expected_result: The reference JSON string that the candidate
315            must match.
316
317        """
318        actual = json.loads(actual_result)
319        expected = json.loads(expected_result)
320
321        def ordered(obj):
322            if isinstance(obj, dict):
323               return sorted((k, ordered(v)) for k, v in obj.items())
324            if isinstance(obj, list):
325               return sorted(ordered(x) for x in obj)
326            else:
327               return obj
328        fail_msg = 'Unexpected result string: %s' % actual_result
329        self.assertEqual(ordered(expected), ordered(actual), msg=fail_msg)
330
331
332    def test_format_for_upload(self):
333        """Verifies format_for_upload generates correct json data."""
334        result = perf_uploader._format_for_upload(
335                'platform', '25.1200.0.0', '25.10.1000.0', 'WINKY E2A-F2K-Q35',
336                'i7', 'test_machine', self._perf_data, self._PRESENT_INFO,
337                '52926644-username/hostname')
338        expected_result_string = (
339          '{"versions":  {'
340             '"cros_version": "25.1200.0.0",'
341             '"chrome_version": "25.10.1000.0"'
342          '},'
343          '"point_id": 10000000120000000,'
344          '"bot": "cros-platform-i7",'
345          '"chart_data": {'
346             '"charts": {'
347               '"metric2": {'
348                 '"summary": {'
349                   '"units": "frames_per_sec",'
350                   '"type": "scalar",'
351                   '"value": 101.35,'
352                   '"improvement_direction": "up"'
353                 '}'
354               '},'
355               '"metric1": {'
356                 '"summary": {'
357                 '"units": "msec",'
358                 '"type": "scalar",'
359                 '"value": 2.7,'
360                 '"improvement_direction": "down"}'
361               '}'
362             '}'
363          '},'
364          '"master": "new_master_name",'
365          '"supplemental": {'
366             '"hardware_identifier": "WINKY E2A-F2K-Q35",'
367             '"jobname": "52926644-username/hostname",'
368             '"hardware_hostname": "test_machine",'
369             '"default_rev": "r_cros_version",'
370             '"variant_name": "i7"}'
371           '}')
372        self._verify_result_string(result['data'], expected_result_string)
373
374
375if __name__ == '__main__':
376    unittest.main()
377