1#  Copyright (C) 2023 The Android Open Source Project
2#
3#  Licensed under the Apache License, Version 2.0 (the "License");
4#  you may not use this file except in compliance with the License.
5#  You may obtain a copy of the License at
6#
7#       http://www.apache.org/licenses/LICENSE-2.0
8#
9#  Unless required by applicable law or agreed to in writing, software
10#  distributed under the License is distributed on an "AS IS" BASIS,
11#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12#  See the License for the specific language governing permissions and
13#  limitations under the License.
14
15"""Stress tests for Neaby Connections used by the quick start flow."""
16
17import datetime
18import logging
19import os
20import sys
21import time
22
23# check the python version
24if sys.version_info < (3,10):
25  logging.error('The test only can run on python 3.10 and above')
26  exit()
27
28from mobly import asserts
29from mobly import base_test
30from mobly import test_runner
31from mobly.controllers import android_device
32
33# Allows local imports to be resolved via relative path, so the test can be run
34# without building.
35_performance_test_dir = os.path.dirname(os.path.dirname(__file__))
36if _performance_test_dir not in sys.path:
37  sys.path.append(_performance_test_dir)
38
39from performance_test import nc_base_test
40from performance_test import nc_constants
41from performance_test import nearby_connection_wrapper
42from performance_test import setup_utils
43
44_TEST_SCRIPT_VERSION = '1.6'
45
46_NEARBY_SNIPPET_2_PACKAGE_NAME = 'com.google.android.nearby.mobly.snippet.second'
47
48_PERFORMANCE_TEST_REPEAT_COUNT = 100
49_PERFORMANCE_TEST_MAX_CONSECUTIVE_ERROR = 5
50
51_DELAY_BETWEEN_EACH_TEST_CYCLE = datetime.timedelta(seconds=5)
52_TRANSFER_FILE_SIZE_1MB = 1024
53_TRANSFER_FILE_SIZE_1GB = 1024 * 1024
54
55
56class QuickStartStressTest(nc_base_test.NCBaseTestClass):
57  """Nearby Connection E2E stress tests for quick start flow."""
58
59  performance_test_iterations: int
60
61  # @typing.override
62  def __init__(self, configs):
63    super().__init__(configs)
64    self._nearby_snippet_2_apk_path: str = None
65    self._test_result: nc_constants.SingleTestResult = (
66        nc_constants.SingleTestResult())
67    self._quick_start_test_metrics: nc_constants.QuickStartTestMetrics = (
68        nc_constants.QuickStartTestMetrics())
69    self._test_script_version = _TEST_SCRIPT_VERSION
70
71  # @typing.override
72  def setup_test(self):
73    self.discoverer.nearby2.stopDiscovery()
74    self.discoverer.nearby2.stopAllEndpoints()
75    self.advertiser.nearby2.stopAdvertising()
76    self.advertiser.nearby2.stopAllEndpoints()
77
78    super().setup_test()
79
80  # @typing.override
81  def setup_class(self):
82    self._nearby_snippet_2_apk_path = self.user_params.get('files', {}).get(
83        'nearby_snippet_2', [''])[0]
84    self.performance_test_iterations = getattr(
85        self.test_quick_start_performance, base_test.ATTR_REPEAT_CNT)
86    logging.info('performance test iterations: %s',
87                 self.performance_test_iterations)
88
89    super().setup_class()
90
91  # @typing.override
92  def _setup_android_device(self, ad: android_device.AndroidDevice) -> None:
93    super()._setup_android_device(ad)
94    ad.log.info('try to install nearby_snippet_2_apk')
95    if self._nearby_snippet_2_apk_path:
96      setup_utils.install_apk(ad, self._nearby_snippet_2_apk_path)
97    else:
98      ad.log.warning('nearby_snippet_2 apk is not specified, '
99                     'make sure it is installed in the device')
100    ad.load_snippet('nearby2', _NEARBY_SNIPPET_2_PACKAGE_NAME)
101    setup_utils.grant_manage_external_storage_permission(
102        ad, _NEARBY_SNIPPET_2_PACKAGE_NAME
103    )
104
105    setup_utils.enable_bluetooth_multiplex(ad)
106
107
108  # @typing.override
109  def _teardown_device(self, ad: android_device.AndroidDevice) -> None:
110    super()._teardown_device(ad)
111    ad.nearby2.transferFilesCleanup()
112    ad.unload_snippet('nearby2')
113
114  @base_test.repeat(
115      count=_PERFORMANCE_TEST_REPEAT_COUNT,
116      max_consecutive_error=_PERFORMANCE_TEST_MAX_CONSECUTIVE_ERROR)
117  def test_quick_start_performance(self) -> None:
118    """Stress test for quick start flow."""
119    try:
120      self._mimic_quick_start_test(
121          self.discoverer,
122          self.advertiser,
123          wifi_ssid=self.test_parameters.wifi_ssid,
124          wifi_password=self.test_parameters.wifi_password,
125      )
126    finally:
127      self._write_current_test_report()
128      self._collect_current_test_metrics()
129      time.sleep(_DELAY_BETWEEN_EACH_TEST_CYCLE.total_seconds())
130
131  def _mimic_quick_start_test(
132      self,
133      discoverer,
134      advertiser,
135      wifi_ssid: str = '',
136      wifi_password: str = '',
137  ) -> None:
138    """Mimics quick start flow test with 2 nearby connections."""
139    if self.test_parameters.toggle_airplane_mode_target_side:
140      setup_utils.toggle_airplane_mode(self.advertiser)
141    if self.test_parameters.reset_wifi_connection:
142      self._reset_wifi_connection()
143    # 1. discoverer connect to wifi wlan
144    self._test_result = nc_constants.SingleTestResult()
145    if wifi_ssid:
146      discoverer_wifi_latency = setup_utils.connect_to_wifi_wlan_till_success(
147          discoverer, wifi_ssid, wifi_password
148      )
149      discoverer.log.info(
150          'connecting to wifi in '
151          f'{round(discoverer_wifi_latency.total_seconds())} s'
152      )
153      self._test_result.discoverer_wifi_wlan_expected = True
154      self._test_result.discoverer_wifi_wlan_latency = discoverer_wifi_latency
155
156    # 2. set up 1st connection
157    advertising_discovery_medium = nc_constants.NearbyMedium(
158        self.test_parameters.advertising_discovery_medium
159    )
160    nearby_snippet_1 = nearby_connection_wrapper.NearbyConnectionWrapper(
161        advertiser,
162        discoverer,
163        advertiser.nearby,
164        discoverer.nearby,
165        advertising_discovery_medium=advertising_discovery_medium,
166        connection_medium=nc_constants.NearbyMedium.BT_ONLY,
167        upgrade_medium=nc_constants.NearbyMedium.BT_ONLY,
168    )
169    first_connection_setup_timeouts = nc_constants.ConnectionSetupTimeouts(
170        nc_constants.FIRST_DISCOVERY_TIMEOUT,
171        nc_constants.FIRST_CONNECTION_INIT_TIMEOUT,
172        nc_constants.FIRST_CONNECTION_RESULT_TIMEOUT)
173
174    try:
175      nearby_snippet_1.start_nearby_connection(
176          timeouts=first_connection_setup_timeouts,
177          medium_upgrade_type=nc_constants.MediumUpgradeType.NON_DISRUPTIVE)
178    finally:
179      self._test_result.connection_setup_quality_info = (
180          nearby_snippet_1.connection_quality_info
181      )
182
183    # 3. transfer file through bluetooth
184    file_1_mb = _TRANSFER_FILE_SIZE_1MB
185    self._test_result.bt_transfer_throughput_kbps = (
186        nearby_snippet_1.transfer_file(
187            file_1_mb, nc_constants.FILE_1M_PAYLOAD_TRANSFER_TIMEOUT,
188            nc_constants.PayloadType.FILE))
189
190    # second Wifi connection and transfer
191    # 4. advertiser connect to wifi wlan
192    if wifi_ssid:
193      advertiser_wlan_latency = setup_utils.connect_to_wifi_wlan_till_success(
194          advertiser, wifi_ssid, wifi_password)
195      advertiser.log.info('connecting to wifi in '
196                          f'{round(advertiser_wlan_latency.total_seconds())} s')
197      advertiser.log.info(
198          advertiser.nearby.wifiGetConnectionInfo().get('mFrequency')
199      )
200      self._test_result.advertiser_wifi_wlan_expected = True
201      self._test_result.advertiser_wifi_wlan_latency = advertiser_wlan_latency
202      time.sleep(
203          self.test_parameters.target_post_wifi_connection_idle_time_sec
204      )
205
206    # 5. set up 2nd connection
207    nearby_snippet_2 = nearby_connection_wrapper.NearbyConnectionWrapper(
208        advertiser,
209        discoverer,
210        advertiser.nearby2,
211        discoverer.nearby2,
212        advertising_discovery_medium=advertising_discovery_medium,
213        connection_medium=nc_constants.NearbyMedium.BT_ONLY,
214        upgrade_medium=nc_constants.NearbyMedium(
215            self.test_parameters.upgrade_medium
216        ),
217    )
218    second_connection_setup_timeouts = nc_constants.ConnectionSetupTimeouts(
219        nc_constants.SECOND_DISCOVERY_TIMEOUT,
220        nc_constants.SECOND_CONNECTION_INIT_TIMEOUT,
221        nc_constants.SECOND_CONNECTION_RESULT_TIMEOUT)
222    try:
223      nearby_snippet_2.start_nearby_connection(
224          timeouts=second_connection_setup_timeouts,
225          medium_upgrade_type=nc_constants.MediumUpgradeType.DISRUPTIVE,
226          keep_alive_timeout_ms=self.test_parameters.keep_alive_timeout_ms,
227          keep_alive_interval_ms=self.test_parameters.keep_alive_interval_ms,
228      )
229    finally:
230      self._test_result.second_connection_setup_quality_info = (
231          nearby_snippet_2.connection_quality_info
232      )
233      self._test_result.second_connection_setup_quality_info.medium_upgrade_expected = (
234          True
235      )
236
237    # 6. transfer file through wifi
238    file_1_gb = _TRANSFER_FILE_SIZE_1GB
239    self._test_result.wifi_transfer_throughput_kbps = (
240        nearby_snippet_2.transfer_file(
241            file_1_gb, nc_constants.FILE_1G_PAYLOAD_TRANSFER_TIMEOUT,
242            self.test_parameters.payload_type))
243
244    # 7. disconnect 1st connection
245    nearby_snippet_1.disconnect_endpoint()
246    # 8. disconnect 2nd connection
247    nearby_snippet_2.disconnect_endpoint()
248
249  def _write_current_test_report(self) -> None:
250    """Writes test report for each iteration."""
251
252    quality_info = {
253        '1st connection': (
254            self._test_result.connection_setup_quality_info.get_dict()),
255        'bt_kBps': self._test_result.bt_transfer_throughput_kbps,
256        '2nd connection': (
257            self._test_result.second_connection_setup_quality_info.get_dict()),
258        'wifi_kBps': self._test_result.wifi_transfer_throughput_kbps,
259    }
260
261    if self._test_result.discoverer_wifi_wlan_expected:
262      quality_info['src_wifi_connection'] = str(
263          round(self._test_result.discoverer_wifi_wlan_latency.total_seconds())
264      )
265    if self._test_result.advertiser_wifi_wlan_expected:
266      quality_info['tgt_wifi_connection'] = str(
267          round(self._test_result.advertiser_wifi_wlan_latency.total_seconds())
268      )
269    test_report = {'quality_info': quality_info}
270
271    self.discoverer.log.info(test_report)
272    self.record_data({
273        'Test Class': self.TAG,
274        'Test Name': self.current_test_info.name,
275        'properties': test_report,
276    })
277
278  def _collect_current_test_metrics(self) -> None:
279    """Collects test result metrics for each iteration."""
280    self._quick_start_test_metrics.first_discovery_latencies.append(
281        self._test_result.connection_setup_quality_info.discovery_latency
282    )
283    self._quick_start_test_metrics.first_connection_latencies.append(
284        self._test_result.connection_setup_quality_info.connection_latency
285    )
286    self._quick_start_test_metrics.bt_transfer_throughputs_kbps.append(
287        self._test_result.bt_transfer_throughput_kbps
288    )
289
290    self._quick_start_test_metrics.second_discovery_latencies.append(
291        self._test_result.second_connection_setup_quality_info.discovery_latency
292    )
293    self._quick_start_test_metrics.second_connection_latencies.append(
294        self._test_result.second_connection_setup_quality_info.connection_latency
295    )
296    self._quick_start_test_metrics.medium_upgrade_latencies.append(
297        self._test_result.second_connection_setup_quality_info.medium_upgrade_latency
298    )
299    self._quick_start_test_metrics.upgraded_wifi_transfer_mediums.append(
300        self._test_result.second_connection_setup_quality_info.upgrade_medium)
301    self._quick_start_test_metrics.wifi_transfer_throughputs_kbps.append(
302        self._test_result.wifi_transfer_throughput_kbps
303    )
304    self._quick_start_test_metrics.discoverer_wifi_wlan_latencies.append(
305        self._test_result.discoverer_wifi_wlan_latency)
306    self._quick_start_test_metrics.advertiser_wifi_wlan_latencies.append(
307        self._test_result.advertiser_wifi_wlan_latency)
308
309  def _summary_upgraded_wifi_transfer_mediums(self) -> dict[str, int]:
310    medium_counts = {}
311    for (upgraded_medium
312         ) in self._quick_start_test_metrics.upgraded_wifi_transfer_mediums:
313      if upgraded_medium:
314        medium_counts[upgraded_medium.name] = medium_counts.get(
315            upgraded_medium.name, 0) + 1
316    return medium_counts
317
318  # @typing.override
319  def _summary_test_results(self) -> None:
320    """Summarizes test results of all iterations."""
321    first_bt_transfer_stats = self._stats_throughput_result(
322        'BT',
323        self._quick_start_test_metrics.bt_transfer_throughputs_kbps,
324        nc_constants.BT_TRANSFER_SUCCESS_RATE_TARGET_PERCENTAGE,
325        self.test_parameters.bt_transfer_throughput_median_benchmark_kbps)
326
327    second_wifi_transfer_stats = self._stats_throughput_result(
328        'Wi-Fi',
329        self._quick_start_test_metrics.wifi_transfer_throughputs_kbps,
330        nc_constants.WIFI_TRANSFER_SUCCESS_RATE_TARGET_PERCENTAGE,
331        self.test_parameters.wifi_transfer_throughput_median_benchmark_kbps)
332
333    first_discovery_stats = self._stats_latency_result(
334        self._quick_start_test_metrics.first_discovery_latencies)
335    first_connection_stats = self._stats_latency_result(
336        self._quick_start_test_metrics.first_connection_latencies)
337    second_discovery_stats = self._stats_latency_result(
338        self._quick_start_test_metrics.second_discovery_latencies)
339    second_connection_stats = self._stats_latency_result(
340        self._quick_start_test_metrics.second_connection_latencies)
341    medium_upgrade_stats = self._stats_latency_result(
342        self._quick_start_test_metrics.medium_upgrade_latencies)
343
344    passed = True
345    result_message = 'Passed'
346    fail_message = ''
347    if first_bt_transfer_stats.fail_targets:
348      fail_message += self._generate_target_fail_message(
349          first_bt_transfer_stats.fail_targets)
350    if second_wifi_transfer_stats.fail_targets:
351      fail_message += self._generate_target_fail_message(
352          second_wifi_transfer_stats.fail_targets)
353    if fail_message:
354      passed = False
355      result_message = 'Test Failed due to:\n' + fail_message
356
357    detailed_stats = {
358        '0 test iterations': self.performance_test_iterations,
359        '1 Completed BT/Wi-Fi transfer': (
360            f'{first_bt_transfer_stats.success_count}'
361            f' / {second_wifi_transfer_stats.success_count}'),
362        '2 BT transfer failures': {
363            '1 discovery': first_discovery_stats.failure_count,
364            '2 connection': first_connection_stats.failure_count,
365            '3 transfer': self.performance_test_iterations - (
366                first_bt_transfer_stats.success_count),
367        },
368        '3 Wi-Fi transfer failures': {
369            '1 discovery': second_discovery_stats.failure_count,
370            '2 connection': second_connection_stats.failure_count,
371            '3 upgrade': medium_upgrade_stats.failure_count,
372            '4 transfer': self.performance_test_iterations - (
373                second_wifi_transfer_stats.success_count),
374        },
375        '4 Medium upgrade count': (
376            self._summary_upgraded_wifi_transfer_mediums()),
377        '5 50% and 95% of BT transfer speed (KBps)': (
378            f'{first_bt_transfer_stats.percentile_50_kbps}'
379            f' / {first_bt_transfer_stats.percentile_95_kbps}'),
380        '6 50% and 95% of Wi-Fi transfer speed(KBps)': (
381            f'{second_wifi_transfer_stats.percentile_50_kbps}'
382            f' / {second_wifi_transfer_stats.percentile_95_kbps}'),
383        '7 50% and 95% of discovery latency(sec)': (
384            f'{first_discovery_stats.percentile_50}'
385            f' / {first_discovery_stats.percentile_95} (1st), '
386            f'{second_discovery_stats.percentile_50}'
387            f' / {second_discovery_stats.percentile_95} (2nd)'),
388        '8 50% and 95% of connection latency(sec)': (
389            f'{first_connection_stats.percentile_50}'
390            f' / {first_connection_stats.percentile_95} (1st), '
391            f'{second_connection_stats.percentile_50}'
392            f' / {second_connection_stats.percentile_95} (2nd)'),
393        '9 50% and 95% of medium upgrade latency(sec)': (
394            f'{medium_upgrade_stats.percentile_50}'
395            f' / {medium_upgrade_stats.percentile_95}'),
396    }
397
398    self.record_data({
399        'Test Class': self.TAG,
400        'properties': {
401            'test_script_version': _TEST_SCRIPT_VERSION,
402            '00_test_report_alias_name': (
403                self.test_parameters.test_report_alias_name),
404            '01_test_result': result_message,
405            '02_source_device_serial': self.discoverer.serial,
406            '03_target_device_serial': self.advertiser.serial,
407            '04_source_GMS_version': setup_utils.dump_gms_version(
408                self.discoverer),
409            '05_target_GMS_version': setup_utils.dump_gms_version(
410                self.advertiser),
411            '06_detailed_stats': detailed_stats
412            }
413        })
414
415    asserts.assert_true(passed, result_message)
416
417
418if __name__ == '__main__':
419  test_runner.main()
420