1#  Copyright (C) 2023 The Android Open Source Project
2#
3#  Licensed under the Apache License, Version 2.0 (the "License");
4#  you may not use this file except in compliance with the License.
5#  You may obtain a copy of the License at
6#
7#       http://www.apache.org/licenses/LICENSE-2.0
8#
9#  Unless required by applicable law or agreed to in writing, software
10#  distributed under the License is distributed on an "AS IS" BASIS,
11#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12#  See the License for the specific language governing permissions and
13#  limitations under the License.
14
15"""Stress tests for Neaby Share/Nearby Connection flow."""
16
17import dataclasses
18import datetime
19import logging
20import os
21import sys
22import time
23
24# check the python version
25if sys.version_info < (3,10):
26  logging.error('The test only can run on python 3.10 and above')
27  exit()
28
29from mobly import asserts
30from mobly import base_test
31from mobly import test_runner
32
33# Allows local imports to be resolved via relative path, so the test can be run
34# without building.
35_performance_test_dir = os.path.dirname(os.path.dirname(__file__))
36if _performance_test_dir not in sys.path:
37  sys.path.append(_performance_test_dir)
38
39from performance_test import nc_base_test
40from performance_test import nc_constants
41from performance_test import nearby_connection_wrapper
42from performance_test import setup_utils
43
44_TEST_SCRIPT_VERSION = '1.6'
45
46_DELAY_BETWEEN_EACH_TEST_CYCLE = datetime.timedelta(seconds=5)
47_TRANSFER_FILE_SIZE_1GB = 1024 * 1024
48
49_PERFORMANCE_TEST_REPEAT_COUNT = 100
50_PERFORMANCE_TEST_MAX_CONSECUTIVE_ERROR = 5
51
52
53class NearbyShareStressTest(nc_base_test.NCBaseTestClass):
54  """Nearby Share stress test."""
55
56  @dataclasses.dataclass(frozen=False)
57  class NearbyShareTestMetrics:
58    """Metrics data for Nearby Share test."""
59
60    discovery_latencies: list[datetime.timedelta] = dataclasses.field(
61        default_factory=list[datetime.timedelta]
62    )
63    connection_latencies: list[datetime.timedelta] = dataclasses.field(
64        default_factory=list[datetime.timedelta]
65    )
66    medium_upgrade_latencies: list[datetime.timedelta] = dataclasses.field(
67        default_factory=list[datetime.timedelta]
68    )
69    wifi_transfer_throughputs_kbps: list[float] = dataclasses.field(
70        default_factory=list[float]
71    )
72
73  # @typing.override
74  def __init__(self, configs):
75    super().__init__(configs)
76    self._test_result = nc_constants.SingleTestResult()
77    self._nearby_share_test_metrics = self.NearbyShareTestMetrics()
78    self._test_script_version = _TEST_SCRIPT_VERSION
79
80  # @typing.override
81  def setup_class(self):
82    super().setup_class()
83    wifi_ssid = self.test_parameters.wifi_ssid
84    wifi_password = self.test_parameters.wifi_password
85    if wifi_ssid:
86      discoverer_wifi_latency = setup_utils.connect_to_wifi_wlan_till_success(
87          self.discoverer, wifi_ssid, wifi_password
88      )
89      self.discoverer.log.info(
90          'connecting to wifi in '
91          f'{round(discoverer_wifi_latency.total_seconds())} s'
92      )
93      advertiser_wlan_latency = setup_utils.connect_to_wifi_wlan_till_success(
94          self.advertiser, wifi_ssid, wifi_password)
95      self.advertiser.log.info(
96          'connecting to wifi in '
97          f'{round(advertiser_wlan_latency.total_seconds())} s')
98      self.advertiser.log.info(
99          self.advertiser.nearby.wifiGetConnectionInfo().get('mFrequency')
100      )
101
102    self.performance_test_iterations = getattr(
103        self.test_nearby_share_performance, base_test.ATTR_REPEAT_CNT)
104    logging.info('performance test iterations: %s',
105                 self.performance_test_iterations)
106
107  @base_test.repeat(
108      count=_PERFORMANCE_TEST_REPEAT_COUNT,
109      max_consecutive_error=_PERFORMANCE_TEST_MAX_CONSECUTIVE_ERROR)
110  def test_nearby_share_performance(self):
111    """Nearby Share stress test, which only transfer data through WiFi."""
112    try:
113      self._mimic_nearby_share()
114    finally:
115      self._write_current_test_report()
116      self._collect_current_test_metrics()
117      time.sleep(_DELAY_BETWEEN_EACH_TEST_CYCLE.total_seconds())
118
119  def _mimic_nearby_share(self):
120    """Mimics Nearby Share stress test, which only transfer data through WiFi."""
121    self._test_result = nc_constants.SingleTestResult()
122
123    # 1. set up BT and WiFi connection
124    advertising_discovery_medium = nc_constants.NearbyMedium(
125        self.test_parameters.advertising_discovery_medium
126    )
127    nearby_snippet_1 = nearby_connection_wrapper.NearbyConnectionWrapper(
128        self.advertiser,
129        self.discoverer,
130        self.advertiser.nearby,
131        self.discoverer.nearby,
132        advertising_discovery_medium=advertising_discovery_medium,
133        connection_medium=nc_constants.NearbyMedium(
134            self.test_parameters.connection_medium
135        ),
136        upgrade_medium=nc_constants.NearbyMedium(
137            self.test_parameters.upgrade_medium
138        ),
139    )
140    connection_setup_timeouts = nc_constants.ConnectionSetupTimeouts(
141        nc_constants.FIRST_DISCOVERY_TIMEOUT,
142        nc_constants.FIRST_CONNECTION_INIT_TIMEOUT,
143        nc_constants.FIRST_CONNECTION_RESULT_TIMEOUT)
144
145    try:
146      nearby_snippet_1.start_nearby_connection(
147          timeouts=connection_setup_timeouts,
148          medium_upgrade_type=nc_constants.MediumUpgradeType.NON_DISRUPTIVE)
149    finally:
150      self._test_result.connection_setup_quality_info = (
151          nearby_snippet_1.connection_quality_info
152      )
153      self._test_result.connection_setup_quality_info.medium_upgrade_expected = (
154          True
155      )
156
157    # 2. transfer file through WiFi
158    file_1_gb = _TRANSFER_FILE_SIZE_1GB
159    self._test_result.wifi_transfer_throughput_kbps = (
160        nearby_snippet_1.transfer_file(
161            file_1_gb, nc_constants.FILE_1G_PAYLOAD_TRANSFER_TIMEOUT,
162            nc_constants.PayloadType.FILE))
163    # 3. disconnect
164    nearby_snippet_1.disconnect_endpoint()
165
166  def _write_current_test_report(self):
167    """Writes test report for each iteration."""
168
169    quality_info = {
170        'Latency (sec)': (
171            self._test_result.connection_setup_quality_info.get_dict()),
172        'Speed (kByte/sec)': self._test_result.wifi_transfer_throughput_kbps,
173    }
174    test_report = {'quality_info': quality_info}
175
176    self.discoverer.log.info(test_report)
177    self.record_data({
178        'Test Class': self.TAG,
179        'Test Name': self.current_test_info.name,
180        'properties': test_report,
181    })
182
183  def _collect_current_test_metrics(self):
184    """Collects test result metrics for each iteration."""
185    self._nearby_share_test_metrics.discovery_latencies.append(
186        self._test_result.connection_setup_quality_info.discovery_latency
187    )
188    self._nearby_share_test_metrics.connection_latencies.append(
189        self._test_result.connection_setup_quality_info.connection_latency
190    )
191    self._nearby_share_test_metrics.medium_upgrade_latencies.append(
192        self._test_result.connection_setup_quality_info.medium_upgrade_latency
193    )
194    self._nearby_share_test_metrics.wifi_transfer_throughputs_kbps.append(
195        self._test_result.wifi_transfer_throughput_kbps
196    )
197
198  # @typing.override
199  def _summary_test_results(self):
200    """Summarizes test results of all iterations."""
201    wifi_transfer_stats = self._stats_throughput_result(
202        'WiFi',
203        self._nearby_share_test_metrics.wifi_transfer_throughputs_kbps,
204        nc_constants.WIFI_TRANSFER_SUCCESS_RATE_TARGET_PERCENTAGE,
205        self.test_parameters.wifi_transfer_throughput_median_benchmark_kbps)
206
207    discovery_stats = self._stats_latency_result(
208        self._nearby_share_test_metrics.discovery_latencies)
209    connection_stats = self._stats_latency_result(
210        self._nearby_share_test_metrics.connection_latencies)
211    medium_upgrade_stats = self._stats_latency_result(
212        self._nearby_share_test_metrics.medium_upgrade_latencies
213    )
214
215    passed = True
216    result_message = 'Passed'
217    fail_message = ''
218    if wifi_transfer_stats.fail_targets:
219      fail_message += self._generate_target_fail_message(
220          wifi_transfer_stats.fail_targets)
221    if fail_message:
222      passed = False
223      result_message = 'Test Failed due to:\n' + fail_message
224
225    detailed_stats = {
226        '0 test iterations': self.performance_test_iterations,
227        '1 Completed WiFi transfer': f'{wifi_transfer_stats.success_count}',
228        '2 failure counts': {
229            '1 discovery': discovery_stats.failure_count,
230            '2 connection': connection_stats.failure_count,
231            '3 upgrade': medium_upgrade_stats.failure_count,
232            '4 transfer': self.performance_test_iterations - (
233                wifi_transfer_stats.success_count),
234        },
235        '3 50% and 95% of WiFi transfer speed (KBps)': (
236            f'{wifi_transfer_stats.percentile_50_kbps}'
237            f' / {wifi_transfer_stats.percentile_95_kbps}'),
238        '4 50% and 95% of discovery latency(sec)': (
239            f'{discovery_stats.percentile_50}'
240            f' / {discovery_stats.percentile_95}'),
241        '5 50% and 95% of connection latency(sec)': (
242            f'{connection_stats.percentile_50}'
243            f' / {connection_stats.percentile_95}'),
244        '6 50% and 95% of upgrade latency(sec)': (
245            f'{medium_upgrade_stats.percentile_50}'
246            f' / {medium_upgrade_stats.percentile_95}'),
247    }
248
249    self.record_data({
250        'Test Class': self.TAG,
251        'properties': {
252            'test_script_version': self._test_script_version,
253            '00_test_report_alias_name': (
254                self.test_parameters.test_report_alias_name),
255            '01_test_result': result_message,
256            '02_source_device_serial': self.discoverer.serial,
257            '03_target_device_serial': self.advertiser.serial,
258            '04_source_GMS_version': setup_utils.dump_gms_version(
259                self.discoverer),
260            '05_target_GMS_version': setup_utils.dump_gms_version(
261                self.advertiser),
262            '06_detailed_stats': detailed_stats
263            }
264        })
265
266    asserts.assert_true(passed, result_message)
267
268
269if __name__ == '__main__':
270  test_runner.main()
271