1#!/usr/bin/env python
2
3import argparse
4from datetime import datetime
5import os
6from pathlib import Path
7import shutil
8import subprocess
9import sys
10import xml.etree.ElementTree as ET
11
12JAVA_UNIT_TESTS = 'test/mts/tools/mts-tradefed/res/config/mts-bluetooth-tests-list-shard-01.xml'
13NATIVE_UNIT_TESTS = 'test/mts/tools/mts-tradefed/res/config/mts-bluetooth-tests-list-shard-02.xml'
14DO_NOT_RETRY_TESTS = {
15  'CtsBluetoothTestCases',
16  'GoogleBluetoothInstrumentationTests',
17}
18MAX_TRIES = 3
19
20
21def run_pts_bot(logs_out):
22  run_pts_bot_cmd = [
23      # atest command with verbose mode.
24      'atest',
25      '-d',
26      '-v',
27      'pts-bot',
28      # Coverage tool chains and specify that coverage should be flush to the
29      # disk between each tests.
30      '--',
31      '--coverage',
32      '--coverage-toolchain JACOCO',
33      '--coverage-toolchain CLANG',
34      '--coverage-flush',
35  ]
36  with open(f'{logs_out}/pts_bot.txt', 'w') as f:
37    subprocess.run(run_pts_bot_cmd, stdout=f, stderr=subprocess.STDOUT)
38
39
40def list_unit_tests():
41  android_build_top = os.getenv('ANDROID_BUILD_TOP')
42
43  unit_tests = []
44  java_unit_xml = ET.parse(f'{android_build_top}/{JAVA_UNIT_TESTS}')
45  for child in java_unit_xml.getroot():
46    value = child.attrib['value']
47    if 'enable:true' in value:
48      test = value.replace(':enable:true', '')
49      unit_tests.append(test)
50
51  native_unit_xml = ET.parse(f'{android_build_top}/{NATIVE_UNIT_TESTS}')
52  for child in native_unit_xml.getroot():
53    value = child.attrib['value']
54    if 'enable:true' in value:
55      test = value.replace(':enable:true', '')
56      unit_tests.append(test)
57
58  return unit_tests
59
60
61def run_unit_test(test, logs_out):
62  print(f'Test started: {test}')
63
64  # Env variables necessary for native unit tests.
65  env = os.environ.copy()
66  env['CLANG_COVERAGE_CONTINUOUS_MODE'] = 'true'
67  env['CLANG_COVERAGE'] = 'true'
68  env['NATIVE_COVERAGE_PATHS'] = 'packages/modules/Bluetooth'
69  run_test_cmd = [
70      # atest command with verbose mode.
71      'atest',
72      '-d',
73      '-v',
74      test,
75      # Coverage tool chains and specify that coverage should be flush to the
76      # disk between each tests.
77      '--',
78      '--coverage',
79      '--coverage-toolchain JACOCO',
80      '--coverage-toolchain CLANG',
81      '--coverage-flush',
82      # Allows tests to use hidden APIs.
83      '--test-arg ',
84      'com.android.compatibility.testtype.LibcoreTest:hidden-api-checks:false',
85      '--test-arg ',
86      'com.android.tradefed.testtype.AndroidJUnitTest:hidden-api-checks:false',
87      '--test-arg ',
88      'com.android.tradefed.testtype.InstrumentationTest:hidden-api-checks:false',
89      '--skip-system-status-check ',
90      'com.android.tradefed.suite.checker.ShellStatusChecker',
91  ]
92
93  try_count = 1
94  while (try_count == 1 or test not in DO_NOT_RETRY_TESTS) and try_count <= MAX_TRIES:
95    with open(f'{logs_out}/{test}_{try_count}.txt', 'w') as f:
96      if try_count > 1: print(f'Retrying {test}: count = {try_count}')
97      returncode = subprocess.run(
98          run_test_cmd, env=env, stdout=f, stderr=subprocess.STDOUT).returncode
99      if returncode == 0: break
100      try_count += 1
101
102  print(
103      f'Test ended [{"Success" if returncode == 0 else "Failed"}]: {test}')
104
105
106def pull_and_rename_trace_for_test(test, trace):
107  date = datetime.now().strftime("%Y%m%d")
108  temp_trace = Path('temp_trace')
109  subprocess.run(['adb', 'pull', '/data/misc/trace', temp_trace])
110  for child in temp_trace.iterdir():
111    child = child.rename(f'{child.parent}/{date}_{test}_{child.name}')
112    shutil.copy(child, trace)
113  shutil.rmtree(temp_trace, ignore_errors=True)
114
115
116def generate_java_coverage(bt_apex_name, trace_path, coverage_out):
117
118  out = os.getenv('OUT')
119  android_host_out = os.getenv('ANDROID_HOST_OUT')
120
121  java_coverage_out = Path(f'{coverage_out}/java')
122  temp_path = Path(f'{coverage_out}/temp')
123  if temp_path.exists():
124    shutil.rmtree(temp_path, ignore_errors=True)
125  temp_path.mkdir()
126
127  framework_jar_path = Path(
128      f'{out}/obj/PACKAGING/jacoco_intermediates/JAVA_LIBRARIES/framework-bluetooth.{bt_apex_name}_intermediates'
129  )
130  service_jar_path = Path(
131      f'{out}/obj/PACKAGING/jacoco_intermediates/JAVA_LIBRARIES/service-bluetooth.{bt_apex_name}_intermediates'
132  )
133  app_jar_path = Path(
134      f'{out}/obj/PACKAGING/jacoco_intermediates/ETC/Bluetooth{"Google" if "com.google" in bt_apex_name else ""}.{bt_apex_name}_intermediates'
135  )
136
137  # From google3/configs/wireless/android/testing/atp/prod/mainline-engprod/templates/modules/bluetooth.gcl.
138  framework_exclude_classes = [
139      # Exclude statically linked & jarjar'ed classes.
140      '**/com/android/bluetooth/jarjar/**/*.class',
141      # Exclude AIDL generated interfaces.
142      '**/android/bluetooth/I*$Default.class',
143      '**/android/bluetooth/**/I*$Default.class',
144      '**/android/bluetooth/I*$Stub.class',
145      '**/android/bluetooth/**/I*$Stub.class',
146      '**/android/bluetooth/I*$Stub$Proxy.class',
147      '**/android/bluetooth/**/I*$Stub$Proxy.class',
148      # Exclude annotations.
149      '**/android/bluetooth/annotation/**/*.class',
150  ]
151  service_exclude_classes = [
152      # Exclude statically linked & jarjar'ed classes.
153      '**/android/support/**/*.class',
154      '**/androidx/**/*.class',
155      '**/com/android/bluetooth/jarjar/**/*.class',
156      '**/com/android/internal/**/*.class',
157      '**/com/google/**/*.class',
158      '**/kotlin/**/*.class',
159      '**/kotlinx/**/*.class',
160      '**/org/**/*.class',
161  ]
162  app_exclude_classes = [
163      # Exclude statically linked & jarjar'ed classes.
164      '**/android/hardware/**/*.class',
165      '**/android/hidl/**/*.class',
166      '**/android/net/**/*.class',
167      '**/android/support/**/*.class',
168      '**/androidx/**/*.class',
169      '**/com/android/bluetooth/jarjar/**/*.class',
170      '**/com/android/internal/**/*.class',
171      '**/com/android/obex/**/*.class',
172      '**/com/android/vcard/**/*.class',
173      '**/com/google/**/*.class',
174      '**/kotlin/**/*.class',
175      '**/kotlinx/**/*.class',
176      '**/javax/**/*.class',
177      '**/org/**/*.class',
178      # Exclude SIM Access Profile (SAP) which is being deprecated.
179      '**/com/android/bluetooth/sap/*.class',
180      # Added for local runs.
181      '**/com/android/bluetooth/**/BluetoothMetrics*.class',
182      '**/com/android/bluetooth/**/R*.class',
183  ]
184
185  # Merged ec files.
186  merged_ec_path = Path(f'{temp_path}/merged.ec')
187  subprocess.run((
188      f'java -jar {android_host_out}/framework/jacoco-cli.jar merge {trace_path.absolute()}/*.ec '
189      f'--destfile {merged_ec_path.absolute()}'),
190                 shell=True)
191
192  # Copy and extract jar files.
193  framework_temp_path = Path(f'{temp_path}/{framework_jar_path.name}')
194  service_temp_path = Path(f'{temp_path}/{service_jar_path.name}')
195  app_temp_path = Path(f'{temp_path}/{app_jar_path.name}')
196
197  shutil.copytree(framework_jar_path, framework_temp_path)
198  shutil.copytree(service_jar_path, service_temp_path)
199  shutil.copytree(app_jar_path, app_temp_path)
200
201  current_dir_path = Path.cwd()
202  for p in [framework_temp_path, service_temp_path, app_temp_path]:
203    os.chdir(p.absolute())
204    os.system('jar xf jacoco-report-classes.jar')
205    os.chdir(current_dir_path)
206
207  os.remove(f'{framework_temp_path}/jacoco-report-classes.jar')
208  os.remove(f'{service_temp_path}/jacoco-report-classes.jar')
209  os.remove(f'{app_temp_path}/jacoco-report-classes.jar')
210
211  # Generate coverage report.
212  exclude_classes = []
213  for glob in framework_exclude_classes:
214    exclude_classes.extend(list(framework_temp_path.glob(glob)))
215  for glob in service_exclude_classes:
216    exclude_classes.extend(list(service_temp_path.glob(glob)))
217  for glob in app_exclude_classes:
218    exclude_classes.extend(list(app_temp_path.glob(glob)))
219
220  for c in exclude_classes:
221    if c.exists():
222      os.remove(c.absolute())
223
224  gen_java_cov_report_cmd = [
225      f'java',
226      f'-jar',
227      f'{android_host_out}/framework/jacoco-cli.jar',
228      f'report',
229      f'{merged_ec_path.absolute()}',
230      f'--classfiles',
231      f'{temp_path.absolute()}',
232      f'--html',
233      f'{java_coverage_out.absolute()}',
234      f'--name',
235      f'{java_coverage_out.absolute()}.html',
236  ]
237  subprocess.run(gen_java_cov_report_cmd)
238
239  # Cleanup.
240  shutil.rmtree(temp_path, ignore_errors=True)
241
242
243def generate_native_coverage(bt_apex_name, trace_path, coverage_out):
244
245  out = os.getenv('OUT')
246  android_build_top = os.getenv('ANDROID_BUILD_TOP')
247
248  native_coverage_out = Path(f'{coverage_out}/native')
249  temp_path = Path(f'{coverage_out}/temp')
250  if temp_path.exists():
251    shutil.rmtree(temp_path, ignore_errors=True)
252  temp_path.mkdir()
253
254  # From google3/configs/wireless/android/testing/atp/prod/mainline-engprod/templates/modules/bluetooth.gcl.
255  exclude_files = {
256      'android/',
257      # Exclude AIDLs definition and generated interfaces.
258      'system/.*_aidl.*',
259      'system/binder/',
260      # Exclude tests.
261      'system/.*_test.*',
262      'system/.*_mock.*',
263      'system/.*_unittest.*',
264      'system/blueberry/',
265      'system/test/',
266      # Exclude config and doc.
267      'system/build/',
268      'system/conf/',
269      'system/doc/',
270      # Exclude (currently) unused GD code.
271      'system/gd/att/',
272      'system/gd/l2cap/',
273      'system/gd/neighbor/',
274      'system/gd/rust/',
275      'system/gd/security/',
276      # Exclude legacy AVRCP implementation (to be removed, current AVRCP
277      # implementation is in packages/modules/Bluetooth/system/profile/avrcp)
278      'system/stack/avrc/',
279      # Exclude audio HIDL since AIDL is used instead today (in
280      # packages/modules/Bluetooth/system/audio_hal_interface/aidl)
281      'system/audio_hal_interface/hidl/',
282  }
283
284  # Merge profdata files.
285  profdata_path = Path(f'{temp_path}/coverage.profdata')
286  subprocess.run(
287      f'llvm-profdata merge --sparse -o {profdata_path.absolute()} {trace_path.absolute()}/*.profraw',
288      shell=True)
289
290  gen_native_cov_report_cmd = [
291      f'llvm-cov',
292      f'show',
293      f'-format=html',
294      f'-output-dir={native_coverage_out.absolute()}',
295      f'-instr-profile={profdata_path.absolute()}',
296      f'{out}/symbols/apex/{bt_apex_name}/lib64/libbluetooth_jni.so',
297      f'-path-equivalence=/proc/self/cwd,{android_build_top}',
298      f'/proc/self/cwd/packages/modules/Bluetooth',
299  ]
300  for f in exclude_files:
301    gen_native_cov_report_cmd.append(f'-ignore-filename-regex={f}')
302  subprocess.run(gen_native_cov_report_cmd, cwd=android_build_top)
303
304  # Cleanup.
305  shutil.rmtree(temp_path, ignore_errors=True)
306
307
308if __name__ == '__main__':
309
310  parser = argparse.ArgumentParser()
311  parser.add_argument(
312      '--apex-name',
313      default='com.android.btservices',
314      help='bluetooth apex name. Default: com.android.btservices')
315  parser.add_argument(
316      '--java', action='store_true', help='generate Java coverage')
317  parser.add_argument(
318      '--native', action='store_true', help='generate native coverage')
319  parser.add_argument(
320      '--out',
321      type=str,
322      default='out_coverage',
323      help='out directory for coverage reports. Default: ./out_coverage')
324  parser.add_argument(
325      '--trace',
326      type=str,
327      default='trace',
328      help='trace directory with .ec and .profraw files. Default: ./trace')
329  parser.add_argument(
330      '--full-report',
331      action='store_true',
332      help='run all tests and compute coverage report')
333  args = parser.parse_args()
334
335  coverage_out = Path(args.out)
336  shutil.rmtree(coverage_out, ignore_errors=True)
337  coverage_out.mkdir()
338
339  if not args.full_report:
340    trace_path = Path(args.trace)
341    if (not trace_path.exists() or not trace_path.is_dir()):
342      sys.exit('Trace directory does not exist')
343
344    if (args.java):
345      generate_java_coverage(args.apex_name, trace_path, coverage_out)
346    if (args.native):
347      generate_native_coverage(args.apex_name, trace_path, coverage_out)
348
349  else:
350
351    # Output logs directory
352    logs_out = Path('logs_bt_tests')
353    logs_out.mkdir(exist_ok=True)
354
355    # Compute Pandora tests coverage
356    coverage_out_pandora = Path(f'{coverage_out}/pandora')
357    coverage_out_pandora.mkdir()
358    trace_pandora = Path('trace_pandora')
359    shutil.rmtree(trace_pandora, ignore_errors=True)
360    trace_pandora.mkdir()
361    subprocess.run(['adb', 'shell', 'rm', '/data/misc/trace/*'])
362    run_pts_bot(logs_out)
363    pull_and_rename_trace_for_test('pts_bot', trace_pandora)
364
365    generate_java_coverage(args.apex_name, trace_pandora, coverage_out_pandora)
366    generate_native_coverage(args.apex_name, trace_pandora, coverage_out_pandora)
367
368    # Compute unit tests coverage
369    coverage_out_unit = Path(f'{coverage_out}/unit')
370    coverage_out_unit.mkdir()
371    trace_unit = Path('trace_unit')
372    shutil.rmtree(trace_unit, ignore_errors=True)
373    trace_unit.mkdir()
374
375    unit_tests = list_unit_tests()
376    for test in unit_tests:
377      subprocess.run(['adb', 'shell', 'rm', '/data/misc/trace/*'])
378      run_unit_test(test, logs_out)
379      pull_and_rename_trace_for_test(test, trace_unit)
380
381    generate_java_coverage(args.apex_name, trace_unit, coverage_out_unit)
382    generate_native_coverage(args.apex_name, trace_unit, coverage_out_unit)
383
384    # Compute all tests coverage
385    coverage_out_mainline = Path(f'{coverage_out}/mainline')
386    coverage_out_mainline.mkdir()
387    trace_mainline = Path('trace_mainline')
388    shutil.rmtree(trace_mainline, ignore_errors=True)
389    trace_mainline.mkdir()
390    for child in trace_pandora.iterdir():
391      shutil.copy(child, trace_mainline)
392    for child in trace_unit.iterdir():
393      shutil.copy(child, trace_mainline)
394
395    generate_java_coverage(args.apex_name, trace_mainline, coverage_out_mainline)
396    generate_native_coverage(args.apex_name, trace_mainline, coverage_out_mainline)
397