1#!/usr/bin/env python 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18# Parses the output of parse_ltp_{make,make_install} and generates a 19# corresponding Android.mk. 20# 21# This process is split into two steps so this second step can later be replaced 22# with an Android.bp generator. 23 24import argparse 25import fileinput 26import json 27import os.path 28import re 29 30import make_parser 31import make_install_parser 32 33MAKE_DRY_RUN_FILE_NAME = 'make_dry_run.dump' 34MAKE_INSTALL_DRY_RUN_FILE_NAME = 'make_install_dry_run.dump' 35DISABLED_TESTS_FILE_NAME = 'disabled_tests.txt' 36DISABLED_LIBS_FILE_NAME = 'disabled_libs.txt' 37DISABLED_CFLAGS_FILE_NAME = 'disabled_cflags.txt' 38 39 40class BuildGenerator(object): 41 '''A class to parse make output and convert the result to Android.ltp.mk modules. 42 43 Attributes: 44 _bp_result: list of strings for blueprint file 45 _mk_result: list of strings for makefile 46 _custom_cflags: dict of string (module name) to lists of strings (cflags 47 to add for said module) 48 _unused_custom_cflags: set of strings; tracks the modules with custom 49 cflags that we haven't yet seen 50 _packages: list of strings of packages for package list file 51 ''' 52 53 def __init__(self, custom_cflags): 54 self._bp_result = [] 55 self._mk_result = [] 56 self._custom_cflags = custom_cflags 57 self._unused_custom_cflags = set(custom_cflags) 58 self._packages = [] 59 60 def UniqueKeepOrder(self, sequence): 61 '''Get a copy of list where items are unique and order is preserved. 62 63 Args: 64 sequence: a sequence, can be a list, tuple, or other iterable 65 66 Returns: 67 a list where items copied from input sequence are unique 68 and order is preserved. 69 ''' 70 seen = set() 71 return [x for x in sequence if not (x in seen or seen.add(x))] 72 73 def ReadCommentedText(self, file_path): 74 '''Read pound commented text file into a list of lines. 75 76 Comments or empty lines will be excluded 77 78 Args: 79 file_path: string 80 ''' 81 ret = set() 82 with open(file_path, 'r') as f: 83 lines = [line.strip() for line in f.readlines()] 84 ret = set([s for s in lines if s and not s.startswith('#')]) 85 86 return ret 87 88 def ArTargetToLibraryName(self, ar_target): 89 '''Convert ar target to library name. 90 91 Args: 92 ar_target: string 93 ''' 94 return os.path.basename(ar_target)[len('lib'):-len('.a')] 95 96 def BuildExecutable(self, cc_target, local_src_files, local_cflags, 97 local_c_includes, local_libraries, ltp_libs, 98 ltp_libs_used, ltp_names_used): 99 '''Build a test module. 100 101 Args: 102 cc_target: string 103 local_src_files: list of string 104 local_cflags: list of string 105 local_c_includes: list of string 106 local_libraries: list of string 107 ltp_libs: list of string 108 ltp_libs_used: set of string 109 ltp_names_used: set of string, set of already used cc_target basenames 110 ''' 111 base_name = os.path.basename(cc_target) 112 if base_name in ltp_names_used: 113 print 'ERROR: base name {} of cc_target {} already used. Skipping...'.format( 114 base_name, cc_target) 115 return 116 ltp_names_used.add(base_name) 117 118 if cc_target in self._custom_cflags: 119 local_cflags.extend(self._custom_cflags[cc_target]) 120 self._unused_custom_cflags.remove(cc_target) 121 122 # ltp_defaults already adds the include directory 123 local_c_includes = [i for i in local_c_includes if i != 'include'] 124 125 self._packages.append('ltp_%s' % base_name) 126 127 self._bp_result.append('cc_test {') 128 self._bp_result.append(' name: "ltp_%s",' % base_name) 129 self._bp_result.append(' stem: "%s",' % base_name) 130 self._bp_result.append(' defaults: ["ltp_test_defaults"],') 131 132 if len(local_src_files) == 1: 133 self._bp_result.append(' srcs: ["%s"],' % list(local_src_files)[0]) 134 else: 135 self._bp_result.append(' srcs: [') 136 for src in local_src_files: 137 self._bp_result.append(' "%s",' % src) 138 self._bp_result.append(' ],') 139 140 if len(local_cflags) == 1: 141 self._bp_result.append(' cflags: ["%s"],' % list(local_cflags)[0]) 142 elif len(local_cflags) > 1: 143 self._bp_result.append(' cflags: [') 144 for cflag in local_cflags: 145 self._bp_result.append(' "%s",' % cflag) 146 self._bp_result.append(' ],') 147 148 if len(local_c_includes) == 1: 149 self._bp_result.append(' local_include_dirs: ["%s"],' % list(local_c_includes)[0]) 150 elif len(local_c_includes) > 1: 151 self._bp_result.append(' local_include_dirs: [') 152 for d in local_c_includes: 153 self._bp_result.append(' "%s",' % d) 154 self._bp_result.append(' ],') 155 156 bionic_builtin_libs = set(['m', 'rt', 'pthread']) 157 filtered_libs = set(local_libraries).difference(bionic_builtin_libs) 158 159 static_libraries = set(i for i in local_libraries if i in ltp_libs) 160 if len(static_libraries) == 1: 161 self._bp_result.append(' static_libs: ["libltp_%s"],' % list(static_libraries)[0]) 162 elif len(static_libraries) > 1: 163 self._bp_result.append(' static_libs: [') 164 for lib in static_libraries: 165 self._bp_result.append(' "libltp_%s",' % lib) 166 self._bp_result.append(' ],') 167 168 for lib in static_libraries: 169 ltp_libs_used.add(lib) 170 171 shared_libraries = set(i for i in filtered_libs if i not in ltp_libs) 172 if len(shared_libraries) == 1: 173 self._bp_result.append(' shared_libs: ["lib%s"],' % list(shared_libraries)[0]) 174 elif len(shared_libraries) > 1: 175 self._bp_result.append(' shared_libs: [') 176 for lib in shared_libraries: 177 self._bp_result.append(' "lib%s",' % lib) 178 self._bp_result.append(' ],') 179 180 self._bp_result.append('}') 181 self._bp_result.append('') 182 183 def BuildStaticLibrary(self, ar_target, local_src_files, local_cflags, 184 local_c_includes): 185 '''Build a library module. 186 187 Args: 188 ar_target: string 189 local_src_files: list of string 190 local_cflags: list of string 191 local_c_includes: list of string 192 ''' 193 self._bp_result.append('cc_library_static {') 194 self._bp_result.append(' name: "libltp_%s",' % 195 self.ArTargetToLibraryName(ar_target)) 196 self._bp_result.append(' defaults: ["ltp_defaults"],') 197 198 if len(local_c_includes): 199 self._bp_result.append(' local_include_dirs: [') 200 for d in local_c_includes: 201 self._bp_result.append(' "%s",' % d) 202 self._bp_result.append(' ],') 203 204 if len(local_cflags): 205 self._bp_result.append(' cflags: [') 206 for cflag in local_cflags: 207 self._bp_result.append(' "%s",' % cflag) 208 self._bp_result.append(' ],') 209 210 self._bp_result.append(' srcs: [') 211 for src in local_src_files: 212 self._bp_result.append(' "%s",' % src) 213 self._bp_result.append(' ],') 214 215 self._bp_result.append('}') 216 self._bp_result.append('') 217 218 def BuildPrebuilt(self, install_target, local_src_file): 219 '''Build a prebuild module. 220 221 Args: 222 install_target: string 223 local_src_file: string 224 ''' 225 self._mk_result.append('module_prebuilt := %s' % install_target) 226 self._mk_result.append('module_src_files := %s' % local_src_file) 227 module_dir = os.path.dirname(install_target) 228 module_stem = os.path.basename(install_target) 229 module = 'ltp_%s' % install_target.replace('/', '_') 230 self._packages.append(module) 231 232 self._mk_result.append('include $(ltp_build_prebuilt)') 233 self._mk_result.append('') 234 235 def HandleParsedRule(self, line, rules): 236 '''Prepare parse rules. 237 238 Args: 239 line: string 240 rules: dictionary {string, dictionary} 241 ''' 242 groups = re.match(r'(.*)\[\'(.*)\'\] = \[(.*)\]', line).groups() 243 rule = groups[0] 244 rule_key = groups[1] 245 if groups[2] == '': 246 rule_value = [] 247 else: 248 rule_value = list(i.strip()[1:-1] for i in groups[2].split(',')) 249 250 rule_value = self.UniqueKeepOrder(rule_value) 251 rules.setdefault(rule, {})[rule_key] = rule_value 252 253 def ParseInput(self, input_list, ltp_root): 254 '''Parse a interpreted make output and produce Android.ltp.mk module. 255 256 Args: 257 input_list: list of string 258 ''' 259 disabled_tests = self.ReadCommentedText(DISABLED_TESTS_FILE_NAME) 260 disabled_libs = self.ReadCommentedText(DISABLED_LIBS_FILE_NAME) 261 disabled_cflags = self.ReadCommentedText(DISABLED_CFLAGS_FILE_NAME) 262 263 rules = {} 264 for line in input_list: 265 self.HandleParsedRule(line.strip(), rules) 266 267 # .a target -> .o files 268 ar = rules.get('ar', {}) 269 # executable target -> .o files 270 cc_link = rules.get('cc_link', {}) 271 # .o target -> .c file 272 cc_compile = rules.get('cc_compile', {}) 273 # executable target -> .c files 274 cc_compilelink = rules.get('cc_compilelink', {}) 275 # Target name -> CFLAGS passed to gcc 276 cc_flags = rules.get('cc_flags', {}) 277 # Target name -> -I paths passed to gcc 278 cc_includes = rules.get('cc_includes', {}) 279 # Target name -> -l paths passed to gcc 280 cc_libraries = rules.get('cc_libraries', {}) 281 # target -> prebuilt source 282 install = rules.get('install', {}) 283 284 # All libraries used by any LTP test (built or not) 285 ltp_libs = set(self.ArTargetToLibraryName(i) for i in ar.keys()) 286 # All libraries used by the LTP tests we actually build 287 ltp_libs_used = set() 288 ltp_names_used = set() 289 290 print( 291 "Disabled lib tests: Test cases listed here are" 292 "suggested to be disabled since they require a disabled library. " 293 "Please copy and paste them into disabled_tests.txt\n") 294 for i in cc_libraries: 295 if len(set(cc_libraries[i]).intersection(disabled_libs)) > 0: 296 print os.path.basename(i) 297 298 print("Disabled_cflag tests: Test cases listed here are" 299 "suggested to be disabled since they require a disabled cflag. " 300 "Please copy and paste them into disabled_tests.txt\n") 301 for i in cc_flags: 302 if len(set(cc_flags[i]).intersection(disabled_cflags)) > 0: 303 module_name = os.path.basename(i) 304 idx = module_name.find('_') 305 if idx > 0: 306 module_name = module_name[:idx] 307 print module_name 308 309 # Remove include directories that don't exist. They're an error in 310 # Soong. 311 for target in cc_includes: 312 cc_includes[target] = [i for i in cc_includes[target] if os.path.isdir(os.path.join(ltp_root, i))] 313 314 for target in cc_compilelink: 315 module_name = os.path.basename(target) 316 if module_name in disabled_tests: 317 continue 318 local_src_files = cc_compilelink[target] 319 local_cflags = cc_flags[target] 320 local_c_includes = cc_includes[target] 321 local_libraries = cc_libraries[target] 322 if len(set(local_libraries).intersection(disabled_libs)) > 0: 323 continue 324 if len(set(local_cflags).intersection(disabled_cflags)) > 0: 325 continue 326 self.BuildExecutable(target, local_src_files, local_cflags, 327 local_c_includes, local_libraries, ltp_libs, 328 ltp_libs_used, ltp_names_used) 329 330 for target in cc_link: 331 if os.path.basename(target) in disabled_tests: 332 continue 333 local_src_files = set() 334 local_cflags = set() 335 local_c_includes = set() 336 local_libraries = cc_libraries[target] 337 # Accumulate flags for all .c files needed to build the .o files. 338 # (Android.mk requires a consistent set of flags across a given target. 339 # Thankfully using the superset of all flags in the target works fine 340 # with LTP tests.) 341 for obj in cc_link[target]: 342 for i in cc_compile[obj]: 343 local_src_files.add(i) 344 for i in cc_flags[obj]: 345 local_cflags.add(i) 346 for i in cc_includes[obj]: 347 local_c_includes.add(i) 348 if len(set(local_libraries).intersection(disabled_libs)) > 0: 349 continue 350 if len(set(local_cflags).intersection(disabled_cflags)) > 0: 351 continue 352 353 self.BuildExecutable(target, local_src_files, local_cflags, 354 local_c_includes, local_libraries, ltp_libs, 355 ltp_libs_used, ltp_names_used) 356 357 for target in ar: 358 # Disabled ltp library is already excluded 359 # since it won't be in ltp_libs_used 360 if not self.ArTargetToLibraryName(target) in ltp_libs_used: 361 continue 362 363 local_src_files = set() 364 local_cflags = set() 365 local_c_includes = set() 366 367 # TODO: disabled cflags 368 369 for obj in ar[target]: 370 for i in cc_compile[obj]: 371 local_src_files.add(i) 372 for i in cc_flags[obj]: 373 local_cflags.add(i) 374 for i in cc_includes[obj]: 375 local_c_includes.add(i) 376 377 if len(set(local_cflags).intersection(disabled_cflags)) > 0: 378 continue 379 380 self.BuildStaticLibrary(target, local_src_files, local_cflags, 381 local_c_includes) 382 383 for target in install: 384 if os.path.basename(target) in disabled_tests: 385 continue 386 local_src_files = install[target] 387 assert len(local_src_files) == 1 388 389 self.BuildPrebuilt(target, local_src_files[0]) 390 391 def WriteAndroidBp(self, output_path): 392 '''Write parse result to blueprint file. 393 394 Args: 395 output_path: string 396 ''' 397 with open(output_path, 'a') as f: 398 f.write('\n'.join(self._bp_result)) 399 self._bp_result = [] 400 401 def WriteAndroidMk(self, output_path): 402 '''Write parse result to make file. 403 404 Args: 405 output_path: string 406 ''' 407 with open(output_path, 'a') as f: 408 f.write('\n'.join(self._mk_result)) 409 self._mk_result = [] 410 411 def WritePackageList(self, output_path): 412 '''Write parse result to package list file. 413 414 Args: 415 output_path: string 416 ''' 417 with open(output_path, 'a') as f: 418 f.write('ltp_packages := \\\n ') 419 f.write(' \\\n '.join(sorted(self._packages))) 420 self._packages = [] 421 422 def ParseAll(self, ltp_root): 423 '''Parse outputs from both 'make' and 'make install'. 424 425 Args: 426 ltp_root: string 427 ''' 428 parser = make_parser.MakeParser(ltp_root) 429 self.ParseInput(parser.ParseFile(MAKE_DRY_RUN_FILE_NAME), ltp_root) 430 parser = make_install_parser.MakeInstallParser(ltp_root) 431 self.ParseInput(parser.ParseFile(MAKE_INSTALL_DRY_RUN_FILE_NAME), ltp_root) 432 433 def GetUnusedCustomCFlagsTargets(self): 434 '''Get targets that have custom cflags, but that weren't built.''' 435 return list(self._unused_custom_cflags) 436 437 438def main(): 439 parser = argparse.ArgumentParser( 440 description='Generate Android.mk from parsed LTP make output') 441 parser.add_argument( 442 '--ltp_root', dest='ltp_root', required=True, help='LTP root dir') 443 parser.add_argument( 444 '--output_mk_path', 445 dest='output_mk_path', 446 required=True, 447 help='output makefile path') 448 parser.add_argument( 449 '--output_bp_path', 450 dest='output_bp_path', 451 required=True, 452 help='output blueprint path') 453 parser.add_argument( 454 '--output_plist_path', 455 required=True, 456 help='output package list path') 457 parser.add_argument( 458 '--custom_cflags_file', 459 dest='custom_cflags_file', 460 required=True, 461 help='file with custom per-module cflags. empty means no file.') 462 args = parser.parse_args() 463 464 custom_cflags = {} 465 if args.custom_cflags_file: 466 # The file is expected to just be a JSON map of string -> [string], e.g. 467 # {"testcases/kernel/syscalls/getcwd/getcwd02": ["-DFOO", "-O3"]} 468 with open(args.custom_cflags_file) as f: 469 custom_cflags = json.load(f) 470 471 generator = BuildGenerator(custom_cflags) 472 generator.ParseAll(args.ltp_root) 473 generator.WriteAndroidMk(args.output_mk_path) 474 generator.WriteAndroidBp(args.output_bp_path) 475 generator.WritePackageList(args.output_plist_path) 476 477 unused_cflags_targs = generator.GetUnusedCustomCFlagsTargets() 478 if unused_cflags_targs: 479 print 'NOTE: Tests had custom cflags, but were never seen: {}'.format( 480 ', '.join(unused_cflags_targs)) 481 482 print 'Finished!' 483 484 485if __name__ == '__main__': 486 main() 487