1#!/usr/bin/env python 2# 3# Copyright (C) 2017 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18"""pprof_proto_generator.py: read perf.data, generate pprof.profile, which can be 19 used by pprof. 20 21 Example: 22 python app_profiler.py 23 python pprof_proto_generator.py 24 pprof -text pprof.profile 25""" 26 27from __future__ import print_function 28import argparse 29import os 30import os.path 31 32from simpleperf_report_lib import ReportLib 33from utils import Addr2Nearestline, extant_dir, find_real_dso_path, find_tool_path, flatten_arg_list 34from utils import log_info, log_exit, ReadElf 35try: 36 import profile_pb2 37except ImportError: 38 log_exit('google.protobuf module is missing. Please install it first.') 39 40def load_pprof_profile(filename): 41 profile = profile_pb2.Profile() 42 with open(filename, "rb") as f: 43 profile.ParseFromString(f.read()) 44 return profile 45 46 47def store_pprof_profile(filename, profile): 48 with open(filename, 'wb') as f: 49 f.write(profile.SerializeToString()) 50 51 52class PprofProfilePrinter(object): 53 54 def __init__(self, profile): 55 self.profile = profile 56 self.string_table = profile.string_table 57 58 def show(self): 59 p = self.profile 60 sub_space = ' ' 61 print('Profile {') 62 print('%d sample_types' % len(p.sample_type)) 63 for i in range(len(p.sample_type)): 64 print('sample_type[%d] = ' % i, end='') 65 self.show_value_type(p.sample_type[i]) 66 print('%d samples' % len(p.sample)) 67 for i in range(len(p.sample)): 68 print('sample[%d]:' % i) 69 self.show_sample(p.sample[i], sub_space) 70 print('%d mappings' % len(p.mapping)) 71 for i in range(len(p.mapping)): 72 print('mapping[%d]:' % i) 73 self.show_mapping(p.mapping[i], sub_space) 74 print('%d locations' % len(p.location)) 75 for i in range(len(p.location)): 76 print('location[%d]:' % i) 77 self.show_location(p.location[i], sub_space) 78 for i in range(len(p.function)): 79 print('function[%d]:' % i) 80 self.show_function(p.function[i], sub_space) 81 print('%d strings' % len(p.string_table)) 82 for i in range(len(p.string_table)): 83 print('string[%d]: %s' % (i, p.string_table[i])) 84 print('drop_frames: %s' % self.string(p.drop_frames)) 85 print('keep_frames: %s' % self.string(p.keep_frames)) 86 print('time_nanos: %u' % p.time_nanos) 87 print('duration_nanos: %u' % p.duration_nanos) 88 print('period_type: ', end='') 89 self.show_value_type(p.period_type) 90 print('period: %u' % p.period) 91 for i in range(len(p.comment)): 92 print('comment[%d] = %s' % (i, self.string(p.comment[i]))) 93 print('default_sample_type: %d' % p.default_sample_type) 94 print('} // Profile') 95 print() 96 97 def show_value_type(self, value_type, space=''): 98 print('%sValueType(typeID=%d, unitID=%d, type=%s, unit=%s)' % 99 (space, value_type.type, value_type.unit, 100 self.string(value_type.type), self.string(value_type.unit))) 101 102 def show_sample(self, sample, space=''): 103 sub_space = space + ' ' 104 for i in range(len(sample.location_id)): 105 print('%slocation_id[%d]: id %d' % (space, i, sample.location_id[i])) 106 self.show_location_id(sample.location_id[i], sub_space) 107 for i in range(len(sample.value)): 108 print('%svalue[%d] = %d' % (space, i, sample.value[i])) 109 for i in range(len(sample.label)): 110 print('%slabel[%d] = ', (space, i)) 111 112 def show_location_id(self, location_id, space=''): 113 location = self.profile.location[location_id - 1] 114 self.show_location(location, space) 115 116 def show_location(self, location, space=''): 117 sub_space = space + ' ' 118 print('%sid: %d' % (space, location.id)) 119 print('%smapping_id: %d' % (space, location.mapping_id)) 120 self.show_mapping_id(location.mapping_id, sub_space) 121 print('%saddress: %x' % (space, location.address)) 122 for i in range(len(location.line)): 123 print('%sline[%d]:' % (space, i)) 124 self.show_line(location.line[i], sub_space) 125 126 def show_mapping_id(self, mapping_id, space=''): 127 mapping = self.profile.mapping[mapping_id - 1] 128 self.show_mapping(mapping, space) 129 130 def show_mapping(self, mapping, space=''): 131 print('%sid: %d' % (space, mapping.id)) 132 print('%smemory_start: %x' % (space, mapping.memory_start)) 133 print('%smemory_limit: %x' % (space, mapping.memory_limit)) 134 print('%sfile_offset: %x' % (space, mapping.file_offset)) 135 print('%sfilename: %s(%d)' % (space, self.string(mapping.filename), 136 mapping.filename)) 137 print('%sbuild_id: %s(%d)' % (space, self.string(mapping.build_id), 138 mapping.build_id)) 139 print('%shas_functions: %s' % (space, mapping.has_functions)) 140 print('%shas_filenames: %s' % (space, mapping.has_filenames)) 141 print('%shas_line_numbers: %s' % (space, mapping.has_line_numbers)) 142 print('%shas_inline_frames: %s' % (space, mapping.has_inline_frames)) 143 144 def show_line(self, line, space=''): 145 sub_space = space + ' ' 146 print('%sfunction_id: %d' % (space, line.function_id)) 147 self.show_function_id(line.function_id, sub_space) 148 print('%sline: %d' % (space, line.line)) 149 150 def show_function_id(self, function_id, space=''): 151 function = self.profile.function[function_id - 1] 152 self.show_function(function, space) 153 154 def show_function(self, function, space=''): 155 print('%sid: %d' % (space, function.id)) 156 print('%sname: %s' % (space, self.string(function.name))) 157 print('%ssystem_name: %s' % (space, self.string(function.system_name))) 158 print('%sfilename: %s' % (space, self.string(function.filename))) 159 print('%sstart_line: %d' % (space, function.start_line)) 160 161 def string(self, string_id): 162 return self.string_table[string_id] 163 164 165class Sample(object): 166 167 def __init__(self): 168 self.location_ids = [] 169 self.values = {} 170 171 def add_location_id(self, location_id): 172 self.location_ids.append(location_id) 173 174 def add_value(self, sample_type_id, value): 175 self.values[sample_type_id] = self.values.get(sample_type_id, 0) + value 176 177 def add_values(self, values): 178 for sample_type_id, value in values.items(): 179 self.add_value(sample_type_id, value) 180 181 @property 182 def key(self): 183 return tuple(self.location_ids) 184 185 186class Location(object): 187 188 def __init__(self, mapping_id, address, vaddr_in_dso): 189 self.id = -1 # unset 190 self.mapping_id = mapping_id 191 self.address = address 192 self.vaddr_in_dso = vaddr_in_dso 193 self.lines = [] 194 195 @property 196 def key(self): 197 return (self.mapping_id, self.address) 198 199 200class Line(object): 201 202 def __init__(self): 203 self.function_id = 0 204 self.line = 0 205 206 207class Mapping(object): 208 209 def __init__(self, start, end, pgoff, filename_id, build_id_id): 210 self.id = -1 # unset 211 self.memory_start = start 212 self.memory_limit = end 213 self.file_offset = pgoff 214 self.filename_id = filename_id 215 self.build_id_id = build_id_id 216 217 @property 218 def key(self): 219 return ( 220 self.memory_start, 221 self.memory_limit, 222 self.file_offset, 223 self.filename_id, 224 self.build_id_id) 225 226 227class Function(object): 228 229 def __init__(self, name_id, dso_name_id, vaddr_in_dso): 230 self.id = -1 # unset 231 self.name_id = name_id 232 self.dso_name_id = dso_name_id 233 self.vaddr_in_dso = vaddr_in_dso 234 self.source_filename_id = 0 235 self.start_line = 0 236 237 @property 238 def key(self): 239 return (self.name_id, self.dso_name_id) 240 241 242# pylint: disable=no-member 243class PprofProfileGenerator(object): 244 245 def __init__(self, config): 246 self.config = config 247 self.lib = ReportLib() 248 249 config['binary_cache_dir'] = 'binary_cache' 250 if not os.path.isdir(config['binary_cache_dir']): 251 config['binary_cache_dir'] = None 252 else: 253 self.lib.SetSymfs(config['binary_cache_dir']) 254 if config.get('perf_data_path'): 255 self.lib.SetRecordFile(config['perf_data_path']) 256 kallsyms = 'binary_cache/kallsyms' 257 if os.path.isfile(kallsyms): 258 self.lib.SetKallsymsFile(kallsyms) 259 if config.get('show_art_frames'): 260 self.lib.ShowArtFrames() 261 self.comm_filter = set(config['comm_filters']) if config.get('comm_filters') else None 262 if config.get('pid_filters'): 263 self.pid_filter = {int(x) for x in config['pid_filters']} 264 else: 265 self.pid_filter = None 266 if config.get('tid_filters'): 267 self.tid_filter = {int(x) for x in config['tid_filters']} 268 else: 269 self.tid_filter = None 270 self.dso_filter = set(config['dso_filters']) if config.get('dso_filters') else None 271 self.max_chain_length = config['max_chain_length'] 272 self.profile = profile_pb2.Profile() 273 self.profile.string_table.append('') 274 self.string_table = {} 275 self.sample_types = {} 276 self.sample_map = {} 277 self.sample_list = [] 278 self.location_map = {} 279 self.location_list = [] 280 self.mapping_map = {} 281 self.mapping_list = [] 282 self.function_map = {} 283 self.function_list = [] 284 285 # Map from dso_name in perf.data to (binary path, build_id). 286 self.binary_map = {} 287 self.read_elf = ReadElf(self.config['ndk_path']) 288 289 def gen(self): 290 # 1. Process all samples in perf.data, aggregate samples. 291 while True: 292 report_sample = self.lib.GetNextSample() 293 if report_sample is None: 294 self.lib.Close() 295 break 296 event = self.lib.GetEventOfCurrentSample() 297 symbol = self.lib.GetSymbolOfCurrentSample() 298 callchain = self.lib.GetCallChainOfCurrentSample() 299 300 if not self._filter_report_sample(report_sample): 301 continue 302 303 sample_type_id = self.get_sample_type_id(event.name) 304 sample = Sample() 305 sample.add_value(sample_type_id, 1) 306 sample.add_value(sample_type_id + 1, report_sample.period) 307 if self._filter_symbol(symbol): 308 location_id = self.get_location_id(report_sample.ip, symbol) 309 sample.add_location_id(location_id) 310 for i in range(max(0, callchain.nr - self.max_chain_length), callchain.nr): 311 entry = callchain.entries[i] 312 if self._filter_symbol(symbol): 313 location_id = self.get_location_id(entry.ip, entry.symbol) 314 sample.add_location_id(location_id) 315 if sample.location_ids: 316 self.add_sample(sample) 317 318 # 2. Generate line info for locations and functions. 319 self.gen_source_lines() 320 321 # 3. Produce samples/locations/functions in profile 322 for sample in self.sample_list: 323 self.gen_profile_sample(sample) 324 for mapping in self.mapping_list: 325 self.gen_profile_mapping(mapping) 326 for location in self.location_list: 327 self.gen_profile_location(location) 328 for function in self.function_list: 329 self.gen_profile_function(function) 330 331 return self.profile 332 333 def _filter_report_sample(self, sample): 334 """Return true if the sample can be used.""" 335 if self.comm_filter: 336 if sample.thread_comm not in self.comm_filter: 337 return False 338 if self.pid_filter: 339 if sample.pid not in self.pid_filter: 340 return False 341 if self.tid_filter: 342 if sample.tid not in self.tid_filter: 343 return False 344 return True 345 346 def _filter_symbol(self, symbol): 347 if not self.dso_filter or symbol.dso_name in self.dso_filter: 348 return True 349 return False 350 351 def get_string_id(self, str_value): 352 if not str_value: 353 return 0 354 str_id = self.string_table.get(str_value) 355 if str_id is not None: 356 return str_id 357 str_id = len(self.string_table) + 1 358 self.string_table[str_value] = str_id 359 self.profile.string_table.append(str_value) 360 return str_id 361 362 def get_string(self, str_id): 363 return self.profile.string_table[str_id] 364 365 def get_sample_type_id(self, name): 366 sample_type_id = self.sample_types.get(name) 367 if sample_type_id is not None: 368 return sample_type_id 369 sample_type_id = len(self.profile.sample_type) 370 sample_type = self.profile.sample_type.add() 371 sample_type.type = self.get_string_id('event_' + name + '_samples') 372 sample_type.unit = self.get_string_id('count') 373 sample_type = self.profile.sample_type.add() 374 sample_type.type = self.get_string_id('event_' + name + '_count') 375 sample_type.unit = self.get_string_id('count') 376 self.sample_types[name] = sample_type_id 377 return sample_type_id 378 379 def get_location_id(self, ip, symbol): 380 binary_path, build_id = self.get_binary(symbol.dso_name) 381 mapping_id = self.get_mapping_id(symbol.mapping[0], binary_path, build_id) 382 location = Location(mapping_id, ip, symbol.vaddr_in_file) 383 function_id = self.get_function_id(symbol.symbol_name, binary_path, symbol.symbol_addr) 384 if function_id: 385 # Add Line only when it has a valid function id, see http://b/36988814. 386 # Default line info only contains the function name 387 line = Line() 388 line.function_id = function_id 389 location.lines.append(line) 390 391 exist_location = self.location_map.get(location.key) 392 if exist_location: 393 return exist_location.id 394 # location_id starts from 1 395 location.id = len(self.location_list) + 1 396 self.location_list.append(location) 397 self.location_map[location.key] = location 398 return location.id 399 400 def get_mapping_id(self, report_mapping, filename, build_id): 401 filename_id = self.get_string_id(filename) 402 build_id_id = self.get_string_id(build_id) 403 mapping = Mapping(report_mapping.start, report_mapping.end, 404 report_mapping.pgoff, filename_id, build_id_id) 405 exist_mapping = self.mapping_map.get(mapping.key) 406 if exist_mapping: 407 return exist_mapping.id 408 # mapping_id starts from 1 409 mapping.id = len(self.mapping_list) + 1 410 self.mapping_list.append(mapping) 411 self.mapping_map[mapping.key] = mapping 412 return mapping.id 413 414 def get_binary(self, dso_name): 415 """ Return (binary_path, build_id) for a given dso_name. """ 416 value = self.binary_map.get(dso_name) 417 if value: 418 return value 419 420 binary_path = dso_name 421 build_id = '' 422 423 # The build ids in perf.data are padded to 20 bytes, but pprof needs without padding. 424 # So read build id from the binary in binary_cache, and check it with build id in 425 # perf.data. 426 build_id_in_perf_data = self.lib.GetBuildIdForPath(dso_name) 427 # Try elf_path in binary cache. 428 elf_path = find_real_dso_path(dso_name, self.config['binary_cache_dir']) 429 if elf_path: 430 elf_build_id = self.read_elf.get_build_id(elf_path, False) 431 if build_id_in_perf_data: 432 match = build_id_in_perf_data == self.read_elf.pad_build_id(elf_build_id) 433 else: 434 # odex files generated by ART on Android O don't contain build id. 435 match = not elf_build_id 436 if match: 437 build_id = elf_build_id 438 binary_path = elf_path 439 440 # When there is no matching elf_path, try converting build_id in perf.data. 441 if not build_id and build_id_in_perf_data.startswith('0x'): 442 # Fallback to the way used by TrimZeroesFromBuildIDString() in quipper. 443 build_id = build_id_in_perf_data[2:] # remove '0x' 444 padding = '0' * 8 445 while build_id.endswith(padding): 446 build_id = build_id[:-len(padding)] 447 448 self.binary_map[dso_name] = (binary_path, build_id) 449 return (binary_path, build_id) 450 451 def get_mapping(self, mapping_id): 452 return self.mapping_list[mapping_id - 1] if mapping_id > 0 else None 453 454 def get_function_id(self, name, dso_name, vaddr_in_file): 455 if name == 'unknown': 456 return 0 457 function = Function(self.get_string_id(name), self.get_string_id(dso_name), vaddr_in_file) 458 exist_function = self.function_map.get(function.key) 459 if exist_function: 460 return exist_function.id 461 # function_id starts from 1 462 function.id = len(self.function_list) + 1 463 self.function_list.append(function) 464 self.function_map[function.key] = function 465 return function.id 466 467 def get_function(self, function_id): 468 return self.function_list[function_id - 1] if function_id > 0 else None 469 470 def add_sample(self, sample): 471 exist_sample = self.sample_map.get(sample.key) 472 if exist_sample: 473 exist_sample.add_values(sample.values) 474 else: 475 self.sample_list.append(sample) 476 self.sample_map[sample.key] = sample 477 478 def gen_source_lines(self): 479 # 1. Create Addr2line instance 480 if not self.config.get('binary_cache_dir'): 481 log_info("Can't generate line information because binary_cache is missing.") 482 return 483 if not find_tool_path('llvm-symbolizer', self.config['ndk_path']): 484 log_info("Can't generate line information because can't find llvm-symbolizer.") 485 return 486 # We have changed dso names to paths in binary_cache in self.get_binary(). So no need to 487 # pass binary_cache_dir to addr2line. 488 addr2line = Addr2Nearestline(self.config['ndk_path'], None, True) 489 490 # 2. Put all needed addresses to it. 491 for location in self.location_list: 492 mapping = self.get_mapping(location.mapping_id) 493 dso_name = self.get_string(mapping.filename_id) 494 if location.lines: 495 function = self.get_function(location.lines[0].function_id) 496 addr2line.add_addr(dso_name, function.vaddr_in_dso, location.vaddr_in_dso) 497 for function in self.function_list: 498 dso_name = self.get_string(function.dso_name_id) 499 addr2line.add_addr(dso_name, function.vaddr_in_dso, function.vaddr_in_dso) 500 501 # 3. Generate source lines. 502 addr2line.convert_addrs_to_lines() 503 504 # 4. Annotate locations and functions. 505 for location in self.location_list: 506 if not location.lines: 507 continue 508 mapping = self.get_mapping(location.mapping_id) 509 dso_name = self.get_string(mapping.filename_id) 510 dso = addr2line.get_dso(dso_name) 511 if not dso: 512 continue 513 sources = addr2line.get_addr_source(dso, location.vaddr_in_dso) 514 if not sources: 515 continue 516 for (source_id, source) in enumerate(sources): 517 source_file, source_line, function_name = source 518 function_id = self.get_function_id(function_name, dso_name, 0) 519 if function_id == 0: 520 continue 521 if source_id == 0: 522 # Clear default line info 523 location.lines = [] 524 location.lines.append(self.add_line(source_file, source_line, function_id)) 525 526 for function in self.function_list: 527 dso_name = self.get_string(function.dso_name_id) 528 if function.vaddr_in_dso: 529 dso = addr2line.get_dso(dso_name) 530 if not dso: 531 continue 532 sources = addr2line.get_addr_source(dso, function.vaddr_in_dso) 533 if sources: 534 source_file, source_line, _ = sources[0] 535 function.source_filename_id = self.get_string_id(source_file) 536 function.start_line = source_line 537 538 def add_line(self, source_file, source_line, function_id): 539 line = Line() 540 function = self.get_function(function_id) 541 function.source_filename_id = self.get_string_id(source_file) 542 line.function_id = function_id 543 line.line = source_line 544 return line 545 546 def gen_profile_sample(self, sample): 547 profile_sample = self.profile.sample.add() 548 profile_sample.location_id.extend(sample.location_ids) 549 sample_type_count = len(self.sample_types) * 2 550 values = [0] * sample_type_count 551 for sample_type_id in sample.values: 552 values[sample_type_id] = sample.values[sample_type_id] 553 profile_sample.value.extend(values) 554 555 def gen_profile_mapping(self, mapping): 556 profile_mapping = self.profile.mapping.add() 557 profile_mapping.id = mapping.id 558 profile_mapping.memory_start = mapping.memory_start 559 profile_mapping.memory_limit = mapping.memory_limit 560 profile_mapping.file_offset = mapping.file_offset 561 profile_mapping.filename = mapping.filename_id 562 profile_mapping.build_id = mapping.build_id_id 563 profile_mapping.has_filenames = True 564 profile_mapping.has_functions = True 565 if self.config.get('binary_cache_dir'): 566 profile_mapping.has_line_numbers = True 567 profile_mapping.has_inline_frames = True 568 else: 569 profile_mapping.has_line_numbers = False 570 profile_mapping.has_inline_frames = False 571 572 def gen_profile_location(self, location): 573 profile_location = self.profile.location.add() 574 profile_location.id = location.id 575 profile_location.mapping_id = location.mapping_id 576 profile_location.address = location.address 577 for i in range(len(location.lines)): 578 line = profile_location.line.add() 579 line.function_id = location.lines[i].function_id 580 line.line = location.lines[i].line 581 582 def gen_profile_function(self, function): 583 profile_function = self.profile.function.add() 584 profile_function.id = function.id 585 profile_function.name = function.name_id 586 profile_function.system_name = function.name_id 587 profile_function.filename = function.source_filename_id 588 profile_function.start_line = function.start_line 589 590 591def main(): 592 parser = argparse.ArgumentParser(description='Generate pprof profile data in pprof.profile.') 593 parser.add_argument('--show', nargs='?', action='append', help='print existing pprof.profile.') 594 parser.add_argument('-i', '--perf_data_path', default='perf.data', help=""" 595 The path of profiling data.""") 596 parser.add_argument('-o', '--output_file', default='pprof.profile', help=""" 597 The path of generated pprof profile data.""") 598 parser.add_argument('--comm', nargs='+', action='append', help=""" 599 Use samples only in threads with selected names.""") 600 parser.add_argument('--pid', nargs='+', action='append', help=""" 601 Use samples only in processes with selected process ids.""") 602 parser.add_argument('--tid', nargs='+', action='append', help=""" 603 Use samples only in threads with selected thread ids.""") 604 parser.add_argument('--dso', nargs='+', action='append', help=""" 605 Use samples only in selected binaries.""") 606 parser.add_argument('--max_chain_length', type=int, default=1000000000, help=""" 607 Maximum depth of samples to be converted.""") # Large value as infinity standin. 608 parser.add_argument('--ndk_path', type=extant_dir, help='Set the path of a ndk release.') 609 parser.add_argument('--show_art_frames', action='store_true', 610 help='Show frames of internal methods in the ART Java interpreter.') 611 612 args = parser.parse_args() 613 if args.show: 614 show_file = args.show[0] if args.show[0] else 'pprof.profile' 615 profile = load_pprof_profile(show_file) 616 printer = PprofProfilePrinter(profile) 617 printer.show() 618 return 619 620 config = {} 621 config['perf_data_path'] = args.perf_data_path 622 config['output_file'] = args.output_file 623 config['comm_filters'] = flatten_arg_list(args.comm) 624 config['pid_filters'] = flatten_arg_list(args.pid) 625 config['tid_filters'] = flatten_arg_list(args.tid) 626 config['dso_filters'] = flatten_arg_list(args.dso) 627 config['ndk_path'] = args.ndk_path 628 config['show_art_frames'] = args.show_art_frames 629 config['max_chain_length'] = args.max_chain_length 630 generator = PprofProfileGenerator(config) 631 profile = generator.gen() 632 store_pprof_profile(config['output_file'], profile) 633 634 635if __name__ == '__main__': 636 main() 637