1#!/usr/bin/env python3 2# 3# Copyright (C) 2017 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18from __future__ import annotations 19import argparse 20import collections 21from concurrent.futures import ThreadPoolExecutor 22from dataclasses import dataclass 23import datetime 24import json 25import os 26from pathlib import Path 27import sys 28from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Tuple, Union 29 30from simpleperf_report_lib import ReportLib, SymbolStruct 31from simpleperf_utils import ( 32 Addr2Nearestline, ArgParseFormatter, BinaryFinder, get_script_dir, log_exit, log_info, Objdump, 33 open_report_in_browser, ReadElf, SourceFileSearcher) 34 35MAX_CALLSTACK_LENGTH = 750 36 37 38class HtmlWriter(object): 39 40 def __init__(self, output_path: Union[Path, str]): 41 self.fh = open(output_path, 'w') 42 self.tag_stack = [] 43 44 def close(self): 45 self.fh.close() 46 47 def open_tag(self, tag: str, **attrs: Dict[str, str]) -> HtmlWriter: 48 attr_str = '' 49 for key in attrs: 50 attr_str += ' %s="%s"' % (key, attrs[key]) 51 self.fh.write('<%s%s>' % (tag, attr_str)) 52 self.tag_stack.append(tag) 53 return self 54 55 def close_tag(self, tag: Optional[str] = None): 56 if tag: 57 assert tag == self.tag_stack[-1] 58 self.fh.write('</%s>\n' % self.tag_stack.pop()) 59 60 def add(self, text: str) -> HtmlWriter: 61 self.fh.write(text) 62 return self 63 64 def add_file(self, file_path: Union[Path, str]) -> HtmlWriter: 65 file_path = os.path.join(get_script_dir(), file_path) 66 with open(file_path, 'r') as f: 67 self.add(f.read()) 68 return self 69 70 71def modify_text_for_html(text: str) -> str: 72 return text.replace('>', '>').replace('<', '<') 73 74 75def hex_address_for_json(addr: int) -> str: 76 """ To handle big addrs (nears uint64_max) in Javascript, store addrs as hex strings in Json. 77 """ 78 return '0x%x' % addr 79 80 81class EventScope(object): 82 83 def __init__(self, name: str): 84 self.name = name 85 self.processes: Dict[int, ProcessScope] = {} # map from pid to ProcessScope 86 self.sample_count = 0 87 self.event_count = 0 88 89 def get_process(self, pid: int) -> ProcessScope: 90 process = self.processes.get(pid) 91 if not process: 92 process = self.processes[pid] = ProcessScope(pid) 93 return process 94 95 def get_sample_info(self, gen_addr_hit_map: bool) -> Dict[str, Any]: 96 result = {} 97 result['eventName'] = self.name 98 result['eventCount'] = self.event_count 99 processes = sorted(self.processes.values(), key=lambda a: a.event_count, reverse=True) 100 result['processes'] = [process.get_sample_info(gen_addr_hit_map) 101 for process in processes] 102 return result 103 104 @property 105 def threads(self) -> Iterator[ThreadScope]: 106 for process in self.processes.values(): 107 for thread in process.threads.values(): 108 yield thread 109 110 @property 111 def libraries(self) -> Iterator[LibScope]: 112 for process in self.processes.values(): 113 for thread in process.threads.values(): 114 for lib in thread.libs.values(): 115 yield lib 116 117 118class ProcessScope(object): 119 120 def __init__(self, pid: int): 121 self.pid = pid 122 self.name = '' 123 self.event_count = 0 124 self.threads: Dict[int, ThreadScope] = {} # map from tid to ThreadScope 125 126 def get_thread(self, tid: int, thread_name: str) -> ThreadScope: 127 thread = self.threads.get(tid) 128 if not thread: 129 thread = self.threads[tid] = ThreadScope(tid) 130 thread.name = thread_name 131 if self.pid == tid: 132 self.name = thread_name 133 return thread 134 135 def get_sample_info(self, gen_addr_hit_map: bool) -> Dict[str, Any]: 136 result = {} 137 result['pid'] = self.pid 138 result['eventCount'] = self.event_count 139 threads = sorted(self.threads.values(), key=lambda a: a.event_count, reverse=True) 140 result['threads'] = [thread.get_sample_info(gen_addr_hit_map) 141 for thread in threads] 142 return result 143 144 def merge_by_thread_name(self, process: ProcessScope): 145 self.event_count += process.event_count 146 thread_list: List[ThreadScope] = list( 147 self.threads.values()) + list(process.threads.values()) 148 new_threads: Dict[str, ThreadScope] = {} # map from thread name to ThreadScope 149 for thread in thread_list: 150 cur_thread = new_threads.get(thread.name) 151 if cur_thread is None: 152 new_threads[thread.name] = thread 153 else: 154 cur_thread.merge(thread) 155 self.threads = {} 156 for thread in new_threads.values(): 157 self.threads[thread.tid] = thread 158 159 160class ThreadScope(object): 161 162 def __init__(self, tid: int): 163 self.tid = tid 164 self.name = '' 165 self.event_count = 0 166 self.sample_count = 0 167 self.libs: Dict[int, LibScope] = {} # map from lib_id to LibScope 168 self.call_graph = CallNode(-1) 169 self.reverse_call_graph = CallNode(-1) 170 171 def add_callstack( 172 self, event_count: int, callstack: List[Tuple[int, int, int]], 173 build_addr_hit_map: bool): 174 """ callstack is a list of tuple (lib_id, func_id, addr). 175 For each i > 0, callstack[i] calls callstack[i-1].""" 176 hit_func_ids: Set[int] = set() 177 for i, (lib_id, func_id, addr) in enumerate(callstack): 178 # When a callstack contains recursive function, only add for each function once. 179 if func_id in hit_func_ids: 180 continue 181 hit_func_ids.add(func_id) 182 183 lib = self.libs.get(lib_id) 184 if not lib: 185 lib = self.libs[lib_id] = LibScope(lib_id) 186 function = lib.get_function(func_id) 187 function.subtree_event_count += event_count 188 if i == 0: 189 lib.event_count += event_count 190 function.event_count += event_count 191 function.sample_count += 1 192 if build_addr_hit_map: 193 function.build_addr_hit_map(addr, event_count if i == 0 else 0, event_count) 194 195 # build call graph and reverse call graph 196 node = self.call_graph 197 for item in reversed(callstack): 198 node = node.get_child(item[1]) 199 node.event_count += event_count 200 node = self.reverse_call_graph 201 for item in callstack: 202 node = node.get_child(item[1]) 203 node.event_count += event_count 204 205 def update_subtree_event_count(self): 206 self.call_graph.update_subtree_event_count() 207 self.reverse_call_graph.update_subtree_event_count() 208 209 def limit_percents(self, min_func_limit: float, min_callchain_percent: float, 210 hit_func_ids: Set[int]): 211 for lib in self.libs.values(): 212 to_del_funcs = [] 213 for function in lib.functions.values(): 214 if function.subtree_event_count < min_func_limit: 215 to_del_funcs.append(function.func_id) 216 else: 217 hit_func_ids.add(function.func_id) 218 for func_id in to_del_funcs: 219 del lib.functions[func_id] 220 min_limit = min_callchain_percent * 0.01 * self.call_graph.subtree_event_count 221 self.call_graph.cut_edge(min_limit, hit_func_ids) 222 self.reverse_call_graph.cut_edge(min_limit, hit_func_ids) 223 224 def get_sample_info(self, gen_addr_hit_map: bool) -> Dict[str, Any]: 225 result = {} 226 result['tid'] = self.tid 227 result['eventCount'] = self.event_count 228 result['sampleCount'] = self.sample_count 229 result['libs'] = [lib.gen_sample_info(gen_addr_hit_map) 230 for lib in self.libs.values()] 231 result['g'] = self.call_graph.gen_sample_info() 232 result['rg'] = self.reverse_call_graph.gen_sample_info() 233 return result 234 235 def merge(self, thread: ThreadScope): 236 self.event_count += thread.event_count 237 self.sample_count += thread.sample_count 238 for lib_id, lib in thread.libs.items(): 239 cur_lib = self.libs.get(lib_id) 240 if cur_lib is None: 241 self.libs[lib_id] = lib 242 else: 243 cur_lib.merge(lib) 244 self.call_graph.merge(thread.call_graph) 245 self.reverse_call_graph.merge(thread.reverse_call_graph) 246 247 248class LibScope(object): 249 250 def __init__(self, lib_id: int): 251 self.lib_id = lib_id 252 self.event_count = 0 253 self.functions: Dict[int, FunctionScope] = {} # map from func_id to FunctionScope. 254 255 def get_function(self, func_id: int) -> FunctionScope: 256 function = self.functions.get(func_id) 257 if not function: 258 function = self.functions[func_id] = FunctionScope(func_id) 259 return function 260 261 def gen_sample_info(self, gen_addr_hit_map: bool) -> Dict[str, Any]: 262 result = {} 263 result['libId'] = self.lib_id 264 result['eventCount'] = self.event_count 265 result['functions'] = [func.gen_sample_info(gen_addr_hit_map) 266 for func in self.functions.values()] 267 return result 268 269 def merge(self, lib: LibScope): 270 self.event_count += lib.event_count 271 for func_id, function in lib.functions.items(): 272 cur_function = self.functions.get(func_id) 273 if cur_function is None: 274 self.functions[func_id] = function 275 else: 276 cur_function.merge(function) 277 278 279class FunctionScope(object): 280 281 def __init__(self, func_id: int): 282 self.func_id = func_id 283 self.sample_count = 0 284 self.event_count = 0 285 self.subtree_event_count = 0 286 self.addr_hit_map = None # map from addr to [event_count, subtree_event_count]. 287 # map from (source_file_id, line) to [event_count, subtree_event_count]. 288 self.line_hit_map = None 289 290 def build_addr_hit_map(self, addr: int, event_count: int, subtree_event_count: int): 291 if self.addr_hit_map is None: 292 self.addr_hit_map = {} 293 count_info = self.addr_hit_map.get(addr) 294 if count_info is None: 295 self.addr_hit_map[addr] = [event_count, subtree_event_count] 296 else: 297 count_info[0] += event_count 298 count_info[1] += subtree_event_count 299 300 def build_line_hit_map(self, source_file_id: int, line: int, event_count: int, 301 subtree_event_count: int): 302 if self.line_hit_map is None: 303 self.line_hit_map = {} 304 key = (source_file_id, line) 305 count_info = self.line_hit_map.get(key) 306 if count_info is None: 307 self.line_hit_map[key] = [event_count, subtree_event_count] 308 else: 309 count_info[0] += event_count 310 count_info[1] += subtree_event_count 311 312 def gen_sample_info(self, gen_addr_hit_map: bool) -> Dict[str, Any]: 313 result = {} 314 result['f'] = self.func_id 315 result['c'] = [self.sample_count, self.event_count, self.subtree_event_count] 316 if self.line_hit_map: 317 items = [] 318 for key in self.line_hit_map: 319 count_info = self.line_hit_map[key] 320 item = {'f': key[0], 'l': key[1], 'e': count_info[0], 's': count_info[1]} 321 items.append(item) 322 result['s'] = items 323 if gen_addr_hit_map and self.addr_hit_map: 324 items = [] 325 for addr in sorted(self.addr_hit_map): 326 count_info = self.addr_hit_map[addr] 327 items.append( 328 {'a': hex_address_for_json(addr), 329 'e': count_info[0], 330 's': count_info[1]}) 331 result['a'] = items 332 return result 333 334 def merge(self, function: FunctionScope): 335 self.sample_count += function.sample_count 336 self.event_count += function.event_count 337 self.subtree_event_count += function.subtree_event_count 338 self.addr_hit_map = self.__merge_hit_map(self.addr_hit_map, function.addr_hit_map) 339 self.line_hit_map = self.__merge_hit_map(self.line_hit_map, function.line_hit_map) 340 341 @staticmethod 342 def __merge_hit_map(map1: Optional[Dict[int, List[int]]], 343 map2: Optional[Dict[int, List[int]]]) -> Optional[Dict[int, List[int]]]: 344 if not map1: 345 return map2 346 if not map2: 347 return map1 348 for key, value2 in map2.items(): 349 value1 = map1.get(key) 350 if value1 is None: 351 map1[key] = value2 352 else: 353 value1[0] += value2[0] 354 value1[1] += value2[1] 355 return map1 356 357 358class CallNode(object): 359 360 def __init__(self, func_id: int): 361 self.event_count = 0 362 self.subtree_event_count = 0 363 self.func_id = func_id 364 # map from func_id to CallNode 365 self.children: Dict[int, CallNode] = collections.OrderedDict() 366 367 def get_child(self, func_id: int) -> CallNode: 368 child = self.children.get(func_id) 369 if not child: 370 child = self.children[func_id] = CallNode(func_id) 371 return child 372 373 def update_subtree_event_count(self): 374 self.subtree_event_count = self.event_count 375 for child in self.children.values(): 376 self.subtree_event_count += child.update_subtree_event_count() 377 return self.subtree_event_count 378 379 def cut_edge(self, min_limit: float, hit_func_ids: Set[int]): 380 hit_func_ids.add(self.func_id) 381 to_del_children = [] 382 for key in self.children: 383 child = self.children[key] 384 if child.subtree_event_count < min_limit: 385 to_del_children.append(key) 386 else: 387 child.cut_edge(min_limit, hit_func_ids) 388 for key in to_del_children: 389 del self.children[key] 390 391 def gen_sample_info(self) -> Dict[str, Any]: 392 result = {} 393 result['e'] = self.event_count 394 result['s'] = self.subtree_event_count 395 result['f'] = self.func_id 396 result['c'] = [child.gen_sample_info() for child in self.children.values()] 397 return result 398 399 def merge(self, node: CallNode): 400 self.event_count += node.event_count 401 self.subtree_event_count += node.subtree_event_count 402 for key, child in node.children.items(): 403 cur_child = self.children.get(key) 404 if cur_child is None: 405 self.children[key] = child 406 else: 407 cur_child.merge(child) 408 409 410@dataclass 411class LibInfo: 412 name: str 413 build_id: str 414 415 416class LibSet(object): 417 """ Collection of shared libraries used in perf.data. """ 418 419 def __init__(self): 420 self.lib_name_to_id: Dict[str, int] = {} 421 self.libs: List[LibInfo] = [] 422 423 def get_lib_id(self, lib_name: str) -> Optional[int]: 424 return self.lib_name_to_id.get(lib_name) 425 426 def add_lib(self, lib_name: str, build_id: str) -> int: 427 """ Return lib_id of the newly added lib. """ 428 lib_id = len(self.libs) 429 self.libs.append(LibInfo(lib_name, build_id)) 430 self.lib_name_to_id[lib_name] = lib_id 431 return lib_id 432 433 def get_lib(self, lib_id: int) -> LibInfo: 434 return self.libs[lib_id] 435 436 437class Function(object): 438 """ Represent a function in a shared library. """ 439 440 def __init__(self, lib_id: int, func_name: str, func_id: int, start_addr: int, addr_len: int): 441 self.lib_id = lib_id 442 self.func_name = func_name 443 self.func_id = func_id 444 self.start_addr = start_addr 445 self.addr_len = addr_len 446 self.source_info = None 447 self.disassembly = None 448 449 450class FunctionSet(object): 451 """ Collection of functions used in perf.data. """ 452 453 def __init__(self): 454 self.name_to_func: Dict[Tuple[int, str], Function] = {} 455 self.id_to_func: Dict[int, Function] = {} 456 457 def get_func_id(self, lib_id: int, symbol: SymbolStruct) -> int: 458 key = (lib_id, symbol.symbol_name) 459 function = self.name_to_func.get(key) 460 if function is None: 461 func_id = len(self.id_to_func) 462 function = Function(lib_id, symbol.symbol_name, func_id, symbol.symbol_addr, 463 symbol.symbol_len) 464 self.name_to_func[key] = function 465 self.id_to_func[func_id] = function 466 return function.func_id 467 468 def trim_functions(self, left_func_ids: Set[int]): 469 """ Remove functions excepts those in left_func_ids. """ 470 for function in self.name_to_func.values(): 471 if function.func_id not in left_func_ids: 472 del self.id_to_func[function.func_id] 473 # name_to_func will not be used. 474 self.name_to_func = None 475 476 477class SourceFile(object): 478 """ A source file containing source code hit by samples. """ 479 480 def __init__(self, file_id: int, abstract_path: str): 481 self.file_id = file_id 482 self.abstract_path = abstract_path # path reported by addr2line 483 self.real_path: Optional[str] = None # file path in the file system 484 self.requested_lines: Optional[Set[int]] = set() 485 self.line_to_code: Dict[int, str] = {} # map from line to code in that line. 486 487 def request_lines(self, start_line: int, end_line: int): 488 self.requested_lines |= set(range(start_line, end_line + 1)) 489 490 def add_source_code(self, real_path: str): 491 self.real_path = real_path 492 with open(real_path, 'r') as f: 493 source_code = f.readlines() 494 max_line = len(source_code) 495 for line in self.requested_lines: 496 if line > 0 and line <= max_line: 497 self.line_to_code[line] = source_code[line - 1] 498 # requested_lines is no longer used. 499 self.requested_lines = None 500 501 502class SourceFileSet(object): 503 """ Collection of source files. """ 504 505 def __init__(self): 506 self.path_to_source_files: Dict[str, SourceFile] = {} # map from file path to SourceFile. 507 508 def get_source_file(self, file_path: str) -> SourceFile: 509 source_file = self.path_to_source_files.get(file_path) 510 if not source_file: 511 source_file = SourceFile(len(self.path_to_source_files), file_path) 512 self.path_to_source_files[file_path] = source_file 513 return source_file 514 515 def load_source_code(self, source_dirs: List[str]): 516 file_searcher = SourceFileSearcher(source_dirs) 517 for source_file in self.path_to_source_files.values(): 518 real_path = file_searcher.get_real_path(source_file.abstract_path) 519 if real_path: 520 source_file.add_source_code(real_path) 521 522 523class RecordData(object): 524 525 """RecordData reads perf.data, and generates data used by report_html.js in json format. 526 All generated items are listed as below: 527 1. recordTime: string 528 2. machineType: string 529 3. androidVersion: string 530 4. recordCmdline: string 531 5. totalSamples: int 532 6. processNames: map from pid to processName. 533 7. threadNames: map from tid to threadName. 534 8. libList: an array of libNames, indexed by libId. 535 9. functionMap: map from functionId to funcData. 536 funcData = { 537 l: libId 538 f: functionName 539 s: [sourceFileId, startLine, endLine] [optional] 540 d: [(disassembly, addr)] [optional] 541 } 542 543 10. sampleInfo = [eventInfo] 544 eventInfo = { 545 eventName 546 eventCount 547 processes: [processInfo] 548 } 549 processInfo = { 550 pid 551 eventCount 552 threads: [threadInfo] 553 } 554 threadInfo = { 555 tid 556 eventCount 557 sampleCount 558 libs: [libInfo], 559 g: callGraph, 560 rg: reverseCallgraph 561 } 562 libInfo = { 563 libId, 564 eventCount, 565 functions: [funcInfo] 566 } 567 funcInfo = { 568 f: functionId 569 c: [sampleCount, eventCount, subTreeEventCount] 570 s: [sourceCodeInfo] [optional] 571 a: [addrInfo] (sorted by addrInfo.addr) [optional] 572 } 573 callGraph and reverseCallGraph are both of type CallNode. 574 callGraph shows how a function calls other functions. 575 reverseCallGraph shows how a function is called by other functions. 576 CallNode { 577 e: selfEventCount 578 s: subTreeEventCount 579 f: functionId 580 c: [CallNode] # children 581 } 582 583 sourceCodeInfo { 584 f: sourceFileId 585 l: line 586 e: eventCount 587 s: subtreeEventCount 588 } 589 590 addrInfo { 591 a: addr 592 e: eventCount 593 s: subtreeEventCount 594 } 595 596 11. sourceFiles: an array of sourceFile, indexed by sourceFileId. 597 sourceFile { 598 path 599 code: # a map from line to code for that line. 600 } 601 """ 602 603 def __init__( 604 self, binary_cache_path: Optional[str], 605 ndk_path: Optional[str], 606 build_addr_hit_map: bool, proguard_mapping_files: Optional[List[str]] = None): 607 self.binary_cache_path = binary_cache_path 608 self.ndk_path = ndk_path 609 self.build_addr_hit_map = build_addr_hit_map 610 self.proguard_mapping_files = proguard_mapping_files 611 self.meta_info: Optional[Dict[str, str]] = None 612 self.cmdline: Optional[str] = None 613 self.arch: Optional[str] = None 614 self.events: Dict[str, EventScope] = {} 615 self.libs = LibSet() 616 self.functions = FunctionSet() 617 self.total_samples = 0 618 self.source_files = SourceFileSet() 619 self.gen_addr_hit_map_in_record_info = False 620 self.binary_finder = BinaryFinder(binary_cache_path, ReadElf(ndk_path)) 621 622 def load_record_file(self, record_file: str, show_art_frames: bool): 623 lib = ReportLib() 624 lib.SetRecordFile(record_file) 625 # If not showing ip for unknown symbols, the percent of the unknown symbol may be 626 # accumulated to very big, and ranks first in the sample table. 627 lib.ShowIpForUnknownSymbol() 628 if show_art_frames: 629 lib.ShowArtFrames() 630 if self.binary_cache_path: 631 lib.SetSymfs(self.binary_cache_path) 632 for file_path in self.proguard_mapping_files or []: 633 lib.AddProguardMappingFile(file_path) 634 self.meta_info = lib.MetaInfo() 635 self.cmdline = lib.GetRecordCmd() 636 self.arch = lib.GetArch() 637 while True: 638 raw_sample = lib.GetNextSample() 639 if not raw_sample: 640 lib.Close() 641 break 642 raw_event = lib.GetEventOfCurrentSample() 643 symbol = lib.GetSymbolOfCurrentSample() 644 callchain = lib.GetCallChainOfCurrentSample() 645 event = self._get_event(raw_event.name) 646 self.total_samples += 1 647 event.sample_count += 1 648 event.event_count += raw_sample.period 649 process = event.get_process(raw_sample.pid) 650 process.event_count += raw_sample.period 651 thread = process.get_thread(raw_sample.tid, raw_sample.thread_comm) 652 thread.event_count += raw_sample.period 653 thread.sample_count += 1 654 655 lib_id = self.libs.get_lib_id(symbol.dso_name) 656 if lib_id is None: 657 lib_id = self.libs.add_lib(symbol.dso_name, lib.GetBuildIdForPath(symbol.dso_name)) 658 func_id = self.functions.get_func_id(lib_id, symbol) 659 callstack = [(lib_id, func_id, symbol.vaddr_in_file)] 660 for i in range(callchain.nr): 661 symbol = callchain.entries[i].symbol 662 lib_id = self.libs.get_lib_id(symbol.dso_name) 663 if lib_id is None: 664 lib_id = self.libs.add_lib( 665 symbol.dso_name, lib.GetBuildIdForPath(symbol.dso_name)) 666 func_id = self.functions.get_func_id(lib_id, symbol) 667 callstack.append((lib_id, func_id, symbol.vaddr_in_file)) 668 if len(callstack) > MAX_CALLSTACK_LENGTH: 669 callstack = callstack[:MAX_CALLSTACK_LENGTH] 670 thread.add_callstack(raw_sample.period, callstack, self.build_addr_hit_map) 671 672 for event in self.events.values(): 673 for thread in event.threads: 674 thread.update_subtree_event_count() 675 676 def aggregate_by_thread_name(self): 677 for event in self.events.values(): 678 new_processes = {} # from process name to ProcessScope 679 for process in event.processes.values(): 680 cur_process = new_processes.get(process.name) 681 if cur_process is None: 682 new_processes[process.name] = process 683 else: 684 cur_process.merge_by_thread_name(process) 685 event.processes = {} 686 for process in new_processes.values(): 687 event.processes[process.pid] = process 688 689 def limit_percents(self, min_func_percent: float, min_callchain_percent: float): 690 hit_func_ids: Set[int] = set() 691 for event in self.events.values(): 692 min_limit = event.event_count * min_func_percent * 0.01 693 to_del_processes = [] 694 for process in event.processes.values(): 695 to_del_threads = [] 696 for thread in process.threads.values(): 697 if thread.call_graph.subtree_event_count < min_limit: 698 to_del_threads.append(thread.tid) 699 else: 700 thread.limit_percents(min_limit, min_callchain_percent, hit_func_ids) 701 for thread in to_del_threads: 702 del process.threads[thread] 703 if not process.threads: 704 to_del_processes.append(process.pid) 705 for process in to_del_processes: 706 del event.processes[process] 707 self.functions.trim_functions(hit_func_ids) 708 709 def _get_event(self, event_name: str) -> EventScope: 710 if event_name not in self.events: 711 self.events[event_name] = EventScope(event_name) 712 return self.events[event_name] 713 714 def add_source_code(self, source_dirs: List[str], filter_lib: Callable[[str], bool]): 715 """ Collect source code information: 716 1. Find line ranges for each function in FunctionSet. 717 2. Find line for each addr in FunctionScope.addr_hit_map. 718 3. Collect needed source code in SourceFileSet. 719 """ 720 addr2line = Addr2Nearestline(self.ndk_path, self.binary_finder, False) 721 # Request line range for each function. 722 for function in self.functions.id_to_func.values(): 723 if function.func_name == 'unknown': 724 continue 725 lib_info = self.libs.get_lib(function.lib_id) 726 if filter_lib(lib_info.name): 727 addr2line.add_addr(lib_info.name, lib_info.build_id, 728 function.start_addr, function.start_addr) 729 addr2line.add_addr(lib_info.name, lib_info.build_id, function.start_addr, 730 function.start_addr + function.addr_len - 1) 731 # Request line for each addr in FunctionScope.addr_hit_map. 732 for event in self.events.values(): 733 for lib in event.libraries: 734 lib_info = self.libs.get_lib(lib.lib_id) 735 if filter_lib(lib_info.name): 736 for function in lib.functions.values(): 737 func_addr = self.functions.id_to_func[function.func_id].start_addr 738 for addr in function.addr_hit_map: 739 addr2line.add_addr(lib_info.name, lib_info.build_id, func_addr, addr) 740 addr2line.convert_addrs_to_lines() 741 742 # Set line range for each function. 743 for function in self.functions.id_to_func.values(): 744 if function.func_name == 'unknown': 745 continue 746 dso = addr2line.get_dso(self.libs.get_lib(function.lib_id).name) 747 if not dso: 748 continue 749 start_source = addr2line.get_addr_source(dso, function.start_addr) 750 end_source = addr2line.get_addr_source(dso, function.start_addr + function.addr_len - 1) 751 if not start_source or not end_source: 752 continue 753 start_file_path, start_line = start_source[-1] 754 end_file_path, end_line = end_source[-1] 755 if start_file_path != end_file_path or start_line > end_line: 756 continue 757 source_file = self.source_files.get_source_file(start_file_path) 758 source_file.request_lines(start_line, end_line) 759 function.source_info = (source_file.file_id, start_line, end_line) 760 761 # Build FunctionScope.line_hit_map. 762 for event in self.events.values(): 763 for lib in event.libraries: 764 dso = addr2line.get_dso(self.libs.get_lib(lib.lib_id).name) 765 if not dso: 766 continue 767 for function in lib.functions.values(): 768 for addr in function.addr_hit_map: 769 source = addr2line.get_addr_source(dso, addr) 770 if not source: 771 continue 772 for file_path, line in source: 773 source_file = self.source_files.get_source_file(file_path) 774 # Show [line - 5, line + 5] of the line hit by a sample. 775 source_file.request_lines(line - 5, line + 5) 776 count_info = function.addr_hit_map[addr] 777 function.build_line_hit_map(source_file.file_id, line, count_info[0], 778 count_info[1]) 779 780 # Collect needed source code in SourceFileSet. 781 self.source_files.load_source_code(source_dirs) 782 783 def add_disassembly(self, filter_lib: Callable[[str], bool], jobs: int): 784 """ Collect disassembly information: 785 1. Use objdump to collect disassembly for each function in FunctionSet. 786 2. Set flag to dump addr_hit_map when generating record info. 787 """ 788 objdump = Objdump(self.ndk_path, self.binary_finder) 789 executor = ThreadPoolExecutor(jobs) 790 lib_functions: Dict[int, List[Function]] = collections.defaultdict(list) 791 792 for function in self.functions.id_to_func.values(): 793 if function.func_name == 'unknown': 794 continue 795 lib_functions[function.lib_id].append(function) 796 797 for lib_id, functions in lib_functions.items(): 798 lib = self.libs.get_lib(lib_id) 799 if not filter_lib(lib.name): 800 continue 801 dso_info = objdump.get_dso_info(lib.name, lib.build_id) 802 if not dso_info: 803 continue 804 log_info('Disassemble %s' % dso_info[0]) 805 for function in functions: 806 def task(function, dso_info): 807 function.disassembly = objdump.disassemble_code( 808 dso_info, function.start_addr, function.addr_len) 809 executor.submit(task, function, dso_info) 810 executor.shutdown(wait=True) 811 self.gen_addr_hit_map_in_record_info = True 812 813 def gen_record_info(self) -> Dict[str, Any]: 814 """ Return json data which will be used by report_html.js. """ 815 record_info = {} 816 timestamp = self.meta_info.get('timestamp') 817 if timestamp: 818 t = datetime.datetime.fromtimestamp(int(timestamp)) 819 else: 820 t = datetime.datetime.now() 821 record_info['recordTime'] = t.strftime('%Y-%m-%d (%A) %H:%M:%S') 822 823 product_props = self.meta_info.get('product_props') 824 machine_type = self.arch 825 if product_props: 826 manufacturer, model, name = product_props.split(':') 827 machine_type = '%s (%s) by %s, arch %s' % (model, name, manufacturer, self.arch) 828 record_info['machineType'] = machine_type 829 record_info['androidVersion'] = self.meta_info.get('android_version', '') 830 record_info['recordCmdline'] = self.cmdline 831 record_info['totalSamples'] = self.total_samples 832 record_info['processNames'] = self._gen_process_names() 833 record_info['threadNames'] = self._gen_thread_names() 834 record_info['libList'] = self._gen_lib_list() 835 record_info['functionMap'] = self._gen_function_map() 836 record_info['sampleInfo'] = self._gen_sample_info() 837 record_info['sourceFiles'] = self._gen_source_files() 838 return record_info 839 840 def _gen_process_names(self) -> Dict[int, str]: 841 process_names: Dict[int, str] = {} 842 for event in self.events.values(): 843 for process in event.processes.values(): 844 process_names[process.pid] = process.name 845 return process_names 846 847 def _gen_thread_names(self) -> Dict[int, str]: 848 thread_names: Dict[int, str] = {} 849 for event in self.events.values(): 850 for process in event.processes.values(): 851 for thread in process.threads.values(): 852 thread_names[thread.tid] = thread.name 853 return thread_names 854 855 def _gen_lib_list(self) -> List[str]: 856 return [modify_text_for_html(lib.name) for lib in self.libs.libs] 857 858 def _gen_function_map(self) -> Dict[int, Any]: 859 func_map: Dict[int, Any] = {} 860 for func_id in sorted(self.functions.id_to_func): 861 function = self.functions.id_to_func[func_id] 862 func_data = {} 863 func_data['l'] = function.lib_id 864 func_data['f'] = modify_text_for_html(function.func_name) 865 if function.source_info: 866 func_data['s'] = function.source_info 867 if function.disassembly: 868 disassembly_list = [] 869 for code, addr in function.disassembly: 870 disassembly_list.append( 871 [modify_text_for_html(code), 872 hex_address_for_json(addr)]) 873 func_data['d'] = disassembly_list 874 func_map[func_id] = func_data 875 return func_map 876 877 def _gen_sample_info(self) -> List[Dict[str, Any]]: 878 return [event.get_sample_info(self.gen_addr_hit_map_in_record_info) 879 for event in self.events.values()] 880 881 def _gen_source_files(self) -> List[Dict[str, Any]]: 882 source_files = sorted(self.source_files.path_to_source_files.values(), 883 key=lambda x: x.file_id) 884 file_list = [] 885 for source_file in source_files: 886 file_data = {} 887 if not source_file.real_path: 888 file_data['path'] = '' 889 file_data['code'] = {} 890 else: 891 file_data['path'] = source_file.real_path 892 code_map = {} 893 for line in source_file.line_to_code: 894 code_map[line] = modify_text_for_html(source_file.line_to_code[line]) 895 file_data['code'] = code_map 896 file_list.append(file_data) 897 return file_list 898 899 900URLS = { 901 'jquery': 'https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js', 902 'bootstrap4-css': 'https://stackpath.bootstrapcdn.com/bootstrap/4.1.2/css/bootstrap.min.css', 903 'bootstrap4-popper': 904 'https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js', 905 'bootstrap4': 'https://stackpath.bootstrapcdn.com/bootstrap/4.1.2/js/bootstrap.min.js', 906 'dataTable': 'https://cdn.datatables.net/1.10.19/js/jquery.dataTables.min.js', 907 'dataTable-bootstrap4': 'https://cdn.datatables.net/1.10.19/js/dataTables.bootstrap4.min.js', 908 'dataTable-css': 'https://cdn.datatables.net/1.10.19/css/dataTables.bootstrap4.min.css', 909 'gstatic-charts': 'https://www.gstatic.com/charts/loader.js', 910} 911 912 913class ReportGenerator(object): 914 915 def __init__(self, html_path: Union[Path, str]): 916 self.hw = HtmlWriter(html_path) 917 self.hw.open_tag('html') 918 self.hw.open_tag('head') 919 for css in ['bootstrap4-css', 'dataTable-css']: 920 self.hw.open_tag('link', rel='stylesheet', type='text/css', href=URLS[css]).close_tag() 921 for js in ['jquery', 'bootstrap4-popper', 'bootstrap4', 'dataTable', 'dataTable-bootstrap4', 922 'gstatic-charts']: 923 self.hw.open_tag('script', src=URLS[js]).close_tag() 924 925 self.hw.open_tag('script').add( 926 "google.charts.load('current', {'packages': ['corechart', 'table']});").close_tag() 927 self.hw.open_tag('style', type='text/css').add(""" 928 .colForLine { width: 50px; } 929 .colForCount { width: 100px; } 930 .tableCell { font-size: 17px; } 931 .boldTableCell { font-weight: bold; font-size: 17px; } 932 """).close_tag() 933 self.hw.close_tag('head') 934 self.hw.open_tag('body') 935 936 def write_content_div(self): 937 self.hw.open_tag('div', id='report_content').close_tag() 938 939 def write_record_data(self, record_data: Dict[str, Any]): 940 self.hw.open_tag('script', id='record_data', type='application/json') 941 self.hw.add(json.dumps(record_data)) 942 self.hw.close_tag() 943 944 def write_script(self): 945 self.hw.open_tag('script').add_file('report_html.js').close_tag() 946 947 def finish(self): 948 self.hw.close_tag('body') 949 self.hw.close_tag('html') 950 self.hw.close() 951 952 953def get_args() -> argparse.Namespace: 954 parser = argparse.ArgumentParser( 955 description='report profiling data', formatter_class=ArgParseFormatter) 956 parser.add_argument('-i', '--record_file', nargs='+', default=['perf.data'], help=""" 957 Set profiling data file to report.""") 958 parser.add_argument('-o', '--report_path', default='report.html', help='Set output html file') 959 parser.add_argument('--min_func_percent', default=0.01, type=float, help=""" 960 Set min percentage of functions shown in the report. 961 For example, when set to 0.01, only functions taking >= 0.01%% of total 962 event count are collected in the report.""") 963 parser.add_argument('--min_callchain_percent', default=0.01, type=float, help=""" 964 Set min percentage of callchains shown in the report. 965 It is used to limit nodes shown in the function flamegraph. For example, 966 when set to 0.01, only callchains taking >= 0.01%% of the event count of 967 the starting function are collected in the report.""") 968 parser.add_argument('--add_source_code', action='store_true', help='Add source code.') 969 parser.add_argument('--source_dirs', nargs='+', help='Source code directories.') 970 parser.add_argument('--add_disassembly', action='store_true', help='Add disassembled code.') 971 parser.add_argument('--binary_filter', nargs='+', help="""Annotate source code and disassembly 972 only for selected binaries.""") 973 parser.add_argument( 974 '-j', '--jobs', type=int, default=os.cpu_count(), 975 help='Use multithreading to speed up disassembly and source code annotation.') 976 parser.add_argument('--ndk_path', nargs=1, help='Find tools in the ndk path.') 977 parser.add_argument('--no_browser', action='store_true', help="Don't open report in browser.") 978 parser.add_argument('--show_art_frames', action='store_true', 979 help='Show frames of internal methods in the ART Java interpreter.') 980 parser.add_argument('--aggregate-by-thread-name', action='store_true', help="""aggregate 981 samples by thread name instead of thread id. This is useful for 982 showing multiple perf.data generated for the same app.""") 983 parser.add_argument( 984 '--proguard-mapping-file', nargs='+', 985 help='Add proguard mapping file to de-obfuscate symbols') 986 return parser.parse_args() 987 988 989def main(): 990 sys.setrecursionlimit(MAX_CALLSTACK_LENGTH * 2 + 50) 991 args = get_args() 992 993 # 1. Process args. 994 binary_cache_path = 'binary_cache' 995 if not os.path.isdir(binary_cache_path): 996 if args.add_source_code or args.add_disassembly: 997 log_exit("""binary_cache/ doesn't exist. Can't add source code or disassembled code 998 without collected binaries. Please run binary_cache_builder.py to 999 collect binaries for current profiling data, or run app_profiler.py 1000 without -nb option.""") 1001 binary_cache_path = None 1002 1003 if args.add_source_code and not args.source_dirs: 1004 log_exit('--source_dirs is needed to add source code.') 1005 build_addr_hit_map = args.add_source_code or args.add_disassembly 1006 ndk_path = None if not args.ndk_path else args.ndk_path[0] 1007 if args.jobs < 1: 1008 log_exit('Invalid --jobs option.') 1009 1010 # 2. Produce record data. 1011 record_data = RecordData(binary_cache_path, ndk_path, 1012 build_addr_hit_map, args.proguard_mapping_file) 1013 for record_file in args.record_file: 1014 record_data.load_record_file(record_file, args.show_art_frames) 1015 if args.aggregate_by_thread_name: 1016 record_data.aggregate_by_thread_name() 1017 record_data.limit_percents(args.min_func_percent, args.min_callchain_percent) 1018 1019 def filter_lib(lib_name: str) -> bool: 1020 if not args.binary_filter: 1021 return True 1022 for binary in args.binary_filter: 1023 if binary in lib_name: 1024 return True 1025 return False 1026 if args.add_source_code: 1027 record_data.add_source_code(args.source_dirs, filter_lib) 1028 if args.add_disassembly: 1029 record_data.add_disassembly(filter_lib, args.jobs) 1030 1031 # 3. Generate report html. 1032 report_generator = ReportGenerator(args.report_path) 1033 report_generator.write_script() 1034 report_generator.write_content_div() 1035 report_generator.write_record_data(record_data.gen_record_info()) 1036 report_generator.finish() 1037 1038 if not args.no_browser: 1039 open_report_in_browser(args.report_path) 1040 log_info("Report generated at '%s'." % args.report_path) 1041 1042 1043if __name__ == '__main__': 1044 main() 1045