1 /*
2 * Copyright (C) 2016 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "errno.h"
18 #include <stdio.h>
19 #include <stdlib.h>
20 #include <sys/file.h>
21 #include <sys/stat.h>
22 #include <unistd.h>
23
24 #include <fstream>
25 #include <iostream>
26 #include <set>
27 #include <string>
28 #include <unordered_set>
29 #include <vector>
30
31 #include "android-base/stringprintf.h"
32 #include "android-base/strings.h"
33
34 #include "base/dumpable.h"
35 #include "base/scoped_flock.h"
36 #include "base/stringpiece.h"
37 #include "base/time_utils.h"
38 #include "base/unix_file/fd_file.h"
39 #include "bytecode_utils.h"
40 #include "dex_file.h"
41 #include "jit/profile_compilation_info.h"
42 #include "runtime.h"
43 #include "utils.h"
44 #include "zip_archive.h"
45 #include "profile_assistant.h"
46
47 namespace art {
48
49 static int original_argc;
50 static char** original_argv;
51
CommandLine()52 static std::string CommandLine() {
53 std::vector<std::string> command;
54 for (int i = 0; i < original_argc; ++i) {
55 command.push_back(original_argv[i]);
56 }
57 return android::base::Join(command, ' ');
58 }
59
60 static constexpr int kInvalidFd = -1;
61
FdIsValid(int fd)62 static bool FdIsValid(int fd) {
63 return fd != kInvalidFd;
64 }
65
UsageErrorV(const char * fmt,va_list ap)66 static void UsageErrorV(const char* fmt, va_list ap) {
67 std::string error;
68 android::base::StringAppendV(&error, fmt, ap);
69 LOG(ERROR) << error;
70 }
71
UsageError(const char * fmt,...)72 static void UsageError(const char* fmt, ...) {
73 va_list ap;
74 va_start(ap, fmt);
75 UsageErrorV(fmt, ap);
76 va_end(ap);
77 }
78
Usage(const char * fmt,...)79 NO_RETURN static void Usage(const char *fmt, ...) {
80 va_list ap;
81 va_start(ap, fmt);
82 UsageErrorV(fmt, ap);
83 va_end(ap);
84
85 UsageError("Command: %s", CommandLine().c_str());
86 UsageError("Usage: profman [options]...");
87 UsageError("");
88 UsageError(" --dump-only: dumps the content of the specified profile files");
89 UsageError(" to standard output (default) in a human readable form.");
90 UsageError("");
91 UsageError(" --dump-output-to-fd=<number>: redirects --dump-only output to a file descriptor.");
92 UsageError("");
93 UsageError(" --dump-classes-and-methods: dumps a sorted list of classes and methods that are");
94 UsageError(" in the specified profile file to standard output (default) in a human");
95 UsageError(" readable form. The output is valid input for --create-profile-from");
96 UsageError("");
97 UsageError(" --profile-file=<filename>: specify profiler output file to use for compilation.");
98 UsageError(" Can be specified multiple time, in which case the data from the different");
99 UsageError(" profiles will be aggregated.");
100 UsageError("");
101 UsageError(" --profile-file-fd=<number>: same as --profile-file but accepts a file descriptor.");
102 UsageError(" Cannot be used together with --profile-file.");
103 UsageError("");
104 UsageError(" --reference-profile-file=<filename>: specify a reference profile.");
105 UsageError(" The data in this file will be compared with the data obtained by merging");
106 UsageError(" all the files specified with --profile-file or --profile-file-fd.");
107 UsageError(" If the exit code is EXIT_COMPILE then all --profile-file will be merged into");
108 UsageError(" --reference-profile-file. ");
109 UsageError("");
110 UsageError(" --reference-profile-file-fd=<number>: same as --reference-profile-file but");
111 UsageError(" accepts a file descriptor. Cannot be used together with");
112 UsageError(" --reference-profile-file.");
113 UsageError("");
114 UsageError(" --generate-test-profile=<filename>: generates a random profile file for testing.");
115 UsageError(" --generate-test-profile-num-dex=<number>: number of dex files that should be");
116 UsageError(" included in the generated profile. Defaults to 20.");
117 UsageError(" --generate-test-profile-method-ratio=<number>: the percentage from the maximum");
118 UsageError(" number of methods that should be generated. Defaults to 5.");
119 UsageError(" --generate-test-profile-class-ratio=<number>: the percentage from the maximum");
120 UsageError(" number of classes that should be generated. Defaults to 5.");
121 UsageError(" --generate-test-profile-seed=<number>: seed for random number generator used when");
122 UsageError(" generating random test profiles. Defaults to using NanoTime.");
123 UsageError("");
124 UsageError(" --create-profile-from=<filename>: creates a profile from a list of classes and");
125 UsageError(" methods.");
126 UsageError("");
127 UsageError(" --dex-location=<string>: location string to use with corresponding");
128 UsageError(" apk-fd to find dex files");
129 UsageError("");
130 UsageError(" --apk-fd=<number>: file descriptor containing an open APK to");
131 UsageError(" search for dex files");
132 UsageError(" --apk-=<filename>: an APK to search for dex files");
133 UsageError("");
134
135 exit(EXIT_FAILURE);
136 }
137
138 // Note: make sure you update the Usage if you change these values.
139 static constexpr uint16_t kDefaultTestProfileNumDex = 20;
140 static constexpr uint16_t kDefaultTestProfileMethodRatio = 5;
141 static constexpr uint16_t kDefaultTestProfileClassRatio = 5;
142
143 // Separators used when parsing human friendly representation of profiles.
144 static const std::string kMethodSep = "->";
145 static const std::string kMissingTypesMarker = "missing_types";
146 static const std::string kInvalidClassDescriptor = "invalid_class";
147 static const std::string kInvalidMethod = "invalid_method";
148 static const std::string kClassAllMethods = "*";
149 static constexpr char kProfileParsingInlineChacheSep = '+';
150 static constexpr char kProfileParsingTypeSep = ',';
151 static constexpr char kProfileParsingFirstCharInSignature = '(';
152
153 // TODO(calin): This class has grown too much from its initial design. Split the functionality
154 // into smaller, more contained pieces.
155 class ProfMan FINAL {
156 public:
ProfMan()157 ProfMan() :
158 reference_profile_file_fd_(kInvalidFd),
159 dump_only_(false),
160 dump_classes_and_methods_(false),
161 dump_output_to_fd_(kInvalidFd),
162 test_profile_num_dex_(kDefaultTestProfileNumDex),
163 test_profile_method_ratio_(kDefaultTestProfileMethodRatio),
164 test_profile_class_ratio_(kDefaultTestProfileClassRatio),
165 test_profile_seed_(NanoTime()),
166 start_ns_(NanoTime()) {}
167
~ProfMan()168 ~ProfMan() {
169 LogCompletionTime();
170 }
171
ParseArgs(int argc,char ** argv)172 void ParseArgs(int argc, char **argv) {
173 original_argc = argc;
174 original_argv = argv;
175
176 InitLogging(argv, Runtime::Aborter);
177
178 // Skip over the command name.
179 argv++;
180 argc--;
181
182 if (argc == 0) {
183 Usage("No arguments specified");
184 }
185
186 for (int i = 0; i < argc; ++i) {
187 const StringPiece option(argv[i]);
188 const bool log_options = false;
189 if (log_options) {
190 LOG(INFO) << "profman: option[" << i << "]=" << argv[i];
191 }
192 if (option == "--dump-only") {
193 dump_only_ = true;
194 } else if (option == "--dump-classes-and-methods") {
195 dump_classes_and_methods_ = true;
196 } else if (option.starts_with("--create-profile-from=")) {
197 create_profile_from_file_ = option.substr(strlen("--create-profile-from=")).ToString();
198 } else if (option.starts_with("--dump-output-to-fd=")) {
199 ParseUintOption(option, "--dump-output-to-fd", &dump_output_to_fd_, Usage);
200 } else if (option.starts_with("--profile-file=")) {
201 profile_files_.push_back(option.substr(strlen("--profile-file=")).ToString());
202 } else if (option.starts_with("--profile-file-fd=")) {
203 ParseFdForCollection(option, "--profile-file-fd", &profile_files_fd_);
204 } else if (option.starts_with("--reference-profile-file=")) {
205 reference_profile_file_ = option.substr(strlen("--reference-profile-file=")).ToString();
206 } else if (option.starts_with("--reference-profile-file-fd=")) {
207 ParseUintOption(option, "--reference-profile-file-fd", &reference_profile_file_fd_, Usage);
208 } else if (option.starts_with("--dex-location=")) {
209 dex_locations_.push_back(option.substr(strlen("--dex-location=")).ToString());
210 } else if (option.starts_with("--apk-fd=")) {
211 ParseFdForCollection(option, "--apk-fd", &apks_fd_);
212 } else if (option.starts_with("--apk=")) {
213 apk_files_.push_back(option.substr(strlen("--apk=")).ToString());
214 } else if (option.starts_with("--generate-test-profile=")) {
215 test_profile_ = option.substr(strlen("--generate-test-profile=")).ToString();
216 } else if (option.starts_with("--generate-test-profile-num-dex=")) {
217 ParseUintOption(option,
218 "--generate-test-profile-num-dex",
219 &test_profile_num_dex_,
220 Usage);
221 } else if (option.starts_with("--generate-test-profile-method-ratio")) {
222 ParseUintOption(option,
223 "--generate-test-profile-method-ratio",
224 &test_profile_method_ratio_,
225 Usage);
226 } else if (option.starts_with("--generate-test-profile-class-ratio")) {
227 ParseUintOption(option,
228 "--generate-test-profile-class-ratio",
229 &test_profile_class_ratio_,
230 Usage);
231 } else if (option.starts_with("--generate-test-profile-seed=")) {
232 ParseUintOption(option, "--generate-test-profile-seed", &test_profile_seed_, Usage);
233 } else {
234 Usage("Unknown argument '%s'", option.data());
235 }
236 }
237
238 // Validate global consistency between file/fd options.
239 if (!profile_files_.empty() && !profile_files_fd_.empty()) {
240 Usage("Profile files should not be specified with both --profile-file-fd and --profile-file");
241 }
242 if (!reference_profile_file_.empty() && FdIsValid(reference_profile_file_fd_)) {
243 Usage("Reference profile should not be specified with both "
244 "--reference-profile-file-fd and --reference-profile-file");
245 }
246 if (!apk_files_.empty() && !apks_fd_.empty()) {
247 Usage("APK files should not be specified with both --apk-fd and --apk");
248 }
249 }
250
ProcessProfiles()251 ProfileAssistant::ProcessingResult ProcessProfiles() {
252 // Validate that at least one profile file was passed, as well as a reference profile.
253 if (profile_files_.empty() && profile_files_fd_.empty()) {
254 Usage("No profile files specified.");
255 }
256 if (reference_profile_file_.empty() && !FdIsValid(reference_profile_file_fd_)) {
257 Usage("No reference profile file specified.");
258 }
259 if ((!profile_files_.empty() && FdIsValid(reference_profile_file_fd_)) ||
260 (!profile_files_fd_.empty() && !FdIsValid(reference_profile_file_fd_))) {
261 Usage("Options --profile-file-fd and --reference-profile-file-fd "
262 "should only be used together");
263 }
264 ProfileAssistant::ProcessingResult result;
265 if (profile_files_.empty()) {
266 // The file doesn't need to be flushed here (ProcessProfiles will do it)
267 // so don't check the usage.
268 File file(reference_profile_file_fd_, false);
269 result = ProfileAssistant::ProcessProfiles(profile_files_fd_, reference_profile_file_fd_);
270 CloseAllFds(profile_files_fd_, "profile_files_fd_");
271 } else {
272 result = ProfileAssistant::ProcessProfiles(profile_files_, reference_profile_file_);
273 }
274 return result;
275 }
276
OpenApkFilesFromLocations(std::vector<std::unique_ptr<const DexFile>> * dex_files)277 void OpenApkFilesFromLocations(std::vector<std::unique_ptr<const DexFile>>* dex_files) {
278 bool use_apk_fd_list = !apks_fd_.empty();
279 if (use_apk_fd_list) {
280 // Get the APKs from the collection of FDs.
281 CHECK_EQ(dex_locations_.size(), apks_fd_.size());
282 } else if (!apk_files_.empty()) {
283 // Get the APKs from the collection of filenames.
284 CHECK_EQ(dex_locations_.size(), apk_files_.size());
285 } else {
286 // No APKs were specified.
287 CHECK(dex_locations_.empty());
288 return;
289 }
290 static constexpr bool kVerifyChecksum = true;
291 for (size_t i = 0; i < dex_locations_.size(); ++i) {
292 std::string error_msg;
293 std::vector<std::unique_ptr<const DexFile>> dex_files_for_location;
294 if (use_apk_fd_list) {
295 if (DexFile::OpenZip(apks_fd_[i],
296 dex_locations_[i],
297 kVerifyChecksum,
298 &error_msg,
299 &dex_files_for_location)) {
300 } else {
301 LOG(WARNING) << "OpenZip failed for '" << dex_locations_[i] << "' " << error_msg;
302 continue;
303 }
304 } else {
305 if (DexFile::Open(apk_files_[i].c_str(),
306 dex_locations_[i],
307 kVerifyChecksum,
308 &error_msg,
309 &dex_files_for_location)) {
310 } else {
311 LOG(WARNING) << "Open failed for '" << dex_locations_[i] << "' " << error_msg;
312 continue;
313 }
314 }
315 for (std::unique_ptr<const DexFile>& dex_file : dex_files_for_location) {
316 dex_files->emplace_back(std::move(dex_file));
317 }
318 }
319 }
320
DumpOneProfile(const std::string & banner,const std::string & filename,int fd,const std::vector<std::unique_ptr<const DexFile>> * dex_files,std::string * dump)321 int DumpOneProfile(const std::string& banner,
322 const std::string& filename,
323 int fd,
324 const std::vector<std::unique_ptr<const DexFile>>* dex_files,
325 std::string* dump) {
326 if (!filename.empty()) {
327 fd = open(filename.c_str(), O_RDWR);
328 if (fd < 0) {
329 LOG(ERROR) << "Cannot open " << filename << strerror(errno);
330 return -1;
331 }
332 }
333 ProfileCompilationInfo info;
334 if (!info.Load(fd)) {
335 LOG(ERROR) << "Cannot load profile info from fd=" << fd << "\n";
336 return -1;
337 }
338 std::string this_dump = banner + "\n" + info.DumpInfo(dex_files) + "\n";
339 *dump += this_dump;
340 if (close(fd) < 0) {
341 PLOG(WARNING) << "Failed to close descriptor";
342 }
343 return 0;
344 }
345
DumpProfileInfo()346 int DumpProfileInfo() {
347 // Validate that at least one profile file or reference was specified.
348 if (profile_files_.empty() && profile_files_fd_.empty() &&
349 reference_profile_file_.empty() && !FdIsValid(reference_profile_file_fd_)) {
350 Usage("No profile files or reference profile specified.");
351 }
352 static const char* kEmptyString = "";
353 static const char* kOrdinaryProfile = "=== profile ===";
354 static const char* kReferenceProfile = "=== reference profile ===";
355
356 // Open apk/zip files and and read dex files.
357 MemMap::Init(); // for ZipArchive::OpenFromFd
358 std::vector<std::unique_ptr<const DexFile>> dex_files;
359 OpenApkFilesFromLocations(&dex_files);
360 std::string dump;
361 // Dump individual profile files.
362 if (!profile_files_fd_.empty()) {
363 for (int profile_file_fd : profile_files_fd_) {
364 int ret = DumpOneProfile(kOrdinaryProfile,
365 kEmptyString,
366 profile_file_fd,
367 &dex_files,
368 &dump);
369 if (ret != 0) {
370 return ret;
371 }
372 }
373 }
374 if (!profile_files_.empty()) {
375 for (const std::string& profile_file : profile_files_) {
376 int ret = DumpOneProfile(kOrdinaryProfile, profile_file, kInvalidFd, &dex_files, &dump);
377 if (ret != 0) {
378 return ret;
379 }
380 }
381 }
382 // Dump reference profile file.
383 if (FdIsValid(reference_profile_file_fd_)) {
384 int ret = DumpOneProfile(kReferenceProfile,
385 kEmptyString,
386 reference_profile_file_fd_,
387 &dex_files,
388 &dump);
389 if (ret != 0) {
390 return ret;
391 }
392 }
393 if (!reference_profile_file_.empty()) {
394 int ret = DumpOneProfile(kReferenceProfile,
395 reference_profile_file_,
396 kInvalidFd,
397 &dex_files,
398 &dump);
399 if (ret != 0) {
400 return ret;
401 }
402 }
403 if (!FdIsValid(dump_output_to_fd_)) {
404 std::cout << dump;
405 } else {
406 unix_file::FdFile out_fd(dump_output_to_fd_, false /*check_usage*/);
407 if (!out_fd.WriteFully(dump.c_str(), dump.length())) {
408 return -1;
409 }
410 }
411 return 0;
412 }
413
ShouldOnlyDumpProfile()414 bool ShouldOnlyDumpProfile() {
415 return dump_only_;
416 }
417
GetClassNamesAndMethods(int fd,std::vector<std::unique_ptr<const DexFile>> * dex_files,std::set<std::string> * out_lines)418 bool GetClassNamesAndMethods(int fd,
419 std::vector<std::unique_ptr<const DexFile>>* dex_files,
420 std::set<std::string>* out_lines) {
421 ProfileCompilationInfo profile_info;
422 if (!profile_info.Load(fd)) {
423 LOG(ERROR) << "Cannot load profile info";
424 return false;
425 }
426 for (const std::unique_ptr<const DexFile>& dex_file : *dex_files) {
427 std::set<dex::TypeIndex> class_types;
428 std::set<uint16_t> methods;
429 if (profile_info.GetClassesAndMethods(*dex_file.get(), &class_types, &methods)) {
430 for (const dex::TypeIndex& type_index : class_types) {
431 const DexFile::TypeId& type_id = dex_file->GetTypeId(type_index);
432 out_lines->insert(std::string(dex_file->GetTypeDescriptor(type_id)));
433 }
434 for (uint16_t dex_method_idx : methods) {
435 const DexFile::MethodId& id = dex_file->GetMethodId(dex_method_idx);
436 std::string signature_string(dex_file->GetMethodSignature(id).ToString());
437 std::string type_string(dex_file->GetTypeDescriptor(dex_file->GetTypeId(id.class_idx_)));
438 std::string method_name(dex_file->GetMethodName(id));
439 out_lines->insert(type_string + kMethodSep + method_name + signature_string);
440 }
441 }
442 }
443 return true;
444 }
445
GetClassNamesAndMethods(const std::string & profile_file,std::vector<std::unique_ptr<const DexFile>> * dex_files,std::set<std::string> * out_lines)446 bool GetClassNamesAndMethods(const std::string& profile_file,
447 std::vector<std::unique_ptr<const DexFile>>* dex_files,
448 std::set<std::string>* out_lines) {
449 int fd = open(profile_file.c_str(), O_RDONLY);
450 if (!FdIsValid(fd)) {
451 LOG(ERROR) << "Cannot open " << profile_file << strerror(errno);
452 return false;
453 }
454 if (!GetClassNamesAndMethods(fd, dex_files, out_lines)) {
455 return false;
456 }
457 if (close(fd) < 0) {
458 PLOG(WARNING) << "Failed to close descriptor";
459 }
460 return true;
461 }
462
DumpClasses()463 int DumpClasses() {
464 // Validate that at least one profile file or reference was specified.
465 if (profile_files_.empty() && profile_files_fd_.empty() &&
466 reference_profile_file_.empty() && !FdIsValid(reference_profile_file_fd_)) {
467 Usage("No profile files or reference profile specified.");
468 }
469 // Open apk/zip files and and read dex files.
470 MemMap::Init(); // for ZipArchive::OpenFromFd
471 // Open the dex files to get the names for classes.
472 std::vector<std::unique_ptr<const DexFile>> dex_files;
473 OpenApkFilesFromLocations(&dex_files);
474 // Build a vector of class names from individual profile files.
475 std::set<std::string> class_names;
476 if (!profile_files_fd_.empty()) {
477 for (int profile_file_fd : profile_files_fd_) {
478 if (!GetClassNamesAndMethods(profile_file_fd, &dex_files, &class_names)) {
479 return -1;
480 }
481 }
482 }
483 if (!profile_files_.empty()) {
484 for (const std::string& profile_file : profile_files_) {
485 if (!GetClassNamesAndMethods(profile_file, &dex_files, &class_names)) {
486 return -1;
487 }
488 }
489 }
490 // Concatenate class names from reference profile file.
491 if (FdIsValid(reference_profile_file_fd_)) {
492 if (!GetClassNamesAndMethods(reference_profile_file_fd_, &dex_files, &class_names)) {
493 return -1;
494 }
495 }
496 if (!reference_profile_file_.empty()) {
497 if (!GetClassNamesAndMethods(reference_profile_file_, &dex_files, &class_names)) {
498 return -1;
499 }
500 }
501 // Dump the class names.
502 std::string dump;
503 for (const std::string& class_name : class_names) {
504 dump += class_name + std::string("\n");
505 }
506 if (!FdIsValid(dump_output_to_fd_)) {
507 std::cout << dump;
508 } else {
509 unix_file::FdFile out_fd(dump_output_to_fd_, false /*check_usage*/);
510 if (!out_fd.WriteFully(dump.c_str(), dump.length())) {
511 return -1;
512 }
513 }
514 return 0;
515 }
516
ShouldOnlyDumpClassesAndMethods()517 bool ShouldOnlyDumpClassesAndMethods() {
518 return dump_classes_and_methods_;
519 }
520
521 // Read lines from the given file, dropping comments and empty lines. Post-process each line with
522 // the given function.
523 template <typename T>
ReadCommentedInputFromFile(const char * input_filename,std::function<std::string (const char *)> * process)524 static T* ReadCommentedInputFromFile(
525 const char* input_filename, std::function<std::string(const char*)>* process) {
526 std::unique_ptr<std::ifstream> input_file(new std::ifstream(input_filename, std::ifstream::in));
527 if (input_file.get() == nullptr) {
528 LOG(ERROR) << "Failed to open input file " << input_filename;
529 return nullptr;
530 }
531 std::unique_ptr<T> result(
532 ReadCommentedInputStream<T>(*input_file, process));
533 input_file->close();
534 return result.release();
535 }
536
537 // Read lines from the given stream, dropping comments and empty lines. Post-process each line
538 // with the given function.
539 template <typename T>
ReadCommentedInputStream(std::istream & in_stream,std::function<std::string (const char *)> * process)540 static T* ReadCommentedInputStream(
541 std::istream& in_stream,
542 std::function<std::string(const char*)>* process) {
543 std::unique_ptr<T> output(new T());
544 while (in_stream.good()) {
545 std::string dot;
546 std::getline(in_stream, dot);
547 if (android::base::StartsWith(dot, "#") || dot.empty()) {
548 continue;
549 }
550 if (process != nullptr) {
551 std::string descriptor((*process)(dot.c_str()));
552 output->insert(output->end(), descriptor);
553 } else {
554 output->insert(output->end(), dot);
555 }
556 }
557 return output.release();
558 }
559
560 // Find class klass_descriptor in the given dex_files and store its reference
561 // in the out parameter class_ref.
562 // Return true if the definition of the class was found in any of the dex_files.
FindClass(const std::vector<std::unique_ptr<const DexFile>> & dex_files,const std::string & klass_descriptor,ProfileMethodInfo::ProfileClassReference * class_ref)563 bool FindClass(const std::vector<std::unique_ptr<const DexFile>>& dex_files,
564 const std::string& klass_descriptor,
565 /*out*/ProfileMethodInfo::ProfileClassReference* class_ref) {
566 constexpr uint16_t kInvalidTypeIndex = std::numeric_limits<uint16_t>::max() - 1;
567 for (const std::unique_ptr<const DexFile>& dex_file_ptr : dex_files) {
568 const DexFile* dex_file = dex_file_ptr.get();
569 if (klass_descriptor == kInvalidClassDescriptor) {
570 if (kInvalidTypeIndex >= dex_file->NumTypeIds()) {
571 // The dex file does not contain all possible type ids which leaves us room
572 // to add an "invalid" type id.
573 class_ref->dex_file = dex_file;
574 class_ref->type_index = dex::TypeIndex(kInvalidTypeIndex);
575 return true;
576 } else {
577 // The dex file contains all possible type ids. We don't have any free type id
578 // that we can use as invalid.
579 continue;
580 }
581 }
582
583 const DexFile::TypeId* type_id = dex_file->FindTypeId(klass_descriptor.c_str());
584 if (type_id == nullptr) {
585 continue;
586 }
587 dex::TypeIndex type_index = dex_file->GetIndexForTypeId(*type_id);
588 if (dex_file->FindClassDef(type_index) == nullptr) {
589 // Class is only referenced in the current dex file but not defined in it.
590 continue;
591 }
592 class_ref->dex_file = dex_file;
593 class_ref->type_index = type_index;
594 return true;
595 }
596 return false;
597 }
598
599 // Find the method specified by method_spec in the class class_ref.
FindMethodIndex(const ProfileMethodInfo::ProfileClassReference & class_ref,const std::string & method_spec)600 uint32_t FindMethodIndex(const ProfileMethodInfo::ProfileClassReference& class_ref,
601 const std::string& method_spec) {
602 const DexFile* dex_file = class_ref.dex_file;
603 if (method_spec == kInvalidMethod) {
604 constexpr uint16_t kInvalidMethodIndex = std::numeric_limits<uint16_t>::max() - 1;
605 return kInvalidMethodIndex >= dex_file->NumMethodIds()
606 ? kInvalidMethodIndex
607 : DexFile::kDexNoIndex;
608 }
609
610 std::vector<std::string> name_and_signature;
611 Split(method_spec, kProfileParsingFirstCharInSignature, &name_and_signature);
612 if (name_and_signature.size() != 2) {
613 LOG(ERROR) << "Invalid method name and signature " << method_spec;
614 return DexFile::kDexNoIndex;
615 }
616
617 const std::string& name = name_and_signature[0];
618 const std::string& signature = kProfileParsingFirstCharInSignature + name_and_signature[1];
619
620 const DexFile::StringId* name_id = dex_file->FindStringId(name.c_str());
621 if (name_id == nullptr) {
622 LOG(ERROR) << "Could not find name: " << name;
623 return DexFile::kDexNoIndex;
624 }
625 dex::TypeIndex return_type_idx;
626 std::vector<dex::TypeIndex> param_type_idxs;
627 if (!dex_file->CreateTypeList(signature, &return_type_idx, ¶m_type_idxs)) {
628 LOG(ERROR) << "Could not create type list" << signature;
629 return DexFile::kDexNoIndex;
630 }
631 const DexFile::ProtoId* proto_id = dex_file->FindProtoId(return_type_idx, param_type_idxs);
632 if (proto_id == nullptr) {
633 LOG(ERROR) << "Could not find proto_id: " << name;
634 return DexFile::kDexNoIndex;
635 }
636 const DexFile::MethodId* method_id = dex_file->FindMethodId(
637 dex_file->GetTypeId(class_ref.type_index), *name_id, *proto_id);
638 if (method_id == nullptr) {
639 LOG(ERROR) << "Could not find method_id: " << name;
640 return DexFile::kDexNoIndex;
641 }
642
643 return dex_file->GetIndexForMethodId(*method_id);
644 }
645
646 // Given a method, return true if the method has a single INVOKE_VIRTUAL in its byte code.
647 // Upon success it returns true and stores the method index and the invoke dex pc
648 // in the output parameters.
649 // The format of the method spec is "inlinePolymorphic(LSuper;)I+LSubA;,LSubB;,LSubC;".
650 //
651 // TODO(calin): support INVOKE_INTERFACE and the range variants.
HasSingleInvoke(const ProfileMethodInfo::ProfileClassReference & class_ref,uint16_t method_index,uint32_t * dex_pc)652 bool HasSingleInvoke(const ProfileMethodInfo::ProfileClassReference& class_ref,
653 uint16_t method_index,
654 /*out*/uint32_t* dex_pc) {
655 const DexFile* dex_file = class_ref.dex_file;
656 uint32_t offset = dex_file->FindCodeItemOffset(
657 *dex_file->FindClassDef(class_ref.type_index),
658 method_index);
659 const DexFile::CodeItem* code_item = dex_file->GetCodeItem(offset);
660
661 bool found_invoke = false;
662 for (CodeItemIterator it(*code_item); !it.Done(); it.Advance()) {
663 if (it.CurrentInstruction().Opcode() == Instruction::INVOKE_VIRTUAL) {
664 if (found_invoke) {
665 LOG(ERROR) << "Multiple invoke INVOKE_VIRTUAL found: "
666 << dex_file->PrettyMethod(method_index);
667 return false;
668 }
669 found_invoke = true;
670 *dex_pc = it.CurrentDexPc();
671 }
672 }
673 if (!found_invoke) {
674 LOG(ERROR) << "Could not find any INVOKE_VIRTUAL: " << dex_file->PrettyMethod(method_index);
675 }
676 return found_invoke;
677 }
678
679 // Process a line defining a class or a method and its inline caches.
680 // Upon success return true and add the class or the method info to profile.
681 // The possible line formats are:
682 // "LJustTheCass;".
683 // "LTestInline;->inlinePolymorphic(LSuper;)I+LSubA;,LSubB;,LSubC;".
684 // "LTestInline;->inlinePolymorphic(LSuper;)I+LSubA;,LSubB;,invalid_class".
685 // "LTestInline;->inlineMissingTypes(LSuper;)I+missing_types".
686 // "LTestInline;->inlineNoInlineCaches(LSuper;)I".
687 // "LTestInline;->*".
688 // "invalid_class".
689 // "LTestInline;->invalid_method".
690 // The method and classes are searched only in the given dex files.
ProcessLine(const std::vector<std::unique_ptr<const DexFile>> & dex_files,const std::string & line,ProfileCompilationInfo * profile)691 bool ProcessLine(const std::vector<std::unique_ptr<const DexFile>>& dex_files,
692 const std::string& line,
693 /*out*/ProfileCompilationInfo* profile) {
694 std::string klass;
695 std::string method_str;
696 size_t method_sep_index = line.find(kMethodSep);
697 if (method_sep_index == std::string::npos) {
698 klass = line;
699 } else {
700 klass = line.substr(0, method_sep_index);
701 method_str = line.substr(method_sep_index + kMethodSep.size());
702 }
703
704 ProfileMethodInfo::ProfileClassReference class_ref;
705 if (!FindClass(dex_files, klass, &class_ref)) {
706 LOG(WARNING) << "Could not find class: " << klass;
707 return false;
708 }
709
710 if (method_str.empty() || method_str == kClassAllMethods) {
711 // Start by adding the class.
712 std::set<DexCacheResolvedClasses> resolved_class_set;
713 const DexFile* dex_file = class_ref.dex_file;
714 const auto& dex_resolved_classes = resolved_class_set.emplace(
715 dex_file->GetLocation(),
716 dex_file->GetBaseLocation(),
717 dex_file->GetLocationChecksum());
718 dex_resolved_classes.first->AddClass(class_ref.type_index);
719 std::vector<ProfileMethodInfo> methods;
720 if (method_str == kClassAllMethods) {
721 // Add all of the methods.
722 const DexFile::ClassDef* class_def = dex_file->FindClassDef(class_ref.type_index);
723 const uint8_t* class_data = dex_file->GetClassData(*class_def);
724 if (class_data != nullptr) {
725 ClassDataItemIterator it(*dex_file, class_data);
726 while (it.HasNextStaticField() || it.HasNextInstanceField()) {
727 it.Next();
728 }
729 while (it.HasNextDirectMethod() || it.HasNextVirtualMethod()) {
730 if (it.GetMethodCodeItemOffset() != 0) {
731 // Add all of the methods that have code to the profile.
732 const uint32_t method_idx = it.GetMemberIndex();
733 methods.push_back(ProfileMethodInfo(dex_file, method_idx));
734 }
735 it.Next();
736 }
737 }
738 }
739 profile->AddMethodsAndClasses(methods, resolved_class_set);
740 return true;
741 }
742
743 // Process the method.
744 std::string method_spec;
745 std::vector<std::string> inline_cache_elems;
746
747 std::vector<std::string> method_elems;
748 bool is_missing_types = false;
749 Split(method_str, kProfileParsingInlineChacheSep, &method_elems);
750 if (method_elems.size() == 2) {
751 method_spec = method_elems[0];
752 is_missing_types = method_elems[1] == kMissingTypesMarker;
753 if (!is_missing_types) {
754 Split(method_elems[1], kProfileParsingTypeSep, &inline_cache_elems);
755 }
756 } else if (method_elems.size() == 1) {
757 method_spec = method_elems[0];
758 } else {
759 LOG(ERROR) << "Invalid method line: " << line;
760 return false;
761 }
762
763 const uint32_t method_index = FindMethodIndex(class_ref, method_spec);
764 if (method_index == DexFile::kDexNoIndex) {
765 return false;
766 }
767
768 std::vector<ProfileMethodInfo> pmi;
769 std::vector<ProfileMethodInfo::ProfileInlineCache> inline_caches;
770 if (is_missing_types || !inline_cache_elems.empty()) {
771 uint32_t dex_pc;
772 if (!HasSingleInvoke(class_ref, method_index, &dex_pc)) {
773 return false;
774 }
775 std::vector<ProfileMethodInfo::ProfileClassReference> classes(inline_cache_elems.size());
776 size_t class_it = 0;
777 for (const std::string& ic_class : inline_cache_elems) {
778 if (!FindClass(dex_files, ic_class, &(classes[class_it++]))) {
779 LOG(ERROR) << "Could not find class: " << ic_class;
780 return false;
781 }
782 }
783 inline_caches.emplace_back(dex_pc, is_missing_types, classes);
784 }
785 pmi.emplace_back(class_ref.dex_file, method_index, inline_caches);
786 profile->AddMethodsAndClasses(pmi, std::set<DexCacheResolvedClasses>());
787 return true;
788 }
789
790 // Creates a profile from a human friendly textual representation.
791 // The expected input format is:
792 // # Classes
793 // Ljava/lang/Comparable;
794 // Ljava/lang/Math;
795 // # Methods with inline caches
796 // LTestInline;->inlinePolymorphic(LSuper;)I+LSubA;,LSubB;,LSubC;
797 // LTestInline;->noInlineCache(LSuper;)I
CreateProfile()798 int CreateProfile() {
799 // Validate parameters for this command.
800 if (apk_files_.empty() && apks_fd_.empty()) {
801 Usage("APK files must be specified");
802 }
803 if (dex_locations_.empty()) {
804 Usage("DEX locations must be specified");
805 }
806 if (reference_profile_file_.empty() && !FdIsValid(reference_profile_file_fd_)) {
807 Usage("Reference profile must be specified with --reference-profile-file or "
808 "--reference-profile-file-fd");
809 }
810 if (!profile_files_.empty() || !profile_files_fd_.empty()) {
811 Usage("Profile must be specified with --reference-profile-file or "
812 "--reference-profile-file-fd");
813 }
814 // for ZipArchive::OpenFromFd
815 MemMap::Init();
816 // Open the profile output file if needed.
817 int fd = reference_profile_file_fd_;
818 if (!FdIsValid(fd)) {
819 CHECK(!reference_profile_file_.empty());
820 fd = open(reference_profile_file_.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0644);
821 if (fd < 0) {
822 LOG(ERROR) << "Cannot open " << reference_profile_file_ << strerror(errno);
823 return -1;
824 }
825 }
826 // Read the user-specified list of classes and methods.
827 std::unique_ptr<std::unordered_set<std::string>>
828 user_lines(ReadCommentedInputFromFile<std::unordered_set<std::string>>(
829 create_profile_from_file_.c_str(), nullptr)); // No post-processing.
830
831 // Open the dex files to look up classes and methods.
832 std::vector<std::unique_ptr<const DexFile>> dex_files;
833 OpenApkFilesFromLocations(&dex_files);
834
835 // Process the lines one by one and add the successful ones to the profile.
836 ProfileCompilationInfo info;
837
838 for (const auto& line : *user_lines) {
839 ProcessLine(dex_files, line, &info);
840 }
841
842 // Write the profile file.
843 CHECK(info.Save(fd));
844 if (close(fd) < 0) {
845 PLOG(WARNING) << "Failed to close descriptor";
846 }
847 return 0;
848 }
849
ShouldCreateProfile()850 bool ShouldCreateProfile() {
851 return !create_profile_from_file_.empty();
852 }
853
GenerateTestProfile()854 int GenerateTestProfile() {
855 // Validate parameters for this command.
856 if (test_profile_method_ratio_ > 100) {
857 Usage("Invalid ratio for --generate-test-profile-method-ratio");
858 }
859 if (test_profile_class_ratio_ > 100) {
860 Usage("Invalid ratio for --generate-test-profile-class-ratio");
861 }
862 // If given APK files or DEX locations, check that they're ok.
863 if (!apk_files_.empty() || !apks_fd_.empty() || !dex_locations_.empty()) {
864 if (apk_files_.empty() && apks_fd_.empty()) {
865 Usage("APK files must be specified when passing DEX locations to --generate-test-profile");
866 }
867 if (dex_locations_.empty()) {
868 Usage("DEX locations must be specified when passing APK files to --generate-test-profile");
869 }
870 }
871 // ShouldGenerateTestProfile confirms !test_profile_.empty().
872 int profile_test_fd = open(test_profile_.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0644);
873 if (profile_test_fd < 0) {
874 LOG(ERROR) << "Cannot open " << test_profile_ << strerror(errno);
875 return -1;
876 }
877 bool result;
878 if (apk_files_.empty() && apks_fd_.empty() && dex_locations_.empty()) {
879 result = ProfileCompilationInfo::GenerateTestProfile(profile_test_fd,
880 test_profile_num_dex_,
881 test_profile_method_ratio_,
882 test_profile_class_ratio_,
883 test_profile_seed_);
884 } else {
885 // Initialize MemMap for ZipArchive::OpenFromFd.
886 MemMap::Init();
887 // Open the dex files to look up classes and methods.
888 std::vector<std::unique_ptr<const DexFile>> dex_files;
889 OpenApkFilesFromLocations(&dex_files);
890 // Create a random profile file based on the set of dex files.
891 result = ProfileCompilationInfo::GenerateTestProfile(profile_test_fd,
892 dex_files,
893 test_profile_seed_);
894 }
895 close(profile_test_fd); // ignore close result.
896 return result ? 0 : -1;
897 }
898
ShouldGenerateTestProfile()899 bool ShouldGenerateTestProfile() {
900 return !test_profile_.empty();
901 }
902
903 private:
ParseFdForCollection(const StringPiece & option,const char * arg_name,std::vector<int> * fds)904 static void ParseFdForCollection(const StringPiece& option,
905 const char* arg_name,
906 std::vector<int>* fds) {
907 int fd;
908 ParseUintOption(option, arg_name, &fd, Usage);
909 fds->push_back(fd);
910 }
911
CloseAllFds(const std::vector<int> & fds,const char * descriptor)912 static void CloseAllFds(const std::vector<int>& fds, const char* descriptor) {
913 for (size_t i = 0; i < fds.size(); i++) {
914 if (close(fds[i]) < 0) {
915 PLOG(WARNING) << "Failed to close descriptor for " << descriptor << " at index " << i;
916 }
917 }
918 }
919
LogCompletionTime()920 void LogCompletionTime() {
921 static constexpr uint64_t kLogThresholdTime = MsToNs(100); // 100ms
922 uint64_t time_taken = NanoTime() - start_ns_;
923 if (time_taken > kLogThresholdTime) {
924 LOG(WARNING) << "profman took " << PrettyDuration(time_taken);
925 }
926 }
927
928 std::vector<std::string> profile_files_;
929 std::vector<int> profile_files_fd_;
930 std::vector<std::string> dex_locations_;
931 std::vector<std::string> apk_files_;
932 std::vector<int> apks_fd_;
933 std::string reference_profile_file_;
934 int reference_profile_file_fd_;
935 bool dump_only_;
936 bool dump_classes_and_methods_;
937 int dump_output_to_fd_;
938 std::string test_profile_;
939 std::string create_profile_from_file_;
940 uint16_t test_profile_num_dex_;
941 uint16_t test_profile_method_ratio_;
942 uint16_t test_profile_class_ratio_;
943 uint32_t test_profile_seed_;
944 uint64_t start_ns_;
945 };
946
947 // See ProfileAssistant::ProcessingResult for return codes.
profman(int argc,char ** argv)948 static int profman(int argc, char** argv) {
949 ProfMan profman;
950
951 // Parse arguments. Argument mistakes will lead to exit(EXIT_FAILURE) in UsageError.
952 profman.ParseArgs(argc, argv);
953
954 if (profman.ShouldGenerateTestProfile()) {
955 return profman.GenerateTestProfile();
956 }
957 if (profman.ShouldOnlyDumpProfile()) {
958 return profman.DumpProfileInfo();
959 }
960 if (profman.ShouldOnlyDumpClassesAndMethods()) {
961 return profman.DumpClasses();
962 }
963 if (profman.ShouldCreateProfile()) {
964 return profman.CreateProfile();
965 }
966 // Process profile information and assess if we need to do a profile guided compilation.
967 // This operation involves I/O.
968 return profman.ProcessProfiles();
969 }
970
971 } // namespace art
972
main(int argc,char ** argv)973 int main(int argc, char **argv) {
974 return art::profman(argc, argv);
975 }
976
977