1 /*
2  * Copyright (C) 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "optimizing_compiler.h"
18 
19 #include <fstream>
20 #include <memory>
21 #include <sstream>
22 
23 #include <stdint.h>
24 
25 #include "art_method-inl.h"
26 #include "base/arena_allocator.h"
27 #include "base/arena_containers.h"
28 #include "base/dumpable.h"
29 #include "base/logging.h"
30 #include "base/macros.h"
31 #include "base/mutex.h"
32 #include "base/scoped_arena_allocator.h"
33 #include "base/timing_logger.h"
34 #include "builder.h"
35 #include "code_generator.h"
36 #include "compiled_method.h"
37 #include "compiler.h"
38 #include "debug/elf_debug_writer.h"
39 #include "debug/method_debug_info.h"
40 #include "dex/dex_file_types.h"
41 #include "dex/verification_results.h"
42 #include "dex/verified_method.h"
43 #include "driver/compiled_method_storage.h"
44 #include "driver/compiler_options.h"
45 #include "driver/dex_compilation_unit.h"
46 #include "graph_checker.h"
47 #include "graph_visualizer.h"
48 #include "inliner.h"
49 #include "jit/debugger_interface.h"
50 #include "jit/jit.h"
51 #include "jit/jit_code_cache.h"
52 #include "jit/jit_logger.h"
53 #include "jni/quick/jni_compiler.h"
54 #include "linker/linker_patch.h"
55 #include "nodes.h"
56 #include "oat_quick_method_header.h"
57 #include "prepare_for_register_allocation.h"
58 #include "reference_type_propagation.h"
59 #include "register_allocator_linear_scan.h"
60 #include "select_generator.h"
61 #include "ssa_builder.h"
62 #include "ssa_liveness_analysis.h"
63 #include "ssa_phi_elimination.h"
64 #include "stack_map_stream.h"
65 #include "utils/assembler.h"
66 #include "verifier/verifier_compiler_binding.h"
67 
68 namespace art {
69 
70 static constexpr size_t kArenaAllocatorMemoryReportThreshold = 8 * MB;
71 
72 static constexpr const char* kPassNameSeparator = "$";
73 
74 /**
75  * Used by the code generator, to allocate the code in a vector.
76  */
77 class CodeVectorAllocator final : public CodeAllocator {
78  public:
CodeVectorAllocator(ArenaAllocator * allocator)79   explicit CodeVectorAllocator(ArenaAllocator* allocator)
80       : memory_(allocator->Adapter(kArenaAllocCodeBuffer)) {}
81 
Allocate(size_t size)82   uint8_t* Allocate(size_t size) override {
83     memory_.resize(size);
84     return &memory_[0];
85   }
86 
GetMemory() const87   ArrayRef<const uint8_t> GetMemory() const override { return ArrayRef<const uint8_t>(memory_); }
GetData()88   uint8_t* GetData() { return memory_.data(); }
89 
90  private:
91   ArenaVector<uint8_t> memory_;
92 
93   DISALLOW_COPY_AND_ASSIGN(CodeVectorAllocator);
94 };
95 
96 /**
97  * Filter to apply to the visualizer. Methods whose name contain that filter will
98  * be dumped.
99  */
100 static constexpr const char kStringFilter[] = "";
101 
102 class PassScope;
103 
104 class PassObserver : public ValueObject {
105  public:
PassObserver(HGraph * graph,CodeGenerator * codegen,std::ostream * visualizer_output,const CompilerOptions & compiler_options,Mutex & dump_mutex)106   PassObserver(HGraph* graph,
107                CodeGenerator* codegen,
108                std::ostream* visualizer_output,
109                const CompilerOptions& compiler_options,
110                Mutex& dump_mutex)
111       : graph_(graph),
112         last_seen_graph_size_(0),
113         cached_method_name_(),
114         timing_logger_enabled_(compiler_options.GetDumpPassTimings()),
115         timing_logger_(timing_logger_enabled_ ? GetMethodName() : "", true, true),
116         disasm_info_(graph->GetAllocator()),
117         visualizer_oss_(),
118         visualizer_output_(visualizer_output),
119         visualizer_enabled_(!compiler_options.GetDumpCfgFileName().empty()),
120         visualizer_(&visualizer_oss_, graph, codegen),
121         codegen_(codegen),
122         visualizer_dump_mutex_(dump_mutex),
123         graph_in_bad_state_(false) {
124     if (timing_logger_enabled_ || visualizer_enabled_) {
125       if (!IsVerboseMethod(compiler_options, GetMethodName())) {
126         timing_logger_enabled_ = visualizer_enabled_ = false;
127       }
128       if (visualizer_enabled_) {
129         visualizer_.PrintHeader(GetMethodName());
130         codegen->SetDisassemblyInformation(&disasm_info_);
131       }
132     }
133   }
134 
~PassObserver()135   ~PassObserver() {
136     if (timing_logger_enabled_) {
137       LOG(INFO) << "TIMINGS " << GetMethodName();
138       LOG(INFO) << Dumpable<TimingLogger>(timing_logger_);
139     }
140     if (visualizer_enabled_) {
141       FlushVisualizer();
142     }
143     DCHECK(visualizer_oss_.str().empty());
144   }
145 
DumpDisassembly()146   void DumpDisassembly() {
147     if (visualizer_enabled_) {
148       visualizer_.DumpGraphWithDisassembly();
149     }
150   }
151 
SetGraphInBadState()152   void SetGraphInBadState() { graph_in_bad_state_ = true; }
153 
GetMethodName()154   const char* GetMethodName() {
155     // PrettyMethod() is expensive, so we delay calling it until we actually have to.
156     if (cached_method_name_.empty()) {
157       cached_method_name_ = graph_->GetDexFile().PrettyMethod(graph_->GetMethodIdx());
158     }
159     return cached_method_name_.c_str();
160   }
161 
162  private:
StartPass(const char * pass_name)163   void StartPass(const char* pass_name) {
164     VLOG(compiler) << "Starting pass: " << pass_name;
165     // Dump graph first, then start timer.
166     if (visualizer_enabled_) {
167       visualizer_.DumpGraph(pass_name, /* is_after_pass= */ false, graph_in_bad_state_);
168     }
169     if (timing_logger_enabled_) {
170       timing_logger_.StartTiming(pass_name);
171     }
172   }
173 
FlushVisualizer()174   void FlushVisualizer() REQUIRES(!visualizer_dump_mutex_) {
175     MutexLock mu(Thread::Current(), visualizer_dump_mutex_);
176     *visualizer_output_ << visualizer_oss_.str();
177     visualizer_output_->flush();
178     visualizer_oss_.str("");
179     visualizer_oss_.clear();
180   }
181 
EndPass(const char * pass_name,bool pass_change)182   void EndPass(const char* pass_name, bool pass_change) {
183     // Pause timer first, then dump graph.
184     if (timing_logger_enabled_) {
185       timing_logger_.EndTiming();
186     }
187     if (visualizer_enabled_) {
188       visualizer_.DumpGraph(pass_name, /* is_after_pass= */ true, graph_in_bad_state_);
189     }
190 
191     // Validate the HGraph if running in debug mode.
192     if (kIsDebugBuild) {
193       if (!graph_in_bad_state_) {
194         GraphChecker checker(graph_, codegen_);
195         last_seen_graph_size_ = checker.Run(pass_change, last_seen_graph_size_);
196         if (!checker.IsValid()) {
197           LOG(FATAL_WITHOUT_ABORT) << "Error after " << pass_name << "(" << graph_->PrettyMethod()
198                                    << "): " << *graph_;
199           LOG(FATAL) << "(" << pass_name <<  "): " << Dumpable<GraphChecker>(checker);
200         }
201       }
202     }
203   }
204 
IsVerboseMethod(const CompilerOptions & compiler_options,const char * method_name)205   static bool IsVerboseMethod(const CompilerOptions& compiler_options, const char* method_name) {
206     // Test an exact match to --verbose-methods. If verbose-methods is set, this overrides an
207     // empty kStringFilter matching all methods.
208     if (compiler_options.HasVerboseMethods()) {
209       return compiler_options.IsVerboseMethod(method_name);
210     }
211 
212     // Test the kStringFilter sub-string. constexpr helper variable to silence unreachable-code
213     // warning when the string is empty.
214     constexpr bool kStringFilterEmpty = arraysize(kStringFilter) <= 1;
215     if (kStringFilterEmpty || strstr(method_name, kStringFilter) != nullptr) {
216       return true;
217     }
218 
219     return false;
220   }
221 
222   HGraph* const graph_;
223   size_t last_seen_graph_size_;
224 
225   std::string cached_method_name_;
226 
227   bool timing_logger_enabled_;
228   TimingLogger timing_logger_;
229 
230   DisassemblyInformation disasm_info_;
231 
232   std::ostringstream visualizer_oss_;
233   std::ostream* visualizer_output_;
234   bool visualizer_enabled_;
235   HGraphVisualizer visualizer_;
236   CodeGenerator* codegen_;
237   Mutex& visualizer_dump_mutex_;
238 
239   // Flag to be set by the compiler if the pass failed and the graph is not
240   // expected to validate.
241   bool graph_in_bad_state_;
242 
243   friend PassScope;
244 
245   DISALLOW_COPY_AND_ASSIGN(PassObserver);
246 };
247 
248 class PassScope : public ValueObject {
249  public:
PassScope(const char * pass_name,PassObserver * pass_observer)250   PassScope(const char *pass_name, PassObserver* pass_observer)
251       : pass_name_(pass_name),
252         pass_change_(true),  // assume change
253         pass_observer_(pass_observer) {
254     pass_observer_->StartPass(pass_name_);
255   }
256 
SetPassNotChanged()257   void SetPassNotChanged() {
258     pass_change_ = false;
259   }
260 
~PassScope()261   ~PassScope() {
262     pass_observer_->EndPass(pass_name_, pass_change_);
263   }
264 
265  private:
266   const char* const pass_name_;
267   bool pass_change_;
268   PassObserver* const pass_observer_;
269 };
270 
271 class OptimizingCompiler final : public Compiler {
272  public:
273   explicit OptimizingCompiler(const CompilerOptions& compiler_options,
274                               CompiledMethodStorage* storage);
275   ~OptimizingCompiler() override;
276 
277   bool CanCompileMethod(uint32_t method_idx, const DexFile& dex_file) const override;
278 
279   CompiledMethod* Compile(const dex::CodeItem* code_item,
280                           uint32_t access_flags,
281                           InvokeType invoke_type,
282                           uint16_t class_def_idx,
283                           uint32_t method_idx,
284                           Handle<mirror::ClassLoader> class_loader,
285                           const DexFile& dex_file,
286                           Handle<mirror::DexCache> dex_cache) const override;
287 
288   CompiledMethod* JniCompile(uint32_t access_flags,
289                              uint32_t method_idx,
290                              const DexFile& dex_file,
291                              Handle<mirror::DexCache> dex_cache) const override;
292 
GetEntryPointOf(ArtMethod * method) const293   uintptr_t GetEntryPointOf(ArtMethod* method) const override
294       REQUIRES_SHARED(Locks::mutator_lock_) {
295     return reinterpret_cast<uintptr_t>(method->GetEntryPointFromQuickCompiledCodePtrSize(
296         InstructionSetPointerSize(GetCompilerOptions().GetInstructionSet())));
297   }
298 
299   bool JitCompile(Thread* self,
300                   jit::JitCodeCache* code_cache,
301                   jit::JitMemoryRegion* region,
302                   ArtMethod* method,
303                   CompilationKind compilation_kind,
304                   jit::JitLogger* jit_logger)
305       override
306       REQUIRES_SHARED(Locks::mutator_lock_);
307 
308  private:
RunOptimizations(HGraph * graph,CodeGenerator * codegen,const DexCompilationUnit & dex_compilation_unit,PassObserver * pass_observer,const OptimizationDef definitions[],size_t length) const309   bool RunOptimizations(HGraph* graph,
310                         CodeGenerator* codegen,
311                         const DexCompilationUnit& dex_compilation_unit,
312                         PassObserver* pass_observer,
313                         const OptimizationDef definitions[],
314                         size_t length) const {
315     // Convert definitions to optimization passes.
316     ArenaVector<HOptimization*> optimizations = ConstructOptimizations(
317         definitions,
318         length,
319         graph->GetAllocator(),
320         graph,
321         compilation_stats_.get(),
322         codegen,
323         dex_compilation_unit);
324     DCHECK_EQ(length, optimizations.size());
325     // Run the optimization passes one by one. Any "depends_on" pass refers back to
326     // the most recent occurrence of that pass, skipped or executed.
327     std::bitset<static_cast<size_t>(OptimizationPass::kLast) + 1u> pass_changes;
328     pass_changes[static_cast<size_t>(OptimizationPass::kNone)] = true;
329     bool change = false;
330     for (size_t i = 0; i < length; ++i) {
331       if (pass_changes[static_cast<size_t>(definitions[i].depends_on)]) {
332         // Execute the pass and record whether it changed anything.
333         PassScope scope(optimizations[i]->GetPassName(), pass_observer);
334         bool pass_change = optimizations[i]->Run();
335         pass_changes[static_cast<size_t>(definitions[i].pass)] = pass_change;
336         if (pass_change) {
337           change = true;
338         } else {
339           scope.SetPassNotChanged();
340         }
341       } else {
342         // Skip the pass and record that nothing changed.
343         pass_changes[static_cast<size_t>(definitions[i].pass)] = false;
344       }
345     }
346     return change;
347   }
348 
RunOptimizations(HGraph * graph,CodeGenerator * codegen,const DexCompilationUnit & dex_compilation_unit,PassObserver * pass_observer,const OptimizationDef (& definitions)[length]) const349   template <size_t length> bool RunOptimizations(
350       HGraph* graph,
351       CodeGenerator* codegen,
352       const DexCompilationUnit& dex_compilation_unit,
353       PassObserver* pass_observer,
354       const OptimizationDef (&definitions)[length]) const {
355     return RunOptimizations(
356         graph, codegen, dex_compilation_unit, pass_observer, definitions, length);
357   }
358 
359   void RunOptimizations(HGraph* graph,
360                         CodeGenerator* codegen,
361                         const DexCompilationUnit& dex_compilation_unit,
362                         PassObserver* pass_observer) const;
363 
364  private:
365   // Create a 'CompiledMethod' for an optimized graph.
366   CompiledMethod* Emit(ArenaAllocator* allocator,
367                        CodeVectorAllocator* code_allocator,
368                        CodeGenerator* codegen,
369                        const dex::CodeItem* item) const;
370 
371   // Try compiling a method and return the code generator used for
372   // compiling it.
373   // This method:
374   // 1) Builds the graph. Returns null if it failed to build it.
375   // 2) Transforms the graph to SSA. Returns null if it failed.
376   // 3) Runs optimizations on the graph, including register allocator.
377   // 4) Generates code with the `code_allocator` provided.
378   CodeGenerator* TryCompile(ArenaAllocator* allocator,
379                             ArenaStack* arena_stack,
380                             CodeVectorAllocator* code_allocator,
381                             const DexCompilationUnit& dex_compilation_unit,
382                             ArtMethod* method,
383                             CompilationKind compilation_kind,
384                             VariableSizedHandleScope* handles) const;
385 
386   CodeGenerator* TryCompileIntrinsic(ArenaAllocator* allocator,
387                                      ArenaStack* arena_stack,
388                                      CodeVectorAllocator* code_allocator,
389                                      const DexCompilationUnit& dex_compilation_unit,
390                                      ArtMethod* method,
391                                      VariableSizedHandleScope* handles) const;
392 
393   bool RunArchOptimizations(HGraph* graph,
394                             CodeGenerator* codegen,
395                             const DexCompilationUnit& dex_compilation_unit,
396                             PassObserver* pass_observer) const;
397 
398   bool RunBaselineOptimizations(HGraph* graph,
399                                 CodeGenerator* codegen,
400                                 const DexCompilationUnit& dex_compilation_unit,
401                                 PassObserver* pass_observer) const;
402 
403   std::vector<uint8_t> GenerateJitDebugInfo(const debug::MethodDebugInfo& method_debug_info);
404 
405   // This must be called before any other function that dumps data to the cfg
406   void DumpInstructionSetFeaturesToCfg() const;
407 
408   std::unique_ptr<OptimizingCompilerStats> compilation_stats_;
409 
410   std::unique_ptr<std::ostream> visualizer_output_;
411 
412   mutable Mutex dump_mutex_;  // To synchronize visualizer writing.
413 
414   DISALLOW_COPY_AND_ASSIGN(OptimizingCompiler);
415 };
416 
417 static const int kMaximumCompilationTimeBeforeWarning = 100; /* ms */
418 
OptimizingCompiler(const CompilerOptions & compiler_options,CompiledMethodStorage * storage)419 OptimizingCompiler::OptimizingCompiler(const CompilerOptions& compiler_options,
420                                        CompiledMethodStorage* storage)
421     : Compiler(compiler_options, storage, kMaximumCompilationTimeBeforeWarning),
422       dump_mutex_("Visualizer dump lock") {
423   // Enable C1visualizer output.
424   const std::string& cfg_file_name = compiler_options.GetDumpCfgFileName();
425   if (!cfg_file_name.empty()) {
426     std::ios_base::openmode cfg_file_mode =
427         compiler_options.GetDumpCfgAppend() ? std::ofstream::app : std::ofstream::out;
428     visualizer_output_.reset(new std::ofstream(cfg_file_name, cfg_file_mode));
429     DumpInstructionSetFeaturesToCfg();
430   }
431   if (compiler_options.GetDumpStats()) {
432     compilation_stats_.reset(new OptimizingCompilerStats());
433   }
434 }
435 
~OptimizingCompiler()436 OptimizingCompiler::~OptimizingCompiler() {
437   if (compilation_stats_.get() != nullptr) {
438     compilation_stats_->Log();
439   }
440 }
441 
DumpInstructionSetFeaturesToCfg() const442 void OptimizingCompiler::DumpInstructionSetFeaturesToCfg() const {
443   const CompilerOptions& compiler_options = GetCompilerOptions();
444   const InstructionSetFeatures* features = compiler_options.GetInstructionSetFeatures();
445   std::string isa_string =
446       std::string("isa:") + GetInstructionSetString(features->GetInstructionSet());
447   std::string features_string = "isa_features:" + features->GetFeatureString();
448   // It is assumed that visualizer_output_ is empty when calling this function, hence the fake
449   // compilation block containing the ISA features will be printed at the beginning of the .cfg
450   // file.
451   *visualizer_output_
452       << HGraphVisualizer::InsertMetaDataAsCompilationBlock(isa_string + ' ' + features_string);
453 }
454 
CanCompileMethod(uint32_t method_idx ATTRIBUTE_UNUSED,const DexFile & dex_file ATTRIBUTE_UNUSED) const455 bool OptimizingCompiler::CanCompileMethod(uint32_t method_idx ATTRIBUTE_UNUSED,
456                                           const DexFile& dex_file ATTRIBUTE_UNUSED) const {
457   return true;
458 }
459 
IsInstructionSetSupported(InstructionSet instruction_set)460 static bool IsInstructionSetSupported(InstructionSet instruction_set) {
461   return instruction_set == InstructionSet::kArm
462       || instruction_set == InstructionSet::kArm64
463       || instruction_set == InstructionSet::kThumb2
464       || instruction_set == InstructionSet::kX86
465       || instruction_set == InstructionSet::kX86_64;
466 }
467 
RunBaselineOptimizations(HGraph * graph,CodeGenerator * codegen,const DexCompilationUnit & dex_compilation_unit,PassObserver * pass_observer) const468 bool OptimizingCompiler::RunBaselineOptimizations(HGraph* graph,
469                                                   CodeGenerator* codegen,
470                                                   const DexCompilationUnit& dex_compilation_unit,
471                                                   PassObserver* pass_observer) const {
472   switch (codegen->GetCompilerOptions().GetInstructionSet()) {
473 #if defined(ART_ENABLE_CODEGEN_arm)
474     case InstructionSet::kThumb2:
475     case InstructionSet::kArm: {
476       OptimizationDef arm_optimizations[] = {
477         OptDef(OptimizationPass::kCriticalNativeAbiFixupArm),
478       };
479       return RunOptimizations(graph,
480                               codegen,
481                               dex_compilation_unit,
482                               pass_observer,
483                               arm_optimizations);
484     }
485 #endif
486 #ifdef ART_ENABLE_CODEGEN_x86
487     case InstructionSet::kX86: {
488       OptimizationDef x86_optimizations[] = {
489         OptDef(OptimizationPass::kPcRelativeFixupsX86),
490       };
491       return RunOptimizations(graph,
492                               codegen,
493                               dex_compilation_unit,
494                               pass_observer,
495                               x86_optimizations);
496     }
497 #endif
498     default:
499       UNUSED(graph);
500       UNUSED(codegen);
501       UNUSED(dex_compilation_unit);
502       UNUSED(pass_observer);
503       return false;
504   }
505 }
506 
RunArchOptimizations(HGraph * graph,CodeGenerator * codegen,const DexCompilationUnit & dex_compilation_unit,PassObserver * pass_observer) const507 bool OptimizingCompiler::RunArchOptimizations(HGraph* graph,
508                                               CodeGenerator* codegen,
509                                               const DexCompilationUnit& dex_compilation_unit,
510                                               PassObserver* pass_observer) const {
511   switch (codegen->GetCompilerOptions().GetInstructionSet()) {
512 #if defined(ART_ENABLE_CODEGEN_arm)
513     case InstructionSet::kThumb2:
514     case InstructionSet::kArm: {
515       OptimizationDef arm_optimizations[] = {
516         OptDef(OptimizationPass::kInstructionSimplifierArm),
517         OptDef(OptimizationPass::kSideEffectsAnalysis),
518         OptDef(OptimizationPass::kGlobalValueNumbering, "GVN$after_arch"),
519         OptDef(OptimizationPass::kCriticalNativeAbiFixupArm),
520         OptDef(OptimizationPass::kScheduling)
521       };
522       return RunOptimizations(graph,
523                               codegen,
524                               dex_compilation_unit,
525                               pass_observer,
526                               arm_optimizations);
527     }
528 #endif
529 #ifdef ART_ENABLE_CODEGEN_arm64
530     case InstructionSet::kArm64: {
531       OptimizationDef arm64_optimizations[] = {
532         OptDef(OptimizationPass::kInstructionSimplifierArm64),
533         OptDef(OptimizationPass::kSideEffectsAnalysis),
534         OptDef(OptimizationPass::kGlobalValueNumbering, "GVN$after_arch"),
535         OptDef(OptimizationPass::kScheduling)
536       };
537       return RunOptimizations(graph,
538                               codegen,
539                               dex_compilation_unit,
540                               pass_observer,
541                               arm64_optimizations);
542     }
543 #endif
544 #ifdef ART_ENABLE_CODEGEN_x86
545     case InstructionSet::kX86: {
546       OptimizationDef x86_optimizations[] = {
547         OptDef(OptimizationPass::kInstructionSimplifierX86),
548         OptDef(OptimizationPass::kSideEffectsAnalysis),
549         OptDef(OptimizationPass::kGlobalValueNumbering, "GVN$after_arch"),
550         OptDef(OptimizationPass::kPcRelativeFixupsX86),
551         OptDef(OptimizationPass::kX86MemoryOperandGeneration)
552       };
553       return RunOptimizations(graph,
554                               codegen,
555                               dex_compilation_unit,
556                               pass_observer,
557                               x86_optimizations);
558     }
559 #endif
560 #ifdef ART_ENABLE_CODEGEN_x86_64
561     case InstructionSet::kX86_64: {
562       OptimizationDef x86_64_optimizations[] = {
563         OptDef(OptimizationPass::kInstructionSimplifierX86_64),
564         OptDef(OptimizationPass::kSideEffectsAnalysis),
565         OptDef(OptimizationPass::kGlobalValueNumbering, "GVN$after_arch"),
566         OptDef(OptimizationPass::kX86MemoryOperandGeneration)
567       };
568       return RunOptimizations(graph,
569                               codegen,
570                               dex_compilation_unit,
571                               pass_observer,
572                               x86_64_optimizations);
573     }
574 #endif
575     default:
576       return false;
577   }
578 }
579 
580 NO_INLINE  // Avoid increasing caller's frame size by large stack-allocated objects.
AllocateRegisters(HGraph * graph,CodeGenerator * codegen,PassObserver * pass_observer,RegisterAllocator::Strategy strategy,OptimizingCompilerStats * stats)581 static void AllocateRegisters(HGraph* graph,
582                               CodeGenerator* codegen,
583                               PassObserver* pass_observer,
584                               RegisterAllocator::Strategy strategy,
585                               OptimizingCompilerStats* stats) {
586   {
587     PassScope scope(PrepareForRegisterAllocation::kPrepareForRegisterAllocationPassName,
588                     pass_observer);
589     PrepareForRegisterAllocation(graph, codegen->GetCompilerOptions(), stats).Run();
590   }
591   // Use local allocator shared by SSA liveness analysis and register allocator.
592   // (Register allocator creates new objects in the liveness data.)
593   ScopedArenaAllocator local_allocator(graph->GetArenaStack());
594   SsaLivenessAnalysis liveness(graph, codegen, &local_allocator);
595   {
596     PassScope scope(SsaLivenessAnalysis::kLivenessPassName, pass_observer);
597     liveness.Analyze();
598   }
599   {
600     PassScope scope(RegisterAllocator::kRegisterAllocatorPassName, pass_observer);
601     std::unique_ptr<RegisterAllocator> register_allocator =
602         RegisterAllocator::Create(&local_allocator, codegen, liveness, strategy);
603     register_allocator->AllocateRegisters();
604   }
605 }
606 
607 // Strip pass name suffix to get optimization name.
ConvertPassNameToOptimizationName(const std::string & pass_name)608 static std::string ConvertPassNameToOptimizationName(const std::string& pass_name) {
609   size_t pos = pass_name.find(kPassNameSeparator);
610   return pos == std::string::npos ? pass_name : pass_name.substr(0, pos);
611 }
612 
RunOptimizations(HGraph * graph,CodeGenerator * codegen,const DexCompilationUnit & dex_compilation_unit,PassObserver * pass_observer) const613 void OptimizingCompiler::RunOptimizations(HGraph* graph,
614                                           CodeGenerator* codegen,
615                                           const DexCompilationUnit& dex_compilation_unit,
616                                           PassObserver* pass_observer) const {
617   const std::vector<std::string>* pass_names = GetCompilerOptions().GetPassesToRun();
618   if (pass_names != nullptr) {
619     // If passes were defined on command-line, build the optimization
620     // passes and run these instead of the built-in optimizations.
621     // TODO: a way to define depends_on via command-line?
622     const size_t length = pass_names->size();
623     std::vector<OptimizationDef> optimizations;
624     for (const std::string& pass_name : *pass_names) {
625       std::string opt_name = ConvertPassNameToOptimizationName(pass_name);
626       optimizations.push_back(OptDef(OptimizationPassByName(opt_name), pass_name.c_str()));
627     }
628     RunOptimizations(graph,
629                      codegen,
630                      dex_compilation_unit,
631                      pass_observer,
632                      optimizations.data(),
633                      length);
634     return;
635   }
636 
637   OptimizationDef optimizations[] = {
638     // Initial optimizations.
639     OptDef(OptimizationPass::kConstantFolding),
640     OptDef(OptimizationPass::kInstructionSimplifier),
641     OptDef(OptimizationPass::kDeadCodeElimination,
642            "dead_code_elimination$initial"),
643     // Inlining.
644     OptDef(OptimizationPass::kInliner),
645     // Simplification (only if inlining occurred).
646     OptDef(OptimizationPass::kConstantFolding,
647            "constant_folding$after_inlining",
648            OptimizationPass::kInliner),
649     OptDef(OptimizationPass::kInstructionSimplifier,
650            "instruction_simplifier$after_inlining",
651            OptimizationPass::kInliner),
652     OptDef(OptimizationPass::kDeadCodeElimination,
653            "dead_code_elimination$after_inlining",
654            OptimizationPass::kInliner),
655     // GVN.
656     OptDef(OptimizationPass::kSideEffectsAnalysis,
657            "side_effects$before_gvn"),
658     OptDef(OptimizationPass::kGlobalValueNumbering),
659     // Simplification (TODO: only if GVN occurred).
660     OptDef(OptimizationPass::kSelectGenerator),
661     OptDef(OptimizationPass::kConstantFolding,
662            "constant_folding$after_gvn"),
663     OptDef(OptimizationPass::kInstructionSimplifier,
664            "instruction_simplifier$after_gvn"),
665     OptDef(OptimizationPass::kDeadCodeElimination,
666            "dead_code_elimination$after_gvn"),
667     // High-level optimizations.
668     OptDef(OptimizationPass::kSideEffectsAnalysis,
669            "side_effects$before_licm"),
670     OptDef(OptimizationPass::kInvariantCodeMotion),
671     OptDef(OptimizationPass::kInductionVarAnalysis),
672     OptDef(OptimizationPass::kBoundsCheckElimination),
673     OptDef(OptimizationPass::kLoopOptimization),
674     // Simplification.
675     OptDef(OptimizationPass::kConstantFolding,
676            "constant_folding$after_bce"),
677     OptDef(OptimizationPass::kAggressiveInstructionSimplifier,
678            "instruction_simplifier$after_bce"),
679     // Other high-level optimizations.
680     OptDef(OptimizationPass::kLoadStoreElimination),
681     OptDef(OptimizationPass::kCHAGuardOptimization),
682     OptDef(OptimizationPass::kDeadCodeElimination,
683            "dead_code_elimination$final"),
684     OptDef(OptimizationPass::kCodeSinking),
685     // The codegen has a few assumptions that only the instruction simplifier
686     // can satisfy. For example, the code generator does not expect to see a
687     // HTypeConversion from a type to the same type.
688     OptDef(OptimizationPass::kAggressiveInstructionSimplifier,
689            "instruction_simplifier$before_codegen"),
690     // Eliminate constructor fences after code sinking to avoid
691     // complicated sinking logic to split a fence with many inputs.
692     OptDef(OptimizationPass::kConstructorFenceRedundancyElimination)
693   };
694   RunOptimizations(graph,
695                    codegen,
696                    dex_compilation_unit,
697                    pass_observer,
698                    optimizations);
699 
700   RunArchOptimizations(graph, codegen, dex_compilation_unit, pass_observer);
701 }
702 
EmitAndSortLinkerPatches(CodeGenerator * codegen)703 static ArenaVector<linker::LinkerPatch> EmitAndSortLinkerPatches(CodeGenerator* codegen) {
704   ArenaVector<linker::LinkerPatch> linker_patches(codegen->GetGraph()->GetAllocator()->Adapter());
705   codegen->EmitLinkerPatches(&linker_patches);
706 
707   // Sort patches by literal offset. Required for .oat_patches encoding.
708   std::sort(linker_patches.begin(), linker_patches.end(),
709             [](const linker::LinkerPatch& lhs, const linker::LinkerPatch& rhs) {
710     return lhs.LiteralOffset() < rhs.LiteralOffset();
711   });
712 
713   return linker_patches;
714 }
715 
Emit(ArenaAllocator * allocator,CodeVectorAllocator * code_allocator,CodeGenerator * codegen,const dex::CodeItem * code_item_for_osr_check) const716 CompiledMethod* OptimizingCompiler::Emit(ArenaAllocator* allocator,
717                                          CodeVectorAllocator* code_allocator,
718                                          CodeGenerator* codegen,
719                                          const dex::CodeItem* code_item_for_osr_check) const {
720   ArenaVector<linker::LinkerPatch> linker_patches = EmitAndSortLinkerPatches(codegen);
721   ScopedArenaVector<uint8_t> stack_map = codegen->BuildStackMaps(code_item_for_osr_check);
722 
723   CompiledMethodStorage* storage = GetCompiledMethodStorage();
724   CompiledMethod* compiled_method = CompiledMethod::SwapAllocCompiledMethod(
725       storage,
726       codegen->GetInstructionSet(),
727       code_allocator->GetMemory(),
728       ArrayRef<const uint8_t>(stack_map),
729       ArrayRef<const uint8_t>(*codegen->GetAssembler()->cfi().data()),
730       ArrayRef<const linker::LinkerPatch>(linker_patches));
731 
732   for (const linker::LinkerPatch& patch : linker_patches) {
733     if (codegen->NeedsThunkCode(patch) && storage->GetThunkCode(patch).empty()) {
734       ArenaVector<uint8_t> code(allocator->Adapter());
735       std::string debug_name;
736       codegen->EmitThunkCode(patch, &code, &debug_name);
737       storage->SetThunkCode(patch, ArrayRef<const uint8_t>(code), debug_name);
738     }
739   }
740 
741   return compiled_method;
742 }
743 
TryCompile(ArenaAllocator * allocator,ArenaStack * arena_stack,CodeVectorAllocator * code_allocator,const DexCompilationUnit & dex_compilation_unit,ArtMethod * method,CompilationKind compilation_kind,VariableSizedHandleScope * handles) const744 CodeGenerator* OptimizingCompiler::TryCompile(ArenaAllocator* allocator,
745                                               ArenaStack* arena_stack,
746                                               CodeVectorAllocator* code_allocator,
747                                               const DexCompilationUnit& dex_compilation_unit,
748                                               ArtMethod* method,
749                                               CompilationKind compilation_kind,
750                                               VariableSizedHandleScope* handles) const {
751   MaybeRecordStat(compilation_stats_.get(), MethodCompilationStat::kAttemptBytecodeCompilation);
752   const CompilerOptions& compiler_options = GetCompilerOptions();
753   InstructionSet instruction_set = compiler_options.GetInstructionSet();
754   const DexFile& dex_file = *dex_compilation_unit.GetDexFile();
755   uint32_t method_idx = dex_compilation_unit.GetDexMethodIndex();
756   const dex::CodeItem* code_item = dex_compilation_unit.GetCodeItem();
757 
758   // Always use the Thumb-2 assembler: some runtime functionality
759   // (like implicit stack overflow checks) assume Thumb-2.
760   DCHECK_NE(instruction_set, InstructionSet::kArm);
761 
762   // Do not attempt to compile on architectures we do not support.
763   if (!IsInstructionSetSupported(instruction_set)) {
764     MaybeRecordStat(compilation_stats_.get(),
765                     MethodCompilationStat::kNotCompiledUnsupportedIsa);
766     return nullptr;
767   }
768 
769   if (Compiler::IsPathologicalCase(*code_item, method_idx, dex_file)) {
770     MaybeRecordStat(compilation_stats_.get(), MethodCompilationStat::kNotCompiledPathological);
771     return nullptr;
772   }
773 
774   // Implementation of the space filter: do not compile a code item whose size in
775   // code units is bigger than 128.
776   static constexpr size_t kSpaceFilterOptimizingThreshold = 128;
777   if ((compiler_options.GetCompilerFilter() == CompilerFilter::kSpace)
778       && (CodeItemInstructionAccessor(dex_file, code_item).InsnsSizeInCodeUnits() >
779           kSpaceFilterOptimizingThreshold)) {
780     MaybeRecordStat(compilation_stats_.get(), MethodCompilationStat::kNotCompiledSpaceFilter);
781     return nullptr;
782   }
783 
784   CodeItemDebugInfoAccessor code_item_accessor(dex_file, code_item, method_idx);
785 
786   bool dead_reference_safe;
787   // For AOT compilation, we may not get a method, for example if its class is erroneous,
788   // possibly due to an unavailable superclass.  JIT should always have a method.
789   DCHECK(Runtime::Current()->IsAotCompiler() || method != nullptr);
790   if (method != nullptr) {
791     const dex::ClassDef* containing_class;
792     {
793       ScopedObjectAccess soa(Thread::Current());
794       containing_class = &method->GetClassDef();
795     }
796     // MethodContainsRSensitiveAccess is currently slow, but HasDeadReferenceSafeAnnotation()
797     // is currently rarely true.
798     dead_reference_safe =
799         annotations::HasDeadReferenceSafeAnnotation(dex_file, *containing_class)
800         && !annotations::MethodContainsRSensitiveAccess(dex_file, *containing_class, method_idx);
801   } else {
802     // If we could not resolve the class, conservatively assume it's dead-reference unsafe.
803     dead_reference_safe = false;
804   }
805 
806   HGraph* graph = new (allocator) HGraph(
807       allocator,
808       arena_stack,
809       handles,
810       dex_file,
811       method_idx,
812       compiler_options.GetInstructionSet(),
813       kInvalidInvokeType,
814       dead_reference_safe,
815       compiler_options.GetDebuggable(),
816       compilation_kind);
817 
818   if (method != nullptr) {
819     graph->SetArtMethod(method);
820   }
821 
822   std::unique_ptr<CodeGenerator> codegen(
823       CodeGenerator::Create(graph,
824                             compiler_options,
825                             compilation_stats_.get()));
826   if (codegen.get() == nullptr) {
827     MaybeRecordStat(compilation_stats_.get(), MethodCompilationStat::kNotCompiledNoCodegen);
828     return nullptr;
829   }
830   codegen->GetAssembler()->cfi().SetEnabled(compiler_options.GenerateAnyDebugInfo());
831 
832   PassObserver pass_observer(graph,
833                              codegen.get(),
834                              visualizer_output_.get(),
835                              compiler_options,
836                              dump_mutex_);
837 
838   {
839     VLOG(compiler) << "Building " << pass_observer.GetMethodName();
840     PassScope scope(HGraphBuilder::kBuilderPassName, &pass_observer);
841     HGraphBuilder builder(graph,
842                           code_item_accessor,
843                           &dex_compilation_unit,
844                           &dex_compilation_unit,
845                           codegen.get(),
846                           compilation_stats_.get());
847     GraphAnalysisResult result = builder.BuildGraph();
848     if (result != kAnalysisSuccess) {
849       switch (result) {
850         case kAnalysisSkipped: {
851           MaybeRecordStat(compilation_stats_.get(),
852                           MethodCompilationStat::kNotCompiledSkipped);
853           break;
854         }
855         case kAnalysisInvalidBytecode: {
856           MaybeRecordStat(compilation_stats_.get(),
857                           MethodCompilationStat::kNotCompiledInvalidBytecode);
858           break;
859         }
860         case kAnalysisFailThrowCatchLoop: {
861           MaybeRecordStat(compilation_stats_.get(),
862                           MethodCompilationStat::kNotCompiledThrowCatchLoop);
863           break;
864         }
865         case kAnalysisFailAmbiguousArrayOp: {
866           MaybeRecordStat(compilation_stats_.get(),
867                           MethodCompilationStat::kNotCompiledAmbiguousArrayOp);
868           break;
869         }
870         case kAnalysisFailIrreducibleLoopAndStringInit: {
871           MaybeRecordStat(compilation_stats_.get(),
872                           MethodCompilationStat::kNotCompiledIrreducibleLoopAndStringInit);
873           break;
874         }
875         case kAnalysisFailPhiEquivalentInOsr: {
876           MaybeRecordStat(compilation_stats_.get(),
877                           MethodCompilationStat::kNotCompiledPhiEquivalentInOsr);
878           break;
879         }
880         case kAnalysisSuccess:
881           UNREACHABLE();
882       }
883       pass_observer.SetGraphInBadState();
884       return nullptr;
885     }
886   }
887 
888   if (compilation_kind == CompilationKind::kBaseline) {
889     RunBaselineOptimizations(graph, codegen.get(), dex_compilation_unit, &pass_observer);
890   } else {
891     RunOptimizations(graph, codegen.get(), dex_compilation_unit, &pass_observer);
892   }
893 
894   RegisterAllocator::Strategy regalloc_strategy =
895     compiler_options.GetRegisterAllocationStrategy();
896   AllocateRegisters(graph,
897                     codegen.get(),
898                     &pass_observer,
899                     regalloc_strategy,
900                     compilation_stats_.get());
901 
902   codegen->Compile(code_allocator);
903   pass_observer.DumpDisassembly();
904 
905   MaybeRecordStat(compilation_stats_.get(), MethodCompilationStat::kCompiledBytecode);
906   return codegen.release();
907 }
908 
TryCompileIntrinsic(ArenaAllocator * allocator,ArenaStack * arena_stack,CodeVectorAllocator * code_allocator,const DexCompilationUnit & dex_compilation_unit,ArtMethod * method,VariableSizedHandleScope * handles) const909 CodeGenerator* OptimizingCompiler::TryCompileIntrinsic(
910     ArenaAllocator* allocator,
911     ArenaStack* arena_stack,
912     CodeVectorAllocator* code_allocator,
913     const DexCompilationUnit& dex_compilation_unit,
914     ArtMethod* method,
915     VariableSizedHandleScope* handles) const {
916   MaybeRecordStat(compilation_stats_.get(), MethodCompilationStat::kAttemptIntrinsicCompilation);
917   const CompilerOptions& compiler_options = GetCompilerOptions();
918   InstructionSet instruction_set = compiler_options.GetInstructionSet();
919   const DexFile& dex_file = *dex_compilation_unit.GetDexFile();
920   uint32_t method_idx = dex_compilation_unit.GetDexMethodIndex();
921 
922   // Always use the Thumb-2 assembler: some runtime functionality
923   // (like implicit stack overflow checks) assume Thumb-2.
924   DCHECK_NE(instruction_set, InstructionSet::kArm);
925 
926   // Do not attempt to compile on architectures we do not support.
927   if (!IsInstructionSetSupported(instruction_set)) {
928     return nullptr;
929   }
930 
931   HGraph* graph = new (allocator) HGraph(
932       allocator,
933       arena_stack,
934       handles,
935       dex_file,
936       method_idx,
937       compiler_options.GetInstructionSet(),
938       kInvalidInvokeType,
939       /* dead_reference_safe= */ true,  // Intrinsics don't affect dead reference safety.
940       compiler_options.GetDebuggable(),
941       CompilationKind::kOptimized);
942 
943   DCHECK(Runtime::Current()->IsAotCompiler());
944   DCHECK(method != nullptr);
945   graph->SetArtMethod(method);
946 
947   std::unique_ptr<CodeGenerator> codegen(
948       CodeGenerator::Create(graph,
949                             compiler_options,
950                             compilation_stats_.get()));
951   if (codegen.get() == nullptr) {
952     return nullptr;
953   }
954   codegen->GetAssembler()->cfi().SetEnabled(compiler_options.GenerateAnyDebugInfo());
955 
956   PassObserver pass_observer(graph,
957                              codegen.get(),
958                              visualizer_output_.get(),
959                              compiler_options,
960                              dump_mutex_);
961 
962   {
963     VLOG(compiler) << "Building intrinsic graph " << pass_observer.GetMethodName();
964     PassScope scope(HGraphBuilder::kBuilderPassName, &pass_observer);
965     HGraphBuilder builder(graph,
966                           CodeItemDebugInfoAccessor(),  // Null code item.
967                           &dex_compilation_unit,
968                           &dex_compilation_unit,
969                           codegen.get(),
970                           compilation_stats_.get());
971     builder.BuildIntrinsicGraph(method);
972   }
973 
974   OptimizationDef optimizations[] = {
975     // The codegen has a few assumptions that only the instruction simplifier
976     // can satisfy.
977     OptDef(OptimizationPass::kInstructionSimplifier),
978   };
979   RunOptimizations(graph,
980                    codegen.get(),
981                    dex_compilation_unit,
982                    &pass_observer,
983                    optimizations);
984 
985   RunArchOptimizations(graph, codegen.get(), dex_compilation_unit, &pass_observer);
986 
987   AllocateRegisters(graph,
988                     codegen.get(),
989                     &pass_observer,
990                     compiler_options.GetRegisterAllocationStrategy(),
991                     compilation_stats_.get());
992   if (!codegen->IsLeafMethod()) {
993     VLOG(compiler) << "Intrinsic method is not leaf: " << method->GetIntrinsic()
994         << " " << graph->PrettyMethod();
995     return nullptr;
996   }
997 
998   codegen->Compile(code_allocator);
999   pass_observer.DumpDisassembly();
1000 
1001   VLOG(compiler) << "Compiled intrinsic: " << method->GetIntrinsic()
1002       << " " << graph->PrettyMethod();
1003   MaybeRecordStat(compilation_stats_.get(), MethodCompilationStat::kCompiledIntrinsic);
1004   return codegen.release();
1005 }
1006 
Compile(const dex::CodeItem * code_item,uint32_t access_flags,InvokeType invoke_type,uint16_t class_def_idx,uint32_t method_idx,Handle<mirror::ClassLoader> jclass_loader,const DexFile & dex_file,Handle<mirror::DexCache> dex_cache) const1007 CompiledMethod* OptimizingCompiler::Compile(const dex::CodeItem* code_item,
1008                                             uint32_t access_flags,
1009                                             InvokeType invoke_type,
1010                                             uint16_t class_def_idx,
1011                                             uint32_t method_idx,
1012                                             Handle<mirror::ClassLoader> jclass_loader,
1013                                             const DexFile& dex_file,
1014                                             Handle<mirror::DexCache> dex_cache) const {
1015   const CompilerOptions& compiler_options = GetCompilerOptions();
1016   DCHECK(compiler_options.IsAotCompiler());
1017   CompiledMethod* compiled_method = nullptr;
1018   Runtime* runtime = Runtime::Current();
1019   DCHECK(runtime->IsAotCompiler());
1020   const VerifiedMethod* verified_method = compiler_options.GetVerifiedMethod(&dex_file, method_idx);
1021   DCHECK(!verified_method->HasRuntimeThrow());
1022   if (compiler_options.IsMethodVerifiedWithoutFailures(method_idx, class_def_idx, dex_file) ||
1023       verifier::CanCompilerHandleVerificationFailure(
1024           verified_method->GetEncounteredVerificationFailures())) {
1025     ArenaAllocator allocator(runtime->GetArenaPool());
1026     ArenaStack arena_stack(runtime->GetArenaPool());
1027     CodeVectorAllocator code_allocator(&allocator);
1028     std::unique_ptr<CodeGenerator> codegen;
1029     bool compiled_intrinsic = false;
1030     {
1031       ScopedObjectAccess soa(Thread::Current());
1032       ArtMethod* method =
1033           runtime->GetClassLinker()->ResolveMethod<ClassLinker::ResolveMode::kCheckICCEAndIAE>(
1034               method_idx, dex_cache, jclass_loader, /*referrer=*/ nullptr, invoke_type);
1035       DCHECK_EQ(method == nullptr, soa.Self()->IsExceptionPending());
1036       soa.Self()->ClearException();  // Suppress exception if any.
1037       VariableSizedHandleScope handles(soa.Self());
1038       Handle<mirror::Class> compiling_class =
1039           handles.NewHandle(method != nullptr ? method->GetDeclaringClass() : nullptr);
1040       DexCompilationUnit dex_compilation_unit(
1041           jclass_loader,
1042           runtime->GetClassLinker(),
1043           dex_file,
1044           code_item,
1045           class_def_idx,
1046           method_idx,
1047           access_flags,
1048           /*verified_method=*/ nullptr,  // Not needed by the Optimizing compiler.
1049           dex_cache,
1050           compiling_class);
1051       // All signature polymorphic methods are native.
1052       DCHECK(method == nullptr || !method->IsSignaturePolymorphic());
1053       // Go to native so that we don't block GC during compilation.
1054       ScopedThreadSuspension sts(soa.Self(), kNative);
1055       // Try to compile a fully intrinsified implementation.
1056       if (method != nullptr && UNLIKELY(method->IsIntrinsic())) {
1057         DCHECK(compiler_options.IsBootImage());
1058         codegen.reset(
1059             TryCompileIntrinsic(&allocator,
1060                                 &arena_stack,
1061                                 &code_allocator,
1062                                 dex_compilation_unit,
1063                                 method,
1064                                 &handles));
1065         if (codegen != nullptr) {
1066           compiled_intrinsic = true;
1067         }
1068       }
1069       if (codegen == nullptr) {
1070         codegen.reset(
1071             TryCompile(&allocator,
1072                        &arena_stack,
1073                        &code_allocator,
1074                        dex_compilation_unit,
1075                        method,
1076                        compiler_options.IsBaseline()
1077                           ? CompilationKind::kBaseline
1078                           : CompilationKind::kOptimized,
1079                        &handles));
1080       }
1081     }
1082     if (codegen.get() != nullptr) {
1083       compiled_method = Emit(&allocator,
1084                              &code_allocator,
1085                              codegen.get(),
1086                              compiled_intrinsic ? nullptr : code_item);
1087       if (compiled_intrinsic) {
1088         compiled_method->MarkAsIntrinsic();
1089       }
1090 
1091       if (kArenaAllocatorCountAllocations) {
1092         codegen.reset();  // Release codegen's ScopedArenaAllocator for memory accounting.
1093         size_t total_allocated = allocator.BytesAllocated() + arena_stack.PeakBytesAllocated();
1094         if (total_allocated > kArenaAllocatorMemoryReportThreshold) {
1095           MemStats mem_stats(allocator.GetMemStats());
1096           MemStats peak_stats(arena_stack.GetPeakStats());
1097           LOG(INFO) << "Used " << total_allocated << " bytes of arena memory for compiling "
1098                     << dex_file.PrettyMethod(method_idx)
1099                     << "\n" << Dumpable<MemStats>(mem_stats)
1100                     << "\n" << Dumpable<MemStats>(peak_stats);
1101         }
1102       }
1103     }
1104   } else {
1105     MethodCompilationStat method_stat;
1106     if (compiler_options.VerifyAtRuntime()) {
1107       method_stat = MethodCompilationStat::kNotCompiledVerifyAtRuntime;
1108     } else {
1109       method_stat = MethodCompilationStat::kNotCompiledVerificationError;
1110     }
1111     MaybeRecordStat(compilation_stats_.get(), method_stat);
1112   }
1113 
1114   if (kIsDebugBuild &&
1115       compiler_options.CompileArtTest() &&
1116       IsInstructionSetSupported(compiler_options.GetInstructionSet())) {
1117     // For testing purposes, we put a special marker on method names
1118     // that should be compiled with this compiler (when the
1119     // instruction set is supported). This makes sure we're not
1120     // regressing.
1121     std::string method_name = dex_file.PrettyMethod(method_idx);
1122     bool shouldCompile = method_name.find("$opt$") != std::string::npos;
1123     DCHECK((compiled_method != nullptr) || !shouldCompile) << "Didn't compile " << method_name;
1124   }
1125 
1126   return compiled_method;
1127 }
1128 
CreateJniStackMap(ScopedArenaAllocator * allocator,const JniCompiledMethod & jni_compiled_method,size_t code_size)1129 static ScopedArenaVector<uint8_t> CreateJniStackMap(ScopedArenaAllocator* allocator,
1130                                                     const JniCompiledMethod& jni_compiled_method,
1131                                                     size_t code_size) {
1132   // StackMapStream is quite large, so allocate it using the ScopedArenaAllocator
1133   // to stay clear of the frame size limit.
1134   std::unique_ptr<StackMapStream> stack_map_stream(
1135       new (allocator) StackMapStream(allocator, jni_compiled_method.GetInstructionSet()));
1136   stack_map_stream->BeginMethod(
1137       jni_compiled_method.GetFrameSize(),
1138       jni_compiled_method.GetCoreSpillMask(),
1139       jni_compiled_method.GetFpSpillMask(),
1140       /* num_dex_registers= */ 0,
1141       /* baseline= */ false);
1142   stack_map_stream->EndMethod(code_size);
1143   return stack_map_stream->Encode();
1144 }
1145 
JniCompile(uint32_t access_flags,uint32_t method_idx,const DexFile & dex_file,Handle<mirror::DexCache> dex_cache) const1146 CompiledMethod* OptimizingCompiler::JniCompile(uint32_t access_flags,
1147                                                uint32_t method_idx,
1148                                                const DexFile& dex_file,
1149                                                Handle<mirror::DexCache> dex_cache) const {
1150   Runtime* runtime = Runtime::Current();
1151   ArenaAllocator allocator(runtime->GetArenaPool());
1152   ArenaStack arena_stack(runtime->GetArenaPool());
1153 
1154   const CompilerOptions& compiler_options = GetCompilerOptions();
1155   if (compiler_options.IsBootImage()) {
1156     ScopedObjectAccess soa(Thread::Current());
1157     ArtMethod* method = runtime->GetClassLinker()->LookupResolvedMethod(
1158         method_idx, dex_cache.Get(), /*class_loader=*/ nullptr);
1159     // Try to compile a fully intrinsified implementation. Do not try to do this for
1160     // signature polymorphic methods as the InstructionBuilder cannot handle them;
1161     // and it would be useless as they always have a slow path for type conversions.
1162     if (method != nullptr && UNLIKELY(method->IsIntrinsic()) && !method->IsSignaturePolymorphic()) {
1163       VariableSizedHandleScope handles(soa.Self());
1164       ScopedNullHandle<mirror::ClassLoader> class_loader;  // null means boot class path loader.
1165       Handle<mirror::Class> compiling_class = handles.NewHandle(method->GetDeclaringClass());
1166       DexCompilationUnit dex_compilation_unit(
1167           class_loader,
1168           runtime->GetClassLinker(),
1169           dex_file,
1170           /*code_item=*/ nullptr,
1171           /*class_def_idx=*/ DexFile::kDexNoIndex16,
1172           method_idx,
1173           access_flags,
1174           /*verified_method=*/ nullptr,
1175           dex_cache,
1176           compiling_class);
1177       CodeVectorAllocator code_allocator(&allocator);
1178       // Go to native so that we don't block GC during compilation.
1179       ScopedThreadSuspension sts(soa.Self(), kNative);
1180       std::unique_ptr<CodeGenerator> codegen(
1181           TryCompileIntrinsic(&allocator,
1182                               &arena_stack,
1183                               &code_allocator,
1184                               dex_compilation_unit,
1185                               method,
1186                               &handles));
1187       if (codegen != nullptr) {
1188         CompiledMethod* compiled_method = Emit(&allocator,
1189                                                &code_allocator,
1190                                                codegen.get(),
1191                                                /* item= */ nullptr);
1192         compiled_method->MarkAsIntrinsic();
1193         return compiled_method;
1194       }
1195     }
1196   }
1197 
1198   JniCompiledMethod jni_compiled_method = ArtQuickJniCompileMethod(
1199       compiler_options, access_flags, method_idx, dex_file);
1200   MaybeRecordStat(compilation_stats_.get(), MethodCompilationStat::kCompiledNativeStub);
1201 
1202   ScopedArenaAllocator stack_map_allocator(&arena_stack);  // Will hold the stack map.
1203   ScopedArenaVector<uint8_t> stack_map = CreateJniStackMap(
1204       &stack_map_allocator, jni_compiled_method, jni_compiled_method.GetCode().size());
1205   return CompiledMethod::SwapAllocCompiledMethod(
1206       GetCompiledMethodStorage(),
1207       jni_compiled_method.GetInstructionSet(),
1208       jni_compiled_method.GetCode(),
1209       ArrayRef<const uint8_t>(stack_map),
1210       jni_compiled_method.GetCfi(),
1211       /* patches= */ ArrayRef<const linker::LinkerPatch>());
1212 }
1213 
CreateOptimizingCompiler(const CompilerOptions & compiler_options,CompiledMethodStorage * storage)1214 Compiler* CreateOptimizingCompiler(const CompilerOptions& compiler_options,
1215                                    CompiledMethodStorage* storage) {
1216   return new OptimizingCompiler(compiler_options, storage);
1217 }
1218 
EncodeArtMethodInInlineInfo(ArtMethod * method ATTRIBUTE_UNUSED)1219 bool EncodeArtMethodInInlineInfo(ArtMethod* method ATTRIBUTE_UNUSED) {
1220   // Note: the runtime is null only for unit testing.
1221   return Runtime::Current() == nullptr || !Runtime::Current()->IsAotCompiler();
1222 }
1223 
JitCompile(Thread * self,jit::JitCodeCache * code_cache,jit::JitMemoryRegion * region,ArtMethod * method,CompilationKind compilation_kind,jit::JitLogger * jit_logger)1224 bool OptimizingCompiler::JitCompile(Thread* self,
1225                                     jit::JitCodeCache* code_cache,
1226                                     jit::JitMemoryRegion* region,
1227                                     ArtMethod* method,
1228                                     CompilationKind compilation_kind,
1229                                     jit::JitLogger* jit_logger) {
1230   const CompilerOptions& compiler_options = GetCompilerOptions();
1231   // If the baseline flag was explicitly passed, change the compilation kind
1232   // from optimized to baseline.
1233   if (compiler_options.IsBaseline() && compilation_kind == CompilationKind::kOptimized) {
1234     compilation_kind = CompilationKind::kBaseline;
1235   }
1236   DCHECK(compiler_options.IsJitCompiler());
1237   DCHECK_EQ(compiler_options.IsJitCompilerForSharedCode(), code_cache->IsSharedRegion(*region));
1238   StackHandleScope<3> hs(self);
1239   Handle<mirror::ClassLoader> class_loader(hs.NewHandle(
1240       method->GetDeclaringClass()->GetClassLoader()));
1241   Handle<mirror::DexCache> dex_cache(hs.NewHandle(method->GetDexCache()));
1242   DCHECK(method->IsCompilable());
1243 
1244   const DexFile* dex_file = method->GetDexFile();
1245   const uint16_t class_def_idx = method->GetClassDefIndex();
1246   const dex::CodeItem* code_item = method->GetCodeItem();
1247   const uint32_t method_idx = method->GetDexMethodIndex();
1248   const uint32_t access_flags = method->GetAccessFlags();
1249 
1250   Runtime* runtime = Runtime::Current();
1251   ArenaAllocator allocator(runtime->GetJitArenaPool());
1252 
1253   if (UNLIKELY(method->IsNative())) {
1254     JniCompiledMethod jni_compiled_method = ArtQuickJniCompileMethod(
1255         compiler_options, access_flags, method_idx, *dex_file);
1256     std::vector<Handle<mirror::Object>> roots;
1257     ArenaSet<ArtMethod*, std::less<ArtMethod*>> cha_single_implementation_list(
1258         allocator.Adapter(kArenaAllocCHA));
1259     ArenaStack arena_stack(runtime->GetJitArenaPool());
1260     // StackMapStream is large and it does not fit into this frame, so we need helper method.
1261     ScopedArenaAllocator stack_map_allocator(&arena_stack);  // Will hold the stack map.
1262     ScopedArenaVector<uint8_t> stack_map = CreateJniStackMap(
1263         &stack_map_allocator, jni_compiled_method, jni_compiled_method.GetCode().size());
1264 
1265     ArrayRef<const uint8_t> reserved_code;
1266     ArrayRef<const uint8_t> reserved_data;
1267     if (!code_cache->Reserve(self,
1268                              region,
1269                              jni_compiled_method.GetCode().size(),
1270                              stack_map.size(),
1271                              /* number_of_roots= */ 0,
1272                              method,
1273                              /*out*/ &reserved_code,
1274                              /*out*/ &reserved_data)) {
1275       MaybeRecordStat(compilation_stats_.get(), MethodCompilationStat::kJitOutOfMemoryForCommit);
1276       return false;
1277     }
1278     const uint8_t* code = reserved_code.data() + OatQuickMethodHeader::InstructionAlignedSize();
1279 
1280     // Add debug info after we know the code location but before we update entry-point.
1281     std::vector<uint8_t> debug_info;
1282     if (compiler_options.GenerateAnyDebugInfo()) {
1283       debug::MethodDebugInfo info = {};
1284       // Simpleperf relies on art_jni_trampoline to detect jni methods.
1285       info.custom_name = "art_jni_trampoline";
1286       info.dex_file = dex_file;
1287       info.class_def_index = class_def_idx;
1288       info.dex_method_index = method_idx;
1289       info.access_flags = access_flags;
1290       info.code_item = code_item;
1291       info.isa = jni_compiled_method.GetInstructionSet();
1292       info.deduped = false;
1293       info.is_native_debuggable = compiler_options.GetNativeDebuggable();
1294       info.is_optimized = true;
1295       info.is_code_address_text_relative = false;
1296       info.code_address = reinterpret_cast<uintptr_t>(code);
1297       info.code_size = jni_compiled_method.GetCode().size();
1298       info.frame_size_in_bytes = jni_compiled_method.GetFrameSize();
1299       info.code_info = nullptr;
1300       info.cfi = jni_compiled_method.GetCfi();
1301       debug_info = GenerateJitDebugInfo(info);
1302     }
1303 
1304     if (!code_cache->Commit(self,
1305                             region,
1306                             method,
1307                             reserved_code,
1308                             jni_compiled_method.GetCode(),
1309                             reserved_data,
1310                             roots,
1311                             ArrayRef<const uint8_t>(stack_map),
1312                             debug_info,
1313                             /* is_full_debug_info= */ compiler_options.GetGenerateDebugInfo(),
1314                             compilation_kind,
1315                             /* has_should_deoptimize_flag= */ false,
1316                             cha_single_implementation_list)) {
1317       code_cache->Free(self, region, reserved_code.data(), reserved_data.data());
1318       return false;
1319     }
1320 
1321     Runtime::Current()->GetJit()->AddMemoryUsage(method, allocator.BytesUsed());
1322     if (jit_logger != nullptr) {
1323       jit_logger->WriteLog(code, jni_compiled_method.GetCode().size(), method);
1324     }
1325     return true;
1326   }
1327 
1328   ArenaStack arena_stack(runtime->GetJitArenaPool());
1329   CodeVectorAllocator code_allocator(&allocator);
1330   VariableSizedHandleScope handles(self);
1331 
1332   std::unique_ptr<CodeGenerator> codegen;
1333   {
1334     Handle<mirror::Class> compiling_class = handles.NewHandle(method->GetDeclaringClass());
1335     DexCompilationUnit dex_compilation_unit(
1336         class_loader,
1337         runtime->GetClassLinker(),
1338         *dex_file,
1339         code_item,
1340         class_def_idx,
1341         method_idx,
1342         access_flags,
1343         /*verified_method=*/ nullptr,
1344         dex_cache,
1345         compiling_class);
1346 
1347     // Go to native so that we don't block GC during compilation.
1348     ScopedThreadSuspension sts(self, kNative);
1349     codegen.reset(
1350         TryCompile(&allocator,
1351                    &arena_stack,
1352                    &code_allocator,
1353                    dex_compilation_unit,
1354                    method,
1355                    compilation_kind,
1356                    &handles));
1357     if (codegen.get() == nullptr) {
1358       return false;
1359     }
1360   }
1361 
1362   ScopedArenaVector<uint8_t> stack_map = codegen->BuildStackMaps(code_item);
1363 
1364   ArrayRef<const uint8_t> reserved_code;
1365   ArrayRef<const uint8_t> reserved_data;
1366   if (!code_cache->Reserve(self,
1367                            region,
1368                            code_allocator.GetMemory().size(),
1369                            stack_map.size(),
1370                            /*number_of_roots=*/codegen->GetNumberOfJitRoots(),
1371                            method,
1372                            /*out*/ &reserved_code,
1373                            /*out*/ &reserved_data)) {
1374     MaybeRecordStat(compilation_stats_.get(), MethodCompilationStat::kJitOutOfMemoryForCommit);
1375     return false;
1376   }
1377   const uint8_t* code = reserved_code.data() + OatQuickMethodHeader::InstructionAlignedSize();
1378   const uint8_t* roots_data = reserved_data.data();
1379 
1380   std::vector<Handle<mirror::Object>> roots;
1381   codegen->EmitJitRoots(code_allocator.GetData(), roots_data, &roots);
1382   // The root Handle<>s filled by the codegen reference entries in the VariableSizedHandleScope.
1383   DCHECK(std::all_of(roots.begin(),
1384                      roots.end(),
1385                      [&handles](Handle<mirror::Object> root){
1386                        return handles.Contains(root.GetReference());
1387                      }));
1388 
1389   // Add debug info after we know the code location but before we update entry-point.
1390   std::vector<uint8_t> debug_info;
1391   if (compiler_options.GenerateAnyDebugInfo()) {
1392     debug::MethodDebugInfo info = {};
1393     DCHECK(info.custom_name.empty());
1394     info.dex_file = dex_file;
1395     info.class_def_index = class_def_idx;
1396     info.dex_method_index = method_idx;
1397     info.access_flags = access_flags;
1398     info.code_item = code_item;
1399     info.isa = codegen->GetInstructionSet();
1400     info.deduped = false;
1401     info.is_native_debuggable = compiler_options.GetNativeDebuggable();
1402     info.is_optimized = true;
1403     info.is_code_address_text_relative = false;
1404     info.code_address = reinterpret_cast<uintptr_t>(code);
1405     info.code_size = code_allocator.GetMemory().size();
1406     info.frame_size_in_bytes = codegen->GetFrameSize();
1407     info.code_info = stack_map.size() == 0 ? nullptr : stack_map.data();
1408     info.cfi = ArrayRef<const uint8_t>(*codegen->GetAssembler()->cfi().data());
1409     debug_info = GenerateJitDebugInfo(info);
1410   }
1411 
1412   if (!code_cache->Commit(self,
1413                           region,
1414                           method,
1415                           reserved_code,
1416                           code_allocator.GetMemory(),
1417                           reserved_data,
1418                           roots,
1419                           ArrayRef<const uint8_t>(stack_map),
1420                           debug_info,
1421                           /* is_full_debug_info= */ compiler_options.GetGenerateDebugInfo(),
1422                           compilation_kind,
1423                           codegen->GetGraph()->HasShouldDeoptimizeFlag(),
1424                           codegen->GetGraph()->GetCHASingleImplementationList())) {
1425     code_cache->Free(self, region, reserved_code.data(), reserved_data.data());
1426     return false;
1427   }
1428 
1429   Runtime::Current()->GetJit()->AddMemoryUsage(method, allocator.BytesUsed());
1430   if (jit_logger != nullptr) {
1431     jit_logger->WriteLog(code, code_allocator.GetMemory().size(), method);
1432   }
1433 
1434   if (kArenaAllocatorCountAllocations) {
1435     codegen.reset();  // Release codegen's ScopedArenaAllocator for memory accounting.
1436     size_t total_allocated = allocator.BytesAllocated() + arena_stack.PeakBytesAllocated();
1437     if (total_allocated > kArenaAllocatorMemoryReportThreshold) {
1438       MemStats mem_stats(allocator.GetMemStats());
1439       MemStats peak_stats(arena_stack.GetPeakStats());
1440       LOG(INFO) << "Used " << total_allocated << " bytes of arena memory for compiling "
1441                 << dex_file->PrettyMethod(method_idx)
1442                 << "\n" << Dumpable<MemStats>(mem_stats)
1443                 << "\n" << Dumpable<MemStats>(peak_stats);
1444     }
1445   }
1446 
1447   return true;
1448 }
1449 
GenerateJitDebugInfo(const debug::MethodDebugInfo & info)1450 std::vector<uint8_t> OptimizingCompiler::GenerateJitDebugInfo(const debug::MethodDebugInfo& info) {
1451   const CompilerOptions& compiler_options = GetCompilerOptions();
1452   if (compiler_options.GenerateAnyDebugInfo()) {
1453     // If both flags are passed, generate full debug info.
1454     const bool mini_debug_info = !compiler_options.GetGenerateDebugInfo();
1455 
1456     // Create entry for the single method that we just compiled.
1457     InstructionSet isa = compiler_options.GetInstructionSet();
1458     const InstructionSetFeatures* features = compiler_options.GetInstructionSetFeatures();
1459     return debug::MakeElfFileForJIT(isa, features, mini_debug_info, info);
1460   }
1461   return std::vector<uint8_t>();
1462 }
1463 
1464 }  // namespace art
1465