1 /*
2  * Copyright (C) 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "inliner.h"
18 
19 #include "art_method-inl.h"
20 #include "base/enums.h"
21 #include "builder.h"
22 #include "class_linker.h"
23 #include "constant_folding.h"
24 #include "data_type-inl.h"
25 #include "dead_code_elimination.h"
26 #include "dex/inline_method_analyser.h"
27 #include "dex/verification_results.h"
28 #include "dex/verified_method.h"
29 #include "driver/compiler_driver-inl.h"
30 #include "driver/compiler_options.h"
31 #include "driver/dex_compilation_unit.h"
32 #include "instruction_simplifier.h"
33 #include "intrinsics.h"
34 #include "jit/jit.h"
35 #include "jit/jit_code_cache.h"
36 #include "mirror/class_loader.h"
37 #include "mirror/dex_cache.h"
38 #include "nodes.h"
39 #include "optimizing_compiler.h"
40 #include "reference_type_propagation.h"
41 #include "register_allocator_linear_scan.h"
42 #include "scoped_thread_state_change-inl.h"
43 #include "sharpening.h"
44 #include "ssa_builder.h"
45 #include "ssa_phi_elimination.h"
46 #include "thread.h"
47 
48 namespace art {
49 
50 // Instruction limit to control memory.
51 static constexpr size_t kMaximumNumberOfTotalInstructions = 1024;
52 
53 // Maximum number of instructions for considering a method small,
54 // which we will always try to inline if the other non-instruction limits
55 // are not reached.
56 static constexpr size_t kMaximumNumberOfInstructionsForSmallMethod = 3;
57 
58 // Limit the number of dex registers that we accumulate while inlining
59 // to avoid creating large amount of nested environments.
60 static constexpr size_t kMaximumNumberOfCumulatedDexRegisters = 32;
61 
62 // Limit recursive call inlining, which do not benefit from too
63 // much inlining compared to code locality.
64 static constexpr size_t kMaximumNumberOfRecursiveCalls = 4;
65 
66 // Controls the use of inline caches in AOT mode.
67 static constexpr bool kUseAOTInlineCaches = true;
68 
69 // We check for line numbers to make sure the DepthString implementation
70 // aligns the output nicely.
71 #define LOG_INTERNAL(msg) \
72   static_assert(__LINE__ > 10, "Unhandled line number"); \
73   static_assert(__LINE__ < 10000, "Unhandled line number"); \
74   VLOG(compiler) << DepthString(__LINE__) << msg
75 
76 #define LOG_TRY() LOG_INTERNAL("Try inlinining call: ")
77 #define LOG_NOTE() LOG_INTERNAL("Note: ")
78 #define LOG_SUCCESS() LOG_INTERNAL("Success: ")
79 #define LOG_FAIL(stats_ptr, stat) MaybeRecordStat(stats_ptr, stat); LOG_INTERNAL("Fail: ")
80 #define LOG_FAIL_NO_STAT() LOG_INTERNAL("Fail: ")
81 
DepthString(int line) const82 std::string HInliner::DepthString(int line) const {
83   std::string value;
84   // Indent according to the inlining depth.
85   size_t count = depth_;
86   // Line numbers get printed in the log, so add a space if the log's line is less
87   // than 1000, and two if less than 100. 10 cannot be reached as it's the copyright.
88   if (!kIsTargetBuild) {
89     if (line < 100) {
90       value += " ";
91     }
92     if (line < 1000) {
93       value += " ";
94     }
95     // Safeguard if this file reaches more than 10000 lines.
96     DCHECK_LT(line, 10000);
97   }
98   for (size_t i = 0; i < count; ++i) {
99     value += "  ";
100   }
101   return value;
102 }
103 
CountNumberOfInstructions(HGraph * graph)104 static size_t CountNumberOfInstructions(HGraph* graph) {
105   size_t number_of_instructions = 0;
106   for (HBasicBlock* block : graph->GetReversePostOrderSkipEntryBlock()) {
107     for (HInstructionIterator instr_it(block->GetInstructions());
108          !instr_it.Done();
109          instr_it.Advance()) {
110       ++number_of_instructions;
111     }
112   }
113   return number_of_instructions;
114 }
115 
UpdateInliningBudget()116 void HInliner::UpdateInliningBudget() {
117   if (total_number_of_instructions_ >= kMaximumNumberOfTotalInstructions) {
118     // Always try to inline small methods.
119     inlining_budget_ = kMaximumNumberOfInstructionsForSmallMethod;
120   } else {
121     inlining_budget_ = std::max(
122         kMaximumNumberOfInstructionsForSmallMethod,
123         kMaximumNumberOfTotalInstructions - total_number_of_instructions_);
124   }
125 }
126 
Run()127 void HInliner::Run() {
128   if (graph_->IsDebuggable()) {
129     // For simplicity, we currently never inline when the graph is debuggable. This avoids
130     // doing some logic in the runtime to discover if a method could have been inlined.
131     return;
132   }
133 
134   // Initialize the number of instructions for the method being compiled. Recursive calls
135   // to HInliner::Run have already updated the instruction count.
136   if (outermost_graph_ == graph_) {
137     total_number_of_instructions_ = CountNumberOfInstructions(graph_);
138   }
139 
140   UpdateInliningBudget();
141   DCHECK_NE(total_number_of_instructions_, 0u);
142   DCHECK_NE(inlining_budget_, 0u);
143 
144   // If we're compiling with a core image (which is only used for
145   // test purposes), honor inlining directives in method names:
146   // - if a method's name contains the substring "$inline$", ensure
147   //   that this method is actually inlined;
148   // - if a method's name contains the substring "$noinline$", do not
149   //   inline that method.
150   // We limit this to AOT compilation, as the JIT may or may not inline
151   // depending on the state of classes at runtime.
152   const bool honor_inlining_directives =
153       IsCompilingWithCoreImage() && Runtime::Current()->IsAotCompiler();
154 
155   // Keep a copy of all blocks when starting the visit.
156   ArenaVector<HBasicBlock*> blocks = graph_->GetReversePostOrder();
157   DCHECK(!blocks.empty());
158   // Because we are changing the graph when inlining,
159   // we just iterate over the blocks of the outer method.
160   // This avoids doing the inlining work again on the inlined blocks.
161   for (HBasicBlock* block : blocks) {
162     for (HInstruction* instruction = block->GetFirstInstruction(); instruction != nullptr;) {
163       HInstruction* next = instruction->GetNext();
164       HInvoke* call = instruction->AsInvoke();
165       // As long as the call is not intrinsified, it is worth trying to inline.
166       if (call != nullptr && call->GetIntrinsic() == Intrinsics::kNone) {
167         if (honor_inlining_directives) {
168           // Debugging case: directives in method names control or assert on inlining.
169           std::string callee_name = outer_compilation_unit_.GetDexFile()->PrettyMethod(
170               call->GetDexMethodIndex(), /* with_signature */ false);
171           // Tests prevent inlining by having $noinline$ in their method names.
172           if (callee_name.find("$noinline$") == std::string::npos) {
173             if (!TryInline(call)) {
174               bool should_have_inlined = (callee_name.find("$inline$") != std::string::npos);
175               CHECK(!should_have_inlined) << "Could not inline " << callee_name;
176             }
177           }
178         } else {
179           // Normal case: try to inline.
180           TryInline(call);
181         }
182       }
183       instruction = next;
184     }
185   }
186 }
187 
IsMethodOrDeclaringClassFinal(ArtMethod * method)188 static bool IsMethodOrDeclaringClassFinal(ArtMethod* method)
189     REQUIRES_SHARED(Locks::mutator_lock_) {
190   return method->IsFinal() || method->GetDeclaringClass()->IsFinal();
191 }
192 
193 /**
194  * Given the `resolved_method` looked up in the dex cache, try to find
195  * the actual runtime target of an interface or virtual call.
196  * Return nullptr if the runtime target cannot be proven.
197  */
FindVirtualOrInterfaceTarget(HInvoke * invoke,ArtMethod * resolved_method)198 static ArtMethod* FindVirtualOrInterfaceTarget(HInvoke* invoke, ArtMethod* resolved_method)
199     REQUIRES_SHARED(Locks::mutator_lock_) {
200   if (IsMethodOrDeclaringClassFinal(resolved_method)) {
201     // No need to lookup further, the resolved method will be the target.
202     return resolved_method;
203   }
204 
205   HInstruction* receiver = invoke->InputAt(0);
206   if (receiver->IsNullCheck()) {
207     // Due to multiple levels of inlining within the same pass, it might be that
208     // null check does not have the reference type of the actual receiver.
209     receiver = receiver->InputAt(0);
210   }
211   ReferenceTypeInfo info = receiver->GetReferenceTypeInfo();
212   DCHECK(info.IsValid()) << "Invalid RTI for " << receiver->DebugName();
213   if (!info.IsExact()) {
214     // We currently only support inlining with known receivers.
215     // TODO: Remove this check, we should be able to inline final methods
216     // on unknown receivers.
217     return nullptr;
218   } else if (info.GetTypeHandle()->IsInterface()) {
219     // Statically knowing that the receiver has an interface type cannot
220     // help us find what is the target method.
221     return nullptr;
222   } else if (!resolved_method->GetDeclaringClass()->IsAssignableFrom(info.GetTypeHandle().Get())) {
223     // The method that we're trying to call is not in the receiver's class or super classes.
224     return nullptr;
225   } else if (info.GetTypeHandle()->IsErroneous()) {
226     // If the type is erroneous, do not go further, as we are going to query the vtable or
227     // imt table, that we can only safely do on non-erroneous classes.
228     return nullptr;
229   }
230 
231   ClassLinker* cl = Runtime::Current()->GetClassLinker();
232   PointerSize pointer_size = cl->GetImagePointerSize();
233   if (invoke->IsInvokeInterface()) {
234     resolved_method = info.GetTypeHandle()->FindVirtualMethodForInterface(
235         resolved_method, pointer_size);
236   } else {
237     DCHECK(invoke->IsInvokeVirtual());
238     resolved_method = info.GetTypeHandle()->FindVirtualMethodForVirtual(
239         resolved_method, pointer_size);
240   }
241 
242   if (resolved_method == nullptr) {
243     // The information we had on the receiver was not enough to find
244     // the target method. Since we check above the exact type of the receiver,
245     // the only reason this can happen is an IncompatibleClassChangeError.
246     return nullptr;
247   } else if (!resolved_method->IsInvokable()) {
248     // The information we had on the receiver was not enough to find
249     // the target method. Since we check above the exact type of the receiver,
250     // the only reason this can happen is an IncompatibleClassChangeError.
251     return nullptr;
252   } else if (IsMethodOrDeclaringClassFinal(resolved_method)) {
253     // A final method has to be the target method.
254     return resolved_method;
255   } else if (info.IsExact()) {
256     // If we found a method and the receiver's concrete type is statically
257     // known, we know for sure the target.
258     return resolved_method;
259   } else {
260     // Even if we did find a method, the receiver type was not enough to
261     // statically find the runtime target.
262     return nullptr;
263   }
264 }
265 
FindMethodIndexIn(ArtMethod * method,const DexFile & dex_file,uint32_t name_and_signature_index)266 static uint32_t FindMethodIndexIn(ArtMethod* method,
267                                   const DexFile& dex_file,
268                                   uint32_t name_and_signature_index)
269     REQUIRES_SHARED(Locks::mutator_lock_) {
270   if (IsSameDexFile(*method->GetDexFile(), dex_file)) {
271     return method->GetDexMethodIndex();
272   } else {
273     return method->FindDexMethodIndexInOtherDexFile(dex_file, name_and_signature_index);
274   }
275 }
276 
FindClassIndexIn(mirror::Class * cls,const DexCompilationUnit & compilation_unit)277 static dex::TypeIndex FindClassIndexIn(mirror::Class* cls,
278                                        const DexCompilationUnit& compilation_unit)
279     REQUIRES_SHARED(Locks::mutator_lock_) {
280   const DexFile& dex_file = *compilation_unit.GetDexFile();
281   dex::TypeIndex index;
282   if (cls->GetDexCache() == nullptr) {
283     DCHECK(cls->IsArrayClass()) << cls->PrettyClass();
284     index = cls->FindTypeIndexInOtherDexFile(dex_file);
285   } else if (!cls->GetDexTypeIndex().IsValid()) {
286     DCHECK(cls->IsProxyClass()) << cls->PrettyClass();
287     // TODO: deal with proxy classes.
288   } else if (IsSameDexFile(cls->GetDexFile(), dex_file)) {
289     DCHECK_EQ(cls->GetDexCache(), compilation_unit.GetDexCache().Get());
290     index = cls->GetDexTypeIndex();
291   } else {
292     index = cls->FindTypeIndexInOtherDexFile(dex_file);
293     // We cannot guarantee the entry will resolve to the same class,
294     // as there may be different class loaders. So only return the index if it's
295     // the right class already resolved with the class loader.
296     if (index.IsValid()) {
297       ObjPtr<mirror::Class> resolved = compilation_unit.GetClassLinker()->LookupResolvedType(
298           index, compilation_unit.GetDexCache().Get(), compilation_unit.GetClassLoader().Get());
299       if (resolved != cls) {
300         index = dex::TypeIndex::Invalid();
301       }
302     }
303   }
304 
305   return index;
306 }
307 
308 class ScopedProfilingInfoInlineUse {
309  public:
ScopedProfilingInfoInlineUse(ArtMethod * method,Thread * self)310   explicit ScopedProfilingInfoInlineUse(ArtMethod* method, Thread* self)
311       : method_(method),
312         self_(self),
313         // Fetch the profiling info ahead of using it. If it's null when fetching,
314         // we should not call JitCodeCache::DoneInlining.
315         profiling_info_(
316             Runtime::Current()->GetJit()->GetCodeCache()->NotifyCompilerUse(method, self)) {
317   }
318 
~ScopedProfilingInfoInlineUse()319   ~ScopedProfilingInfoInlineUse() {
320     if (profiling_info_ != nullptr) {
321       PointerSize pointer_size = Runtime::Current()->GetClassLinker()->GetImagePointerSize();
322       DCHECK_EQ(profiling_info_, method_->GetProfilingInfo(pointer_size));
323       Runtime::Current()->GetJit()->GetCodeCache()->DoneCompilerUse(method_, self_);
324     }
325   }
326 
GetProfilingInfo() const327   ProfilingInfo* GetProfilingInfo() const { return profiling_info_; }
328 
329  private:
330   ArtMethod* const method_;
331   Thread* const self_;
332   ProfilingInfo* const profiling_info_;
333 };
334 
GetInlineCacheType(const Handle<mirror::ObjectArray<mirror::Class>> & classes)335 HInliner::InlineCacheType HInliner::GetInlineCacheType(
336     const Handle<mirror::ObjectArray<mirror::Class>>& classes)
337   REQUIRES_SHARED(Locks::mutator_lock_) {
338   uint8_t number_of_types = 0;
339   for (; number_of_types < InlineCache::kIndividualCacheSize; ++number_of_types) {
340     if (classes->Get(number_of_types) == nullptr) {
341       break;
342     }
343   }
344 
345   if (number_of_types == 0) {
346     return kInlineCacheUninitialized;
347   } else if (number_of_types == 1) {
348     return kInlineCacheMonomorphic;
349   } else if (number_of_types == InlineCache::kIndividualCacheSize) {
350     return kInlineCacheMegamorphic;
351   } else {
352     return kInlineCachePolymorphic;
353   }
354 }
355 
GetMonomorphicType(Handle<mirror::ObjectArray<mirror::Class>> classes)356 static mirror::Class* GetMonomorphicType(Handle<mirror::ObjectArray<mirror::Class>> classes)
357     REQUIRES_SHARED(Locks::mutator_lock_) {
358   DCHECK(classes->Get(0) != nullptr);
359   return classes->Get(0);
360 }
361 
TryCHADevirtualization(ArtMethod * resolved_method)362 ArtMethod* HInliner::TryCHADevirtualization(ArtMethod* resolved_method) {
363   if (!resolved_method->HasSingleImplementation()) {
364     return nullptr;
365   }
366   if (Runtime::Current()->IsAotCompiler()) {
367     // No CHA-based devirtulization for AOT compiler (yet).
368     return nullptr;
369   }
370   if (outermost_graph_->IsCompilingOsr()) {
371     // We do not support HDeoptimize in OSR methods.
372     return nullptr;
373   }
374   PointerSize pointer_size = caller_compilation_unit_.GetClassLinker()->GetImagePointerSize();
375   ArtMethod* single_impl = resolved_method->GetSingleImplementation(pointer_size);
376   if (single_impl == nullptr) {
377     return nullptr;
378   }
379   if (single_impl->IsProxyMethod()) {
380     // Proxy method is a generic invoker that's not worth
381     // devirtualizing/inlining. It also causes issues when the proxy
382     // method is in another dex file if we try to rewrite invoke-interface to
383     // invoke-virtual because a proxy method doesn't have a real dex file.
384     return nullptr;
385   }
386   if (!single_impl->GetDeclaringClass()->IsResolved()) {
387     // There's a race with the class loading, which updates the CHA info
388     // before setting the class to resolved. So we just bail for this
389     // rare occurence.
390     return nullptr;
391   }
392   return single_impl;
393 }
394 
IsMethodUnverified(CompilerDriver * const compiler_driver,ArtMethod * method)395 static bool IsMethodUnverified(CompilerDriver* const compiler_driver, ArtMethod* method)
396     REQUIRES_SHARED(Locks::mutator_lock_) {
397   if (!method->GetDeclaringClass()->IsVerified()) {
398     if (Runtime::Current()->UseJitCompilation()) {
399       // We're at runtime, we know this is cold code if the class
400       // is not verified, so don't bother analyzing.
401       return true;
402     }
403     uint16_t class_def_idx = method->GetDeclaringClass()->GetDexClassDefIndex();
404     if (!compiler_driver->IsMethodVerifiedWithoutFailures(
405         method->GetDexMethodIndex(), class_def_idx, *method->GetDexFile())) {
406       // Method has soft or hard failures, don't analyze.
407       return true;
408     }
409   }
410   return false;
411 }
412 
AlwaysThrows(CompilerDriver * const compiler_driver,ArtMethod * method)413 static bool AlwaysThrows(CompilerDriver* const compiler_driver, ArtMethod* method)
414     REQUIRES_SHARED(Locks::mutator_lock_) {
415   DCHECK(method != nullptr);
416   // Skip non-compilable and unverified methods.
417   if (!method->IsCompilable() || IsMethodUnverified(compiler_driver, method)) {
418     return false;
419   }
420   // Skip native methods, methods with try blocks, and methods that are too large.
421   CodeItemDataAccessor accessor(method->DexInstructionData());
422   if (!accessor.HasCodeItem() ||
423       accessor.TriesSize() != 0 ||
424       accessor.InsnsSizeInCodeUnits() > kMaximumNumberOfTotalInstructions) {
425     return false;
426   }
427   // Scan for exits.
428   bool throw_seen = false;
429   for (const DexInstructionPcPair& pair : accessor) {
430     switch (pair.Inst().Opcode()) {
431       case Instruction::RETURN:
432       case Instruction::RETURN_VOID:
433       case Instruction::RETURN_WIDE:
434       case Instruction::RETURN_OBJECT:
435       case Instruction::RETURN_VOID_NO_BARRIER:
436         return false;  // found regular control flow back
437       case Instruction::THROW:
438         throw_seen = true;
439         break;
440       default:
441         break;
442     }
443   }
444   return throw_seen;
445 }
446 
TryInline(HInvoke * invoke_instruction)447 bool HInliner::TryInline(HInvoke* invoke_instruction) {
448   if (invoke_instruction->IsInvokeUnresolved() ||
449       invoke_instruction->IsInvokePolymorphic()) {
450     return false;  // Don't bother to move further if we know the method is unresolved or an
451                    // invoke-polymorphic.
452   }
453 
454   ScopedObjectAccess soa(Thread::Current());
455   uint32_t method_index = invoke_instruction->GetDexMethodIndex();
456   const DexFile& caller_dex_file = *caller_compilation_unit_.GetDexFile();
457   LOG_TRY() << caller_dex_file.PrettyMethod(method_index);
458 
459   ArtMethod* resolved_method = invoke_instruction->GetResolvedMethod();
460   if (resolved_method == nullptr) {
461     DCHECK(invoke_instruction->IsInvokeStaticOrDirect());
462     DCHECK(invoke_instruction->AsInvokeStaticOrDirect()->IsStringInit());
463     LOG_FAIL_NO_STAT() << "Not inlining a String.<init> method";
464     return false;
465   }
466   ArtMethod* actual_method = nullptr;
467 
468   if (invoke_instruction->IsInvokeStaticOrDirect()) {
469     actual_method = resolved_method;
470   } else {
471     // Check if we can statically find the method.
472     actual_method = FindVirtualOrInterfaceTarget(invoke_instruction, resolved_method);
473   }
474 
475   bool cha_devirtualize = false;
476   if (actual_method == nullptr) {
477     ArtMethod* method = TryCHADevirtualization(resolved_method);
478     if (method != nullptr) {
479       cha_devirtualize = true;
480       actual_method = method;
481       LOG_NOTE() << "Try CHA-based inlining of " << actual_method->PrettyMethod();
482     }
483   }
484 
485   if (actual_method != nullptr) {
486     // Single target.
487     bool result = TryInlineAndReplace(invoke_instruction,
488                                       actual_method,
489                                       ReferenceTypeInfo::CreateInvalid(),
490                                       /* do_rtp */ true,
491                                       cha_devirtualize);
492     if (result) {
493       // Successfully inlined.
494       if (!invoke_instruction->IsInvokeStaticOrDirect()) {
495         if (cha_devirtualize) {
496           // Add dependency due to devirtualization. We've assumed resolved_method
497           // has single implementation.
498           outermost_graph_->AddCHASingleImplementationDependency(resolved_method);
499           MaybeRecordStat(stats_, MethodCompilationStat::kCHAInline);
500         } else {
501           MaybeRecordStat(stats_, MethodCompilationStat::kInlinedInvokeVirtualOrInterface);
502         }
503       }
504     } else if (!cha_devirtualize && AlwaysThrows(compiler_driver_, actual_method)) {
505       // Set always throws property for non-inlined method call with single target
506       // (unless it was obtained through CHA, because that would imply we have
507       // to add the CHA dependency, which seems not worth it).
508       invoke_instruction->SetAlwaysThrows(true);
509     }
510     return result;
511   }
512   DCHECK(!invoke_instruction->IsInvokeStaticOrDirect());
513 
514   // Try using inline caches.
515   return TryInlineFromInlineCache(caller_dex_file, invoke_instruction, resolved_method);
516 }
517 
AllocateInlineCacheHolder(const DexCompilationUnit & compilation_unit,StackHandleScope<1> * hs)518 static Handle<mirror::ObjectArray<mirror::Class>> AllocateInlineCacheHolder(
519     const DexCompilationUnit& compilation_unit,
520     StackHandleScope<1>* hs)
521     REQUIRES_SHARED(Locks::mutator_lock_) {
522   Thread* self = Thread::Current();
523   ClassLinker* class_linker = compilation_unit.GetClassLinker();
524   Handle<mirror::ObjectArray<mirror::Class>> inline_cache = hs->NewHandle(
525       mirror::ObjectArray<mirror::Class>::Alloc(
526           self,
527           class_linker->GetClassRoot(ClassLinker::kClassArrayClass),
528           InlineCache::kIndividualCacheSize));
529   if (inline_cache == nullptr) {
530     // We got an OOME. Just clear the exception, and don't inline.
531     DCHECK(self->IsExceptionPending());
532     self->ClearException();
533     VLOG(compiler) << "Out of memory in the compiler when trying to inline";
534   }
535   return inline_cache;
536 }
537 
UseOnlyPolymorphicInliningWithNoDeopt()538 bool HInliner::UseOnlyPolymorphicInliningWithNoDeopt() {
539   // If we are compiling AOT or OSR, pretend the call using inline caches is polymorphic and
540   // do not generate a deopt.
541   //
542   // For AOT:
543   //    Generating a deopt does not ensure that we will actually capture the new types;
544   //    and the danger is that we could be stuck in a loop with "forever" deoptimizations.
545   //    Take for example the following scenario:
546   //      - we capture the inline cache in one run
547   //      - the next run, we deoptimize because we miss a type check, but the method
548   //        never becomes hot again
549   //    In this case, the inline cache will not be updated in the profile and the AOT code
550   //    will keep deoptimizing.
551   //    Another scenario is if we use profile compilation for a process which is not allowed
552   //    to JIT (e.g. system server). If we deoptimize we will run interpreted code for the
553   //    rest of the lifetime.
554   // TODO(calin):
555   //    This is a compromise because we will most likely never update the inline cache
556   //    in the profile (unless there's another reason to deopt). So we might be stuck with
557   //    a sub-optimal inline cache.
558   //    We could be smarter when capturing inline caches to mitigate this.
559   //    (e.g. by having different thresholds for new and old methods).
560   //
561   // For OSR:
562   //     We may come from the interpreter and it may have seen different receiver types.
563   return Runtime::Current()->IsAotCompiler() || outermost_graph_->IsCompilingOsr();
564 }
TryInlineFromInlineCache(const DexFile & caller_dex_file,HInvoke * invoke_instruction,ArtMethod * resolved_method)565 bool HInliner::TryInlineFromInlineCache(const DexFile& caller_dex_file,
566                                         HInvoke* invoke_instruction,
567                                         ArtMethod* resolved_method)
568     REQUIRES_SHARED(Locks::mutator_lock_) {
569   if (Runtime::Current()->IsAotCompiler() && !kUseAOTInlineCaches) {
570     return false;
571   }
572 
573   StackHandleScope<1> hs(Thread::Current());
574   Handle<mirror::ObjectArray<mirror::Class>> inline_cache;
575   InlineCacheType inline_cache_type = Runtime::Current()->IsAotCompiler()
576       ? GetInlineCacheAOT(caller_dex_file, invoke_instruction, &hs, &inline_cache)
577       : GetInlineCacheJIT(invoke_instruction, &hs, &inline_cache);
578 
579   switch (inline_cache_type) {
580     case kInlineCacheNoData: {
581       LOG_FAIL_NO_STAT()
582           << "Interface or virtual call to "
583           << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex())
584           << " could not be statically determined";
585       return false;
586     }
587 
588     case kInlineCacheUninitialized: {
589       LOG_FAIL_NO_STAT()
590           << "Interface or virtual call to "
591           << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex())
592           << " is not hit and not inlined";
593       return false;
594     }
595 
596     case kInlineCacheMonomorphic: {
597       MaybeRecordStat(stats_, MethodCompilationStat::kMonomorphicCall);
598       if (UseOnlyPolymorphicInliningWithNoDeopt()) {
599         return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache);
600       } else {
601         return TryInlineMonomorphicCall(invoke_instruction, resolved_method, inline_cache);
602       }
603     }
604 
605     case kInlineCachePolymorphic: {
606       MaybeRecordStat(stats_, MethodCompilationStat::kPolymorphicCall);
607       return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache);
608     }
609 
610     case kInlineCacheMegamorphic: {
611       LOG_FAIL_NO_STAT()
612           << "Interface or virtual call to "
613           << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex())
614           << " is megamorphic and not inlined";
615       MaybeRecordStat(stats_, MethodCompilationStat::kMegamorphicCall);
616       return false;
617     }
618 
619     case kInlineCacheMissingTypes: {
620       LOG_FAIL_NO_STAT()
621           << "Interface or virtual call to "
622           << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex())
623           << " is missing types and not inlined";
624       return false;
625     }
626   }
627   UNREACHABLE();
628 }
629 
GetInlineCacheJIT(HInvoke * invoke_instruction,StackHandleScope<1> * hs,Handle<mirror::ObjectArray<mirror::Class>> * inline_cache)630 HInliner::InlineCacheType HInliner::GetInlineCacheJIT(
631     HInvoke* invoke_instruction,
632     StackHandleScope<1>* hs,
633     /*out*/Handle<mirror::ObjectArray<mirror::Class>>* inline_cache)
634     REQUIRES_SHARED(Locks::mutator_lock_) {
635   DCHECK(Runtime::Current()->UseJitCompilation());
636 
637   ArtMethod* caller = graph_->GetArtMethod();
638   // Under JIT, we should always know the caller.
639   DCHECK(caller != nullptr);
640   ScopedProfilingInfoInlineUse spiis(caller, Thread::Current());
641   ProfilingInfo* profiling_info = spiis.GetProfilingInfo();
642 
643   if (profiling_info == nullptr) {
644     return kInlineCacheNoData;
645   }
646 
647   *inline_cache = AllocateInlineCacheHolder(caller_compilation_unit_, hs);
648   if (inline_cache->Get() == nullptr) {
649     // We can't extract any data if we failed to allocate;
650     return kInlineCacheNoData;
651   } else {
652     Runtime::Current()->GetJit()->GetCodeCache()->CopyInlineCacheInto(
653         *profiling_info->GetInlineCache(invoke_instruction->GetDexPc()),
654         *inline_cache);
655     return GetInlineCacheType(*inline_cache);
656   }
657 }
658 
GetInlineCacheAOT(const DexFile & caller_dex_file,HInvoke * invoke_instruction,StackHandleScope<1> * hs,Handle<mirror::ObjectArray<mirror::Class>> * inline_cache)659 HInliner::InlineCacheType HInliner::GetInlineCacheAOT(
660     const DexFile& caller_dex_file,
661     HInvoke* invoke_instruction,
662     StackHandleScope<1>* hs,
663     /*out*/Handle<mirror::ObjectArray<mirror::Class>>* inline_cache)
664     REQUIRES_SHARED(Locks::mutator_lock_) {
665   DCHECK(Runtime::Current()->IsAotCompiler());
666   const ProfileCompilationInfo* pci = compiler_driver_->GetProfileCompilationInfo();
667   if (pci == nullptr) {
668     return kInlineCacheNoData;
669   }
670 
671   std::unique_ptr<ProfileCompilationInfo::OfflineProfileMethodInfo> offline_profile =
672       pci->GetMethod(caller_dex_file.GetLocation(),
673                      caller_dex_file.GetLocationChecksum(),
674                      caller_compilation_unit_.GetDexMethodIndex());
675   if (offline_profile == nullptr) {
676     return kInlineCacheNoData;  // no profile information for this invocation.
677   }
678 
679   *inline_cache = AllocateInlineCacheHolder(caller_compilation_unit_, hs);
680   if (inline_cache == nullptr) {
681     // We can't extract any data if we failed to allocate;
682     return kInlineCacheNoData;
683   } else {
684     return ExtractClassesFromOfflineProfile(invoke_instruction,
685                                             *(offline_profile.get()),
686                                             *inline_cache);
687   }
688 }
689 
ExtractClassesFromOfflineProfile(const HInvoke * invoke_instruction,const ProfileCompilationInfo::OfflineProfileMethodInfo & offline_profile,Handle<mirror::ObjectArray<mirror::Class>> inline_cache)690 HInliner::InlineCacheType HInliner::ExtractClassesFromOfflineProfile(
691     const HInvoke* invoke_instruction,
692     const ProfileCompilationInfo::OfflineProfileMethodInfo& offline_profile,
693     /*out*/Handle<mirror::ObjectArray<mirror::Class>> inline_cache)
694     REQUIRES_SHARED(Locks::mutator_lock_) {
695   const auto it = offline_profile.inline_caches->find(invoke_instruction->GetDexPc());
696   if (it == offline_profile.inline_caches->end()) {
697     return kInlineCacheUninitialized;
698   }
699 
700   const ProfileCompilationInfo::DexPcData& dex_pc_data = it->second;
701 
702   if (dex_pc_data.is_missing_types) {
703     return kInlineCacheMissingTypes;
704   }
705   if (dex_pc_data.is_megamorphic) {
706     return kInlineCacheMegamorphic;
707   }
708 
709   DCHECK_LE(dex_pc_data.classes.size(), InlineCache::kIndividualCacheSize);
710   Thread* self = Thread::Current();
711   // We need to resolve the class relative to the containing dex file.
712   // So first, build a mapping from the index of dex file in the profile to
713   // its dex cache. This will avoid repeating the lookup when walking over
714   // the inline cache types.
715   std::vector<ObjPtr<mirror::DexCache>> dex_profile_index_to_dex_cache(
716         offline_profile.dex_references.size());
717   for (size_t i = 0; i < offline_profile.dex_references.size(); i++) {
718     bool found = false;
719     for (const DexFile* dex_file : compiler_driver_->GetDexFilesForOatFile()) {
720       if (offline_profile.dex_references[i].MatchesDex(dex_file)) {
721         dex_profile_index_to_dex_cache[i] =
722             caller_compilation_unit_.GetClassLinker()->FindDexCache(self, *dex_file);
723         found = true;
724       }
725     }
726     if (!found) {
727       VLOG(compiler) << "Could not find profiled dex file: "
728           << offline_profile.dex_references[i].dex_location;
729       return kInlineCacheMissingTypes;
730     }
731   }
732 
733   // Walk over the classes and resolve them. If we cannot find a type we return
734   // kInlineCacheMissingTypes.
735   int ic_index = 0;
736   for (const ProfileCompilationInfo::ClassReference& class_ref : dex_pc_data.classes) {
737     ObjPtr<mirror::DexCache> dex_cache =
738         dex_profile_index_to_dex_cache[class_ref.dex_profile_index];
739     DCHECK(dex_cache != nullptr);
740 
741     if (!dex_cache->GetDexFile()->IsTypeIndexValid(class_ref.type_index)) {
742       VLOG(compiler) << "Profile data corrupt: type index " << class_ref.type_index
743             << "is invalid in location" << dex_cache->GetDexFile()->GetLocation();
744       return kInlineCacheNoData;
745     }
746     ObjPtr<mirror::Class> clazz = caller_compilation_unit_.GetClassLinker()->LookupResolvedType(
747           class_ref.type_index,
748           dex_cache,
749           caller_compilation_unit_.GetClassLoader().Get());
750     if (clazz != nullptr) {
751       inline_cache->Set(ic_index++, clazz);
752     } else {
753       VLOG(compiler) << "Could not resolve class from inline cache in AOT mode "
754           << caller_compilation_unit_.GetDexFile()->PrettyMethod(
755               invoke_instruction->GetDexMethodIndex()) << " : "
756           << caller_compilation_unit_
757               .GetDexFile()->StringByTypeIdx(class_ref.type_index);
758       return kInlineCacheMissingTypes;
759     }
760   }
761   return GetInlineCacheType(inline_cache);
762 }
763 
BuildGetReceiverClass(ClassLinker * class_linker,HInstruction * receiver,uint32_t dex_pc) const764 HInstanceFieldGet* HInliner::BuildGetReceiverClass(ClassLinker* class_linker,
765                                                    HInstruction* receiver,
766                                                    uint32_t dex_pc) const {
767   ArtField* field = class_linker->GetClassRoot(ClassLinker::kJavaLangObject)->GetInstanceField(0);
768   DCHECK_EQ(std::string(field->GetName()), "shadow$_klass_");
769   HInstanceFieldGet* result = new (graph_->GetAllocator()) HInstanceFieldGet(
770       receiver,
771       field,
772       DataType::Type::kReference,
773       field->GetOffset(),
774       field->IsVolatile(),
775       field->GetDexFieldIndex(),
776       field->GetDeclaringClass()->GetDexClassDefIndex(),
777       *field->GetDexFile(),
778       dex_pc);
779   // The class of a field is effectively final, and does not have any memory dependencies.
780   result->SetSideEffects(SideEffects::None());
781   return result;
782 }
783 
ResolveMethodFromInlineCache(Handle<mirror::Class> klass,ArtMethod * resolved_method,HInstruction * invoke_instruction,PointerSize pointer_size)784 static ArtMethod* ResolveMethodFromInlineCache(Handle<mirror::Class> klass,
785                                                ArtMethod* resolved_method,
786                                                HInstruction* invoke_instruction,
787                                                PointerSize pointer_size)
788     REQUIRES_SHARED(Locks::mutator_lock_) {
789   if (Runtime::Current()->IsAotCompiler()) {
790     // We can get unrelated types when working with profiles (corruption,
791     // systme updates, or anyone can write to it). So first check if the class
792     // actually implements the declaring class of the method that is being
793     // called in bytecode.
794     // Note: the lookup methods used below require to have assignable types.
795     if (!resolved_method->GetDeclaringClass()->IsAssignableFrom(klass.Get())) {
796       return nullptr;
797     }
798   }
799 
800   if (invoke_instruction->IsInvokeInterface()) {
801     resolved_method = klass->FindVirtualMethodForInterface(resolved_method, pointer_size);
802   } else {
803     DCHECK(invoke_instruction->IsInvokeVirtual());
804     resolved_method = klass->FindVirtualMethodForVirtual(resolved_method, pointer_size);
805   }
806   DCHECK(resolved_method != nullptr);
807   return resolved_method;
808 }
809 
TryInlineMonomorphicCall(HInvoke * invoke_instruction,ArtMethod * resolved_method,Handle<mirror::ObjectArray<mirror::Class>> classes)810 bool HInliner::TryInlineMonomorphicCall(HInvoke* invoke_instruction,
811                                         ArtMethod* resolved_method,
812                                         Handle<mirror::ObjectArray<mirror::Class>> classes) {
813   DCHECK(invoke_instruction->IsInvokeVirtual() || invoke_instruction->IsInvokeInterface())
814       << invoke_instruction->DebugName();
815 
816   dex::TypeIndex class_index = FindClassIndexIn(
817       GetMonomorphicType(classes), caller_compilation_unit_);
818   if (!class_index.IsValid()) {
819     LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedDexCache)
820         << "Call to " << ArtMethod::PrettyMethod(resolved_method)
821         << " from inline cache is not inlined because its class is not"
822         << " accessible to the caller";
823     return false;
824   }
825 
826   ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker();
827   PointerSize pointer_size = class_linker->GetImagePointerSize();
828   Handle<mirror::Class> monomorphic_type = handles_->NewHandle(GetMonomorphicType(classes));
829   resolved_method = ResolveMethodFromInlineCache(
830       monomorphic_type, resolved_method, invoke_instruction, pointer_size);
831 
832   LOG_NOTE() << "Try inline monomorphic call to " << resolved_method->PrettyMethod();
833   if (resolved_method == nullptr) {
834     // Bogus AOT profile, bail.
835     DCHECK(Runtime::Current()->IsAotCompiler());
836     return false;
837   }
838 
839   HInstruction* receiver = invoke_instruction->InputAt(0);
840   HInstruction* cursor = invoke_instruction->GetPrevious();
841   HBasicBlock* bb_cursor = invoke_instruction->GetBlock();
842   if (!TryInlineAndReplace(invoke_instruction,
843                            resolved_method,
844                            ReferenceTypeInfo::Create(monomorphic_type, /* is_exact */ true),
845                            /* do_rtp */ false,
846                            /* cha_devirtualize */ false)) {
847     return false;
848   }
849 
850   // We successfully inlined, now add a guard.
851   AddTypeGuard(receiver,
852                cursor,
853                bb_cursor,
854                class_index,
855                monomorphic_type,
856                invoke_instruction,
857                /* with_deoptimization */ true);
858 
859   // Run type propagation to get the guard typed, and eventually propagate the
860   // type of the receiver.
861   ReferenceTypePropagation rtp_fixup(graph_,
862                                      outer_compilation_unit_.GetClassLoader(),
863                                      outer_compilation_unit_.GetDexCache(),
864                                      handles_,
865                                      /* is_first_run */ false);
866   rtp_fixup.Run();
867 
868   MaybeRecordStat(stats_, MethodCompilationStat::kInlinedMonomorphicCall);
869   return true;
870 }
871 
AddCHAGuard(HInstruction * invoke_instruction,uint32_t dex_pc,HInstruction * cursor,HBasicBlock * bb_cursor)872 void HInliner::AddCHAGuard(HInstruction* invoke_instruction,
873                            uint32_t dex_pc,
874                            HInstruction* cursor,
875                            HBasicBlock* bb_cursor) {
876   HShouldDeoptimizeFlag* deopt_flag = new (graph_->GetAllocator())
877       HShouldDeoptimizeFlag(graph_->GetAllocator(), dex_pc);
878   HInstruction* compare = new (graph_->GetAllocator()) HNotEqual(
879       deopt_flag, graph_->GetIntConstant(0, dex_pc));
880   HInstruction* deopt = new (graph_->GetAllocator()) HDeoptimize(
881       graph_->GetAllocator(), compare, DeoptimizationKind::kCHA, dex_pc);
882 
883   if (cursor != nullptr) {
884     bb_cursor->InsertInstructionAfter(deopt_flag, cursor);
885   } else {
886     bb_cursor->InsertInstructionBefore(deopt_flag, bb_cursor->GetFirstInstruction());
887   }
888   bb_cursor->InsertInstructionAfter(compare, deopt_flag);
889   bb_cursor->InsertInstructionAfter(deopt, compare);
890 
891   // Add receiver as input to aid CHA guard optimization later.
892   deopt_flag->AddInput(invoke_instruction->InputAt(0));
893   DCHECK_EQ(deopt_flag->InputCount(), 1u);
894   deopt->CopyEnvironmentFrom(invoke_instruction->GetEnvironment());
895   outermost_graph_->IncrementNumberOfCHAGuards();
896 }
897 
AddTypeGuard(HInstruction * receiver,HInstruction * cursor,HBasicBlock * bb_cursor,dex::TypeIndex class_index,Handle<mirror::Class> klass,HInstruction * invoke_instruction,bool with_deoptimization)898 HInstruction* HInliner::AddTypeGuard(HInstruction* receiver,
899                                      HInstruction* cursor,
900                                      HBasicBlock* bb_cursor,
901                                      dex::TypeIndex class_index,
902                                      Handle<mirror::Class> klass,
903                                      HInstruction* invoke_instruction,
904                                      bool with_deoptimization) {
905   ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker();
906   HInstanceFieldGet* receiver_class = BuildGetReceiverClass(
907       class_linker, receiver, invoke_instruction->GetDexPc());
908   if (cursor != nullptr) {
909     bb_cursor->InsertInstructionAfter(receiver_class, cursor);
910   } else {
911     bb_cursor->InsertInstructionBefore(receiver_class, bb_cursor->GetFirstInstruction());
912   }
913 
914   const DexFile& caller_dex_file = *caller_compilation_unit_.GetDexFile();
915   bool is_referrer;
916   ArtMethod* outermost_art_method = outermost_graph_->GetArtMethod();
917   if (outermost_art_method == nullptr) {
918     DCHECK(Runtime::Current()->IsAotCompiler());
919     // We are in AOT mode and we don't have an ART method to determine
920     // if the inlined method belongs to the referrer. Assume it doesn't.
921     is_referrer = false;
922   } else {
923     is_referrer = klass.Get() == outermost_art_method->GetDeclaringClass();
924   }
925 
926   // Note that we will just compare the classes, so we don't need Java semantics access checks.
927   // Note that the type index and the dex file are relative to the method this type guard is
928   // inlined into.
929   HLoadClass* load_class = new (graph_->GetAllocator()) HLoadClass(graph_->GetCurrentMethod(),
930                                                                    class_index,
931                                                                    caller_dex_file,
932                                                                    klass,
933                                                                    is_referrer,
934                                                                    invoke_instruction->GetDexPc(),
935                                                                    /* needs_access_check */ false);
936   HLoadClass::LoadKind kind = HSharpening::ComputeLoadClassKind(
937       load_class, codegen_, compiler_driver_, caller_compilation_unit_);
938   DCHECK(kind != HLoadClass::LoadKind::kInvalid)
939       << "We should always be able to reference a class for inline caches";
940   // Load kind must be set before inserting the instruction into the graph.
941   load_class->SetLoadKind(kind);
942   bb_cursor->InsertInstructionAfter(load_class, receiver_class);
943   // In AOT mode, we will most likely load the class from BSS, which will involve a call
944   // to the runtime. In this case, the load instruction will need an environment so copy
945   // it from the invoke instruction.
946   if (load_class->NeedsEnvironment()) {
947     DCHECK(Runtime::Current()->IsAotCompiler());
948     load_class->CopyEnvironmentFrom(invoke_instruction->GetEnvironment());
949   }
950 
951   HNotEqual* compare = new (graph_->GetAllocator()) HNotEqual(load_class, receiver_class);
952   bb_cursor->InsertInstructionAfter(compare, load_class);
953   if (with_deoptimization) {
954     HDeoptimize* deoptimize = new (graph_->GetAllocator()) HDeoptimize(
955         graph_->GetAllocator(),
956         compare,
957         receiver,
958         Runtime::Current()->IsAotCompiler()
959             ? DeoptimizationKind::kAotInlineCache
960             : DeoptimizationKind::kJitInlineCache,
961         invoke_instruction->GetDexPc());
962     bb_cursor->InsertInstructionAfter(deoptimize, compare);
963     deoptimize->CopyEnvironmentFrom(invoke_instruction->GetEnvironment());
964     DCHECK_EQ(invoke_instruction->InputAt(0), receiver);
965     receiver->ReplaceUsesDominatedBy(deoptimize, deoptimize);
966     deoptimize->SetReferenceTypeInfo(receiver->GetReferenceTypeInfo());
967   }
968   return compare;
969 }
970 
TryInlinePolymorphicCall(HInvoke * invoke_instruction,ArtMethod * resolved_method,Handle<mirror::ObjectArray<mirror::Class>> classes)971 bool HInliner::TryInlinePolymorphicCall(HInvoke* invoke_instruction,
972                                         ArtMethod* resolved_method,
973                                         Handle<mirror::ObjectArray<mirror::Class>> classes) {
974   DCHECK(invoke_instruction->IsInvokeVirtual() || invoke_instruction->IsInvokeInterface())
975       << invoke_instruction->DebugName();
976 
977   if (TryInlinePolymorphicCallToSameTarget(invoke_instruction, resolved_method, classes)) {
978     return true;
979   }
980 
981   ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker();
982   PointerSize pointer_size = class_linker->GetImagePointerSize();
983 
984   bool all_targets_inlined = true;
985   bool one_target_inlined = false;
986   for (size_t i = 0; i < InlineCache::kIndividualCacheSize; ++i) {
987     if (classes->Get(i) == nullptr) {
988       break;
989     }
990     ArtMethod* method = nullptr;
991 
992     Handle<mirror::Class> handle = handles_->NewHandle(classes->Get(i));
993     method = ResolveMethodFromInlineCache(
994         handle, resolved_method, invoke_instruction, pointer_size);
995     if (method == nullptr) {
996       DCHECK(Runtime::Current()->IsAotCompiler());
997       // AOT profile is bogus. This loop expects to iterate over all entries,
998       // so just just continue.
999       all_targets_inlined = false;
1000       continue;
1001     }
1002 
1003     HInstruction* receiver = invoke_instruction->InputAt(0);
1004     HInstruction* cursor = invoke_instruction->GetPrevious();
1005     HBasicBlock* bb_cursor = invoke_instruction->GetBlock();
1006 
1007     dex::TypeIndex class_index = FindClassIndexIn(handle.Get(), caller_compilation_unit_);
1008     HInstruction* return_replacement = nullptr;
1009     LOG_NOTE() << "Try inline polymorphic call to " << method->PrettyMethod();
1010     if (!class_index.IsValid() ||
1011         !TryBuildAndInline(invoke_instruction,
1012                            method,
1013                            ReferenceTypeInfo::Create(handle, /* is_exact */ true),
1014                            &return_replacement)) {
1015       all_targets_inlined = false;
1016     } else {
1017       one_target_inlined = true;
1018 
1019       LOG_SUCCESS() << "Polymorphic call to " << ArtMethod::PrettyMethod(resolved_method)
1020                     << " has inlined " << ArtMethod::PrettyMethod(method);
1021 
1022       // If we have inlined all targets before, and this receiver is the last seen,
1023       // we deoptimize instead of keeping the original invoke instruction.
1024       bool deoptimize = !UseOnlyPolymorphicInliningWithNoDeopt() &&
1025           all_targets_inlined &&
1026           (i != InlineCache::kIndividualCacheSize - 1) &&
1027           (classes->Get(i + 1) == nullptr);
1028 
1029       HInstruction* compare = AddTypeGuard(receiver,
1030                                            cursor,
1031                                            bb_cursor,
1032                                            class_index,
1033                                            handle,
1034                                            invoke_instruction,
1035                                            deoptimize);
1036       if (deoptimize) {
1037         if (return_replacement != nullptr) {
1038           invoke_instruction->ReplaceWith(return_replacement);
1039         }
1040         invoke_instruction->GetBlock()->RemoveInstruction(invoke_instruction);
1041         // Because the inline cache data can be populated concurrently, we force the end of the
1042         // iteration. Otherwise, we could see a new receiver type.
1043         break;
1044       } else {
1045         CreateDiamondPatternForPolymorphicInline(compare, return_replacement, invoke_instruction);
1046       }
1047     }
1048   }
1049 
1050   if (!one_target_inlined) {
1051     LOG_FAIL_NO_STAT()
1052         << "Call to " << ArtMethod::PrettyMethod(resolved_method)
1053         << " from inline cache is not inlined because none"
1054         << " of its targets could be inlined";
1055     return false;
1056   }
1057 
1058   MaybeRecordStat(stats_, MethodCompilationStat::kInlinedPolymorphicCall);
1059 
1060   // Run type propagation to get the guards typed.
1061   ReferenceTypePropagation rtp_fixup(graph_,
1062                                      outer_compilation_unit_.GetClassLoader(),
1063                                      outer_compilation_unit_.GetDexCache(),
1064                                      handles_,
1065                                      /* is_first_run */ false);
1066   rtp_fixup.Run();
1067   return true;
1068 }
1069 
CreateDiamondPatternForPolymorphicInline(HInstruction * compare,HInstruction * return_replacement,HInstruction * invoke_instruction)1070 void HInliner::CreateDiamondPatternForPolymorphicInline(HInstruction* compare,
1071                                                         HInstruction* return_replacement,
1072                                                         HInstruction* invoke_instruction) {
1073   uint32_t dex_pc = invoke_instruction->GetDexPc();
1074   HBasicBlock* cursor_block = compare->GetBlock();
1075   HBasicBlock* original_invoke_block = invoke_instruction->GetBlock();
1076   ArenaAllocator* allocator = graph_->GetAllocator();
1077 
1078   // Spit the block after the compare: `cursor_block` will now be the start of the diamond,
1079   // and the returned block is the start of the then branch (that could contain multiple blocks).
1080   HBasicBlock* then = cursor_block->SplitAfterForInlining(compare);
1081 
1082   // Split the block containing the invoke before and after the invoke. The returned block
1083   // of the split before will contain the invoke and will be the otherwise branch of
1084   // the diamond. The returned block of the split after will be the merge block
1085   // of the diamond.
1086   HBasicBlock* end_then = invoke_instruction->GetBlock();
1087   HBasicBlock* otherwise = end_then->SplitBeforeForInlining(invoke_instruction);
1088   HBasicBlock* merge = otherwise->SplitAfterForInlining(invoke_instruction);
1089 
1090   // If the methods we are inlining return a value, we create a phi in the merge block
1091   // that will have the `invoke_instruction and the `return_replacement` as inputs.
1092   if (return_replacement != nullptr) {
1093     HPhi* phi = new (allocator) HPhi(
1094         allocator, kNoRegNumber, 0, HPhi::ToPhiType(invoke_instruction->GetType()), dex_pc);
1095     merge->AddPhi(phi);
1096     invoke_instruction->ReplaceWith(phi);
1097     phi->AddInput(return_replacement);
1098     phi->AddInput(invoke_instruction);
1099   }
1100 
1101   // Add the control flow instructions.
1102   otherwise->AddInstruction(new (allocator) HGoto(dex_pc));
1103   end_then->AddInstruction(new (allocator) HGoto(dex_pc));
1104   cursor_block->AddInstruction(new (allocator) HIf(compare, dex_pc));
1105 
1106   // Add the newly created blocks to the graph.
1107   graph_->AddBlock(then);
1108   graph_->AddBlock(otherwise);
1109   graph_->AddBlock(merge);
1110 
1111   // Set up successor (and implictly predecessor) relations.
1112   cursor_block->AddSuccessor(otherwise);
1113   cursor_block->AddSuccessor(then);
1114   end_then->AddSuccessor(merge);
1115   otherwise->AddSuccessor(merge);
1116 
1117   // Set up dominance information.
1118   then->SetDominator(cursor_block);
1119   cursor_block->AddDominatedBlock(then);
1120   otherwise->SetDominator(cursor_block);
1121   cursor_block->AddDominatedBlock(otherwise);
1122   merge->SetDominator(cursor_block);
1123   cursor_block->AddDominatedBlock(merge);
1124 
1125   // Update the revert post order.
1126   size_t index = IndexOfElement(graph_->reverse_post_order_, cursor_block);
1127   MakeRoomFor(&graph_->reverse_post_order_, 1, index);
1128   graph_->reverse_post_order_[++index] = then;
1129   index = IndexOfElement(graph_->reverse_post_order_, end_then);
1130   MakeRoomFor(&graph_->reverse_post_order_, 2, index);
1131   graph_->reverse_post_order_[++index] = otherwise;
1132   graph_->reverse_post_order_[++index] = merge;
1133 
1134 
1135   graph_->UpdateLoopAndTryInformationOfNewBlock(
1136       then, original_invoke_block, /* replace_if_back_edge */ false);
1137   graph_->UpdateLoopAndTryInformationOfNewBlock(
1138       otherwise, original_invoke_block, /* replace_if_back_edge */ false);
1139 
1140   // In case the original invoke location was a back edge, we need to update
1141   // the loop to now have the merge block as a back edge.
1142   graph_->UpdateLoopAndTryInformationOfNewBlock(
1143       merge, original_invoke_block, /* replace_if_back_edge */ true);
1144 }
1145 
TryInlinePolymorphicCallToSameTarget(HInvoke * invoke_instruction,ArtMethod * resolved_method,Handle<mirror::ObjectArray<mirror::Class>> classes)1146 bool HInliner::TryInlinePolymorphicCallToSameTarget(
1147     HInvoke* invoke_instruction,
1148     ArtMethod* resolved_method,
1149     Handle<mirror::ObjectArray<mirror::Class>> classes) {
1150   // This optimization only works under JIT for now.
1151   if (!Runtime::Current()->UseJitCompilation()) {
1152     return false;
1153   }
1154 
1155   ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker();
1156   PointerSize pointer_size = class_linker->GetImagePointerSize();
1157 
1158   DCHECK(resolved_method != nullptr);
1159   ArtMethod* actual_method = nullptr;
1160   size_t method_index = invoke_instruction->IsInvokeVirtual()
1161       ? invoke_instruction->AsInvokeVirtual()->GetVTableIndex()
1162       : invoke_instruction->AsInvokeInterface()->GetImtIndex();
1163 
1164   // Check whether we are actually calling the same method among
1165   // the different types seen.
1166   for (size_t i = 0; i < InlineCache::kIndividualCacheSize; ++i) {
1167     if (classes->Get(i) == nullptr) {
1168       break;
1169     }
1170     ArtMethod* new_method = nullptr;
1171     if (invoke_instruction->IsInvokeInterface()) {
1172       new_method = classes->Get(i)->GetImt(pointer_size)->Get(
1173           method_index, pointer_size);
1174       if (new_method->IsRuntimeMethod()) {
1175         // Bail out as soon as we see a conflict trampoline in one of the target's
1176         // interface table.
1177         return false;
1178       }
1179     } else {
1180       DCHECK(invoke_instruction->IsInvokeVirtual());
1181       new_method = classes->Get(i)->GetEmbeddedVTableEntry(method_index, pointer_size);
1182     }
1183     DCHECK(new_method != nullptr);
1184     if (actual_method == nullptr) {
1185       actual_method = new_method;
1186     } else if (actual_method != new_method) {
1187       // Different methods, bailout.
1188       return false;
1189     }
1190   }
1191 
1192   HInstruction* receiver = invoke_instruction->InputAt(0);
1193   HInstruction* cursor = invoke_instruction->GetPrevious();
1194   HBasicBlock* bb_cursor = invoke_instruction->GetBlock();
1195 
1196   HInstruction* return_replacement = nullptr;
1197   if (!TryBuildAndInline(invoke_instruction,
1198                          actual_method,
1199                          ReferenceTypeInfo::CreateInvalid(),
1200                          &return_replacement)) {
1201     return false;
1202   }
1203 
1204   // We successfully inlined, now add a guard.
1205   HInstanceFieldGet* receiver_class = BuildGetReceiverClass(
1206       class_linker, receiver, invoke_instruction->GetDexPc());
1207 
1208   DataType::Type type = Is64BitInstructionSet(graph_->GetInstructionSet())
1209       ? DataType::Type::kInt64
1210       : DataType::Type::kInt32;
1211   HClassTableGet* class_table_get = new (graph_->GetAllocator()) HClassTableGet(
1212       receiver_class,
1213       type,
1214       invoke_instruction->IsInvokeVirtual() ? HClassTableGet::TableKind::kVTable
1215                                             : HClassTableGet::TableKind::kIMTable,
1216       method_index,
1217       invoke_instruction->GetDexPc());
1218 
1219   HConstant* constant;
1220   if (type == DataType::Type::kInt64) {
1221     constant = graph_->GetLongConstant(
1222         reinterpret_cast<intptr_t>(actual_method), invoke_instruction->GetDexPc());
1223   } else {
1224     constant = graph_->GetIntConstant(
1225         reinterpret_cast<intptr_t>(actual_method), invoke_instruction->GetDexPc());
1226   }
1227 
1228   HNotEqual* compare = new (graph_->GetAllocator()) HNotEqual(class_table_get, constant);
1229   if (cursor != nullptr) {
1230     bb_cursor->InsertInstructionAfter(receiver_class, cursor);
1231   } else {
1232     bb_cursor->InsertInstructionBefore(receiver_class, bb_cursor->GetFirstInstruction());
1233   }
1234   bb_cursor->InsertInstructionAfter(class_table_get, receiver_class);
1235   bb_cursor->InsertInstructionAfter(compare, class_table_get);
1236 
1237   if (outermost_graph_->IsCompilingOsr()) {
1238     CreateDiamondPatternForPolymorphicInline(compare, return_replacement, invoke_instruction);
1239   } else {
1240     HDeoptimize* deoptimize = new (graph_->GetAllocator()) HDeoptimize(
1241         graph_->GetAllocator(),
1242         compare,
1243         receiver,
1244         DeoptimizationKind::kJitSameTarget,
1245         invoke_instruction->GetDexPc());
1246     bb_cursor->InsertInstructionAfter(deoptimize, compare);
1247     deoptimize->CopyEnvironmentFrom(invoke_instruction->GetEnvironment());
1248     if (return_replacement != nullptr) {
1249       invoke_instruction->ReplaceWith(return_replacement);
1250     }
1251     receiver->ReplaceUsesDominatedBy(deoptimize, deoptimize);
1252     invoke_instruction->GetBlock()->RemoveInstruction(invoke_instruction);
1253     deoptimize->SetReferenceTypeInfo(receiver->GetReferenceTypeInfo());
1254   }
1255 
1256   // Run type propagation to get the guard typed.
1257   ReferenceTypePropagation rtp_fixup(graph_,
1258                                      outer_compilation_unit_.GetClassLoader(),
1259                                      outer_compilation_unit_.GetDexCache(),
1260                                      handles_,
1261                                      /* is_first_run */ false);
1262   rtp_fixup.Run();
1263 
1264   MaybeRecordStat(stats_, MethodCompilationStat::kInlinedPolymorphicCall);
1265 
1266   LOG_SUCCESS() << "Inlined same polymorphic target " << actual_method->PrettyMethod();
1267   return true;
1268 }
1269 
TryInlineAndReplace(HInvoke * invoke_instruction,ArtMethod * method,ReferenceTypeInfo receiver_type,bool do_rtp,bool cha_devirtualize)1270 bool HInliner::TryInlineAndReplace(HInvoke* invoke_instruction,
1271                                    ArtMethod* method,
1272                                    ReferenceTypeInfo receiver_type,
1273                                    bool do_rtp,
1274                                    bool cha_devirtualize) {
1275   DCHECK(!invoke_instruction->IsIntrinsic());
1276   HInstruction* return_replacement = nullptr;
1277   uint32_t dex_pc = invoke_instruction->GetDexPc();
1278   HInstruction* cursor = invoke_instruction->GetPrevious();
1279   HBasicBlock* bb_cursor = invoke_instruction->GetBlock();
1280   bool should_remove_invoke_instruction = false;
1281 
1282   // If invoke_instruction is devirtualized to a different method, give intrinsics
1283   // another chance before we try to inline it.
1284   bool wrong_invoke_type = false;
1285   if (invoke_instruction->GetResolvedMethod() != method &&
1286       IntrinsicsRecognizer::Recognize(invoke_instruction, method, &wrong_invoke_type)) {
1287     MaybeRecordStat(stats_, MethodCompilationStat::kIntrinsicRecognized);
1288     if (invoke_instruction->IsInvokeInterface()) {
1289       // We don't intrinsify an invoke-interface directly.
1290       // Replace the invoke-interface with an invoke-virtual.
1291       HInvokeVirtual* new_invoke = new (graph_->GetAllocator()) HInvokeVirtual(
1292           graph_->GetAllocator(),
1293           invoke_instruction->GetNumberOfArguments(),
1294           invoke_instruction->GetType(),
1295           invoke_instruction->GetDexPc(),
1296           invoke_instruction->GetDexMethodIndex(),  // Use interface method's dex method index.
1297           method,
1298           method->GetMethodIndex());
1299       HInputsRef inputs = invoke_instruction->GetInputs();
1300       for (size_t index = 0; index != inputs.size(); ++index) {
1301         new_invoke->SetArgumentAt(index, inputs[index]);
1302       }
1303       invoke_instruction->GetBlock()->InsertInstructionBefore(new_invoke, invoke_instruction);
1304       new_invoke->CopyEnvironmentFrom(invoke_instruction->GetEnvironment());
1305       if (invoke_instruction->GetType() == DataType::Type::kReference) {
1306         new_invoke->SetReferenceTypeInfo(invoke_instruction->GetReferenceTypeInfo());
1307       }
1308       // Run intrinsic recognizer again to set new_invoke's intrinsic.
1309       IntrinsicsRecognizer::Recognize(new_invoke, method, &wrong_invoke_type);
1310       DCHECK_NE(new_invoke->GetIntrinsic(), Intrinsics::kNone);
1311       return_replacement = new_invoke;
1312       // invoke_instruction is replaced with new_invoke.
1313       should_remove_invoke_instruction = true;
1314     } else {
1315       // invoke_instruction is intrinsified and stays.
1316     }
1317   } else if (!TryBuildAndInline(invoke_instruction, method, receiver_type, &return_replacement)) {
1318     if (invoke_instruction->IsInvokeInterface()) {
1319       DCHECK(!method->IsProxyMethod());
1320       // Turn an invoke-interface into an invoke-virtual. An invoke-virtual is always
1321       // better than an invoke-interface because:
1322       // 1) In the best case, the interface call has one more indirection (to fetch the IMT).
1323       // 2) We will not go to the conflict trampoline with an invoke-virtual.
1324       // TODO: Consider sharpening once it is not dependent on the compiler driver.
1325 
1326       if (method->IsDefault() && !method->IsCopied()) {
1327         // Changing to invoke-virtual cannot be done on an original default method
1328         // since it's not in any vtable. Devirtualization by exact type/inline-cache
1329         // always uses a method in the iftable which is never an original default
1330         // method.
1331         // On the other hand, inlining an original default method by CHA is fine.
1332         DCHECK(cha_devirtualize);
1333         return false;
1334       }
1335 
1336       const DexFile& caller_dex_file = *caller_compilation_unit_.GetDexFile();
1337       uint32_t dex_method_index = FindMethodIndexIn(
1338           method, caller_dex_file, invoke_instruction->GetDexMethodIndex());
1339       if (dex_method_index == dex::kDexNoIndex) {
1340         return false;
1341       }
1342       HInvokeVirtual* new_invoke = new (graph_->GetAllocator()) HInvokeVirtual(
1343           graph_->GetAllocator(),
1344           invoke_instruction->GetNumberOfArguments(),
1345           invoke_instruction->GetType(),
1346           invoke_instruction->GetDexPc(),
1347           dex_method_index,
1348           method,
1349           method->GetMethodIndex());
1350       HInputsRef inputs = invoke_instruction->GetInputs();
1351       for (size_t index = 0; index != inputs.size(); ++index) {
1352         new_invoke->SetArgumentAt(index, inputs[index]);
1353       }
1354       invoke_instruction->GetBlock()->InsertInstructionBefore(new_invoke, invoke_instruction);
1355       new_invoke->CopyEnvironmentFrom(invoke_instruction->GetEnvironment());
1356       if (invoke_instruction->GetType() == DataType::Type::kReference) {
1357         new_invoke->SetReferenceTypeInfo(invoke_instruction->GetReferenceTypeInfo());
1358       }
1359       return_replacement = new_invoke;
1360       // invoke_instruction is replaced with new_invoke.
1361       should_remove_invoke_instruction = true;
1362     } else {
1363       // TODO: Consider sharpening an invoke virtual once it is not dependent on the
1364       // compiler driver.
1365       return false;
1366     }
1367   } else {
1368     // invoke_instruction is inlined.
1369     should_remove_invoke_instruction = true;
1370   }
1371 
1372   if (cha_devirtualize) {
1373     AddCHAGuard(invoke_instruction, dex_pc, cursor, bb_cursor);
1374   }
1375   if (return_replacement != nullptr) {
1376     invoke_instruction->ReplaceWith(return_replacement);
1377   }
1378   if (should_remove_invoke_instruction) {
1379     invoke_instruction->GetBlock()->RemoveInstruction(invoke_instruction);
1380   }
1381   FixUpReturnReferenceType(method, return_replacement);
1382   if (do_rtp && ReturnTypeMoreSpecific(invoke_instruction, return_replacement)) {
1383     // Actual return value has a more specific type than the method's declared
1384     // return type. Run RTP again on the outer graph to propagate it.
1385     ReferenceTypePropagation(graph_,
1386                              outer_compilation_unit_.GetClassLoader(),
1387                              outer_compilation_unit_.GetDexCache(),
1388                              handles_,
1389                              /* is_first_run */ false).Run();
1390   }
1391   return true;
1392 }
1393 
CountRecursiveCallsOf(ArtMethod * method) const1394 size_t HInliner::CountRecursiveCallsOf(ArtMethod* method) const {
1395   const HInliner* current = this;
1396   size_t count = 0;
1397   do {
1398     if (current->graph_->GetArtMethod() == method) {
1399       ++count;
1400     }
1401     current = current->parent_;
1402   } while (current != nullptr);
1403   return count;
1404 }
1405 
TryBuildAndInline(HInvoke * invoke_instruction,ArtMethod * method,ReferenceTypeInfo receiver_type,HInstruction ** return_replacement)1406 bool HInliner::TryBuildAndInline(HInvoke* invoke_instruction,
1407                                  ArtMethod* method,
1408                                  ReferenceTypeInfo receiver_type,
1409                                  HInstruction** return_replacement) {
1410   if (method->IsProxyMethod()) {
1411     LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedProxy)
1412         << "Method " << method->PrettyMethod()
1413         << " is not inlined because of unimplemented inline support for proxy methods.";
1414     return false;
1415   }
1416 
1417   if (CountRecursiveCallsOf(method) > kMaximumNumberOfRecursiveCalls) {
1418     LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedRecursiveBudget)
1419         << "Method "
1420         << method->PrettyMethod()
1421         << " is not inlined because it has reached its recursive call budget.";
1422     return false;
1423   }
1424 
1425   // Check whether we're allowed to inline. The outermost compilation unit is the relevant
1426   // dex file here (though the transitivity of an inline chain would allow checking the calller).
1427   if (!compiler_driver_->MayInline(method->GetDexFile(),
1428                                    outer_compilation_unit_.GetDexFile())) {
1429     if (TryPatternSubstitution(invoke_instruction, method, return_replacement)) {
1430       LOG_SUCCESS() << "Successfully replaced pattern of invoke "
1431                     << method->PrettyMethod();
1432       MaybeRecordStat(stats_, MethodCompilationStat::kReplacedInvokeWithSimplePattern);
1433       return true;
1434     }
1435     LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedWont)
1436         << "Won't inline " << method->PrettyMethod() << " in "
1437         << outer_compilation_unit_.GetDexFile()->GetLocation() << " ("
1438         << caller_compilation_unit_.GetDexFile()->GetLocation() << ") from "
1439         << method->GetDexFile()->GetLocation();
1440     return false;
1441   }
1442 
1443   bool same_dex_file = IsSameDexFile(*outer_compilation_unit_.GetDexFile(), *method->GetDexFile());
1444 
1445   CodeItemDataAccessor accessor(method->DexInstructionData());
1446 
1447   if (!accessor.HasCodeItem()) {
1448     LOG_FAIL_NO_STAT()
1449         << "Method " << method->PrettyMethod() << " is not inlined because it is native";
1450     return false;
1451   }
1452 
1453   size_t inline_max_code_units = compiler_driver_->GetCompilerOptions().GetInlineMaxCodeUnits();
1454   if (accessor.InsnsSizeInCodeUnits() > inline_max_code_units) {
1455     LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedCodeItem)
1456         << "Method " << method->PrettyMethod()
1457         << " is not inlined because its code item is too big: "
1458         << accessor.InsnsSizeInCodeUnits()
1459         << " > "
1460         << inline_max_code_units;
1461     return false;
1462   }
1463 
1464   if (accessor.TriesSize() != 0) {
1465     LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedTryCatch)
1466         << "Method " << method->PrettyMethod() << " is not inlined because of try block";
1467     return false;
1468   }
1469 
1470   if (!method->IsCompilable()) {
1471     LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedNotVerified)
1472         << "Method " << method->PrettyMethod()
1473         << " has soft failures un-handled by the compiler, so it cannot be inlined";
1474     return false;
1475   }
1476 
1477   if (IsMethodUnverified(compiler_driver_, method)) {
1478     LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedNotVerified)
1479         << "Method " << method->PrettyMethod()
1480         << " couldn't be verified, so it cannot be inlined";
1481     return false;
1482   }
1483 
1484   if (invoke_instruction->IsInvokeStaticOrDirect() &&
1485       invoke_instruction->AsInvokeStaticOrDirect()->IsStaticWithImplicitClinitCheck()) {
1486     // Case of a static method that cannot be inlined because it implicitly
1487     // requires an initialization check of its declaring class.
1488     LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedDexCache)
1489         << "Method " << method->PrettyMethod()
1490         << " is not inlined because it is static and requires a clinit"
1491         << " check that cannot be emitted due to Dex cache limitations";
1492     return false;
1493   }
1494 
1495   if (!TryBuildAndInlineHelper(
1496           invoke_instruction, method, receiver_type, same_dex_file, return_replacement)) {
1497     return false;
1498   }
1499 
1500   LOG_SUCCESS() << method->PrettyMethod();
1501   MaybeRecordStat(stats_, MethodCompilationStat::kInlinedInvoke);
1502   return true;
1503 }
1504 
GetInvokeInputForArgVRegIndex(HInvoke * invoke_instruction,size_t arg_vreg_index)1505 static HInstruction* GetInvokeInputForArgVRegIndex(HInvoke* invoke_instruction,
1506                                                    size_t arg_vreg_index)
1507     REQUIRES_SHARED(Locks::mutator_lock_) {
1508   size_t input_index = 0;
1509   for (size_t i = 0; i < arg_vreg_index; ++i, ++input_index) {
1510     DCHECK_LT(input_index, invoke_instruction->GetNumberOfArguments());
1511     if (DataType::Is64BitType(invoke_instruction->InputAt(input_index)->GetType())) {
1512       ++i;
1513       DCHECK_NE(i, arg_vreg_index);
1514     }
1515   }
1516   DCHECK_LT(input_index, invoke_instruction->GetNumberOfArguments());
1517   return invoke_instruction->InputAt(input_index);
1518 }
1519 
1520 // Try to recognize known simple patterns and replace invoke call with appropriate instructions.
TryPatternSubstitution(HInvoke * invoke_instruction,ArtMethod * resolved_method,HInstruction ** return_replacement)1521 bool HInliner::TryPatternSubstitution(HInvoke* invoke_instruction,
1522                                       ArtMethod* resolved_method,
1523                                       HInstruction** return_replacement) {
1524   InlineMethod inline_method;
1525   if (!InlineMethodAnalyser::AnalyseMethodCode(resolved_method, &inline_method)) {
1526     return false;
1527   }
1528 
1529   switch (inline_method.opcode) {
1530     case kInlineOpNop:
1531       DCHECK_EQ(invoke_instruction->GetType(), DataType::Type::kVoid);
1532       *return_replacement = nullptr;
1533       break;
1534     case kInlineOpReturnArg:
1535       *return_replacement = GetInvokeInputForArgVRegIndex(invoke_instruction,
1536                                                           inline_method.d.return_data.arg);
1537       break;
1538     case kInlineOpNonWideConst:
1539       if (resolved_method->GetShorty()[0] == 'L') {
1540         DCHECK_EQ(inline_method.d.data, 0u);
1541         *return_replacement = graph_->GetNullConstant();
1542       } else {
1543         *return_replacement = graph_->GetIntConstant(static_cast<int32_t>(inline_method.d.data));
1544       }
1545       break;
1546     case kInlineOpIGet: {
1547       const InlineIGetIPutData& data = inline_method.d.ifield_data;
1548       if (data.method_is_static || data.object_arg != 0u) {
1549         // TODO: Needs null check.
1550         return false;
1551       }
1552       HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, data.object_arg);
1553       HInstanceFieldGet* iget = CreateInstanceFieldGet(data.field_idx, resolved_method, obj);
1554       DCHECK_EQ(iget->GetFieldOffset().Uint32Value(), data.field_offset);
1555       DCHECK_EQ(iget->IsVolatile() ? 1u : 0u, data.is_volatile);
1556       invoke_instruction->GetBlock()->InsertInstructionBefore(iget, invoke_instruction);
1557       *return_replacement = iget;
1558       break;
1559     }
1560     case kInlineOpIPut: {
1561       const InlineIGetIPutData& data = inline_method.d.ifield_data;
1562       if (data.method_is_static || data.object_arg != 0u) {
1563         // TODO: Needs null check.
1564         return false;
1565       }
1566       HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, data.object_arg);
1567       HInstruction* value = GetInvokeInputForArgVRegIndex(invoke_instruction, data.src_arg);
1568       HInstanceFieldSet* iput = CreateInstanceFieldSet(data.field_idx, resolved_method, obj, value);
1569       DCHECK_EQ(iput->GetFieldOffset().Uint32Value(), data.field_offset);
1570       DCHECK_EQ(iput->IsVolatile() ? 1u : 0u, data.is_volatile);
1571       invoke_instruction->GetBlock()->InsertInstructionBefore(iput, invoke_instruction);
1572       if (data.return_arg_plus1 != 0u) {
1573         size_t return_arg = data.return_arg_plus1 - 1u;
1574         *return_replacement = GetInvokeInputForArgVRegIndex(invoke_instruction, return_arg);
1575       }
1576       break;
1577     }
1578     case kInlineOpConstructor: {
1579       const InlineConstructorData& data = inline_method.d.constructor_data;
1580       // Get the indexes to arrays for easier processing.
1581       uint16_t iput_field_indexes[] = {
1582           data.iput0_field_index, data.iput1_field_index, data.iput2_field_index
1583       };
1584       uint16_t iput_args[] = { data.iput0_arg, data.iput1_arg, data.iput2_arg };
1585       static_assert(arraysize(iput_args) == arraysize(iput_field_indexes), "Size mismatch");
1586       // Count valid field indexes.
1587       size_t number_of_iputs = 0u;
1588       while (number_of_iputs != arraysize(iput_field_indexes) &&
1589           iput_field_indexes[number_of_iputs] != DexFile::kDexNoIndex16) {
1590         // Check that there are no duplicate valid field indexes.
1591         DCHECK_EQ(0, std::count(iput_field_indexes + number_of_iputs + 1,
1592                                 iput_field_indexes + arraysize(iput_field_indexes),
1593                                 iput_field_indexes[number_of_iputs]));
1594         ++number_of_iputs;
1595       }
1596       // Check that there are no valid field indexes in the rest of the array.
1597       DCHECK_EQ(0, std::count_if(iput_field_indexes + number_of_iputs,
1598                                  iput_field_indexes + arraysize(iput_field_indexes),
1599                                  [](uint16_t index) { return index != DexFile::kDexNoIndex16; }));
1600 
1601       // Create HInstanceFieldSet for each IPUT that stores non-zero data.
1602       HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, /* this */ 0u);
1603       bool needs_constructor_barrier = false;
1604       for (size_t i = 0; i != number_of_iputs; ++i) {
1605         HInstruction* value = GetInvokeInputForArgVRegIndex(invoke_instruction, iput_args[i]);
1606         if (!value->IsConstant() || !value->AsConstant()->IsZeroBitPattern()) {
1607           uint16_t field_index = iput_field_indexes[i];
1608           bool is_final;
1609           HInstanceFieldSet* iput =
1610               CreateInstanceFieldSet(field_index, resolved_method, obj, value, &is_final);
1611           invoke_instruction->GetBlock()->InsertInstructionBefore(iput, invoke_instruction);
1612 
1613           // Check whether the field is final. If it is, we need to add a barrier.
1614           if (is_final) {
1615             needs_constructor_barrier = true;
1616           }
1617         }
1618       }
1619       if (needs_constructor_barrier) {
1620         // See CompilerDriver::RequiresConstructorBarrier for more details.
1621         DCHECK(obj != nullptr) << "only non-static methods can have a constructor fence";
1622 
1623         HConstructorFence* constructor_fence =
1624             new (graph_->GetAllocator()) HConstructorFence(obj, kNoDexPc, graph_->GetAllocator());
1625         invoke_instruction->GetBlock()->InsertInstructionBefore(constructor_fence,
1626                                                                 invoke_instruction);
1627       }
1628       *return_replacement = nullptr;
1629       break;
1630     }
1631     default:
1632       LOG(FATAL) << "UNREACHABLE";
1633       UNREACHABLE();
1634   }
1635   return true;
1636 }
1637 
CreateInstanceFieldGet(uint32_t field_index,ArtMethod * referrer,HInstruction * obj)1638 HInstanceFieldGet* HInliner::CreateInstanceFieldGet(uint32_t field_index,
1639                                                     ArtMethod* referrer,
1640                                                     HInstruction* obj)
1641     REQUIRES_SHARED(Locks::mutator_lock_) {
1642   ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
1643   ArtField* resolved_field =
1644       class_linker->LookupResolvedField(field_index, referrer, /* is_static */ false);
1645   DCHECK(resolved_field != nullptr);
1646   HInstanceFieldGet* iget = new (graph_->GetAllocator()) HInstanceFieldGet(
1647       obj,
1648       resolved_field,
1649       DataType::FromShorty(resolved_field->GetTypeDescriptor()[0]),
1650       resolved_field->GetOffset(),
1651       resolved_field->IsVolatile(),
1652       field_index,
1653       resolved_field->GetDeclaringClass()->GetDexClassDefIndex(),
1654       *referrer->GetDexFile(),
1655       // Read barrier generates a runtime call in slow path and we need a valid
1656       // dex pc for the associated stack map. 0 is bogus but valid. Bug: 26854537.
1657       /* dex_pc */ 0);
1658   if (iget->GetType() == DataType::Type::kReference) {
1659     // Use the same dex_cache that we used for field lookup as the hint_dex_cache.
1660     Handle<mirror::DexCache> dex_cache = handles_->NewHandle(referrer->GetDexCache());
1661     ReferenceTypePropagation rtp(graph_,
1662                                  outer_compilation_unit_.GetClassLoader(),
1663                                  dex_cache,
1664                                  handles_,
1665                                  /* is_first_run */ false);
1666     rtp.Visit(iget);
1667   }
1668   return iget;
1669 }
1670 
CreateInstanceFieldSet(uint32_t field_index,ArtMethod * referrer,HInstruction * obj,HInstruction * value,bool * is_final)1671 HInstanceFieldSet* HInliner::CreateInstanceFieldSet(uint32_t field_index,
1672                                                     ArtMethod* referrer,
1673                                                     HInstruction* obj,
1674                                                     HInstruction* value,
1675                                                     bool* is_final)
1676     REQUIRES_SHARED(Locks::mutator_lock_) {
1677   ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
1678   ArtField* resolved_field =
1679       class_linker->LookupResolvedField(field_index, referrer, /* is_static */ false);
1680   DCHECK(resolved_field != nullptr);
1681   if (is_final != nullptr) {
1682     // This information is needed only for constructors.
1683     DCHECK(referrer->IsConstructor());
1684     *is_final = resolved_field->IsFinal();
1685   }
1686   HInstanceFieldSet* iput = new (graph_->GetAllocator()) HInstanceFieldSet(
1687       obj,
1688       value,
1689       resolved_field,
1690       DataType::FromShorty(resolved_field->GetTypeDescriptor()[0]),
1691       resolved_field->GetOffset(),
1692       resolved_field->IsVolatile(),
1693       field_index,
1694       resolved_field->GetDeclaringClass()->GetDexClassDefIndex(),
1695       *referrer->GetDexFile(),
1696       // Read barrier generates a runtime call in slow path and we need a valid
1697       // dex pc for the associated stack map. 0 is bogus but valid. Bug: 26854537.
1698       /* dex_pc */ 0);
1699   return iput;
1700 }
1701 
1702 template <typename T>
NewHandleIfDifferent(T * object,Handle<T> hint,VariableSizedHandleScope * handles)1703 static inline Handle<T> NewHandleIfDifferent(T* object,
1704                                              Handle<T> hint,
1705                                              VariableSizedHandleScope* handles)
1706     REQUIRES_SHARED(Locks::mutator_lock_) {
1707   return (object != hint.Get()) ? handles->NewHandle(object) : hint;
1708 }
1709 
TryBuildAndInlineHelper(HInvoke * invoke_instruction,ArtMethod * resolved_method,ReferenceTypeInfo receiver_type,bool same_dex_file,HInstruction ** return_replacement)1710 bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction,
1711                                        ArtMethod* resolved_method,
1712                                        ReferenceTypeInfo receiver_type,
1713                                        bool same_dex_file,
1714                                        HInstruction** return_replacement) {
1715   DCHECK(!(resolved_method->IsStatic() && receiver_type.IsValid()));
1716   ScopedObjectAccess soa(Thread::Current());
1717   const DexFile::CodeItem* code_item = resolved_method->GetCodeItem();
1718   const DexFile& callee_dex_file = *resolved_method->GetDexFile();
1719   uint32_t method_index = resolved_method->GetDexMethodIndex();
1720   CodeItemDebugInfoAccessor code_item_accessor(resolved_method->DexInstructionDebugInfo());
1721   ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker();
1722   Handle<mirror::DexCache> dex_cache = NewHandleIfDifferent(resolved_method->GetDexCache(),
1723                                                             caller_compilation_unit_.GetDexCache(),
1724                                                             handles_);
1725   Handle<mirror::ClassLoader> class_loader =
1726       NewHandleIfDifferent(resolved_method->GetDeclaringClass()->GetClassLoader(),
1727                            caller_compilation_unit_.GetClassLoader(),
1728                            handles_);
1729 
1730   DexCompilationUnit dex_compilation_unit(
1731       class_loader,
1732       class_linker,
1733       callee_dex_file,
1734       code_item,
1735       resolved_method->GetDeclaringClass()->GetDexClassDefIndex(),
1736       method_index,
1737       resolved_method->GetAccessFlags(),
1738       /* verified_method */ nullptr,
1739       dex_cache);
1740 
1741   InvokeType invoke_type = invoke_instruction->GetInvokeType();
1742   if (invoke_type == kInterface) {
1743     // We have statically resolved the dispatch. To please the class linker
1744     // at runtime, we change this call as if it was a virtual call.
1745     invoke_type = kVirtual;
1746   }
1747 
1748   const int32_t caller_instruction_counter = graph_->GetCurrentInstructionId();
1749   HGraph* callee_graph = new (graph_->GetAllocator()) HGraph(
1750       graph_->GetAllocator(),
1751       graph_->GetArenaStack(),
1752       callee_dex_file,
1753       method_index,
1754       compiler_driver_->GetInstructionSet(),
1755       invoke_type,
1756       graph_->IsDebuggable(),
1757       /* osr */ false,
1758       caller_instruction_counter);
1759   callee_graph->SetArtMethod(resolved_method);
1760 
1761   // When they are needed, allocate `inline_stats_` on the Arena instead
1762   // of on the stack, as Clang might produce a stack frame too large
1763   // for this function, that would not fit the requirements of the
1764   // `-Wframe-larger-than` option.
1765   if (stats_ != nullptr) {
1766     // Reuse one object for all inline attempts from this caller to keep Arena memory usage low.
1767     if (inline_stats_ == nullptr) {
1768       void* storage = graph_->GetAllocator()->Alloc<OptimizingCompilerStats>(kArenaAllocMisc);
1769       inline_stats_ = new (storage) OptimizingCompilerStats;
1770     } else {
1771       inline_stats_->Reset();
1772     }
1773   }
1774   HGraphBuilder builder(callee_graph,
1775                         code_item_accessor,
1776                         &dex_compilation_unit,
1777                         &outer_compilation_unit_,
1778                         compiler_driver_,
1779                         codegen_,
1780                         inline_stats_,
1781                         resolved_method->GetQuickenedInfo(),
1782                         handles_);
1783 
1784   if (builder.BuildGraph() != kAnalysisSuccess) {
1785     LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedCannotBuild)
1786         << "Method " << callee_dex_file.PrettyMethod(method_index)
1787         << " could not be built, so cannot be inlined";
1788     return false;
1789   }
1790 
1791   if (!RegisterAllocator::CanAllocateRegistersFor(*callee_graph,
1792                                                   compiler_driver_->GetInstructionSet())) {
1793     LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedRegisterAllocator)
1794         << "Method " << callee_dex_file.PrettyMethod(method_index)
1795         << " cannot be inlined because of the register allocator";
1796     return false;
1797   }
1798 
1799   size_t parameter_index = 0;
1800   bool run_rtp = false;
1801   for (HInstructionIterator instructions(callee_graph->GetEntryBlock()->GetInstructions());
1802        !instructions.Done();
1803        instructions.Advance()) {
1804     HInstruction* current = instructions.Current();
1805     if (current->IsParameterValue()) {
1806       HInstruction* argument = invoke_instruction->InputAt(parameter_index);
1807       if (argument->IsNullConstant()) {
1808         current->ReplaceWith(callee_graph->GetNullConstant());
1809       } else if (argument->IsIntConstant()) {
1810         current->ReplaceWith(callee_graph->GetIntConstant(argument->AsIntConstant()->GetValue()));
1811       } else if (argument->IsLongConstant()) {
1812         current->ReplaceWith(callee_graph->GetLongConstant(argument->AsLongConstant()->GetValue()));
1813       } else if (argument->IsFloatConstant()) {
1814         current->ReplaceWith(
1815             callee_graph->GetFloatConstant(argument->AsFloatConstant()->GetValue()));
1816       } else if (argument->IsDoubleConstant()) {
1817         current->ReplaceWith(
1818             callee_graph->GetDoubleConstant(argument->AsDoubleConstant()->GetValue()));
1819       } else if (argument->GetType() == DataType::Type::kReference) {
1820         if (!resolved_method->IsStatic() && parameter_index == 0 && receiver_type.IsValid()) {
1821           run_rtp = true;
1822           current->SetReferenceTypeInfo(receiver_type);
1823         } else {
1824           current->SetReferenceTypeInfo(argument->GetReferenceTypeInfo());
1825         }
1826         current->AsParameterValue()->SetCanBeNull(argument->CanBeNull());
1827       }
1828       ++parameter_index;
1829     }
1830   }
1831 
1832   // We have replaced formal arguments with actual arguments. If actual types
1833   // are more specific than the declared ones, run RTP again on the inner graph.
1834   if (run_rtp || ArgumentTypesMoreSpecific(invoke_instruction, resolved_method)) {
1835     ReferenceTypePropagation(callee_graph,
1836                              outer_compilation_unit_.GetClassLoader(),
1837                              dex_compilation_unit.GetDexCache(),
1838                              handles_,
1839                              /* is_first_run */ false).Run();
1840   }
1841 
1842   RunOptimizations(callee_graph, code_item, dex_compilation_unit);
1843 
1844   HBasicBlock* exit_block = callee_graph->GetExitBlock();
1845   if (exit_block == nullptr) {
1846     LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedInfiniteLoop)
1847         << "Method " << callee_dex_file.PrettyMethod(method_index)
1848         << " could not be inlined because it has an infinite loop";
1849     return false;
1850   }
1851 
1852   bool has_one_return = false;
1853   for (HBasicBlock* predecessor : exit_block->GetPredecessors()) {
1854     if (predecessor->GetLastInstruction()->IsThrow()) {
1855       if (invoke_instruction->GetBlock()->IsTryBlock()) {
1856         // TODO(ngeoffray): Support adding HTryBoundary in Hgraph::InlineInto.
1857         LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedTryCatch)
1858             << "Method " << callee_dex_file.PrettyMethod(method_index)
1859             << " could not be inlined because one branch always throws and"
1860             << " caller is in a try/catch block";
1861         return false;
1862       } else if (graph_->GetExitBlock() == nullptr) {
1863         // TODO(ngeoffray): Support adding HExit in the caller graph.
1864         LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedInfiniteLoop)
1865             << "Method " << callee_dex_file.PrettyMethod(method_index)
1866             << " could not be inlined because one branch always throws and"
1867             << " caller does not have an exit block";
1868         return false;
1869       } else if (graph_->HasIrreducibleLoops()) {
1870         // TODO(ngeoffray): Support re-computing loop information to graphs with
1871         // irreducible loops?
1872         VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index)
1873                        << " could not be inlined because one branch always throws and"
1874                        << " caller has irreducible loops";
1875         return false;
1876       }
1877     } else {
1878       has_one_return = true;
1879     }
1880   }
1881 
1882   if (!has_one_return) {
1883     LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedAlwaysThrows)
1884         << "Method " << callee_dex_file.PrettyMethod(method_index)
1885         << " could not be inlined because it always throws";
1886     return false;
1887   }
1888 
1889   size_t number_of_instructions = 0;
1890   // Skip the entry block, it does not contain instructions that prevent inlining.
1891   for (HBasicBlock* block : callee_graph->GetReversePostOrderSkipEntryBlock()) {
1892     if (block->IsLoopHeader()) {
1893       if (block->GetLoopInformation()->IsIrreducible()) {
1894         // Don't inline methods with irreducible loops, they could prevent some
1895         // optimizations to run.
1896         LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedIrreducibleLoop)
1897             << "Method " << callee_dex_file.PrettyMethod(method_index)
1898             << " could not be inlined because it contains an irreducible loop";
1899         return false;
1900       }
1901       if (!block->GetLoopInformation()->HasExitEdge()) {
1902         // Don't inline methods with loops without exit, since they cause the
1903         // loop information to be computed incorrectly when updating after
1904         // inlining.
1905         LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedLoopWithoutExit)
1906             << "Method " << callee_dex_file.PrettyMethod(method_index)
1907             << " could not be inlined because it contains a loop with no exit";
1908         return false;
1909       }
1910     }
1911 
1912     for (HInstructionIterator instr_it(block->GetInstructions());
1913          !instr_it.Done();
1914          instr_it.Advance()) {
1915       if (++number_of_instructions >= inlining_budget_) {
1916         LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedInstructionBudget)
1917             << "Method " << callee_dex_file.PrettyMethod(method_index)
1918             << " is not inlined because the outer method has reached"
1919             << " its instruction budget limit.";
1920         return false;
1921       }
1922       HInstruction* current = instr_it.Current();
1923       if (current->NeedsEnvironment() &&
1924           (total_number_of_dex_registers_ >= kMaximumNumberOfCumulatedDexRegisters)) {
1925         LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedEnvironmentBudget)
1926             << "Method " << callee_dex_file.PrettyMethod(method_index)
1927             << " is not inlined because its caller has reached"
1928             << " its environment budget limit.";
1929         return false;
1930       }
1931 
1932       if (current->NeedsEnvironment() &&
1933           !CanEncodeInlinedMethodInStackMap(*caller_compilation_unit_.GetDexFile(),
1934                                             resolved_method)) {
1935         LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedStackMaps)
1936             << "Method " << callee_dex_file.PrettyMethod(method_index)
1937             << " could not be inlined because " << current->DebugName()
1938             << " needs an environment, is in a different dex file"
1939             << ", and cannot be encoded in the stack maps.";
1940         return false;
1941       }
1942 
1943       if (!same_dex_file && current->NeedsDexCacheOfDeclaringClass()) {
1944         LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedDexCache)
1945             << "Method " << callee_dex_file.PrettyMethod(method_index)
1946             << " could not be inlined because " << current->DebugName()
1947             << " it is in a different dex file and requires access to the dex cache";
1948         return false;
1949       }
1950 
1951       if (current->IsUnresolvedStaticFieldGet() ||
1952           current->IsUnresolvedInstanceFieldGet() ||
1953           current->IsUnresolvedStaticFieldSet() ||
1954           current->IsUnresolvedInstanceFieldSet()) {
1955         // Entrypoint for unresolved fields does not handle inlined frames.
1956         LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedUnresolvedEntrypoint)
1957             << "Method " << callee_dex_file.PrettyMethod(method_index)
1958             << " could not be inlined because it is using an unresolved"
1959             << " entrypoint";
1960         return false;
1961       }
1962     }
1963   }
1964   DCHECK_EQ(caller_instruction_counter, graph_->GetCurrentInstructionId())
1965       << "No instructions can be added to the outer graph while inner graph is being built";
1966 
1967   // Inline the callee graph inside the caller graph.
1968   const int32_t callee_instruction_counter = callee_graph->GetCurrentInstructionId();
1969   graph_->SetCurrentInstructionId(callee_instruction_counter);
1970   *return_replacement = callee_graph->InlineInto(graph_, invoke_instruction);
1971   // Update our budget for other inlining attempts in `caller_graph`.
1972   total_number_of_instructions_ += number_of_instructions;
1973   UpdateInliningBudget();
1974 
1975   DCHECK_EQ(callee_instruction_counter, callee_graph->GetCurrentInstructionId())
1976       << "No instructions can be added to the inner graph during inlining into the outer graph";
1977 
1978   if (stats_ != nullptr) {
1979     DCHECK(inline_stats_ != nullptr);
1980     inline_stats_->AddTo(stats_);
1981   }
1982 
1983   return true;
1984 }
1985 
RunOptimizations(HGraph * callee_graph,const DexFile::CodeItem * code_item,const DexCompilationUnit & dex_compilation_unit)1986 void HInliner::RunOptimizations(HGraph* callee_graph,
1987                                 const DexFile::CodeItem* code_item,
1988                                 const DexCompilationUnit& dex_compilation_unit) {
1989   // Note: if the outermost_graph_ is being compiled OSR, we should not run any
1990   // optimization that could lead to a HDeoptimize. The following optimizations do not.
1991   HDeadCodeElimination dce(callee_graph, inline_stats_, "dead_code_elimination$inliner");
1992   HConstantFolding fold(callee_graph, "constant_folding$inliner");
1993   HSharpening sharpening(callee_graph, codegen_, compiler_driver_);
1994   InstructionSimplifier simplify(callee_graph, codegen_, compiler_driver_, inline_stats_);
1995   IntrinsicsRecognizer intrinsics(callee_graph, inline_stats_);
1996 
1997   HOptimization* optimizations[] = {
1998     &intrinsics,
1999     &sharpening,
2000     &simplify,
2001     &fold,
2002     &dce,
2003   };
2004 
2005   for (size_t i = 0; i < arraysize(optimizations); ++i) {
2006     HOptimization* optimization = optimizations[i];
2007     optimization->Run();
2008   }
2009 
2010   // Bail early for pathological cases on the environment (for example recursive calls,
2011   // or too large environment).
2012   if (total_number_of_dex_registers_ >= kMaximumNumberOfCumulatedDexRegisters) {
2013     LOG_NOTE() << "Calls in " << callee_graph->GetArtMethod()->PrettyMethod()
2014              << " will not be inlined because the outer method has reached"
2015              << " its environment budget limit.";
2016     return;
2017   }
2018 
2019   // Bail early if we know we already are over the limit.
2020   size_t number_of_instructions = CountNumberOfInstructions(callee_graph);
2021   if (number_of_instructions > inlining_budget_) {
2022     LOG_NOTE() << "Calls in " << callee_graph->GetArtMethod()->PrettyMethod()
2023              << " will not be inlined because the outer method has reached"
2024              << " its instruction budget limit. " << number_of_instructions;
2025     return;
2026   }
2027 
2028   CodeItemDataAccessor accessor(callee_graph->GetDexFile(), code_item);
2029   HInliner inliner(callee_graph,
2030                    outermost_graph_,
2031                    codegen_,
2032                    outer_compilation_unit_,
2033                    dex_compilation_unit,
2034                    compiler_driver_,
2035                    handles_,
2036                    inline_stats_,
2037                    total_number_of_dex_registers_ + accessor.RegistersSize(),
2038                    total_number_of_instructions_ + number_of_instructions,
2039                    this,
2040                    depth_ + 1);
2041   inliner.Run();
2042 }
2043 
IsReferenceTypeRefinement(ReferenceTypeInfo declared_rti,bool declared_can_be_null,HInstruction * actual_obj)2044 static bool IsReferenceTypeRefinement(ReferenceTypeInfo declared_rti,
2045                                       bool declared_can_be_null,
2046                                       HInstruction* actual_obj)
2047     REQUIRES_SHARED(Locks::mutator_lock_) {
2048   if (declared_can_be_null && !actual_obj->CanBeNull()) {
2049     return true;
2050   }
2051 
2052   ReferenceTypeInfo actual_rti = actual_obj->GetReferenceTypeInfo();
2053   return (actual_rti.IsExact() && !declared_rti.IsExact()) ||
2054           declared_rti.IsStrictSupertypeOf(actual_rti);
2055 }
2056 
GetClassRTI(ObjPtr<mirror::Class> klass)2057 ReferenceTypeInfo HInliner::GetClassRTI(ObjPtr<mirror::Class> klass) {
2058   return ReferenceTypePropagation::IsAdmissible(klass)
2059       ? ReferenceTypeInfo::Create(handles_->NewHandle(klass))
2060       : graph_->GetInexactObjectRti();
2061 }
2062 
ArgumentTypesMoreSpecific(HInvoke * invoke_instruction,ArtMethod * resolved_method)2063 bool HInliner::ArgumentTypesMoreSpecific(HInvoke* invoke_instruction, ArtMethod* resolved_method) {
2064   // If this is an instance call, test whether the type of the `this` argument
2065   // is more specific than the class which declares the method.
2066   if (!resolved_method->IsStatic()) {
2067     if (IsReferenceTypeRefinement(GetClassRTI(resolved_method->GetDeclaringClass()),
2068                                   /* declared_can_be_null */ false,
2069                                   invoke_instruction->InputAt(0u))) {
2070       return true;
2071     }
2072   }
2073 
2074   // Iterate over the list of parameter types and test whether any of the
2075   // actual inputs has a more specific reference type than the type declared in
2076   // the signature.
2077   const DexFile::TypeList* param_list = resolved_method->GetParameterTypeList();
2078   for (size_t param_idx = 0,
2079               input_idx = resolved_method->IsStatic() ? 0 : 1,
2080               e = (param_list == nullptr ? 0 : param_list->Size());
2081        param_idx < e;
2082        ++param_idx, ++input_idx) {
2083     HInstruction* input = invoke_instruction->InputAt(input_idx);
2084     if (input->GetType() == DataType::Type::kReference) {
2085       ObjPtr<mirror::Class> param_cls = resolved_method->LookupResolvedClassFromTypeIndex(
2086           param_list->GetTypeItem(param_idx).type_idx_);
2087       if (IsReferenceTypeRefinement(GetClassRTI(param_cls),
2088                                     /* declared_can_be_null */ true,
2089                                     input)) {
2090         return true;
2091       }
2092     }
2093   }
2094 
2095   return false;
2096 }
2097 
ReturnTypeMoreSpecific(HInvoke * invoke_instruction,HInstruction * return_replacement)2098 bool HInliner::ReturnTypeMoreSpecific(HInvoke* invoke_instruction,
2099                                       HInstruction* return_replacement) {
2100   // Check the integrity of reference types and run another type propagation if needed.
2101   if (return_replacement != nullptr) {
2102     if (return_replacement->GetType() == DataType::Type::kReference) {
2103       // Test if the return type is a refinement of the declared return type.
2104       if (IsReferenceTypeRefinement(invoke_instruction->GetReferenceTypeInfo(),
2105                                     /* declared_can_be_null */ true,
2106                                     return_replacement)) {
2107         return true;
2108       } else if (return_replacement->IsInstanceFieldGet()) {
2109         HInstanceFieldGet* field_get = return_replacement->AsInstanceFieldGet();
2110         ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
2111         if (field_get->GetFieldInfo().GetField() ==
2112               class_linker->GetClassRoot(ClassLinker::kJavaLangObject)->GetInstanceField(0)) {
2113           return true;
2114         }
2115       }
2116     } else if (return_replacement->IsInstanceOf()) {
2117       // Inlining InstanceOf into an If may put a tighter bound on reference types.
2118       return true;
2119     }
2120   }
2121 
2122   return false;
2123 }
2124 
FixUpReturnReferenceType(ArtMethod * resolved_method,HInstruction * return_replacement)2125 void HInliner::FixUpReturnReferenceType(ArtMethod* resolved_method,
2126                                         HInstruction* return_replacement) {
2127   if (return_replacement != nullptr) {
2128     if (return_replacement->GetType() == DataType::Type::kReference) {
2129       if (!return_replacement->GetReferenceTypeInfo().IsValid()) {
2130         // Make sure that we have a valid type for the return. We may get an invalid one when
2131         // we inline invokes with multiple branches and create a Phi for the result.
2132         // TODO: we could be more precise by merging the phi inputs but that requires
2133         // some functionality from the reference type propagation.
2134         DCHECK(return_replacement->IsPhi());
2135         ObjPtr<mirror::Class> cls = resolved_method->LookupResolvedReturnType();
2136         return_replacement->SetReferenceTypeInfo(GetClassRTI(cls));
2137       }
2138     }
2139   }
2140 }
2141 
2142 }  // namespace art
2143