1 /*
2  * Copyright (C) 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "code_generator.h"
18 
19 #ifdef ART_ENABLE_CODEGEN_arm
20 #include "code_generator_arm.h"
21 #endif
22 
23 #ifdef ART_ENABLE_CODEGEN_arm64
24 #include "code_generator_arm64.h"
25 #endif
26 
27 #ifdef ART_ENABLE_CODEGEN_x86
28 #include "code_generator_x86.h"
29 #endif
30 
31 #ifdef ART_ENABLE_CODEGEN_x86_64
32 #include "code_generator_x86_64.h"
33 #endif
34 
35 #ifdef ART_ENABLE_CODEGEN_mips
36 #include "code_generator_mips.h"
37 #endif
38 
39 #ifdef ART_ENABLE_CODEGEN_mips64
40 #include "code_generator_mips64.h"
41 #endif
42 
43 #include "bytecode_utils.h"
44 #include "compiled_method.h"
45 #include "dex/verified_method.h"
46 #include "driver/compiler_driver.h"
47 #include "graph_visualizer.h"
48 #include "intrinsics.h"
49 #include "leb128.h"
50 #include "mirror/array-inl.h"
51 #include "mirror/object_array-inl.h"
52 #include "mirror/object_reference.h"
53 #include "parallel_move_resolver.h"
54 #include "ssa_liveness_analysis.h"
55 #include "utils/assembler.h"
56 
57 namespace art {
58 
59 // Return whether a location is consistent with a type.
CheckType(Primitive::Type type,Location location)60 static bool CheckType(Primitive::Type type, Location location) {
61   if (location.IsFpuRegister()
62       || (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresFpuRegister))) {
63     return (type == Primitive::kPrimFloat) || (type == Primitive::kPrimDouble);
64   } else if (location.IsRegister() ||
65              (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresRegister))) {
66     return Primitive::IsIntegralType(type) || (type == Primitive::kPrimNot);
67   } else if (location.IsRegisterPair()) {
68     return type == Primitive::kPrimLong;
69   } else if (location.IsFpuRegisterPair()) {
70     return type == Primitive::kPrimDouble;
71   } else if (location.IsStackSlot()) {
72     return (Primitive::IsIntegralType(type) && type != Primitive::kPrimLong)
73            || (type == Primitive::kPrimFloat)
74            || (type == Primitive::kPrimNot);
75   } else if (location.IsDoubleStackSlot()) {
76     return (type == Primitive::kPrimLong) || (type == Primitive::kPrimDouble);
77   } else if (location.IsConstant()) {
78     if (location.GetConstant()->IsIntConstant()) {
79       return Primitive::IsIntegralType(type) && (type != Primitive::kPrimLong);
80     } else if (location.GetConstant()->IsNullConstant()) {
81       return type == Primitive::kPrimNot;
82     } else if (location.GetConstant()->IsLongConstant()) {
83       return type == Primitive::kPrimLong;
84     } else if (location.GetConstant()->IsFloatConstant()) {
85       return type == Primitive::kPrimFloat;
86     } else {
87       return location.GetConstant()->IsDoubleConstant()
88           && (type == Primitive::kPrimDouble);
89     }
90   } else {
91     return location.IsInvalid() || (location.GetPolicy() == Location::kAny);
92   }
93 }
94 
95 // Check that a location summary is consistent with an instruction.
CheckTypeConsistency(HInstruction * instruction)96 static bool CheckTypeConsistency(HInstruction* instruction) {
97   LocationSummary* locations = instruction->GetLocations();
98   if (locations == nullptr) {
99     return true;
100   }
101 
102   if (locations->Out().IsUnallocated()
103       && (locations->Out().GetPolicy() == Location::kSameAsFirstInput)) {
104     DCHECK(CheckType(instruction->GetType(), locations->InAt(0)))
105         << instruction->GetType()
106         << " " << locations->InAt(0);
107   } else {
108     DCHECK(CheckType(instruction->GetType(), locations->Out()))
109         << instruction->GetType()
110         << " " << locations->Out();
111   }
112 
113   for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
114     DCHECK(CheckType(instruction->InputAt(i)->GetType(), locations->InAt(i)))
115       << instruction->InputAt(i)->GetType()
116       << " " << locations->InAt(i);
117   }
118 
119   HEnvironment* environment = instruction->GetEnvironment();
120   for (size_t i = 0; i < instruction->EnvironmentSize(); ++i) {
121     if (environment->GetInstructionAt(i) != nullptr) {
122       Primitive::Type type = environment->GetInstructionAt(i)->GetType();
123       DCHECK(CheckType(type, environment->GetLocationAt(i)))
124         << type << " " << environment->GetLocationAt(i);
125     } else {
126       DCHECK(environment->GetLocationAt(i).IsInvalid())
127         << environment->GetLocationAt(i);
128     }
129   }
130   return true;
131 }
132 
GetCacheOffset(uint32_t index)133 size_t CodeGenerator::GetCacheOffset(uint32_t index) {
134   return sizeof(GcRoot<mirror::Object>) * index;
135 }
136 
GetCachePointerOffset(uint32_t index)137 size_t CodeGenerator::GetCachePointerOffset(uint32_t index) {
138   auto pointer_size = InstructionSetPointerSize(GetInstructionSet());
139   return pointer_size * index;
140 }
141 
GoesToNextBlock(HBasicBlock * current,HBasicBlock * next) const142 bool CodeGenerator::GoesToNextBlock(HBasicBlock* current, HBasicBlock* next) const {
143   DCHECK_EQ((*block_order_)[current_block_index_], current);
144   return GetNextBlockToEmit() == FirstNonEmptyBlock(next);
145 }
146 
GetNextBlockToEmit() const147 HBasicBlock* CodeGenerator::GetNextBlockToEmit() const {
148   for (size_t i = current_block_index_ + 1; i < block_order_->size(); ++i) {
149     HBasicBlock* block = (*block_order_)[i];
150     if (!block->IsSingleJump()) {
151       return block;
152     }
153   }
154   return nullptr;
155 }
156 
FirstNonEmptyBlock(HBasicBlock * block) const157 HBasicBlock* CodeGenerator::FirstNonEmptyBlock(HBasicBlock* block) const {
158   while (block->IsSingleJump()) {
159     block = block->GetSuccessors()[0];
160   }
161   return block;
162 }
163 
164 class DisassemblyScope {
165  public:
DisassemblyScope(HInstruction * instruction,const CodeGenerator & codegen)166   DisassemblyScope(HInstruction* instruction, const CodeGenerator& codegen)
167       : codegen_(codegen), instruction_(instruction), start_offset_(static_cast<size_t>(-1)) {
168     if (codegen_.GetDisassemblyInformation() != nullptr) {
169       start_offset_ = codegen_.GetAssembler().CodeSize();
170     }
171   }
172 
~DisassemblyScope()173   ~DisassemblyScope() {
174     // We avoid building this data when we know it will not be used.
175     if (codegen_.GetDisassemblyInformation() != nullptr) {
176       codegen_.GetDisassemblyInformation()->AddInstructionInterval(
177           instruction_, start_offset_, codegen_.GetAssembler().CodeSize());
178     }
179   }
180 
181  private:
182   const CodeGenerator& codegen_;
183   HInstruction* instruction_;
184   size_t start_offset_;
185 };
186 
187 
GenerateSlowPaths()188 void CodeGenerator::GenerateSlowPaths() {
189   size_t code_start = 0;
190   for (const std::unique_ptr<SlowPathCode>& slow_path_unique_ptr : slow_paths_) {
191     SlowPathCode* slow_path = slow_path_unique_ptr.get();
192     current_slow_path_ = slow_path;
193     if (disasm_info_ != nullptr) {
194       code_start = GetAssembler()->CodeSize();
195     }
196     // Record the dex pc at start of slow path (required for java line number mapping).
197     MaybeRecordNativeDebugInfo(slow_path->GetInstruction(), slow_path->GetDexPc(), slow_path);
198     slow_path->EmitNativeCode(this);
199     if (disasm_info_ != nullptr) {
200       disasm_info_->AddSlowPathInterval(slow_path, code_start, GetAssembler()->CodeSize());
201     }
202   }
203   current_slow_path_ = nullptr;
204 }
205 
Compile(CodeAllocator * allocator)206 void CodeGenerator::Compile(CodeAllocator* allocator) {
207   // The register allocator already called `InitializeCodeGeneration`,
208   // where the frame size has been computed.
209   DCHECK(block_order_ != nullptr);
210   Initialize();
211 
212   HGraphVisitor* instruction_visitor = GetInstructionVisitor();
213   DCHECK_EQ(current_block_index_, 0u);
214 
215   size_t frame_start = GetAssembler()->CodeSize();
216   GenerateFrameEntry();
217   DCHECK_EQ(GetAssembler()->cfi().GetCurrentCFAOffset(), static_cast<int>(frame_size_));
218   if (disasm_info_ != nullptr) {
219     disasm_info_->SetFrameEntryInterval(frame_start, GetAssembler()->CodeSize());
220   }
221 
222   for (size_t e = block_order_->size(); current_block_index_ < e; ++current_block_index_) {
223     HBasicBlock* block = (*block_order_)[current_block_index_];
224     // Don't generate code for an empty block. Its predecessors will branch to its successor
225     // directly. Also, the label of that block will not be emitted, so this helps catch
226     // errors where we reference that label.
227     if (block->IsSingleJump()) continue;
228     Bind(block);
229     // This ensures that we have correct native line mapping for all native instructions.
230     // It is necessary to make stepping over a statement work. Otherwise, any initial
231     // instructions (e.g. moves) would be assumed to be the start of next statement.
232     MaybeRecordNativeDebugInfo(nullptr /* instruction */, block->GetDexPc());
233     for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) {
234       HInstruction* current = it.Current();
235       if (current->HasEnvironment()) {
236         // Create stackmap for HNativeDebugInfo or any instruction which calls native code.
237         // Note that we need correct mapping for the native PC of the call instruction,
238         // so the runtime's stackmap is not sufficient since it is at PC after the call.
239         MaybeRecordNativeDebugInfo(current, block->GetDexPc());
240       }
241       DisassemblyScope disassembly_scope(current, *this);
242       DCHECK(CheckTypeConsistency(current));
243       current->Accept(instruction_visitor);
244     }
245   }
246 
247   GenerateSlowPaths();
248 
249   // Emit catch stack maps at the end of the stack map stream as expected by the
250   // runtime exception handler.
251   if (graph_->HasTryCatch()) {
252     RecordCatchBlockInfo();
253   }
254 
255   // Finalize instructions in assember;
256   Finalize(allocator);
257 }
258 
Finalize(CodeAllocator * allocator)259 void CodeGenerator::Finalize(CodeAllocator* allocator) {
260   size_t code_size = GetAssembler()->CodeSize();
261   uint8_t* buffer = allocator->Allocate(code_size);
262 
263   MemoryRegion code(buffer, code_size);
264   GetAssembler()->FinalizeInstructions(code);
265 }
266 
EmitLinkerPatches(ArenaVector<LinkerPatch> * linker_patches ATTRIBUTE_UNUSED)267 void CodeGenerator::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches ATTRIBUTE_UNUSED) {
268   // No linker patches by default.
269 }
270 
InitializeCodeGeneration(size_t number_of_spill_slots,size_t maximum_number_of_live_core_registers,size_t maximum_number_of_live_fpu_registers,size_t number_of_out_slots,const ArenaVector<HBasicBlock * > & block_order)271 void CodeGenerator::InitializeCodeGeneration(size_t number_of_spill_slots,
272                                              size_t maximum_number_of_live_core_registers,
273                                              size_t maximum_number_of_live_fpu_registers,
274                                              size_t number_of_out_slots,
275                                              const ArenaVector<HBasicBlock*>& block_order) {
276   block_order_ = &block_order;
277   DCHECK(!block_order.empty());
278   DCHECK(block_order[0] == GetGraph()->GetEntryBlock());
279   ComputeSpillMask();
280   first_register_slot_in_slow_path_ = (number_of_out_slots + number_of_spill_slots) * kVRegSize;
281 
282   if (number_of_spill_slots == 0
283       && !HasAllocatedCalleeSaveRegisters()
284       && IsLeafMethod()
285       && !RequiresCurrentMethod()) {
286     DCHECK_EQ(maximum_number_of_live_core_registers, 0u);
287     DCHECK_EQ(maximum_number_of_live_fpu_registers, 0u);
288     SetFrameSize(CallPushesPC() ? GetWordSize() : 0);
289   } else {
290     SetFrameSize(RoundUp(
291         number_of_spill_slots * kVRegSize
292         + number_of_out_slots * kVRegSize
293         + maximum_number_of_live_core_registers * GetWordSize()
294         + maximum_number_of_live_fpu_registers * GetFloatingPointSpillSlotSize()
295         + FrameEntrySpillSize(),
296         kStackAlignment));
297   }
298 }
299 
CreateCommonInvokeLocationSummary(HInvoke * invoke,InvokeDexCallingConventionVisitor * visitor)300 void CodeGenerator::CreateCommonInvokeLocationSummary(
301     HInvoke* invoke, InvokeDexCallingConventionVisitor* visitor) {
302   ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena();
303   LocationSummary* locations = new (allocator) LocationSummary(invoke, LocationSummary::kCall);
304 
305   for (size_t i = 0; i < invoke->GetNumberOfArguments(); i++) {
306     HInstruction* input = invoke->InputAt(i);
307     locations->SetInAt(i, visitor->GetNextLocation(input->GetType()));
308   }
309 
310   locations->SetOut(visitor->GetReturnLocation(invoke->GetType()));
311 
312   if (invoke->IsInvokeStaticOrDirect()) {
313     HInvokeStaticOrDirect* call = invoke->AsInvokeStaticOrDirect();
314     switch (call->GetMethodLoadKind()) {
315       case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
316         locations->SetInAt(call->GetSpecialInputIndex(), visitor->GetMethodLocation());
317         break;
318       case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod:
319         locations->AddTemp(visitor->GetMethodLocation());
320         locations->SetInAt(call->GetSpecialInputIndex(), Location::RequiresRegister());
321         break;
322       default:
323         locations->AddTemp(visitor->GetMethodLocation());
324         break;
325     }
326   } else {
327     locations->AddTemp(visitor->GetMethodLocation());
328   }
329 }
330 
GenerateInvokeUnresolvedRuntimeCall(HInvokeUnresolved * invoke)331 void CodeGenerator::GenerateInvokeUnresolvedRuntimeCall(HInvokeUnresolved* invoke) {
332   MoveConstant(invoke->GetLocations()->GetTemp(0), invoke->GetDexMethodIndex());
333 
334   // Initialize to anything to silent compiler warnings.
335   QuickEntrypointEnum entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
336   switch (invoke->GetOriginalInvokeType()) {
337     case kStatic:
338       entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
339       break;
340     case kDirect:
341       entrypoint = kQuickInvokeDirectTrampolineWithAccessCheck;
342       break;
343     case kVirtual:
344       entrypoint = kQuickInvokeVirtualTrampolineWithAccessCheck;
345       break;
346     case kSuper:
347       entrypoint = kQuickInvokeSuperTrampolineWithAccessCheck;
348       break;
349     case kInterface:
350       entrypoint = kQuickInvokeInterfaceTrampolineWithAccessCheck;
351       break;
352   }
353   InvokeRuntime(entrypoint, invoke, invoke->GetDexPc(), nullptr);
354 }
355 
CreateUnresolvedFieldLocationSummary(HInstruction * field_access,Primitive::Type field_type,const FieldAccessCallingConvention & calling_convention)356 void CodeGenerator::CreateUnresolvedFieldLocationSummary(
357     HInstruction* field_access,
358     Primitive::Type field_type,
359     const FieldAccessCallingConvention& calling_convention) {
360   bool is_instance = field_access->IsUnresolvedInstanceFieldGet()
361       || field_access->IsUnresolvedInstanceFieldSet();
362   bool is_get = field_access->IsUnresolvedInstanceFieldGet()
363       || field_access->IsUnresolvedStaticFieldGet();
364 
365   ArenaAllocator* allocator = field_access->GetBlock()->GetGraph()->GetArena();
366   LocationSummary* locations =
367       new (allocator) LocationSummary(field_access, LocationSummary::kCall);
368 
369   locations->AddTemp(calling_convention.GetFieldIndexLocation());
370 
371   if (is_instance) {
372     // Add the `this` object for instance field accesses.
373     locations->SetInAt(0, calling_convention.GetObjectLocation());
374   }
375 
376   // Note that pSetXXStatic/pGetXXStatic always takes/returns an int or int64
377   // regardless of the the type. Because of that we forced to special case
378   // the access to floating point values.
379   if (is_get) {
380     if (Primitive::IsFloatingPointType(field_type)) {
381       // The return value will be stored in regular registers while register
382       // allocator expects it in a floating point register.
383       // Note We don't need to request additional temps because the return
384       // register(s) are already blocked due the call and they may overlap with
385       // the input or field index.
386       // The transfer between the two will be done at codegen level.
387       locations->SetOut(calling_convention.GetFpuLocation(field_type));
388     } else {
389       locations->SetOut(calling_convention.GetReturnLocation(field_type));
390     }
391   } else {
392      size_t set_index = is_instance ? 1 : 0;
393      if (Primitive::IsFloatingPointType(field_type)) {
394       // The set value comes from a float location while the calling convention
395       // expects it in a regular register location. Allocate a temp for it and
396       // make the transfer at codegen.
397       AddLocationAsTemp(calling_convention.GetSetValueLocation(field_type, is_instance), locations);
398       locations->SetInAt(set_index, calling_convention.GetFpuLocation(field_type));
399     } else {
400       locations->SetInAt(set_index,
401           calling_convention.GetSetValueLocation(field_type, is_instance));
402     }
403   }
404 }
405 
GenerateUnresolvedFieldAccess(HInstruction * field_access,Primitive::Type field_type,uint32_t field_index,uint32_t dex_pc,const FieldAccessCallingConvention & calling_convention)406 void CodeGenerator::GenerateUnresolvedFieldAccess(
407     HInstruction* field_access,
408     Primitive::Type field_type,
409     uint32_t field_index,
410     uint32_t dex_pc,
411     const FieldAccessCallingConvention& calling_convention) {
412   LocationSummary* locations = field_access->GetLocations();
413 
414   MoveConstant(locations->GetTemp(0), field_index);
415 
416   bool is_instance = field_access->IsUnresolvedInstanceFieldGet()
417       || field_access->IsUnresolvedInstanceFieldSet();
418   bool is_get = field_access->IsUnresolvedInstanceFieldGet()
419       || field_access->IsUnresolvedStaticFieldGet();
420 
421   if (!is_get && Primitive::IsFloatingPointType(field_type)) {
422     // Copy the float value to be set into the calling convention register.
423     // Note that using directly the temp location is problematic as we don't
424     // support temp register pairs. To avoid boilerplate conversion code, use
425     // the location from the calling convention.
426     MoveLocation(calling_convention.GetSetValueLocation(field_type, is_instance),
427                  locations->InAt(is_instance ? 1 : 0),
428                  (Primitive::Is64BitType(field_type) ? Primitive::kPrimLong : Primitive::kPrimInt));
429   }
430 
431   QuickEntrypointEnum entrypoint = kQuickSet8Static;  // Initialize to anything to avoid warnings.
432   switch (field_type) {
433     case Primitive::kPrimBoolean:
434       entrypoint = is_instance
435           ? (is_get ? kQuickGetBooleanInstance : kQuickSet8Instance)
436           : (is_get ? kQuickGetBooleanStatic : kQuickSet8Static);
437       break;
438     case Primitive::kPrimByte:
439       entrypoint = is_instance
440           ? (is_get ? kQuickGetByteInstance : kQuickSet8Instance)
441           : (is_get ? kQuickGetByteStatic : kQuickSet8Static);
442       break;
443     case Primitive::kPrimShort:
444       entrypoint = is_instance
445           ? (is_get ? kQuickGetShortInstance : kQuickSet16Instance)
446           : (is_get ? kQuickGetShortStatic : kQuickSet16Static);
447       break;
448     case Primitive::kPrimChar:
449       entrypoint = is_instance
450           ? (is_get ? kQuickGetCharInstance : kQuickSet16Instance)
451           : (is_get ? kQuickGetCharStatic : kQuickSet16Static);
452       break;
453     case Primitive::kPrimInt:
454     case Primitive::kPrimFloat:
455       entrypoint = is_instance
456           ? (is_get ? kQuickGet32Instance : kQuickSet32Instance)
457           : (is_get ? kQuickGet32Static : kQuickSet32Static);
458       break;
459     case Primitive::kPrimNot:
460       entrypoint = is_instance
461           ? (is_get ? kQuickGetObjInstance : kQuickSetObjInstance)
462           : (is_get ? kQuickGetObjStatic : kQuickSetObjStatic);
463       break;
464     case Primitive::kPrimLong:
465     case Primitive::kPrimDouble:
466       entrypoint = is_instance
467           ? (is_get ? kQuickGet64Instance : kQuickSet64Instance)
468           : (is_get ? kQuickGet64Static : kQuickSet64Static);
469       break;
470     default:
471       LOG(FATAL) << "Invalid type " << field_type;
472   }
473   InvokeRuntime(entrypoint, field_access, dex_pc, nullptr);
474 
475   if (is_get && Primitive::IsFloatingPointType(field_type)) {
476     MoveLocation(locations->Out(), calling_convention.GetReturnLocation(field_type), field_type);
477   }
478 }
479 
480 // TODO: Remove argument `code_generator_supports_read_barrier` when
481 // all code generators have read barrier support.
CreateLoadClassLocationSummary(HLoadClass * cls,Location runtime_type_index_location,Location runtime_return_location,bool code_generator_supports_read_barrier)482 void CodeGenerator::CreateLoadClassLocationSummary(HLoadClass* cls,
483                                                    Location runtime_type_index_location,
484                                                    Location runtime_return_location,
485                                                    bool code_generator_supports_read_barrier) {
486   ArenaAllocator* allocator = cls->GetBlock()->GetGraph()->GetArena();
487   LocationSummary::CallKind call_kind = cls->NeedsAccessCheck()
488       ? LocationSummary::kCall
489       : (((code_generator_supports_read_barrier && kEmitCompilerReadBarrier) ||
490           cls->CanCallRuntime())
491             ? LocationSummary::kCallOnSlowPath
492             : LocationSummary::kNoCall);
493   LocationSummary* locations = new (allocator) LocationSummary(cls, call_kind);
494   if (cls->NeedsAccessCheck()) {
495     locations->SetInAt(0, Location::NoLocation());
496     locations->AddTemp(runtime_type_index_location);
497     locations->SetOut(runtime_return_location);
498   } else {
499     locations->SetInAt(0, Location::RequiresRegister());
500     locations->SetOut(Location::RequiresRegister());
501   }
502 }
503 
504 
BlockIfInRegister(Location location,bool is_out) const505 void CodeGenerator::BlockIfInRegister(Location location, bool is_out) const {
506   // The DCHECKS below check that a register is not specified twice in
507   // the summary. The out location can overlap with an input, so we need
508   // to special case it.
509   if (location.IsRegister()) {
510     DCHECK(is_out || !blocked_core_registers_[location.reg()]);
511     blocked_core_registers_[location.reg()] = true;
512   } else if (location.IsFpuRegister()) {
513     DCHECK(is_out || !blocked_fpu_registers_[location.reg()]);
514     blocked_fpu_registers_[location.reg()] = true;
515   } else if (location.IsFpuRegisterPair()) {
516     DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()]);
517     blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()] = true;
518     DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()]);
519     blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()] = true;
520   } else if (location.IsRegisterPair()) {
521     DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairLow<int>()]);
522     blocked_core_registers_[location.AsRegisterPairLow<int>()] = true;
523     DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairHigh<int>()]);
524     blocked_core_registers_[location.AsRegisterPairHigh<int>()] = true;
525   }
526 }
527 
AllocateLocations(HInstruction * instruction)528 void CodeGenerator::AllocateLocations(HInstruction* instruction) {
529   instruction->Accept(GetLocationBuilder());
530   DCHECK(CheckTypeConsistency(instruction));
531   LocationSummary* locations = instruction->GetLocations();
532   if (!instruction->IsSuspendCheckEntry()) {
533     if (locations != nullptr) {
534       if (locations->CanCall()) {
535         MarkNotLeaf();
536       } else if (locations->Intrinsified() &&
537                  instruction->IsInvokeStaticOrDirect() &&
538                  !instruction->AsInvokeStaticOrDirect()->HasCurrentMethodInput()) {
539         // A static method call that has been fully intrinsified, and cannot call on the slow
540         // path or refer to the current method directly, no longer needs current method.
541         return;
542       }
543     }
544     if (instruction->NeedsCurrentMethod()) {
545       SetRequiresCurrentMethod();
546     }
547   }
548 }
549 
MaybeRecordStat(MethodCompilationStat compilation_stat,size_t count) const550 void CodeGenerator::MaybeRecordStat(MethodCompilationStat compilation_stat, size_t count) const {
551   if (stats_ != nullptr) {
552     stats_->RecordStat(compilation_stat, count);
553   }
554 }
555 
Create(HGraph * graph,InstructionSet instruction_set,const InstructionSetFeatures & isa_features,const CompilerOptions & compiler_options,OptimizingCompilerStats * stats)556 std::unique_ptr<CodeGenerator> CodeGenerator::Create(HGraph* graph,
557                                                      InstructionSet instruction_set,
558                                                      const InstructionSetFeatures& isa_features,
559                                                      const CompilerOptions& compiler_options,
560                                                      OptimizingCompilerStats* stats) {
561   ArenaAllocator* arena = graph->GetArena();
562   switch (instruction_set) {
563 #ifdef ART_ENABLE_CODEGEN_arm
564     case kArm:
565     case kThumb2: {
566       return std::unique_ptr<CodeGenerator>(
567           new (arena) arm::CodeGeneratorARM(graph,
568                                             *isa_features.AsArmInstructionSetFeatures(),
569                                             compiler_options,
570                                             stats));
571     }
572 #endif
573 #ifdef ART_ENABLE_CODEGEN_arm64
574     case kArm64: {
575       return std::unique_ptr<CodeGenerator>(
576           new (arena) arm64::CodeGeneratorARM64(graph,
577                                                 *isa_features.AsArm64InstructionSetFeatures(),
578                                                 compiler_options,
579                                                 stats));
580     }
581 #endif
582 #ifdef ART_ENABLE_CODEGEN_mips
583     case kMips: {
584       return std::unique_ptr<CodeGenerator>(
585           new (arena) mips::CodeGeneratorMIPS(graph,
586                                               *isa_features.AsMipsInstructionSetFeatures(),
587                                               compiler_options,
588                                               stats));
589     }
590 #endif
591 #ifdef ART_ENABLE_CODEGEN_mips64
592     case kMips64: {
593       return std::unique_ptr<CodeGenerator>(
594           new (arena) mips64::CodeGeneratorMIPS64(graph,
595                                                   *isa_features.AsMips64InstructionSetFeatures(),
596                                                   compiler_options,
597                                                   stats));
598     }
599 #endif
600 #ifdef ART_ENABLE_CODEGEN_x86
601     case kX86: {
602       return std::unique_ptr<CodeGenerator>(
603           new (arena) x86::CodeGeneratorX86(graph,
604                                             *isa_features.AsX86InstructionSetFeatures(),
605                                             compiler_options,
606                                             stats));
607     }
608 #endif
609 #ifdef ART_ENABLE_CODEGEN_x86_64
610     case kX86_64: {
611       return std::unique_ptr<CodeGenerator>(
612           new (arena) x86_64::CodeGeneratorX86_64(graph,
613                                                   *isa_features.AsX86_64InstructionSetFeatures(),
614                                                   compiler_options,
615                                                   stats));
616     }
617 #endif
618     default:
619       return nullptr;
620   }
621 }
622 
ComputeStackMapsSize()623 size_t CodeGenerator::ComputeStackMapsSize() {
624   return stack_map_stream_.PrepareForFillIn();
625 }
626 
CheckCovers(uint32_t dex_pc,const HGraph & graph,const CodeInfo & code_info,const ArenaVector<HSuspendCheck * > & loop_headers,ArenaVector<size_t> * covered)627 static void CheckCovers(uint32_t dex_pc,
628                         const HGraph& graph,
629                         const CodeInfo& code_info,
630                         const ArenaVector<HSuspendCheck*>& loop_headers,
631                         ArenaVector<size_t>* covered) {
632   CodeInfoEncoding encoding = code_info.ExtractEncoding();
633   for (size_t i = 0; i < loop_headers.size(); ++i) {
634     if (loop_headers[i]->GetDexPc() == dex_pc) {
635       if (graph.IsCompilingOsr()) {
636         DCHECK(code_info.GetOsrStackMapForDexPc(dex_pc, encoding).IsValid());
637       }
638       ++(*covered)[i];
639     }
640   }
641 }
642 
643 // Debug helper to ensure loop entries in compiled code are matched by
644 // dex branch instructions.
CheckLoopEntriesCanBeUsedForOsr(const HGraph & graph,const CodeInfo & code_info,const DexFile::CodeItem & code_item)645 static void CheckLoopEntriesCanBeUsedForOsr(const HGraph& graph,
646                                             const CodeInfo& code_info,
647                                             const DexFile::CodeItem& code_item) {
648   if (graph.HasTryCatch()) {
649     // One can write loops through try/catch, which we do not support for OSR anyway.
650     return;
651   }
652   ArenaVector<HSuspendCheck*> loop_headers(graph.GetArena()->Adapter(kArenaAllocMisc));
653   for (HReversePostOrderIterator it(graph); !it.Done(); it.Advance()) {
654     if (it.Current()->IsLoopHeader()) {
655       HSuspendCheck* suspend_check = it.Current()->GetLoopInformation()->GetSuspendCheck();
656       if (!suspend_check->GetEnvironment()->IsFromInlinedInvoke()) {
657         loop_headers.push_back(suspend_check);
658       }
659     }
660   }
661   ArenaVector<size_t> covered(loop_headers.size(), 0, graph.GetArena()->Adapter(kArenaAllocMisc));
662   const uint16_t* code_ptr = code_item.insns_;
663   const uint16_t* code_end = code_item.insns_ + code_item.insns_size_in_code_units_;
664 
665   size_t dex_pc = 0;
666   while (code_ptr < code_end) {
667     const Instruction& instruction = *Instruction::At(code_ptr);
668     if (instruction.IsBranch()) {
669       uint32_t target = dex_pc + instruction.GetTargetOffset();
670       CheckCovers(target, graph, code_info, loop_headers, &covered);
671     } else if (instruction.IsSwitch()) {
672       DexSwitchTable table(instruction, dex_pc);
673       uint16_t num_entries = table.GetNumEntries();
674       size_t offset = table.GetFirstValueIndex();
675 
676       // Use a larger loop counter type to avoid overflow issues.
677       for (size_t i = 0; i < num_entries; ++i) {
678         // The target of the case.
679         uint32_t target = dex_pc + table.GetEntryAt(i + offset);
680         CheckCovers(target, graph, code_info, loop_headers, &covered);
681       }
682     }
683     dex_pc += instruction.SizeInCodeUnits();
684     code_ptr += instruction.SizeInCodeUnits();
685   }
686 
687   for (size_t i = 0; i < covered.size(); ++i) {
688     DCHECK_NE(covered[i], 0u) << "Loop in compiled code has no dex branch equivalent";
689   }
690 }
691 
BuildStackMaps(MemoryRegion region,const DexFile::CodeItem & code_item)692 void CodeGenerator::BuildStackMaps(MemoryRegion region, const DexFile::CodeItem& code_item) {
693   stack_map_stream_.FillIn(region);
694   if (kIsDebugBuild) {
695     CheckLoopEntriesCanBeUsedForOsr(*graph_, CodeInfo(region), code_item);
696   }
697 }
698 
RecordPcInfo(HInstruction * instruction,uint32_t dex_pc,SlowPathCode * slow_path)699 void CodeGenerator::RecordPcInfo(HInstruction* instruction,
700                                  uint32_t dex_pc,
701                                  SlowPathCode* slow_path) {
702   if (instruction != nullptr) {
703     // The code generated for some type conversions
704     // may call the runtime, thus normally requiring a subsequent
705     // call to this method. However, the method verifier does not
706     // produce PC information for certain instructions, which are
707     // considered "atomic" (they cannot join a GC).
708     // Therefore we do not currently record PC information for such
709     // instructions.  As this may change later, we added this special
710     // case so that code generators may nevertheless call
711     // CodeGenerator::RecordPcInfo without triggering an error in
712     // CodeGenerator::BuildNativeGCMap ("Missing ref for dex pc 0x")
713     // thereafter.
714     if (instruction->IsTypeConversion()) {
715       return;
716     }
717     if (instruction->IsRem()) {
718       Primitive::Type type = instruction->AsRem()->GetResultType();
719       if ((type == Primitive::kPrimFloat) || (type == Primitive::kPrimDouble)) {
720         return;
721       }
722     }
723   }
724 
725   uint32_t outer_dex_pc = dex_pc;
726   uint32_t outer_environment_size = 0;
727   uint32_t inlining_depth = 0;
728   if (instruction != nullptr) {
729     for (HEnvironment* environment = instruction->GetEnvironment();
730          environment != nullptr;
731          environment = environment->GetParent()) {
732       outer_dex_pc = environment->GetDexPc();
733       outer_environment_size = environment->Size();
734       if (environment != instruction->GetEnvironment()) {
735         inlining_depth++;
736       }
737     }
738   }
739 
740   // Collect PC infos for the mapping table.
741   uint32_t native_pc = GetAssembler()->CodeSize();
742 
743   if (instruction == nullptr) {
744     // For stack overflow checks and native-debug-info entries without dex register
745     // mapping (i.e. start of basic block or start of slow path).
746     stack_map_stream_.BeginStackMapEntry(outer_dex_pc, native_pc, 0, 0, 0, 0);
747     stack_map_stream_.EndStackMapEntry();
748     return;
749   }
750   LocationSummary* locations = instruction->GetLocations();
751 
752   uint32_t register_mask = locations->GetRegisterMask();
753   if (locations->OnlyCallsOnSlowPath()) {
754     // In case of slow path, we currently set the location of caller-save registers
755     // to register (instead of their stack location when pushed before the slow-path
756     // call). Therefore register_mask contains both callee-save and caller-save
757     // registers that hold objects. We must remove the caller-save from the mask, since
758     // they will be overwritten by the callee.
759     register_mask &= core_callee_save_mask_;
760   }
761   // The register mask must be a subset of callee-save registers.
762   DCHECK_EQ(register_mask & core_callee_save_mask_, register_mask);
763   stack_map_stream_.BeginStackMapEntry(outer_dex_pc,
764                                        native_pc,
765                                        register_mask,
766                                        locations->GetStackMask(),
767                                        outer_environment_size,
768                                        inlining_depth);
769 
770   EmitEnvironment(instruction->GetEnvironment(), slow_path);
771   stack_map_stream_.EndStackMapEntry();
772 
773   HLoopInformation* info = instruction->GetBlock()->GetLoopInformation();
774   if (instruction->IsSuspendCheck() &&
775       (info != nullptr) &&
776       graph_->IsCompilingOsr() &&
777       (inlining_depth == 0)) {
778     DCHECK_EQ(info->GetSuspendCheck(), instruction);
779     // We duplicate the stack map as a marker that this stack map can be an OSR entry.
780     // Duplicating it avoids having the runtime recognize and skip an OSR stack map.
781     DCHECK(info->IsIrreducible());
782     stack_map_stream_.BeginStackMapEntry(
783         dex_pc, native_pc, register_mask, locations->GetStackMask(), outer_environment_size, 0);
784     EmitEnvironment(instruction->GetEnvironment(), slow_path);
785     stack_map_stream_.EndStackMapEntry();
786     if (kIsDebugBuild) {
787       HEnvironment* environment = instruction->GetEnvironment();
788       for (size_t i = 0, environment_size = environment->Size(); i < environment_size; ++i) {
789         HInstruction* in_environment = environment->GetInstructionAt(i);
790         if (in_environment != nullptr) {
791           DCHECK(in_environment->IsPhi() || in_environment->IsConstant());
792           Location location = environment->GetLocationAt(i);
793           DCHECK(location.IsStackSlot() ||
794                  location.IsDoubleStackSlot() ||
795                  location.IsConstant() ||
796                  location.IsInvalid());
797           if (location.IsStackSlot() || location.IsDoubleStackSlot()) {
798             DCHECK_LT(location.GetStackIndex(), static_cast<int32_t>(GetFrameSize()));
799           }
800         }
801       }
802     }
803   } else if (kIsDebugBuild) {
804     // Ensure stack maps are unique, by checking that the native pc in the stack map
805     // last emitted is different than the native pc of the stack map just emitted.
806     size_t number_of_stack_maps = stack_map_stream_.GetNumberOfStackMaps();
807     if (number_of_stack_maps > 1) {
808       DCHECK_NE(stack_map_stream_.GetStackMap(number_of_stack_maps - 1).native_pc_offset,
809                 stack_map_stream_.GetStackMap(number_of_stack_maps - 2).native_pc_offset);
810     }
811   }
812 }
813 
HasStackMapAtCurrentPc()814 bool CodeGenerator::HasStackMapAtCurrentPc() {
815   uint32_t pc = GetAssembler()->CodeSize();
816   size_t count = stack_map_stream_.GetNumberOfStackMaps();
817   return count > 0 && stack_map_stream_.GetStackMap(count - 1).native_pc_offset == pc;
818 }
819 
MaybeRecordNativeDebugInfo(HInstruction * instruction,uint32_t dex_pc,SlowPathCode * slow_path)820 void CodeGenerator::MaybeRecordNativeDebugInfo(HInstruction* instruction,
821                                                uint32_t dex_pc,
822                                                SlowPathCode* slow_path) {
823   if (GetCompilerOptions().GetNativeDebuggable() && dex_pc != kNoDexPc) {
824     if (HasStackMapAtCurrentPc()) {
825       // Ensure that we do not collide with the stack map of the previous instruction.
826       GenerateNop();
827     }
828     RecordPcInfo(instruction, dex_pc, slow_path);
829   }
830 }
831 
RecordCatchBlockInfo()832 void CodeGenerator::RecordCatchBlockInfo() {
833   ArenaAllocator* arena = graph_->GetArena();
834 
835   for (HBasicBlock* block : *block_order_) {
836     if (!block->IsCatchBlock()) {
837       continue;
838     }
839 
840     uint32_t dex_pc = block->GetDexPc();
841     uint32_t num_vregs = graph_->GetNumberOfVRegs();
842     uint32_t inlining_depth = 0;  // Inlining of catch blocks is not supported at the moment.
843     uint32_t native_pc = GetAddressOf(block);
844     uint32_t register_mask = 0;   // Not used.
845 
846     // The stack mask is not used, so we leave it empty.
847     ArenaBitVector* stack_mask =
848         ArenaBitVector::Create(arena, 0, /* expandable */ true, kArenaAllocCodeGenerator);
849 
850     stack_map_stream_.BeginStackMapEntry(dex_pc,
851                                          native_pc,
852                                          register_mask,
853                                          stack_mask,
854                                          num_vregs,
855                                          inlining_depth);
856 
857     HInstruction* current_phi = block->GetFirstPhi();
858     for (size_t vreg = 0; vreg < num_vregs; ++vreg) {
859     while (current_phi != nullptr && current_phi->AsPhi()->GetRegNumber() < vreg) {
860       HInstruction* next_phi = current_phi->GetNext();
861       DCHECK(next_phi == nullptr ||
862              current_phi->AsPhi()->GetRegNumber() <= next_phi->AsPhi()->GetRegNumber())
863           << "Phis need to be sorted by vreg number to keep this a linear-time loop.";
864       current_phi = next_phi;
865     }
866 
867       if (current_phi == nullptr || current_phi->AsPhi()->GetRegNumber() != vreg) {
868         stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
869       } else {
870         Location location = current_phi->GetLiveInterval()->ToLocation();
871         switch (location.GetKind()) {
872           case Location::kStackSlot: {
873             stack_map_stream_.AddDexRegisterEntry(
874                 DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
875             break;
876           }
877           case Location::kDoubleStackSlot: {
878             stack_map_stream_.AddDexRegisterEntry(
879                 DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
880             stack_map_stream_.AddDexRegisterEntry(
881                 DexRegisterLocation::Kind::kInStack, location.GetHighStackIndex(kVRegSize));
882             ++vreg;
883             DCHECK_LT(vreg, num_vregs);
884             break;
885           }
886           default: {
887             // All catch phis must be allocated to a stack slot.
888             LOG(FATAL) << "Unexpected kind " << location.GetKind();
889             UNREACHABLE();
890           }
891         }
892       }
893     }
894 
895     stack_map_stream_.EndStackMapEntry();
896   }
897 }
898 
EmitEnvironment(HEnvironment * environment,SlowPathCode * slow_path)899 void CodeGenerator::EmitEnvironment(HEnvironment* environment, SlowPathCode* slow_path) {
900   if (environment == nullptr) return;
901 
902   if (environment->GetParent() != nullptr) {
903     // We emit the parent environment first.
904     EmitEnvironment(environment->GetParent(), slow_path);
905     stack_map_stream_.BeginInlineInfoEntry(environment->GetMethodIdx(),
906                                            environment->GetDexPc(),
907                                            environment->GetInvokeType(),
908                                            environment->Size());
909   }
910 
911   // Walk over the environment, and record the location of dex registers.
912   for (size_t i = 0, environment_size = environment->Size(); i < environment_size; ++i) {
913     HInstruction* current = environment->GetInstructionAt(i);
914     if (current == nullptr) {
915       stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
916       continue;
917     }
918 
919     Location location = environment->GetLocationAt(i);
920     switch (location.GetKind()) {
921       case Location::kConstant: {
922         DCHECK_EQ(current, location.GetConstant());
923         if (current->IsLongConstant()) {
924           int64_t value = current->AsLongConstant()->GetValue();
925           stack_map_stream_.AddDexRegisterEntry(
926               DexRegisterLocation::Kind::kConstant, Low32Bits(value));
927           stack_map_stream_.AddDexRegisterEntry(
928               DexRegisterLocation::Kind::kConstant, High32Bits(value));
929           ++i;
930           DCHECK_LT(i, environment_size);
931         } else if (current->IsDoubleConstant()) {
932           int64_t value = bit_cast<int64_t, double>(current->AsDoubleConstant()->GetValue());
933           stack_map_stream_.AddDexRegisterEntry(
934               DexRegisterLocation::Kind::kConstant, Low32Bits(value));
935           stack_map_stream_.AddDexRegisterEntry(
936               DexRegisterLocation::Kind::kConstant, High32Bits(value));
937           ++i;
938           DCHECK_LT(i, environment_size);
939         } else if (current->IsIntConstant()) {
940           int32_t value = current->AsIntConstant()->GetValue();
941           stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, value);
942         } else if (current->IsNullConstant()) {
943           stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, 0);
944         } else {
945           DCHECK(current->IsFloatConstant()) << current->DebugName();
946           int32_t value = bit_cast<int32_t, float>(current->AsFloatConstant()->GetValue());
947           stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, value);
948         }
949         break;
950       }
951 
952       case Location::kStackSlot: {
953         stack_map_stream_.AddDexRegisterEntry(
954             DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
955         break;
956       }
957 
958       case Location::kDoubleStackSlot: {
959         stack_map_stream_.AddDexRegisterEntry(
960             DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
961         stack_map_stream_.AddDexRegisterEntry(
962             DexRegisterLocation::Kind::kInStack, location.GetHighStackIndex(kVRegSize));
963         ++i;
964         DCHECK_LT(i, environment_size);
965         break;
966       }
967 
968       case Location::kRegister : {
969         int id = location.reg();
970         if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(id)) {
971           uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(id);
972           stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
973           if (current->GetType() == Primitive::kPrimLong) {
974             stack_map_stream_.AddDexRegisterEntry(
975                 DexRegisterLocation::Kind::kInStack, offset + kVRegSize);
976             ++i;
977             DCHECK_LT(i, environment_size);
978           }
979         } else {
980           stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, id);
981           if (current->GetType() == Primitive::kPrimLong) {
982             stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegisterHigh, id);
983             ++i;
984             DCHECK_LT(i, environment_size);
985           }
986         }
987         break;
988       }
989 
990       case Location::kFpuRegister : {
991         int id = location.reg();
992         if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(id)) {
993           uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(id);
994           stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
995           if (current->GetType() == Primitive::kPrimDouble) {
996             stack_map_stream_.AddDexRegisterEntry(
997                 DexRegisterLocation::Kind::kInStack, offset + kVRegSize);
998             ++i;
999             DCHECK_LT(i, environment_size);
1000           }
1001         } else {
1002           stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, id);
1003           if (current->GetType() == Primitive::kPrimDouble) {
1004             stack_map_stream_.AddDexRegisterEntry(
1005                 DexRegisterLocation::Kind::kInFpuRegisterHigh, id);
1006             ++i;
1007             DCHECK_LT(i, environment_size);
1008           }
1009         }
1010         break;
1011       }
1012 
1013       case Location::kFpuRegisterPair : {
1014         int low = location.low();
1015         int high = location.high();
1016         if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(low)) {
1017           uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(low);
1018           stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1019         } else {
1020           stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, low);
1021         }
1022         if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(high)) {
1023           uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(high);
1024           stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1025           ++i;
1026         } else {
1027           stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, high);
1028           ++i;
1029         }
1030         DCHECK_LT(i, environment_size);
1031         break;
1032       }
1033 
1034       case Location::kRegisterPair : {
1035         int low = location.low();
1036         int high = location.high();
1037         if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(low)) {
1038           uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(low);
1039           stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1040         } else {
1041           stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, low);
1042         }
1043         if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(high)) {
1044           uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(high);
1045           stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1046         } else {
1047           stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, high);
1048         }
1049         ++i;
1050         DCHECK_LT(i, environment_size);
1051         break;
1052       }
1053 
1054       case Location::kInvalid: {
1055         stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
1056         break;
1057       }
1058 
1059       default:
1060         LOG(FATAL) << "Unexpected kind " << location.GetKind();
1061     }
1062   }
1063 
1064   if (environment->GetParent() != nullptr) {
1065     stack_map_stream_.EndInlineInfoEntry();
1066   }
1067 }
1068 
IsImplicitNullCheckAllowed(HNullCheck * null_check) const1069 bool CodeGenerator::IsImplicitNullCheckAllowed(HNullCheck* null_check) const {
1070   return compiler_options_.GetImplicitNullChecks() &&
1071          // Null checks which might throw into a catch block need to save live
1072          // registers and therefore cannot be done implicitly.
1073          !null_check->CanThrowIntoCatchBlock();
1074 }
1075 
CanMoveNullCheckToUser(HNullCheck * null_check)1076 bool CodeGenerator::CanMoveNullCheckToUser(HNullCheck* null_check) {
1077   HInstruction* first_next_not_move = null_check->GetNextDisregardingMoves();
1078 
1079   return (first_next_not_move != nullptr)
1080       && first_next_not_move->CanDoImplicitNullCheckOn(null_check->InputAt(0));
1081 }
1082 
MaybeRecordImplicitNullCheck(HInstruction * instr)1083 void CodeGenerator::MaybeRecordImplicitNullCheck(HInstruction* instr) {
1084   // If we are from a static path don't record the pc as we can't throw NPE.
1085   // NB: having the checks here makes the code much less verbose in the arch
1086   // specific code generators.
1087   if (instr->IsStaticFieldSet() || instr->IsStaticFieldGet()) {
1088     return;
1089   }
1090 
1091   if (!instr->CanDoImplicitNullCheckOn(instr->InputAt(0))) {
1092     return;
1093   }
1094 
1095   // Find the first previous instruction which is not a move.
1096   HInstruction* first_prev_not_move = instr->GetPreviousDisregardingMoves();
1097 
1098   // If the instruction is a null check it means that `instr` is the first user
1099   // and needs to record the pc.
1100   if (first_prev_not_move != nullptr && first_prev_not_move->IsNullCheck()) {
1101     HNullCheck* null_check = first_prev_not_move->AsNullCheck();
1102     if (IsImplicitNullCheckAllowed(null_check)) {
1103       // TODO: The parallel moves modify the environment. Their changes need to be
1104       // reverted otherwise the stack maps at the throw point will not be correct.
1105       RecordPcInfo(null_check, null_check->GetDexPc());
1106     }
1107   }
1108 }
1109 
GenerateNullCheck(HNullCheck * instruction)1110 void CodeGenerator::GenerateNullCheck(HNullCheck* instruction) {
1111   if (IsImplicitNullCheckAllowed(instruction)) {
1112     MaybeRecordStat(kImplicitNullCheckGenerated);
1113     GenerateImplicitNullCheck(instruction);
1114   } else {
1115     MaybeRecordStat(kExplicitNullCheckGenerated);
1116     GenerateExplicitNullCheck(instruction);
1117   }
1118 }
1119 
ClearSpillSlotsFromLoopPhisInStackMap(HSuspendCheck * suspend_check) const1120 void CodeGenerator::ClearSpillSlotsFromLoopPhisInStackMap(HSuspendCheck* suspend_check) const {
1121   LocationSummary* locations = suspend_check->GetLocations();
1122   HBasicBlock* block = suspend_check->GetBlock();
1123   DCHECK(block->GetLoopInformation()->GetSuspendCheck() == suspend_check);
1124   DCHECK(block->IsLoopHeader());
1125 
1126   for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) {
1127     HInstruction* current = it.Current();
1128     LiveInterval* interval = current->GetLiveInterval();
1129     // We only need to clear bits of loop phis containing objects and allocated in register.
1130     // Loop phis allocated on stack already have the object in the stack.
1131     if (current->GetType() == Primitive::kPrimNot
1132         && interval->HasRegister()
1133         && interval->HasSpillSlot()) {
1134       locations->ClearStackBit(interval->GetSpillSlot() / kVRegSize);
1135     }
1136   }
1137 }
1138 
EmitParallelMoves(Location from1,Location to1,Primitive::Type type1,Location from2,Location to2,Primitive::Type type2)1139 void CodeGenerator::EmitParallelMoves(Location from1,
1140                                       Location to1,
1141                                       Primitive::Type type1,
1142                                       Location from2,
1143                                       Location to2,
1144                                       Primitive::Type type2) {
1145   HParallelMove parallel_move(GetGraph()->GetArena());
1146   parallel_move.AddMove(from1, to1, type1, nullptr);
1147   parallel_move.AddMove(from2, to2, type2, nullptr);
1148   GetMoveResolver()->EmitNativeCode(&parallel_move);
1149 }
1150 
ValidateInvokeRuntime(HInstruction * instruction,SlowPathCode * slow_path)1151 void CodeGenerator::ValidateInvokeRuntime(HInstruction* instruction, SlowPathCode* slow_path) {
1152   // Ensure that the call kind indication given to the register allocator is
1153   // coherent with the runtime call generated, and that the GC side effect is
1154   // set when required.
1155   if (slow_path == nullptr) {
1156     DCHECK(instruction->GetLocations()->WillCall())
1157         << "instruction->DebugName()=" << instruction->DebugName();
1158     DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()))
1159         << "instruction->DebugName()=" << instruction->DebugName()
1160         << " instruction->GetSideEffects().ToString()=" << instruction->GetSideEffects().ToString();
1161   } else {
1162     DCHECK(instruction->GetLocations()->OnlyCallsOnSlowPath() || slow_path->IsFatal())
1163         << "instruction->DebugName()=" << instruction->DebugName()
1164         << " slow_path->GetDescription()=" << slow_path->GetDescription();
1165     DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()) ||
1166            // When read barriers are enabled, some instructions use a
1167            // slow path to emit a read barrier, which does not trigger
1168            // GC, is not fatal, nor is emitted by HDeoptimize
1169            // instructions.
1170            (kEmitCompilerReadBarrier &&
1171             (instruction->IsInstanceFieldGet() ||
1172              instruction->IsStaticFieldGet() ||
1173              instruction->IsArraySet() ||
1174              instruction->IsArrayGet() ||
1175              instruction->IsLoadClass() ||
1176              instruction->IsLoadString() ||
1177              instruction->IsInstanceOf() ||
1178              instruction->IsCheckCast())))
1179         << "instruction->DebugName()=" << instruction->DebugName()
1180         << " instruction->GetSideEffects().ToString()=" << instruction->GetSideEffects().ToString()
1181         << " slow_path->GetDescription()=" << slow_path->GetDescription();
1182   }
1183 
1184   // Check the coherency of leaf information.
1185   DCHECK(instruction->IsSuspendCheck()
1186          || ((slow_path != nullptr) && slow_path->IsFatal())
1187          || instruction->GetLocations()->CanCall()
1188          || !IsLeafMethod())
1189       << instruction->DebugName() << ((slow_path != nullptr) ? slow_path->GetDescription() : "");
1190 }
1191 
SaveLiveRegisters(CodeGenerator * codegen,LocationSummary * locations)1192 void SlowPathCode::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
1193   RegisterSet* live_registers = locations->GetLiveRegisters();
1194   size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
1195 
1196   for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
1197     if (!codegen->IsCoreCalleeSaveRegister(i)) {
1198       if (live_registers->ContainsCoreRegister(i)) {
1199         // If the register holds an object, update the stack mask.
1200         if (locations->RegisterContainsObject(i)) {
1201           locations->SetStackBit(stack_offset / kVRegSize);
1202         }
1203         DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1204         DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1205         saved_core_stack_offsets_[i] = stack_offset;
1206         stack_offset += codegen->SaveCoreRegister(stack_offset, i);
1207       }
1208     }
1209   }
1210 
1211   for (size_t i = 0, e = codegen->GetNumberOfFloatingPointRegisters(); i < e; ++i) {
1212     if (!codegen->IsFloatingPointCalleeSaveRegister(i)) {
1213       if (live_registers->ContainsFloatingPointRegister(i)) {
1214         DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1215         DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1216         saved_fpu_stack_offsets_[i] = stack_offset;
1217         stack_offset += codegen->SaveFloatingPointRegister(stack_offset, i);
1218       }
1219     }
1220   }
1221 }
1222 
RestoreLiveRegisters(CodeGenerator * codegen,LocationSummary * locations)1223 void SlowPathCode::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
1224   RegisterSet* live_registers = locations->GetLiveRegisters();
1225   size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
1226 
1227   for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
1228     if (!codegen->IsCoreCalleeSaveRegister(i)) {
1229       if (live_registers->ContainsCoreRegister(i)) {
1230         DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1231         DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1232         stack_offset += codegen->RestoreCoreRegister(stack_offset, i);
1233       }
1234     }
1235   }
1236 
1237   for (size_t i = 0, e = codegen->GetNumberOfFloatingPointRegisters(); i < e; ++i) {
1238     if (!codegen->IsFloatingPointCalleeSaveRegister(i)) {
1239       if (live_registers->ContainsFloatingPointRegister(i)) {
1240         DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1241         DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1242         stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, i);
1243       }
1244     }
1245   }
1246 }
1247 
CreateSystemArrayCopyLocationSummary(HInvoke * invoke)1248 void CodeGenerator::CreateSystemArrayCopyLocationSummary(HInvoke* invoke) {
1249   // Check to see if we have known failures that will cause us to have to bail out
1250   // to the runtime, and just generate the runtime call directly.
1251   HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant();
1252   HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant();
1253 
1254   // The positions must be non-negative.
1255   if ((src_pos != nullptr && src_pos->GetValue() < 0) ||
1256       (dest_pos != nullptr && dest_pos->GetValue() < 0)) {
1257     // We will have to fail anyways.
1258     return;
1259   }
1260 
1261   // The length must be >= 0.
1262   HIntConstant* length = invoke->InputAt(4)->AsIntConstant();
1263   if (length != nullptr) {
1264     int32_t len = length->GetValue();
1265     if (len < 0) {
1266       // Just call as normal.
1267       return;
1268     }
1269   }
1270 
1271   SystemArrayCopyOptimizations optimizations(invoke);
1272 
1273   if (optimizations.GetDestinationIsSource()) {
1274     if (src_pos != nullptr && dest_pos != nullptr && src_pos->GetValue() < dest_pos->GetValue()) {
1275       // We only support backward copying if source and destination are the same.
1276       return;
1277     }
1278   }
1279 
1280   if (optimizations.GetDestinationIsPrimitiveArray() || optimizations.GetSourceIsPrimitiveArray()) {
1281     // We currently don't intrinsify primitive copying.
1282     return;
1283   }
1284 
1285   ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena();
1286   LocationSummary* locations = new (allocator) LocationSummary(invoke,
1287                                                                LocationSummary::kCallOnSlowPath,
1288                                                                kIntrinsified);
1289   // arraycopy(Object src, int src_pos, Object dest, int dest_pos, int length).
1290   locations->SetInAt(0, Location::RequiresRegister());
1291   locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1)));
1292   locations->SetInAt(2, Location::RequiresRegister());
1293   locations->SetInAt(3, Location::RegisterOrConstant(invoke->InputAt(3)));
1294   locations->SetInAt(4, Location::RegisterOrConstant(invoke->InputAt(4)));
1295 
1296   locations->AddTemp(Location::RequiresRegister());
1297   locations->AddTemp(Location::RequiresRegister());
1298   locations->AddTemp(Location::RequiresRegister());
1299 }
1300 
1301 }  // namespace art
1302