1 /*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "assembler_arm64.h"
18 #include "base/logging.h"
19 #include "entrypoints/quick/quick_entrypoints.h"
20 #include "offsets.h"
21 #include "thread.h"
22
23 using namespace vixl; // NOLINT(build/namespaces)
24
25 namespace art {
26 namespace arm64 {
27
28 #ifdef ___
29 #error "ARM64 Assembler macro already defined."
30 #else
31 #define ___ vixl_masm_->
32 #endif
33
EmitSlowPaths()34 void Arm64Assembler::EmitSlowPaths() {
35 if (!exception_blocks_.empty()) {
36 for (size_t i = 0; i < exception_blocks_.size(); i++) {
37 EmitExceptionPoll(exception_blocks_.at(i));
38 }
39 }
40 ___ FinalizeCode();
41 }
42
CodeSize() const43 size_t Arm64Assembler::CodeSize() const {
44 return vixl_masm_->BufferCapacity() - vixl_masm_->RemainingBufferSpace();
45 }
46
FinalizeInstructions(const MemoryRegion & region)47 void Arm64Assembler::FinalizeInstructions(const MemoryRegion& region) {
48 // Copy the instructions from the buffer.
49 MemoryRegion from(vixl_masm_->GetStartAddress<void*>(), CodeSize());
50 region.CopyFrom(0, from);
51 }
52
GetCurrentThread(ManagedRegister tr)53 void Arm64Assembler::GetCurrentThread(ManagedRegister tr) {
54 ___ Mov(reg_x(tr.AsArm64().AsXRegister()), reg_x(ETR));
55 }
56
GetCurrentThread(FrameOffset offset,ManagedRegister)57 void Arm64Assembler::GetCurrentThread(FrameOffset offset, ManagedRegister /* scratch */) {
58 StoreToOffset(ETR, SP, offset.Int32Value());
59 }
60
61 // See Arm64 PCS Section 5.2.2.1.
IncreaseFrameSize(size_t adjust)62 void Arm64Assembler::IncreaseFrameSize(size_t adjust) {
63 CHECK_ALIGNED(adjust, kStackAlignment);
64 AddConstant(SP, -adjust);
65 cfi().AdjustCFAOffset(adjust);
66 }
67
68 // See Arm64 PCS Section 5.2.2.1.
DecreaseFrameSize(size_t adjust)69 void Arm64Assembler::DecreaseFrameSize(size_t adjust) {
70 CHECK_ALIGNED(adjust, kStackAlignment);
71 AddConstant(SP, adjust);
72 cfi().AdjustCFAOffset(-adjust);
73 }
74
AddConstant(XRegister rd,int32_t value,Condition cond)75 void Arm64Assembler::AddConstant(XRegister rd, int32_t value, Condition cond) {
76 AddConstant(rd, rd, value, cond);
77 }
78
AddConstant(XRegister rd,XRegister rn,int32_t value,Condition cond)79 void Arm64Assembler::AddConstant(XRegister rd, XRegister rn, int32_t value,
80 Condition cond) {
81 if ((cond == al) || (cond == nv)) {
82 // VIXL macro-assembler handles all variants.
83 ___ Add(reg_x(rd), reg_x(rn), value);
84 } else {
85 // temp = rd + value
86 // rd = cond ? temp : rn
87 vixl::UseScratchRegisterScope temps(vixl_masm_);
88 temps.Exclude(reg_x(rd), reg_x(rn));
89 vixl::Register temp = temps.AcquireX();
90 ___ Add(temp, reg_x(rn), value);
91 ___ Csel(reg_x(rd), temp, reg_x(rd), cond);
92 }
93 }
94
StoreWToOffset(StoreOperandType type,WRegister source,XRegister base,int32_t offset)95 void Arm64Assembler::StoreWToOffset(StoreOperandType type, WRegister source,
96 XRegister base, int32_t offset) {
97 switch (type) {
98 case kStoreByte:
99 ___ Strb(reg_w(source), MEM_OP(reg_x(base), offset));
100 break;
101 case kStoreHalfword:
102 ___ Strh(reg_w(source), MEM_OP(reg_x(base), offset));
103 break;
104 case kStoreWord:
105 ___ Str(reg_w(source), MEM_OP(reg_x(base), offset));
106 break;
107 default:
108 LOG(FATAL) << "UNREACHABLE";
109 }
110 }
111
StoreToOffset(XRegister source,XRegister base,int32_t offset)112 void Arm64Assembler::StoreToOffset(XRegister source, XRegister base, int32_t offset) {
113 CHECK_NE(source, SP);
114 ___ Str(reg_x(source), MEM_OP(reg_x(base), offset));
115 }
116
StoreSToOffset(SRegister source,XRegister base,int32_t offset)117 void Arm64Assembler::StoreSToOffset(SRegister source, XRegister base, int32_t offset) {
118 ___ Str(reg_s(source), MEM_OP(reg_x(base), offset));
119 }
120
StoreDToOffset(DRegister source,XRegister base,int32_t offset)121 void Arm64Assembler::StoreDToOffset(DRegister source, XRegister base, int32_t offset) {
122 ___ Str(reg_d(source), MEM_OP(reg_x(base), offset));
123 }
124
Store(FrameOffset offs,ManagedRegister m_src,size_t size)125 void Arm64Assembler::Store(FrameOffset offs, ManagedRegister m_src, size_t size) {
126 Arm64ManagedRegister src = m_src.AsArm64();
127 if (src.IsNoRegister()) {
128 CHECK_EQ(0u, size);
129 } else if (src.IsWRegister()) {
130 CHECK_EQ(4u, size);
131 StoreWToOffset(kStoreWord, src.AsWRegister(), SP, offs.Int32Value());
132 } else if (src.IsXRegister()) {
133 CHECK_EQ(8u, size);
134 StoreToOffset(src.AsXRegister(), SP, offs.Int32Value());
135 } else if (src.IsSRegister()) {
136 StoreSToOffset(src.AsSRegister(), SP, offs.Int32Value());
137 } else {
138 CHECK(src.IsDRegister()) << src;
139 StoreDToOffset(src.AsDRegister(), SP, offs.Int32Value());
140 }
141 }
142
StoreRef(FrameOffset offs,ManagedRegister m_src)143 void Arm64Assembler::StoreRef(FrameOffset offs, ManagedRegister m_src) {
144 Arm64ManagedRegister src = m_src.AsArm64();
145 CHECK(src.IsXRegister()) << src;
146 StoreWToOffset(kStoreWord, src.AsOverlappingWRegister(), SP,
147 offs.Int32Value());
148 }
149
StoreRawPtr(FrameOffset offs,ManagedRegister m_src)150 void Arm64Assembler::StoreRawPtr(FrameOffset offs, ManagedRegister m_src) {
151 Arm64ManagedRegister src = m_src.AsArm64();
152 CHECK(src.IsXRegister()) << src;
153 StoreToOffset(src.AsXRegister(), SP, offs.Int32Value());
154 }
155
StoreImmediateToFrame(FrameOffset offs,uint32_t imm,ManagedRegister m_scratch)156 void Arm64Assembler::StoreImmediateToFrame(FrameOffset offs, uint32_t imm,
157 ManagedRegister m_scratch) {
158 Arm64ManagedRegister scratch = m_scratch.AsArm64();
159 CHECK(scratch.IsXRegister()) << scratch;
160 LoadImmediate(scratch.AsXRegister(), imm);
161 StoreWToOffset(kStoreWord, scratch.AsOverlappingWRegister(), SP,
162 offs.Int32Value());
163 }
164
StoreImmediateToThread64(ThreadOffset<8> offs,uint32_t imm,ManagedRegister m_scratch)165 void Arm64Assembler::StoreImmediateToThread64(ThreadOffset<8> offs, uint32_t imm,
166 ManagedRegister m_scratch) {
167 Arm64ManagedRegister scratch = m_scratch.AsArm64();
168 CHECK(scratch.IsXRegister()) << scratch;
169 LoadImmediate(scratch.AsXRegister(), imm);
170 StoreToOffset(scratch.AsXRegister(), ETR, offs.Int32Value());
171 }
172
StoreStackOffsetToThread64(ThreadOffset<8> tr_offs,FrameOffset fr_offs,ManagedRegister m_scratch)173 void Arm64Assembler::StoreStackOffsetToThread64(ThreadOffset<8> tr_offs,
174 FrameOffset fr_offs,
175 ManagedRegister m_scratch) {
176 Arm64ManagedRegister scratch = m_scratch.AsArm64();
177 CHECK(scratch.IsXRegister()) << scratch;
178 AddConstant(scratch.AsXRegister(), SP, fr_offs.Int32Value());
179 StoreToOffset(scratch.AsXRegister(), ETR, tr_offs.Int32Value());
180 }
181
StoreStackPointerToThread64(ThreadOffset<8> tr_offs)182 void Arm64Assembler::StoreStackPointerToThread64(ThreadOffset<8> tr_offs) {
183 vixl::UseScratchRegisterScope temps(vixl_masm_);
184 vixl::Register temp = temps.AcquireX();
185 ___ Mov(temp, reg_x(SP));
186 ___ Str(temp, MEM_OP(reg_x(ETR), tr_offs.Int32Value()));
187 }
188
StoreSpanning(FrameOffset dest_off,ManagedRegister m_source,FrameOffset in_off,ManagedRegister m_scratch)189 void Arm64Assembler::StoreSpanning(FrameOffset dest_off, ManagedRegister m_source,
190 FrameOffset in_off, ManagedRegister m_scratch) {
191 Arm64ManagedRegister source = m_source.AsArm64();
192 Arm64ManagedRegister scratch = m_scratch.AsArm64();
193 StoreToOffset(source.AsXRegister(), SP, dest_off.Int32Value());
194 LoadFromOffset(scratch.AsXRegister(), SP, in_off.Int32Value());
195 StoreToOffset(scratch.AsXRegister(), SP, dest_off.Int32Value() + 8);
196 }
197
198 // Load routines.
LoadImmediate(XRegister dest,int32_t value,Condition cond)199 void Arm64Assembler::LoadImmediate(XRegister dest, int32_t value,
200 Condition cond) {
201 if ((cond == al) || (cond == nv)) {
202 ___ Mov(reg_x(dest), value);
203 } else {
204 // temp = value
205 // rd = cond ? temp : rd
206 if (value != 0) {
207 vixl::UseScratchRegisterScope temps(vixl_masm_);
208 temps.Exclude(reg_x(dest));
209 vixl::Register temp = temps.AcquireX();
210 ___ Mov(temp, value);
211 ___ Csel(reg_x(dest), temp, reg_x(dest), cond);
212 } else {
213 ___ Csel(reg_x(dest), reg_x(XZR), reg_x(dest), cond);
214 }
215 }
216 }
217
LoadWFromOffset(LoadOperandType type,WRegister dest,XRegister base,int32_t offset)218 void Arm64Assembler::LoadWFromOffset(LoadOperandType type, WRegister dest,
219 XRegister base, int32_t offset) {
220 switch (type) {
221 case kLoadSignedByte:
222 ___ Ldrsb(reg_w(dest), MEM_OP(reg_x(base), offset));
223 break;
224 case kLoadSignedHalfword:
225 ___ Ldrsh(reg_w(dest), MEM_OP(reg_x(base), offset));
226 break;
227 case kLoadUnsignedByte:
228 ___ Ldrb(reg_w(dest), MEM_OP(reg_x(base), offset));
229 break;
230 case kLoadUnsignedHalfword:
231 ___ Ldrh(reg_w(dest), MEM_OP(reg_x(base), offset));
232 break;
233 case kLoadWord:
234 ___ Ldr(reg_w(dest), MEM_OP(reg_x(base), offset));
235 break;
236 default:
237 LOG(FATAL) << "UNREACHABLE";
238 }
239 }
240
241 // Note: We can extend this member by adding load type info - see
242 // sign extended A64 load variants.
LoadFromOffset(XRegister dest,XRegister base,int32_t offset)243 void Arm64Assembler::LoadFromOffset(XRegister dest, XRegister base,
244 int32_t offset) {
245 CHECK_NE(dest, SP);
246 ___ Ldr(reg_x(dest), MEM_OP(reg_x(base), offset));
247 }
248
LoadSFromOffset(SRegister dest,XRegister base,int32_t offset)249 void Arm64Assembler::LoadSFromOffset(SRegister dest, XRegister base,
250 int32_t offset) {
251 ___ Ldr(reg_s(dest), MEM_OP(reg_x(base), offset));
252 }
253
LoadDFromOffset(DRegister dest,XRegister base,int32_t offset)254 void Arm64Assembler::LoadDFromOffset(DRegister dest, XRegister base,
255 int32_t offset) {
256 ___ Ldr(reg_d(dest), MEM_OP(reg_x(base), offset));
257 }
258
Load(Arm64ManagedRegister dest,XRegister base,int32_t offset,size_t size)259 void Arm64Assembler::Load(Arm64ManagedRegister dest, XRegister base,
260 int32_t offset, size_t size) {
261 if (dest.IsNoRegister()) {
262 CHECK_EQ(0u, size) << dest;
263 } else if (dest.IsWRegister()) {
264 CHECK_EQ(4u, size) << dest;
265 ___ Ldr(reg_w(dest.AsWRegister()), MEM_OP(reg_x(base), offset));
266 } else if (dest.IsXRegister()) {
267 CHECK_NE(dest.AsXRegister(), SP) << dest;
268 if (size == 4u) {
269 ___ Ldr(reg_w(dest.AsOverlappingWRegister()), MEM_OP(reg_x(base), offset));
270 } else {
271 CHECK_EQ(8u, size) << dest;
272 ___ Ldr(reg_x(dest.AsXRegister()), MEM_OP(reg_x(base), offset));
273 }
274 } else if (dest.IsSRegister()) {
275 ___ Ldr(reg_s(dest.AsSRegister()), MEM_OP(reg_x(base), offset));
276 } else {
277 CHECK(dest.IsDRegister()) << dest;
278 ___ Ldr(reg_d(dest.AsDRegister()), MEM_OP(reg_x(base), offset));
279 }
280 }
281
Load(ManagedRegister m_dst,FrameOffset src,size_t size)282 void Arm64Assembler::Load(ManagedRegister m_dst, FrameOffset src, size_t size) {
283 return Load(m_dst.AsArm64(), SP, src.Int32Value(), size);
284 }
285
LoadFromThread64(ManagedRegister m_dst,ThreadOffset<8> src,size_t size)286 void Arm64Assembler::LoadFromThread64(ManagedRegister m_dst, ThreadOffset<8> src, size_t size) {
287 return Load(m_dst.AsArm64(), ETR, src.Int32Value(), size);
288 }
289
LoadRef(ManagedRegister m_dst,FrameOffset offs)290 void Arm64Assembler::LoadRef(ManagedRegister m_dst, FrameOffset offs) {
291 Arm64ManagedRegister dst = m_dst.AsArm64();
292 CHECK(dst.IsXRegister()) << dst;
293 LoadWFromOffset(kLoadWord, dst.AsOverlappingWRegister(), SP, offs.Int32Value());
294 }
295
LoadRef(ManagedRegister m_dst,ManagedRegister m_base,MemberOffset offs,bool poison_reference)296 void Arm64Assembler::LoadRef(ManagedRegister m_dst, ManagedRegister m_base, MemberOffset offs,
297 bool poison_reference) {
298 Arm64ManagedRegister dst = m_dst.AsArm64();
299 Arm64ManagedRegister base = m_base.AsArm64();
300 CHECK(dst.IsXRegister() && base.IsXRegister());
301 LoadWFromOffset(kLoadWord, dst.AsOverlappingWRegister(), base.AsXRegister(),
302 offs.Int32Value());
303 if (kPoisonHeapReferences && poison_reference) {
304 WRegister ref_reg = dst.AsOverlappingWRegister();
305 ___ Neg(reg_w(ref_reg), vixl::Operand(reg_w(ref_reg)));
306 }
307 }
308
LoadRawPtr(ManagedRegister m_dst,ManagedRegister m_base,Offset offs)309 void Arm64Assembler::LoadRawPtr(ManagedRegister m_dst, ManagedRegister m_base, Offset offs) {
310 Arm64ManagedRegister dst = m_dst.AsArm64();
311 Arm64ManagedRegister base = m_base.AsArm64();
312 CHECK(dst.IsXRegister() && base.IsXRegister());
313 // Remove dst and base form the temp list - higher level API uses IP1, IP0.
314 vixl::UseScratchRegisterScope temps(vixl_masm_);
315 temps.Exclude(reg_x(dst.AsXRegister()), reg_x(base.AsXRegister()));
316 ___ Ldr(reg_x(dst.AsXRegister()), MEM_OP(reg_x(base.AsXRegister()), offs.Int32Value()));
317 }
318
LoadRawPtrFromThread64(ManagedRegister m_dst,ThreadOffset<8> offs)319 void Arm64Assembler::LoadRawPtrFromThread64(ManagedRegister m_dst, ThreadOffset<8> offs) {
320 Arm64ManagedRegister dst = m_dst.AsArm64();
321 CHECK(dst.IsXRegister()) << dst;
322 LoadFromOffset(dst.AsXRegister(), ETR, offs.Int32Value());
323 }
324
325 // Copying routines.
Move(ManagedRegister m_dst,ManagedRegister m_src,size_t size)326 void Arm64Assembler::Move(ManagedRegister m_dst, ManagedRegister m_src, size_t size) {
327 Arm64ManagedRegister dst = m_dst.AsArm64();
328 Arm64ManagedRegister src = m_src.AsArm64();
329 if (!dst.Equals(src)) {
330 if (dst.IsXRegister()) {
331 if (size == 4) {
332 CHECK(src.IsWRegister());
333 ___ Mov(reg_w(dst.AsOverlappingWRegister()), reg_w(src.AsWRegister()));
334 } else {
335 if (src.IsXRegister()) {
336 ___ Mov(reg_x(dst.AsXRegister()), reg_x(src.AsXRegister()));
337 } else {
338 ___ Mov(reg_x(dst.AsXRegister()), reg_x(src.AsOverlappingXRegister()));
339 }
340 }
341 } else if (dst.IsWRegister()) {
342 CHECK(src.IsWRegister()) << src;
343 ___ Mov(reg_w(dst.AsWRegister()), reg_w(src.AsWRegister()));
344 } else if (dst.IsSRegister()) {
345 CHECK(src.IsSRegister()) << src;
346 ___ Fmov(reg_s(dst.AsSRegister()), reg_s(src.AsSRegister()));
347 } else {
348 CHECK(dst.IsDRegister()) << dst;
349 CHECK(src.IsDRegister()) << src;
350 ___ Fmov(reg_d(dst.AsDRegister()), reg_d(src.AsDRegister()));
351 }
352 }
353 }
354
CopyRawPtrFromThread64(FrameOffset fr_offs,ThreadOffset<8> tr_offs,ManagedRegister m_scratch)355 void Arm64Assembler::CopyRawPtrFromThread64(FrameOffset fr_offs,
356 ThreadOffset<8> tr_offs,
357 ManagedRegister m_scratch) {
358 Arm64ManagedRegister scratch = m_scratch.AsArm64();
359 CHECK(scratch.IsXRegister()) << scratch;
360 LoadFromOffset(scratch.AsXRegister(), ETR, tr_offs.Int32Value());
361 StoreToOffset(scratch.AsXRegister(), SP, fr_offs.Int32Value());
362 }
363
CopyRawPtrToThread64(ThreadOffset<8> tr_offs,FrameOffset fr_offs,ManagedRegister m_scratch)364 void Arm64Assembler::CopyRawPtrToThread64(ThreadOffset<8> tr_offs,
365 FrameOffset fr_offs,
366 ManagedRegister m_scratch) {
367 Arm64ManagedRegister scratch = m_scratch.AsArm64();
368 CHECK(scratch.IsXRegister()) << scratch;
369 LoadFromOffset(scratch.AsXRegister(), SP, fr_offs.Int32Value());
370 StoreToOffset(scratch.AsXRegister(), ETR, tr_offs.Int32Value());
371 }
372
CopyRef(FrameOffset dest,FrameOffset src,ManagedRegister m_scratch)373 void Arm64Assembler::CopyRef(FrameOffset dest, FrameOffset src,
374 ManagedRegister m_scratch) {
375 Arm64ManagedRegister scratch = m_scratch.AsArm64();
376 CHECK(scratch.IsXRegister()) << scratch;
377 LoadWFromOffset(kLoadWord, scratch.AsOverlappingWRegister(),
378 SP, src.Int32Value());
379 StoreWToOffset(kStoreWord, scratch.AsOverlappingWRegister(),
380 SP, dest.Int32Value());
381 }
382
Copy(FrameOffset dest,FrameOffset src,ManagedRegister m_scratch,size_t size)383 void Arm64Assembler::Copy(FrameOffset dest, FrameOffset src,
384 ManagedRegister m_scratch, size_t size) {
385 Arm64ManagedRegister scratch = m_scratch.AsArm64();
386 CHECK(scratch.IsXRegister()) << scratch;
387 CHECK(size == 4 || size == 8) << size;
388 if (size == 4) {
389 LoadWFromOffset(kLoadWord, scratch.AsOverlappingWRegister(), SP, src.Int32Value());
390 StoreWToOffset(kStoreWord, scratch.AsOverlappingWRegister(), SP, dest.Int32Value());
391 } else if (size == 8) {
392 LoadFromOffset(scratch.AsXRegister(), SP, src.Int32Value());
393 StoreToOffset(scratch.AsXRegister(), SP, dest.Int32Value());
394 } else {
395 UNIMPLEMENTED(FATAL) << "We only support Copy() of size 4 and 8";
396 }
397 }
398
Copy(FrameOffset dest,ManagedRegister src_base,Offset src_offset,ManagedRegister m_scratch,size_t size)399 void Arm64Assembler::Copy(FrameOffset dest, ManagedRegister src_base, Offset src_offset,
400 ManagedRegister m_scratch, size_t size) {
401 Arm64ManagedRegister scratch = m_scratch.AsArm64();
402 Arm64ManagedRegister base = src_base.AsArm64();
403 CHECK(base.IsXRegister()) << base;
404 CHECK(scratch.IsXRegister() || scratch.IsWRegister()) << scratch;
405 CHECK(size == 4 || size == 8) << size;
406 if (size == 4) {
407 LoadWFromOffset(kLoadWord, scratch.AsWRegister(), base.AsXRegister(),
408 src_offset.Int32Value());
409 StoreWToOffset(kStoreWord, scratch.AsWRegister(), SP, dest.Int32Value());
410 } else if (size == 8) {
411 LoadFromOffset(scratch.AsXRegister(), base.AsXRegister(), src_offset.Int32Value());
412 StoreToOffset(scratch.AsXRegister(), SP, dest.Int32Value());
413 } else {
414 UNIMPLEMENTED(FATAL) << "We only support Copy() of size 4 and 8";
415 }
416 }
417
Copy(ManagedRegister m_dest_base,Offset dest_offs,FrameOffset src,ManagedRegister m_scratch,size_t size)418 void Arm64Assembler::Copy(ManagedRegister m_dest_base, Offset dest_offs, FrameOffset src,
419 ManagedRegister m_scratch, size_t size) {
420 Arm64ManagedRegister scratch = m_scratch.AsArm64();
421 Arm64ManagedRegister base = m_dest_base.AsArm64();
422 CHECK(base.IsXRegister()) << base;
423 CHECK(scratch.IsXRegister() || scratch.IsWRegister()) << scratch;
424 CHECK(size == 4 || size == 8) << size;
425 if (size == 4) {
426 LoadWFromOffset(kLoadWord, scratch.AsWRegister(), SP, src.Int32Value());
427 StoreWToOffset(kStoreWord, scratch.AsWRegister(), base.AsXRegister(),
428 dest_offs.Int32Value());
429 } else if (size == 8) {
430 LoadFromOffset(scratch.AsXRegister(), SP, src.Int32Value());
431 StoreToOffset(scratch.AsXRegister(), base.AsXRegister(), dest_offs.Int32Value());
432 } else {
433 UNIMPLEMENTED(FATAL) << "We only support Copy() of size 4 and 8";
434 }
435 }
436
Copy(FrameOffset,FrameOffset,Offset,ManagedRegister,size_t)437 void Arm64Assembler::Copy(FrameOffset /*dst*/, FrameOffset /*src_base*/, Offset /*src_offset*/,
438 ManagedRegister /*mscratch*/, size_t /*size*/) {
439 UNIMPLEMENTED(FATAL) << "Unimplemented Copy() variant";
440 }
441
Copy(ManagedRegister m_dest,Offset dest_offset,ManagedRegister m_src,Offset src_offset,ManagedRegister m_scratch,size_t size)442 void Arm64Assembler::Copy(ManagedRegister m_dest, Offset dest_offset,
443 ManagedRegister m_src, Offset src_offset,
444 ManagedRegister m_scratch, size_t size) {
445 Arm64ManagedRegister scratch = m_scratch.AsArm64();
446 Arm64ManagedRegister src = m_src.AsArm64();
447 Arm64ManagedRegister dest = m_dest.AsArm64();
448 CHECK(dest.IsXRegister()) << dest;
449 CHECK(src.IsXRegister()) << src;
450 CHECK(scratch.IsXRegister() || scratch.IsWRegister()) << scratch;
451 CHECK(size == 4 || size == 8) << size;
452 if (size == 4) {
453 if (scratch.IsWRegister()) {
454 LoadWFromOffset(kLoadWord, scratch.AsWRegister(), src.AsXRegister(),
455 src_offset.Int32Value());
456 StoreWToOffset(kStoreWord, scratch.AsWRegister(), dest.AsXRegister(),
457 dest_offset.Int32Value());
458 } else {
459 LoadWFromOffset(kLoadWord, scratch.AsOverlappingWRegister(), src.AsXRegister(),
460 src_offset.Int32Value());
461 StoreWToOffset(kStoreWord, scratch.AsOverlappingWRegister(), dest.AsXRegister(),
462 dest_offset.Int32Value());
463 }
464 } else if (size == 8) {
465 LoadFromOffset(scratch.AsXRegister(), src.AsXRegister(), src_offset.Int32Value());
466 StoreToOffset(scratch.AsXRegister(), dest.AsXRegister(), dest_offset.Int32Value());
467 } else {
468 UNIMPLEMENTED(FATAL) << "We only support Copy() of size 4 and 8";
469 }
470 }
471
Copy(FrameOffset,Offset,FrameOffset,Offset,ManagedRegister,size_t)472 void Arm64Assembler::Copy(FrameOffset /*dst*/, Offset /*dest_offset*/,
473 FrameOffset /*src*/, Offset /*src_offset*/,
474 ManagedRegister /*scratch*/, size_t /*size*/) {
475 UNIMPLEMENTED(FATAL) << "Unimplemented Copy() variant";
476 }
477
MemoryBarrier(ManagedRegister m_scratch ATTRIBUTE_UNUSED)478 void Arm64Assembler::MemoryBarrier(ManagedRegister m_scratch ATTRIBUTE_UNUSED) {
479 // TODO: Should we check that m_scratch is IP? - see arm.
480 ___ Dmb(vixl::InnerShareable, vixl::BarrierAll);
481 }
482
SignExtend(ManagedRegister mreg,size_t size)483 void Arm64Assembler::SignExtend(ManagedRegister mreg, size_t size) {
484 Arm64ManagedRegister reg = mreg.AsArm64();
485 CHECK(size == 1 || size == 2) << size;
486 CHECK(reg.IsWRegister()) << reg;
487 if (size == 1) {
488 ___ Sxtb(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
489 } else {
490 ___ Sxth(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
491 }
492 }
493
ZeroExtend(ManagedRegister mreg,size_t size)494 void Arm64Assembler::ZeroExtend(ManagedRegister mreg, size_t size) {
495 Arm64ManagedRegister reg = mreg.AsArm64();
496 CHECK(size == 1 || size == 2) << size;
497 CHECK(reg.IsWRegister()) << reg;
498 if (size == 1) {
499 ___ Uxtb(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
500 } else {
501 ___ Uxth(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
502 }
503 }
504
VerifyObject(ManagedRegister,bool)505 void Arm64Assembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) {
506 // TODO: not validating references.
507 }
508
VerifyObject(FrameOffset,bool)509 void Arm64Assembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) {
510 // TODO: not validating references.
511 }
512
Call(ManagedRegister m_base,Offset offs,ManagedRegister m_scratch)513 void Arm64Assembler::Call(ManagedRegister m_base, Offset offs, ManagedRegister m_scratch) {
514 Arm64ManagedRegister base = m_base.AsArm64();
515 Arm64ManagedRegister scratch = m_scratch.AsArm64();
516 CHECK(base.IsXRegister()) << base;
517 CHECK(scratch.IsXRegister()) << scratch;
518 LoadFromOffset(scratch.AsXRegister(), base.AsXRegister(), offs.Int32Value());
519 ___ Blr(reg_x(scratch.AsXRegister()));
520 }
521
JumpTo(ManagedRegister m_base,Offset offs,ManagedRegister m_scratch)522 void Arm64Assembler::JumpTo(ManagedRegister m_base, Offset offs, ManagedRegister m_scratch) {
523 Arm64ManagedRegister base = m_base.AsArm64();
524 Arm64ManagedRegister scratch = m_scratch.AsArm64();
525 CHECK(base.IsXRegister()) << base;
526 CHECK(scratch.IsXRegister()) << scratch;
527 // Remove base and scratch form the temp list - higher level API uses IP1, IP0.
528 vixl::UseScratchRegisterScope temps(vixl_masm_);
529 temps.Exclude(reg_x(base.AsXRegister()), reg_x(scratch.AsXRegister()));
530 ___ Ldr(reg_x(scratch.AsXRegister()), MEM_OP(reg_x(base.AsXRegister()), offs.Int32Value()));
531 ___ Br(reg_x(scratch.AsXRegister()));
532 }
533
Call(FrameOffset base,Offset offs,ManagedRegister m_scratch)534 void Arm64Assembler::Call(FrameOffset base, Offset offs, ManagedRegister m_scratch) {
535 Arm64ManagedRegister scratch = m_scratch.AsArm64();
536 CHECK(scratch.IsXRegister()) << scratch;
537 // Call *(*(SP + base) + offset)
538 LoadFromOffset(scratch.AsXRegister(), SP, base.Int32Value());
539 LoadFromOffset(scratch.AsXRegister(), scratch.AsXRegister(), offs.Int32Value());
540 ___ Blr(reg_x(scratch.AsXRegister()));
541 }
542
CallFromThread64(ThreadOffset<8>,ManagedRegister)543 void Arm64Assembler::CallFromThread64(ThreadOffset<8> /*offset*/, ManagedRegister /*scratch*/) {
544 UNIMPLEMENTED(FATAL) << "Unimplemented Call() variant";
545 }
546
CreateHandleScopeEntry(ManagedRegister m_out_reg,FrameOffset handle_scope_offs,ManagedRegister m_in_reg,bool null_allowed)547 void Arm64Assembler::CreateHandleScopeEntry(
548 ManagedRegister m_out_reg, FrameOffset handle_scope_offs, ManagedRegister m_in_reg,
549 bool null_allowed) {
550 Arm64ManagedRegister out_reg = m_out_reg.AsArm64();
551 Arm64ManagedRegister in_reg = m_in_reg.AsArm64();
552 // For now we only hold stale handle scope entries in x registers.
553 CHECK(in_reg.IsNoRegister() || in_reg.IsXRegister()) << in_reg;
554 CHECK(out_reg.IsXRegister()) << out_reg;
555 if (null_allowed) {
556 // Null values get a handle scope entry value of 0. Otherwise, the handle scope entry is
557 // the address in the handle scope holding the reference.
558 // e.g. out_reg = (handle == 0) ? 0 : (SP+handle_offset)
559 if (in_reg.IsNoRegister()) {
560 LoadWFromOffset(kLoadWord, out_reg.AsOverlappingWRegister(), SP,
561 handle_scope_offs.Int32Value());
562 in_reg = out_reg;
563 }
564 ___ Cmp(reg_w(in_reg.AsOverlappingWRegister()), 0);
565 if (!out_reg.Equals(in_reg)) {
566 LoadImmediate(out_reg.AsXRegister(), 0, eq);
567 }
568 AddConstant(out_reg.AsXRegister(), SP, handle_scope_offs.Int32Value(), ne);
569 } else {
570 AddConstant(out_reg.AsXRegister(), SP, handle_scope_offs.Int32Value(), al);
571 }
572 }
573
CreateHandleScopeEntry(FrameOffset out_off,FrameOffset handle_scope_offset,ManagedRegister m_scratch,bool null_allowed)574 void Arm64Assembler::CreateHandleScopeEntry(FrameOffset out_off, FrameOffset handle_scope_offset,
575 ManagedRegister m_scratch, bool null_allowed) {
576 Arm64ManagedRegister scratch = m_scratch.AsArm64();
577 CHECK(scratch.IsXRegister()) << scratch;
578 if (null_allowed) {
579 LoadWFromOffset(kLoadWord, scratch.AsOverlappingWRegister(), SP,
580 handle_scope_offset.Int32Value());
581 // Null values get a handle scope entry value of 0. Otherwise, the handle scope entry is
582 // the address in the handle scope holding the reference.
583 // e.g. scratch = (scratch == 0) ? 0 : (SP+handle_scope_offset)
584 ___ Cmp(reg_w(scratch.AsOverlappingWRegister()), 0);
585 // Move this logic in add constants with flags.
586 AddConstant(scratch.AsXRegister(), SP, handle_scope_offset.Int32Value(), ne);
587 } else {
588 AddConstant(scratch.AsXRegister(), SP, handle_scope_offset.Int32Value(), al);
589 }
590 StoreToOffset(scratch.AsXRegister(), SP, out_off.Int32Value());
591 }
592
LoadReferenceFromHandleScope(ManagedRegister m_out_reg,ManagedRegister m_in_reg)593 void Arm64Assembler::LoadReferenceFromHandleScope(ManagedRegister m_out_reg,
594 ManagedRegister m_in_reg) {
595 Arm64ManagedRegister out_reg = m_out_reg.AsArm64();
596 Arm64ManagedRegister in_reg = m_in_reg.AsArm64();
597 CHECK(out_reg.IsXRegister()) << out_reg;
598 CHECK(in_reg.IsXRegister()) << in_reg;
599 vixl::Label exit;
600 if (!out_reg.Equals(in_reg)) {
601 // FIXME: Who sets the flags here?
602 LoadImmediate(out_reg.AsXRegister(), 0, eq);
603 }
604 ___ Cbz(reg_x(in_reg.AsXRegister()), &exit);
605 LoadFromOffset(out_reg.AsXRegister(), in_reg.AsXRegister(), 0);
606 ___ Bind(&exit);
607 }
608
ExceptionPoll(ManagedRegister m_scratch,size_t stack_adjust)609 void Arm64Assembler::ExceptionPoll(ManagedRegister m_scratch, size_t stack_adjust) {
610 CHECK_ALIGNED(stack_adjust, kStackAlignment);
611 Arm64ManagedRegister scratch = m_scratch.AsArm64();
612 Arm64Exception *current_exception = new Arm64Exception(scratch, stack_adjust);
613 exception_blocks_.push_back(current_exception);
614 LoadFromOffset(scratch.AsXRegister(), ETR, Thread::ExceptionOffset<8>().Int32Value());
615 ___ Cbnz(reg_x(scratch.AsXRegister()), current_exception->Entry());
616 }
617
EmitExceptionPoll(Arm64Exception * exception)618 void Arm64Assembler::EmitExceptionPoll(Arm64Exception *exception) {
619 vixl::UseScratchRegisterScope temps(vixl_masm_);
620 temps.Exclude(reg_x(exception->scratch_.AsXRegister()));
621 vixl::Register temp = temps.AcquireX();
622
623 // Bind exception poll entry.
624 ___ Bind(exception->Entry());
625 if (exception->stack_adjust_ != 0) { // Fix up the frame.
626 DecreaseFrameSize(exception->stack_adjust_);
627 }
628 // Pass exception object as argument.
629 // Don't care about preserving X0 as this won't return.
630 ___ Mov(reg_x(X0), reg_x(exception->scratch_.AsXRegister()));
631 ___ Ldr(temp, MEM_OP(reg_x(ETR), QUICK_ENTRYPOINT_OFFSET(8, pDeliverException).Int32Value()));
632
633 // Move ETR(Callee saved) back to TR(Caller saved) reg. We use ETR on calls
634 // to external functions that might trash TR. We do not need the original
635 // ETR(X21) saved in BuildFrame().
636 ___ Mov(reg_x(TR), reg_x(ETR));
637
638 ___ Blr(temp);
639 // Call should never return.
640 ___ Brk();
641 }
642
DWARFReg(CPURegister reg)643 static inline dwarf::Reg DWARFReg(CPURegister reg) {
644 if (reg.IsFPRegister()) {
645 return dwarf::Reg::Arm64Fp(reg.code());
646 } else {
647 DCHECK_LT(reg.code(), 31u); // X0 - X30.
648 return dwarf::Reg::Arm64Core(reg.code());
649 }
650 }
651
SpillRegisters(vixl::CPURegList registers,int offset)652 void Arm64Assembler::SpillRegisters(vixl::CPURegList registers, int offset) {
653 int size = registers.RegisterSizeInBytes();
654 const Register sp = vixl_masm_->StackPointer();
655 while (registers.Count() >= 2) {
656 const CPURegister& dst0 = registers.PopLowestIndex();
657 const CPURegister& dst1 = registers.PopLowestIndex();
658 ___ Stp(dst0, dst1, MemOperand(sp, offset));
659 cfi_.RelOffset(DWARFReg(dst0), offset);
660 cfi_.RelOffset(DWARFReg(dst1), offset + size);
661 offset += 2 * size;
662 }
663 if (!registers.IsEmpty()) {
664 const CPURegister& dst0 = registers.PopLowestIndex();
665 ___ Str(dst0, MemOperand(sp, offset));
666 cfi_.RelOffset(DWARFReg(dst0), offset);
667 }
668 DCHECK(registers.IsEmpty());
669 }
670
UnspillRegisters(vixl::CPURegList registers,int offset)671 void Arm64Assembler::UnspillRegisters(vixl::CPURegList registers, int offset) {
672 int size = registers.RegisterSizeInBytes();
673 const Register sp = vixl_masm_->StackPointer();
674 while (registers.Count() >= 2) {
675 const CPURegister& dst0 = registers.PopLowestIndex();
676 const CPURegister& dst1 = registers.PopLowestIndex();
677 ___ Ldp(dst0, dst1, MemOperand(sp, offset));
678 cfi_.Restore(DWARFReg(dst0));
679 cfi_.Restore(DWARFReg(dst1));
680 offset += 2 * size;
681 }
682 if (!registers.IsEmpty()) {
683 const CPURegister& dst0 = registers.PopLowestIndex();
684 ___ Ldr(dst0, MemOperand(sp, offset));
685 cfi_.Restore(DWARFReg(dst0));
686 }
687 DCHECK(registers.IsEmpty());
688 }
689
BuildFrame(size_t frame_size,ManagedRegister method_reg,const std::vector<ManagedRegister> & callee_save_regs,const ManagedRegisterEntrySpills & entry_spills)690 void Arm64Assembler::BuildFrame(size_t frame_size, ManagedRegister method_reg,
691 const std::vector<ManagedRegister>& callee_save_regs,
692 const ManagedRegisterEntrySpills& entry_spills) {
693 // Setup VIXL CPURegList for callee-saves.
694 CPURegList core_reg_list(CPURegister::kRegister, kXRegSize, 0);
695 CPURegList fp_reg_list(CPURegister::kFPRegister, kDRegSize, 0);
696 for (auto r : callee_save_regs) {
697 Arm64ManagedRegister reg = r.AsArm64();
698 if (reg.IsXRegister()) {
699 core_reg_list.Combine(reg_x(reg.AsXRegister()).code());
700 } else {
701 DCHECK(reg.IsDRegister());
702 fp_reg_list.Combine(reg_d(reg.AsDRegister()).code());
703 }
704 }
705 size_t core_reg_size = core_reg_list.TotalSizeInBytes();
706 size_t fp_reg_size = fp_reg_list.TotalSizeInBytes();
707
708 // Increase frame to required size.
709 DCHECK_ALIGNED(frame_size, kStackAlignment);
710 DCHECK_GE(frame_size, core_reg_size + fp_reg_size + kArm64PointerSize);
711 IncreaseFrameSize(frame_size);
712
713 // Save callee-saves.
714 SpillRegisters(core_reg_list, frame_size - core_reg_size);
715 SpillRegisters(fp_reg_list, frame_size - core_reg_size - fp_reg_size);
716
717 // Note: This is specific to JNI method frame.
718 // We will need to move TR(Caller saved in AAPCS) to ETR(Callee saved in AAPCS). The original
719 // (ETR)X21 has been saved on stack. In this way, we can restore TR later.
720 DCHECK(!core_reg_list.IncludesAliasOf(reg_x(TR)));
721 DCHECK(core_reg_list.IncludesAliasOf(reg_x(ETR)));
722 ___ Mov(reg_x(ETR), reg_x(TR));
723
724 // Write ArtMethod*
725 DCHECK(X0 == method_reg.AsArm64().AsXRegister());
726 StoreToOffset(X0, SP, 0);
727
728 // Write out entry spills
729 int32_t offset = frame_size + kArm64PointerSize;
730 for (size_t i = 0; i < entry_spills.size(); ++i) {
731 Arm64ManagedRegister reg = entry_spills.at(i).AsArm64();
732 if (reg.IsNoRegister()) {
733 // only increment stack offset.
734 ManagedRegisterSpill spill = entry_spills.at(i);
735 offset += spill.getSize();
736 } else if (reg.IsXRegister()) {
737 StoreToOffset(reg.AsXRegister(), SP, offset);
738 offset += 8;
739 } else if (reg.IsWRegister()) {
740 StoreWToOffset(kStoreWord, reg.AsWRegister(), SP, offset);
741 offset += 4;
742 } else if (reg.IsDRegister()) {
743 StoreDToOffset(reg.AsDRegister(), SP, offset);
744 offset += 8;
745 } else if (reg.IsSRegister()) {
746 StoreSToOffset(reg.AsSRegister(), SP, offset);
747 offset += 4;
748 }
749 }
750 }
751
RemoveFrame(size_t frame_size,const std::vector<ManagedRegister> & callee_save_regs)752 void Arm64Assembler::RemoveFrame(size_t frame_size,
753 const std::vector<ManagedRegister>& callee_save_regs) {
754 // Setup VIXL CPURegList for callee-saves.
755 CPURegList core_reg_list(CPURegister::kRegister, kXRegSize, 0);
756 CPURegList fp_reg_list(CPURegister::kFPRegister, kDRegSize, 0);
757 for (auto r : callee_save_regs) {
758 Arm64ManagedRegister reg = r.AsArm64();
759 if (reg.IsXRegister()) {
760 core_reg_list.Combine(reg_x(reg.AsXRegister()).code());
761 } else {
762 DCHECK(reg.IsDRegister());
763 fp_reg_list.Combine(reg_d(reg.AsDRegister()).code());
764 }
765 }
766 size_t core_reg_size = core_reg_list.TotalSizeInBytes();
767 size_t fp_reg_size = fp_reg_list.TotalSizeInBytes();
768
769 // For now we only check that the size of the frame is large enough to hold spills and method
770 // reference.
771 DCHECK_GE(frame_size, core_reg_size + fp_reg_size + kArm64PointerSize);
772 DCHECK_ALIGNED(frame_size, kStackAlignment);
773
774 // Note: This is specific to JNI method frame.
775 // Restore TR(Caller saved in AAPCS) from ETR(Callee saved in AAPCS).
776 DCHECK(!core_reg_list.IncludesAliasOf(reg_x(TR)));
777 DCHECK(core_reg_list.IncludesAliasOf(reg_x(ETR)));
778 ___ Mov(reg_x(TR), reg_x(ETR));
779
780 cfi_.RememberState();
781
782 // Restore callee-saves.
783 UnspillRegisters(core_reg_list, frame_size - core_reg_size);
784 UnspillRegisters(fp_reg_list, frame_size - core_reg_size - fp_reg_size);
785
786 // Decrease frame size to start of callee saved regs.
787 DecreaseFrameSize(frame_size);
788
789 // Pop callee saved and return to LR.
790 ___ Ret();
791
792 // The CFI should be restored for any code that follows the exit block.
793 cfi_.RestoreState();
794 cfi_.DefCFAOffset(frame_size);
795 }
796
797 } // namespace arm64
798 } // namespace art
799