1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "code_generator_arm64.h"
18 #include "mirror/array-inl.h"
19 
20 using namespace vixl::aarch64;  // NOLINT(build/namespaces)
21 
22 namespace art {
23 namespace arm64 {
24 
25 using helpers::VRegisterFrom;
26 using helpers::HeapOperand;
27 using helpers::InputRegisterAt;
28 using helpers::Int64ConstantFrom;
29 using helpers::XRegisterFrom;
30 using helpers::WRegisterFrom;
31 
32 #define __ GetVIXLAssembler()->
33 
VisitVecReplicateScalar(HVecReplicateScalar * instruction)34 void LocationsBuilderARM64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) {
35   LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
36   switch (instruction->GetPackedType()) {
37     case Primitive::kPrimBoolean:
38     case Primitive::kPrimByte:
39     case Primitive::kPrimChar:
40     case Primitive::kPrimShort:
41     case Primitive::kPrimInt:
42     case Primitive::kPrimLong:
43       locations->SetInAt(0, Location::RequiresRegister());
44       locations->SetOut(Location::RequiresFpuRegister());
45       break;
46     case Primitive::kPrimFloat:
47     case Primitive::kPrimDouble:
48       locations->SetInAt(0, Location::RequiresFpuRegister());
49       locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
50       break;
51     default:
52       LOG(FATAL) << "Unsupported SIMD type";
53       UNREACHABLE();
54   }
55 }
56 
VisitVecReplicateScalar(HVecReplicateScalar * instruction)57 void InstructionCodeGeneratorARM64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) {
58   LocationSummary* locations = instruction->GetLocations();
59   VRegister dst = VRegisterFrom(locations->Out());
60   switch (instruction->GetPackedType()) {
61     case Primitive::kPrimBoolean:
62     case Primitive::kPrimByte:
63       DCHECK_EQ(16u, instruction->GetVectorLength());
64       __ Dup(dst.V16B(), InputRegisterAt(instruction, 0));
65       break;
66     case Primitive::kPrimChar:
67     case Primitive::kPrimShort:
68       DCHECK_EQ(8u, instruction->GetVectorLength());
69       __ Dup(dst.V8H(), InputRegisterAt(instruction, 0));
70       break;
71     case Primitive::kPrimInt:
72       DCHECK_EQ(4u, instruction->GetVectorLength());
73       __ Dup(dst.V4S(), InputRegisterAt(instruction, 0));
74       break;
75     case Primitive::kPrimLong:
76       DCHECK_EQ(2u, instruction->GetVectorLength());
77       __ Dup(dst.V2D(), XRegisterFrom(locations->InAt(0)));
78       break;
79     case Primitive::kPrimFloat:
80       DCHECK_EQ(4u, instruction->GetVectorLength());
81       __ Dup(dst.V4S(), VRegisterFrom(locations->InAt(0)).V4S(), 0);
82       break;
83     case Primitive::kPrimDouble:
84       DCHECK_EQ(2u, instruction->GetVectorLength());
85       __ Dup(dst.V2D(), VRegisterFrom(locations->InAt(0)).V2D(), 0);
86       break;
87     default:
88       LOG(FATAL) << "Unsupported SIMD type";
89       UNREACHABLE();
90   }
91 }
92 
VisitVecSetScalars(HVecSetScalars * instruction)93 void LocationsBuilderARM64::VisitVecSetScalars(HVecSetScalars* instruction) {
94   LOG(FATAL) << "No SIMD for " << instruction->GetId();
95 }
96 
VisitVecSetScalars(HVecSetScalars * instruction)97 void InstructionCodeGeneratorARM64::VisitVecSetScalars(HVecSetScalars* instruction) {
98   LOG(FATAL) << "No SIMD for " << instruction->GetId();
99 }
100 
VisitVecSumReduce(HVecSumReduce * instruction)101 void LocationsBuilderARM64::VisitVecSumReduce(HVecSumReduce* instruction) {
102   LOG(FATAL) << "No SIMD for " << instruction->GetId();
103 }
104 
VisitVecSumReduce(HVecSumReduce * instruction)105 void InstructionCodeGeneratorARM64::VisitVecSumReduce(HVecSumReduce* instruction) {
106   LOG(FATAL) << "No SIMD for " << instruction->GetId();
107 }
108 
109 // Helper to set up locations for vector unary operations.
CreateVecUnOpLocations(ArenaAllocator * arena,HVecUnaryOperation * instruction)110 static void CreateVecUnOpLocations(ArenaAllocator* arena, HVecUnaryOperation* instruction) {
111   LocationSummary* locations = new (arena) LocationSummary(instruction);
112   switch (instruction->GetPackedType()) {
113     case Primitive::kPrimBoolean:
114       locations->SetInAt(0, Location::RequiresFpuRegister());
115       locations->SetOut(Location::RequiresFpuRegister(),
116                         instruction->IsVecNot() ? Location::kOutputOverlap
117                                                 : Location::kNoOutputOverlap);
118       break;
119     case Primitive::kPrimByte:
120     case Primitive::kPrimChar:
121     case Primitive::kPrimShort:
122     case Primitive::kPrimInt:
123     case Primitive::kPrimLong:
124     case Primitive::kPrimFloat:
125     case Primitive::kPrimDouble:
126       locations->SetInAt(0, Location::RequiresFpuRegister());
127       locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
128       break;
129     default:
130       LOG(FATAL) << "Unsupported SIMD type";
131       UNREACHABLE();
132   }
133 }
134 
VisitVecCnv(HVecCnv * instruction)135 void LocationsBuilderARM64::VisitVecCnv(HVecCnv* instruction) {
136   CreateVecUnOpLocations(GetGraph()->GetArena(), instruction);
137 }
138 
VisitVecCnv(HVecCnv * instruction)139 void InstructionCodeGeneratorARM64::VisitVecCnv(HVecCnv* instruction) {
140   LocationSummary* locations = instruction->GetLocations();
141   VRegister src = VRegisterFrom(locations->InAt(0));
142   VRegister dst = VRegisterFrom(locations->Out());
143   Primitive::Type from = instruction->GetInputType();
144   Primitive::Type to = instruction->GetResultType();
145   if (from == Primitive::kPrimInt && to == Primitive::kPrimFloat) {
146     DCHECK_EQ(4u, instruction->GetVectorLength());
147     __ Scvtf(dst.V4S(), src.V4S());
148   } else {
149     LOG(FATAL) << "Unsupported SIMD type";
150   }
151 }
152 
VisitVecNeg(HVecNeg * instruction)153 void LocationsBuilderARM64::VisitVecNeg(HVecNeg* instruction) {
154   CreateVecUnOpLocations(GetGraph()->GetArena(), instruction);
155 }
156 
VisitVecNeg(HVecNeg * instruction)157 void InstructionCodeGeneratorARM64::VisitVecNeg(HVecNeg* instruction) {
158   LocationSummary* locations = instruction->GetLocations();
159   VRegister src = VRegisterFrom(locations->InAt(0));
160   VRegister dst = VRegisterFrom(locations->Out());
161   switch (instruction->GetPackedType()) {
162     case Primitive::kPrimByte:
163       DCHECK_EQ(16u, instruction->GetVectorLength());
164       __ Neg(dst.V16B(), src.V16B());
165       break;
166     case Primitive::kPrimChar:
167     case Primitive::kPrimShort:
168       DCHECK_EQ(8u, instruction->GetVectorLength());
169       __ Neg(dst.V8H(), src.V8H());
170       break;
171     case Primitive::kPrimInt:
172       DCHECK_EQ(4u, instruction->GetVectorLength());
173       __ Neg(dst.V4S(), src.V4S());
174       break;
175     case Primitive::kPrimLong:
176       DCHECK_EQ(2u, instruction->GetVectorLength());
177       __ Neg(dst.V2D(), src.V2D());
178       break;
179     case Primitive::kPrimFloat:
180       DCHECK_EQ(4u, instruction->GetVectorLength());
181       __ Fneg(dst.V4S(), src.V4S());
182       break;
183     case Primitive::kPrimDouble:
184       DCHECK_EQ(2u, instruction->GetVectorLength());
185       __ Fneg(dst.V2D(), src.V2D());
186       break;
187     default:
188       LOG(FATAL) << "Unsupported SIMD type";
189       UNREACHABLE();
190   }
191 }
192 
VisitVecAbs(HVecAbs * instruction)193 void LocationsBuilderARM64::VisitVecAbs(HVecAbs* instruction) {
194   CreateVecUnOpLocations(GetGraph()->GetArena(), instruction);
195 }
196 
VisitVecAbs(HVecAbs * instruction)197 void InstructionCodeGeneratorARM64::VisitVecAbs(HVecAbs* instruction) {
198   LocationSummary* locations = instruction->GetLocations();
199   VRegister src = VRegisterFrom(locations->InAt(0));
200   VRegister dst = VRegisterFrom(locations->Out());
201   switch (instruction->GetPackedType()) {
202     case Primitive::kPrimByte:
203       DCHECK_EQ(16u, instruction->GetVectorLength());
204       __ Abs(dst.V16B(), src.V16B());
205       break;
206     case Primitive::kPrimChar:
207     case Primitive::kPrimShort:
208       DCHECK_EQ(8u, instruction->GetVectorLength());
209       __ Abs(dst.V8H(), src.V8H());
210       break;
211     case Primitive::kPrimInt:
212       DCHECK_EQ(4u, instruction->GetVectorLength());
213       __ Abs(dst.V4S(), src.V4S());
214       break;
215     case Primitive::kPrimLong:
216       DCHECK_EQ(2u, instruction->GetVectorLength());
217       __ Abs(dst.V2D(), src.V2D());
218       break;
219     case Primitive::kPrimFloat:
220       DCHECK_EQ(4u, instruction->GetVectorLength());
221       __ Fabs(dst.V4S(), src.V4S());
222       break;
223     case Primitive::kPrimDouble:
224       DCHECK_EQ(2u, instruction->GetVectorLength());
225       __ Fabs(dst.V2D(), src.V2D());
226       break;
227     default:
228       LOG(FATAL) << "Unsupported SIMD type";
229   }
230 }
231 
VisitVecNot(HVecNot * instruction)232 void LocationsBuilderARM64::VisitVecNot(HVecNot* instruction) {
233   CreateVecUnOpLocations(GetGraph()->GetArena(), instruction);
234 }
235 
VisitVecNot(HVecNot * instruction)236 void InstructionCodeGeneratorARM64::VisitVecNot(HVecNot* instruction) {
237   LocationSummary* locations = instruction->GetLocations();
238   VRegister src = VRegisterFrom(locations->InAt(0));
239   VRegister dst = VRegisterFrom(locations->Out());
240   switch (instruction->GetPackedType()) {
241     case Primitive::kPrimBoolean:  // special case boolean-not
242       DCHECK_EQ(16u, instruction->GetVectorLength());
243       __ Movi(dst.V16B(), 1);
244       __ Eor(dst.V16B(), dst.V16B(), src.V16B());
245       break;
246     case Primitive::kPrimByte:
247     case Primitive::kPrimChar:
248     case Primitive::kPrimShort:
249     case Primitive::kPrimInt:
250     case Primitive::kPrimLong:
251       __ Not(dst.V16B(), src.V16B());  // lanes do not matter
252       break;
253     default:
254       LOG(FATAL) << "Unsupported SIMD type";
255       UNREACHABLE();
256   }
257 }
258 
259 // Helper to set up locations for vector binary operations.
CreateVecBinOpLocations(ArenaAllocator * arena,HVecBinaryOperation * instruction)260 static void CreateVecBinOpLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) {
261   LocationSummary* locations = new (arena) LocationSummary(instruction);
262   switch (instruction->GetPackedType()) {
263     case Primitive::kPrimBoolean:
264     case Primitive::kPrimByte:
265     case Primitive::kPrimChar:
266     case Primitive::kPrimShort:
267     case Primitive::kPrimInt:
268     case Primitive::kPrimLong:
269     case Primitive::kPrimFloat:
270     case Primitive::kPrimDouble:
271       locations->SetInAt(0, Location::RequiresFpuRegister());
272       locations->SetInAt(1, Location::RequiresFpuRegister());
273       locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
274       break;
275     default:
276       LOG(FATAL) << "Unsupported SIMD type";
277       UNREACHABLE();
278   }
279 }
280 
VisitVecAdd(HVecAdd * instruction)281 void LocationsBuilderARM64::VisitVecAdd(HVecAdd* instruction) {
282   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
283 }
284 
VisitVecAdd(HVecAdd * instruction)285 void InstructionCodeGeneratorARM64::VisitVecAdd(HVecAdd* instruction) {
286   LocationSummary* locations = instruction->GetLocations();
287   VRegister lhs = VRegisterFrom(locations->InAt(0));
288   VRegister rhs = VRegisterFrom(locations->InAt(1));
289   VRegister dst = VRegisterFrom(locations->Out());
290   switch (instruction->GetPackedType()) {
291     case Primitive::kPrimByte:
292       DCHECK_EQ(16u, instruction->GetVectorLength());
293       __ Add(dst.V16B(), lhs.V16B(), rhs.V16B());
294       break;
295     case Primitive::kPrimChar:
296     case Primitive::kPrimShort:
297       DCHECK_EQ(8u, instruction->GetVectorLength());
298       __ Add(dst.V8H(), lhs.V8H(), rhs.V8H());
299       break;
300     case Primitive::kPrimInt:
301       DCHECK_EQ(4u, instruction->GetVectorLength());
302       __ Add(dst.V4S(), lhs.V4S(), rhs.V4S());
303       break;
304     case Primitive::kPrimLong:
305       DCHECK_EQ(2u, instruction->GetVectorLength());
306       __ Add(dst.V2D(), lhs.V2D(), rhs.V2D());
307       break;
308     case Primitive::kPrimFloat:
309       DCHECK_EQ(4u, instruction->GetVectorLength());
310       __ Fadd(dst.V4S(), lhs.V4S(), rhs.V4S());
311       break;
312     case Primitive::kPrimDouble:
313       DCHECK_EQ(2u, instruction->GetVectorLength());
314       __ Fadd(dst.V2D(), lhs.V2D(), rhs.V2D());
315       break;
316     default:
317       LOG(FATAL) << "Unsupported SIMD type";
318       UNREACHABLE();
319   }
320 }
321 
VisitVecHalvingAdd(HVecHalvingAdd * instruction)322 void LocationsBuilderARM64::VisitVecHalvingAdd(HVecHalvingAdd* instruction) {
323   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
324 }
325 
VisitVecHalvingAdd(HVecHalvingAdd * instruction)326 void InstructionCodeGeneratorARM64::VisitVecHalvingAdd(HVecHalvingAdd* instruction) {
327   LocationSummary* locations = instruction->GetLocations();
328   VRegister lhs = VRegisterFrom(locations->InAt(0));
329   VRegister rhs = VRegisterFrom(locations->InAt(1));
330   VRegister dst = VRegisterFrom(locations->Out());
331   switch (instruction->GetPackedType()) {
332     case Primitive::kPrimByte:
333       DCHECK_EQ(16u, instruction->GetVectorLength());
334       if (instruction->IsUnsigned()) {
335         instruction->IsRounded()
336             ? __ Urhadd(dst.V16B(), lhs.V16B(), rhs.V16B())
337             : __ Uhadd(dst.V16B(), lhs.V16B(), rhs.V16B());
338       } else {
339         instruction->IsRounded()
340             ? __ Srhadd(dst.V16B(), lhs.V16B(), rhs.V16B())
341             : __ Shadd(dst.V16B(), lhs.V16B(), rhs.V16B());
342       }
343       break;
344     case Primitive::kPrimChar:
345     case Primitive::kPrimShort:
346       DCHECK_EQ(8u, instruction->GetVectorLength());
347       if (instruction->IsUnsigned()) {
348         instruction->IsRounded()
349             ? __ Urhadd(dst.V8H(), lhs.V8H(), rhs.V8H())
350             : __ Uhadd(dst.V8H(), lhs.V8H(), rhs.V8H());
351       } else {
352         instruction->IsRounded()
353             ? __ Srhadd(dst.V8H(), lhs.V8H(), rhs.V8H())
354             : __ Shadd(dst.V8H(), lhs.V8H(), rhs.V8H());
355       }
356       break;
357     default:
358       LOG(FATAL) << "Unsupported SIMD type";
359       UNREACHABLE();
360   }
361 }
362 
VisitVecSub(HVecSub * instruction)363 void LocationsBuilderARM64::VisitVecSub(HVecSub* instruction) {
364   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
365 }
366 
VisitVecSub(HVecSub * instruction)367 void InstructionCodeGeneratorARM64::VisitVecSub(HVecSub* instruction) {
368   LocationSummary* locations = instruction->GetLocations();
369   VRegister lhs = VRegisterFrom(locations->InAt(0));
370   VRegister rhs = VRegisterFrom(locations->InAt(1));
371   VRegister dst = VRegisterFrom(locations->Out());
372   switch (instruction->GetPackedType()) {
373     case Primitive::kPrimByte:
374       DCHECK_EQ(16u, instruction->GetVectorLength());
375       __ Sub(dst.V16B(), lhs.V16B(), rhs.V16B());
376       break;
377     case Primitive::kPrimChar:
378     case Primitive::kPrimShort:
379       DCHECK_EQ(8u, instruction->GetVectorLength());
380       __ Sub(dst.V8H(), lhs.V8H(), rhs.V8H());
381       break;
382     case Primitive::kPrimInt:
383       DCHECK_EQ(4u, instruction->GetVectorLength());
384       __ Sub(dst.V4S(), lhs.V4S(), rhs.V4S());
385       break;
386     case Primitive::kPrimLong:
387       DCHECK_EQ(2u, instruction->GetVectorLength());
388       __ Sub(dst.V2D(), lhs.V2D(), rhs.V2D());
389       break;
390     case Primitive::kPrimFloat:
391       DCHECK_EQ(4u, instruction->GetVectorLength());
392       __ Fsub(dst.V4S(), lhs.V4S(), rhs.V4S());
393       break;
394     case Primitive::kPrimDouble:
395       DCHECK_EQ(2u, instruction->GetVectorLength());
396       __ Fsub(dst.V2D(), lhs.V2D(), rhs.V2D());
397       break;
398     default:
399       LOG(FATAL) << "Unsupported SIMD type";
400       UNREACHABLE();
401   }
402 }
403 
VisitVecMul(HVecMul * instruction)404 void LocationsBuilderARM64::VisitVecMul(HVecMul* instruction) {
405   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
406 }
407 
VisitVecMul(HVecMul * instruction)408 void InstructionCodeGeneratorARM64::VisitVecMul(HVecMul* instruction) {
409   LocationSummary* locations = instruction->GetLocations();
410   VRegister lhs = VRegisterFrom(locations->InAt(0));
411   VRegister rhs = VRegisterFrom(locations->InAt(1));
412   VRegister dst = VRegisterFrom(locations->Out());
413   switch (instruction->GetPackedType()) {
414     case Primitive::kPrimByte:
415       DCHECK_EQ(16u, instruction->GetVectorLength());
416       __ Mul(dst.V16B(), lhs.V16B(), rhs.V16B());
417       break;
418     case Primitive::kPrimChar:
419     case Primitive::kPrimShort:
420       DCHECK_EQ(8u, instruction->GetVectorLength());
421       __ Mul(dst.V8H(), lhs.V8H(), rhs.V8H());
422       break;
423     case Primitive::kPrimInt:
424       DCHECK_EQ(4u, instruction->GetVectorLength());
425       __ Mul(dst.V4S(), lhs.V4S(), rhs.V4S());
426       break;
427     case Primitive::kPrimFloat:
428       DCHECK_EQ(4u, instruction->GetVectorLength());
429       __ Fmul(dst.V4S(), lhs.V4S(), rhs.V4S());
430       break;
431     case Primitive::kPrimDouble:
432       DCHECK_EQ(2u, instruction->GetVectorLength());
433       __ Fmul(dst.V2D(), lhs.V2D(), rhs.V2D());
434       break;
435     default:
436       LOG(FATAL) << "Unsupported SIMD type";
437       UNREACHABLE();
438   }
439 }
440 
VisitVecDiv(HVecDiv * instruction)441 void LocationsBuilderARM64::VisitVecDiv(HVecDiv* instruction) {
442   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
443 }
444 
VisitVecDiv(HVecDiv * instruction)445 void InstructionCodeGeneratorARM64::VisitVecDiv(HVecDiv* instruction) {
446   LocationSummary* locations = instruction->GetLocations();
447   VRegister lhs = VRegisterFrom(locations->InAt(0));
448   VRegister rhs = VRegisterFrom(locations->InAt(1));
449   VRegister dst = VRegisterFrom(locations->Out());
450   switch (instruction->GetPackedType()) {
451     case Primitive::kPrimFloat:
452       DCHECK_EQ(4u, instruction->GetVectorLength());
453       __ Fdiv(dst.V4S(), lhs.V4S(), rhs.V4S());
454       break;
455     case Primitive::kPrimDouble:
456       DCHECK_EQ(2u, instruction->GetVectorLength());
457       __ Fdiv(dst.V2D(), lhs.V2D(), rhs.V2D());
458       break;
459     default:
460       LOG(FATAL) << "Unsupported SIMD type";
461       UNREACHABLE();
462   }
463 }
464 
VisitVecMin(HVecMin * instruction)465 void LocationsBuilderARM64::VisitVecMin(HVecMin* instruction) {
466   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
467 }
468 
VisitVecMin(HVecMin * instruction)469 void InstructionCodeGeneratorARM64::VisitVecMin(HVecMin* instruction) {
470   LOG(FATAL) << "Unsupported SIMD instruction " << instruction->GetId();
471 }
472 
VisitVecMax(HVecMax * instruction)473 void LocationsBuilderARM64::VisitVecMax(HVecMax* instruction) {
474   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
475 }
476 
VisitVecMax(HVecMax * instruction)477 void InstructionCodeGeneratorARM64::VisitVecMax(HVecMax* instruction) {
478   LOG(FATAL) << "Unsupported SIMD instruction " << instruction->GetId();
479 }
480 
VisitVecAnd(HVecAnd * instruction)481 void LocationsBuilderARM64::VisitVecAnd(HVecAnd* instruction) {
482   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
483 }
484 
VisitVecAnd(HVecAnd * instruction)485 void InstructionCodeGeneratorARM64::VisitVecAnd(HVecAnd* instruction) {
486   LocationSummary* locations = instruction->GetLocations();
487   VRegister lhs = VRegisterFrom(locations->InAt(0));
488   VRegister rhs = VRegisterFrom(locations->InAt(1));
489   VRegister dst = VRegisterFrom(locations->Out());
490   switch (instruction->GetPackedType()) {
491     case Primitive::kPrimBoolean:
492     case Primitive::kPrimByte:
493     case Primitive::kPrimChar:
494     case Primitive::kPrimShort:
495     case Primitive::kPrimInt:
496     case Primitive::kPrimLong:
497     case Primitive::kPrimFloat:
498     case Primitive::kPrimDouble:
499       __ And(dst.V16B(), lhs.V16B(), rhs.V16B());  // lanes do not matter
500       break;
501     default:
502       LOG(FATAL) << "Unsupported SIMD type";
503       UNREACHABLE();
504   }
505 }
506 
VisitVecAndNot(HVecAndNot * instruction)507 void LocationsBuilderARM64::VisitVecAndNot(HVecAndNot* instruction) {
508   LOG(FATAL) << "Unsupported SIMD instruction " << instruction->GetId();
509 }
510 
VisitVecAndNot(HVecAndNot * instruction)511 void InstructionCodeGeneratorARM64::VisitVecAndNot(HVecAndNot* instruction) {
512   LOG(FATAL) << "Unsupported SIMD instruction " << instruction->GetId();
513 }
514 
VisitVecOr(HVecOr * instruction)515 void LocationsBuilderARM64::VisitVecOr(HVecOr* instruction) {
516   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
517 }
518 
VisitVecOr(HVecOr * instruction)519 void InstructionCodeGeneratorARM64::VisitVecOr(HVecOr* instruction) {
520   LocationSummary* locations = instruction->GetLocations();
521   VRegister lhs = VRegisterFrom(locations->InAt(0));
522   VRegister rhs = VRegisterFrom(locations->InAt(1));
523   VRegister dst = VRegisterFrom(locations->Out());
524   switch (instruction->GetPackedType()) {
525     case Primitive::kPrimBoolean:
526     case Primitive::kPrimByte:
527     case Primitive::kPrimChar:
528     case Primitive::kPrimShort:
529     case Primitive::kPrimInt:
530     case Primitive::kPrimLong:
531     case Primitive::kPrimFloat:
532     case Primitive::kPrimDouble:
533       __ Orr(dst.V16B(), lhs.V16B(), rhs.V16B());  // lanes do not matter
534       break;
535     default:
536       LOG(FATAL) << "Unsupported SIMD type";
537       UNREACHABLE();
538   }
539 }
540 
VisitVecXor(HVecXor * instruction)541 void LocationsBuilderARM64::VisitVecXor(HVecXor* instruction) {
542   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
543 }
544 
VisitVecXor(HVecXor * instruction)545 void InstructionCodeGeneratorARM64::VisitVecXor(HVecXor* instruction) {
546   LocationSummary* locations = instruction->GetLocations();
547   VRegister lhs = VRegisterFrom(locations->InAt(0));
548   VRegister rhs = VRegisterFrom(locations->InAt(1));
549   VRegister dst = VRegisterFrom(locations->Out());
550   switch (instruction->GetPackedType()) {
551     case Primitive::kPrimBoolean:
552     case Primitive::kPrimByte:
553     case Primitive::kPrimChar:
554     case Primitive::kPrimShort:
555     case Primitive::kPrimInt:
556     case Primitive::kPrimLong:
557     case Primitive::kPrimFloat:
558     case Primitive::kPrimDouble:
559       __ Eor(dst.V16B(), lhs.V16B(), rhs.V16B());  // lanes do not matter
560       break;
561     default:
562       LOG(FATAL) << "Unsupported SIMD type";
563       UNREACHABLE();
564   }
565 }
566 
567 // Helper to set up locations for vector shift operations.
CreateVecShiftLocations(ArenaAllocator * arena,HVecBinaryOperation * instruction)568 static void CreateVecShiftLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) {
569   LocationSummary* locations = new (arena) LocationSummary(instruction);
570   switch (instruction->GetPackedType()) {
571     case Primitive::kPrimByte:
572     case Primitive::kPrimChar:
573     case Primitive::kPrimShort:
574     case Primitive::kPrimInt:
575     case Primitive::kPrimLong:
576       locations->SetInAt(0, Location::RequiresFpuRegister());
577       locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant()));
578       locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
579       break;
580     default:
581       LOG(FATAL) << "Unsupported SIMD type";
582       UNREACHABLE();
583   }
584 }
585 
VisitVecShl(HVecShl * instruction)586 void LocationsBuilderARM64::VisitVecShl(HVecShl* instruction) {
587   CreateVecShiftLocations(GetGraph()->GetArena(), instruction);
588 }
589 
VisitVecShl(HVecShl * instruction)590 void InstructionCodeGeneratorARM64::VisitVecShl(HVecShl* instruction) {
591   LocationSummary* locations = instruction->GetLocations();
592   VRegister lhs = VRegisterFrom(locations->InAt(0));
593   VRegister dst = VRegisterFrom(locations->Out());
594   int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
595   switch (instruction->GetPackedType()) {
596     case Primitive::kPrimByte:
597       DCHECK_EQ(16u, instruction->GetVectorLength());
598       __ Shl(dst.V16B(), lhs.V16B(), value);
599       break;
600     case Primitive::kPrimChar:
601     case Primitive::kPrimShort:
602       DCHECK_EQ(8u, instruction->GetVectorLength());
603       __ Shl(dst.V8H(), lhs.V8H(), value);
604       break;
605     case Primitive::kPrimInt:
606       DCHECK_EQ(4u, instruction->GetVectorLength());
607       __ Shl(dst.V4S(), lhs.V4S(), value);
608       break;
609     case Primitive::kPrimLong:
610       DCHECK_EQ(2u, instruction->GetVectorLength());
611       __ Shl(dst.V2D(), lhs.V2D(), value);
612       break;
613     default:
614       LOG(FATAL) << "Unsupported SIMD type";
615       UNREACHABLE();
616   }
617 }
618 
VisitVecShr(HVecShr * instruction)619 void LocationsBuilderARM64::VisitVecShr(HVecShr* instruction) {
620   CreateVecShiftLocations(GetGraph()->GetArena(), instruction);
621 }
622 
VisitVecShr(HVecShr * instruction)623 void InstructionCodeGeneratorARM64::VisitVecShr(HVecShr* instruction) {
624   LocationSummary* locations = instruction->GetLocations();
625   VRegister lhs = VRegisterFrom(locations->InAt(0));
626   VRegister dst = VRegisterFrom(locations->Out());
627   int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
628   switch (instruction->GetPackedType()) {
629     case Primitive::kPrimByte:
630       DCHECK_EQ(16u, instruction->GetVectorLength());
631       __ Sshr(dst.V16B(), lhs.V16B(), value);
632       break;
633     case Primitive::kPrimChar:
634     case Primitive::kPrimShort:
635       DCHECK_EQ(8u, instruction->GetVectorLength());
636       __ Sshr(dst.V8H(), lhs.V8H(), value);
637       break;
638     case Primitive::kPrimInt:
639       DCHECK_EQ(4u, instruction->GetVectorLength());
640       __ Sshr(dst.V4S(), lhs.V4S(), value);
641       break;
642     case Primitive::kPrimLong:
643       DCHECK_EQ(2u, instruction->GetVectorLength());
644       __ Sshr(dst.V2D(), lhs.V2D(), value);
645       break;
646     default:
647       LOG(FATAL) << "Unsupported SIMD type";
648       UNREACHABLE();
649   }
650 }
651 
VisitVecUShr(HVecUShr * instruction)652 void LocationsBuilderARM64::VisitVecUShr(HVecUShr* instruction) {
653   CreateVecShiftLocations(GetGraph()->GetArena(), instruction);
654 }
655 
VisitVecUShr(HVecUShr * instruction)656 void InstructionCodeGeneratorARM64::VisitVecUShr(HVecUShr* instruction) {
657   LocationSummary* locations = instruction->GetLocations();
658   VRegister lhs = VRegisterFrom(locations->InAt(0));
659   VRegister dst = VRegisterFrom(locations->Out());
660   int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
661   switch (instruction->GetPackedType()) {
662     case Primitive::kPrimByte:
663       DCHECK_EQ(16u, instruction->GetVectorLength());
664       __ Ushr(dst.V16B(), lhs.V16B(), value);
665       break;
666     case Primitive::kPrimChar:
667     case Primitive::kPrimShort:
668       DCHECK_EQ(8u, instruction->GetVectorLength());
669       __ Ushr(dst.V8H(), lhs.V8H(), value);
670       break;
671     case Primitive::kPrimInt:
672       DCHECK_EQ(4u, instruction->GetVectorLength());
673       __ Ushr(dst.V4S(), lhs.V4S(), value);
674       break;
675     case Primitive::kPrimLong:
676       DCHECK_EQ(2u, instruction->GetVectorLength());
677       __ Ushr(dst.V2D(), lhs.V2D(), value);
678       break;
679     default:
680       LOG(FATAL) << "Unsupported SIMD type";
681       UNREACHABLE();
682   }
683 }
684 
VisitVecMultiplyAccumulate(HVecMultiplyAccumulate * instr)685 void LocationsBuilderARM64::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instr) {
686   LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr);
687   switch (instr->GetPackedType()) {
688     case Primitive::kPrimByte:
689     case Primitive::kPrimChar:
690     case Primitive::kPrimShort:
691     case Primitive::kPrimInt:
692       locations->SetInAt(
693           HVecMultiplyAccumulate::kInputAccumulatorIndex, Location::RequiresFpuRegister());
694       locations->SetInAt(
695           HVecMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresFpuRegister());
696       locations->SetInAt(
697           HVecMultiplyAccumulate::kInputMulRightIndex, Location::RequiresFpuRegister());
698       DCHECK_EQ(HVecMultiplyAccumulate::kInputAccumulatorIndex, 0);
699       locations->SetOut(Location::SameAsFirstInput());
700       break;
701     default:
702       LOG(FATAL) << "Unsupported SIMD type";
703       UNREACHABLE();
704   }
705 }
706 
707 // Some early revisions of the Cortex-A53 have an erratum (835769) whereby it is possible for a
708 // 64-bit scalar multiply-accumulate instruction in AArch64 state to generate an incorrect result.
709 // However vector MultiplyAccumulate instruction is not affected.
VisitVecMultiplyAccumulate(HVecMultiplyAccumulate * instr)710 void InstructionCodeGeneratorARM64::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instr) {
711   LocationSummary* locations = instr->GetLocations();
712   VRegister acc = VRegisterFrom(locations->InAt(HVecMultiplyAccumulate::kInputAccumulatorIndex));
713   VRegister left = VRegisterFrom(locations->InAt(HVecMultiplyAccumulate::kInputMulLeftIndex));
714   VRegister right = VRegisterFrom(locations->InAt(HVecMultiplyAccumulate::kInputMulRightIndex));
715   switch (instr->GetPackedType()) {
716     case Primitive::kPrimByte:
717       DCHECK_EQ(16u, instr->GetVectorLength());
718       if (instr->GetOpKind() == HInstruction::kAdd) {
719         __ Mla(acc.V16B(), left.V16B(), right.V16B());
720       } else {
721         __ Mls(acc.V16B(), left.V16B(), right.V16B());
722       }
723       break;
724     case Primitive::kPrimChar:
725     case Primitive::kPrimShort:
726       DCHECK_EQ(8u, instr->GetVectorLength());
727       if (instr->GetOpKind() == HInstruction::kAdd) {
728         __ Mla(acc.V8H(), left.V8H(), right.V8H());
729       } else {
730         __ Mls(acc.V8H(), left.V8H(), right.V8H());
731       }
732       break;
733     case Primitive::kPrimInt:
734       DCHECK_EQ(4u, instr->GetVectorLength());
735       if (instr->GetOpKind() == HInstruction::kAdd) {
736         __ Mla(acc.V4S(), left.V4S(), right.V4S());
737       } else {
738         __ Mls(acc.V4S(), left.V4S(), right.V4S());
739       }
740       break;
741     default:
742       LOG(FATAL) << "Unsupported SIMD type";
743   }
744 }
745 
746 // Helper to set up locations for vector memory operations.
CreateVecMemLocations(ArenaAllocator * arena,HVecMemoryOperation * instruction,bool is_load)747 static void CreateVecMemLocations(ArenaAllocator* arena,
748                                   HVecMemoryOperation* instruction,
749                                   bool is_load) {
750   LocationSummary* locations = new (arena) LocationSummary(instruction);
751   switch (instruction->GetPackedType()) {
752     case Primitive::kPrimBoolean:
753     case Primitive::kPrimByte:
754     case Primitive::kPrimChar:
755     case Primitive::kPrimShort:
756     case Primitive::kPrimInt:
757     case Primitive::kPrimLong:
758     case Primitive::kPrimFloat:
759     case Primitive::kPrimDouble:
760       locations->SetInAt(0, Location::RequiresRegister());
761       locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
762       if (is_load) {
763         locations->SetOut(Location::RequiresFpuRegister());
764       } else {
765         locations->SetInAt(2, Location::RequiresFpuRegister());
766       }
767       break;
768     default:
769       LOG(FATAL) << "Unsupported SIMD type";
770       UNREACHABLE();
771   }
772 }
773 
774 // Helper to set up registers and address for vector memory operations.
CreateVecMemRegisters(HVecMemoryOperation * instruction,Location * reg_loc,bool is_load,UseScratchRegisterScope * temps_scope)775 MemOperand InstructionCodeGeneratorARM64::CreateVecMemRegisters(
776     HVecMemoryOperation* instruction,
777     Location* reg_loc,
778     bool is_load,
779     UseScratchRegisterScope* temps_scope) {
780   LocationSummary* locations = instruction->GetLocations();
781   Register base = InputRegisterAt(instruction, 0);
782   Location index = locations->InAt(1);
783   *reg_loc = is_load ? locations->Out() : locations->InAt(2);
784 
785   Primitive::Type packed_type = instruction->GetPackedType();
786   uint32_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(packed_type)).Uint32Value();
787   size_t shift = Primitive::ComponentSizeShift(packed_type);
788 
789   // HIntermediateAddress optimization is only applied for scalar ArrayGet and ArraySet.
790   DCHECK(!instruction->InputAt(0)->IsIntermediateAddress());
791 
792   if (index.IsConstant()) {
793     offset += Int64ConstantFrom(index) << shift;
794     return HeapOperand(base, offset);
795   } else {
796     Register temp = temps_scope->AcquireSameSizeAs(base);
797     __ Add(temp, base, Operand(WRegisterFrom(index), LSL, shift));
798 
799     return HeapOperand(temp, offset);
800   }
801 }
802 
VisitVecLoad(HVecLoad * instruction)803 void LocationsBuilderARM64::VisitVecLoad(HVecLoad* instruction) {
804   CreateVecMemLocations(GetGraph()->GetArena(), instruction, /*is_load*/ true);
805 }
806 
VisitVecLoad(HVecLoad * instruction)807 void InstructionCodeGeneratorARM64::VisitVecLoad(HVecLoad* instruction) {
808   Location reg_loc = Location::NoLocation();
809   UseScratchRegisterScope temps(GetVIXLAssembler());
810   MemOperand mem = CreateVecMemRegisters(instruction, &reg_loc, /*is_load*/ true, &temps);
811   VRegister reg = VRegisterFrom(reg_loc);
812 
813   switch (instruction->GetPackedType()) {
814     case Primitive::kPrimBoolean:
815     case Primitive::kPrimByte:
816     case Primitive::kPrimChar:
817     case Primitive::kPrimShort:
818     case Primitive::kPrimInt:
819     case Primitive::kPrimFloat:
820     case Primitive::kPrimLong:
821     case Primitive::kPrimDouble:
822       DCHECK_LE(2u, instruction->GetVectorLength());
823       DCHECK_LE(instruction->GetVectorLength(), 16u);
824       __ Ldr(reg, mem);
825       break;
826     default:
827       LOG(FATAL) << "Unsupported SIMD type";
828       UNREACHABLE();
829   }
830 }
831 
VisitVecStore(HVecStore * instruction)832 void LocationsBuilderARM64::VisitVecStore(HVecStore* instruction) {
833   CreateVecMemLocations(GetGraph()->GetArena(), instruction, /*is_load*/ false);
834 }
835 
VisitVecStore(HVecStore * instruction)836 void InstructionCodeGeneratorARM64::VisitVecStore(HVecStore* instruction) {
837   Location reg_loc = Location::NoLocation();
838   UseScratchRegisterScope temps(GetVIXLAssembler());
839   MemOperand mem = CreateVecMemRegisters(instruction, &reg_loc, /*is_load*/ false, &temps);
840   VRegister reg = VRegisterFrom(reg_loc);
841 
842   switch (instruction->GetPackedType()) {
843     case Primitive::kPrimBoolean:
844     case Primitive::kPrimByte:
845     case Primitive::kPrimChar:
846     case Primitive::kPrimShort:
847     case Primitive::kPrimInt:
848     case Primitive::kPrimFloat:
849     case Primitive::kPrimLong:
850     case Primitive::kPrimDouble:
851       DCHECK_LE(2u, instruction->GetVectorLength());
852       DCHECK_LE(instruction->GetVectorLength(), 16u);
853       __ Str(reg, mem);
854       break;
855     default:
856       LOG(FATAL) << "Unsupported SIMD type";
857       UNREACHABLE();
858   }
859 }
860 
861 #undef __
862 
863 }  // namespace arm64
864 }  // namespace art
865