1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "code_generator_x86_64.h"
18 #include "mirror/array-inl.h"
19 
20 namespace art {
21 namespace x86_64 {
22 
23 // NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy.
24 #define __ down_cast<X86_64Assembler*>(GetAssembler())->  // NOLINT
25 
VisitVecReplicateScalar(HVecReplicateScalar * instruction)26 void LocationsBuilderX86_64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) {
27   LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
28   switch (instruction->GetPackedType()) {
29     case Primitive::kPrimBoolean:
30     case Primitive::kPrimByte:
31     case Primitive::kPrimChar:
32     case Primitive::kPrimShort:
33     case Primitive::kPrimInt:
34     case Primitive::kPrimLong:
35       locations->SetInAt(0, Location::RequiresRegister());
36       locations->SetOut(Location::RequiresFpuRegister());
37       break;
38     case Primitive::kPrimFloat:
39     case Primitive::kPrimDouble:
40       locations->SetInAt(0, Location::RequiresFpuRegister());
41       locations->SetOut(Location::SameAsFirstInput());
42       break;
43     default:
44       LOG(FATAL) << "Unsupported SIMD type";
45       UNREACHABLE();
46   }
47 }
48 
VisitVecReplicateScalar(HVecReplicateScalar * instruction)49 void InstructionCodeGeneratorX86_64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) {
50   LocationSummary* locations = instruction->GetLocations();
51   XmmRegister reg = locations->Out().AsFpuRegister<XmmRegister>();
52   switch (instruction->GetPackedType()) {
53     case Primitive::kPrimBoolean:
54     case Primitive::kPrimByte:
55       DCHECK_EQ(16u, instruction->GetVectorLength());
56       __ movd(reg, locations->InAt(0).AsRegister<CpuRegister>());
57       __ punpcklbw(reg, reg);
58       __ punpcklwd(reg, reg);
59       __ pshufd(reg, reg, Immediate(0));
60       break;
61     case Primitive::kPrimChar:
62     case Primitive::kPrimShort:
63       DCHECK_EQ(8u, instruction->GetVectorLength());
64       __ movd(reg, locations->InAt(0).AsRegister<CpuRegister>());
65       __ punpcklwd(reg, reg);
66       __ pshufd(reg, reg, Immediate(0));
67       break;
68     case Primitive::kPrimInt:
69       DCHECK_EQ(4u, instruction->GetVectorLength());
70       __ movd(reg, locations->InAt(0).AsRegister<CpuRegister>());
71       __ pshufd(reg, reg, Immediate(0));
72       break;
73     case Primitive::kPrimLong:
74       DCHECK_EQ(2u, instruction->GetVectorLength());
75       __ movd(reg, locations->InAt(0).AsRegister<CpuRegister>());  // is 64-bit
76       __ punpcklqdq(reg, reg);
77       break;
78     case Primitive::kPrimFloat:
79       DCHECK(locations->InAt(0).Equals(locations->Out()));
80       DCHECK_EQ(4u, instruction->GetVectorLength());
81       __ shufps(reg, reg, Immediate(0));
82       break;
83     case Primitive::kPrimDouble:
84       DCHECK(locations->InAt(0).Equals(locations->Out()));
85       DCHECK_EQ(2u, instruction->GetVectorLength());
86       __ shufpd(reg, reg, Immediate(0));
87       break;
88     default:
89       LOG(FATAL) << "Unsupported SIMD type";
90       UNREACHABLE();
91   }
92 }
93 
VisitVecSetScalars(HVecSetScalars * instruction)94 void LocationsBuilderX86_64::VisitVecSetScalars(HVecSetScalars* instruction) {
95   LOG(FATAL) << "No SIMD for " << instruction->GetId();
96 }
97 
VisitVecSetScalars(HVecSetScalars * instruction)98 void InstructionCodeGeneratorX86_64::VisitVecSetScalars(HVecSetScalars* instruction) {
99   LOG(FATAL) << "No SIMD for " << instruction->GetId();
100 }
101 
VisitVecSumReduce(HVecSumReduce * instruction)102 void LocationsBuilderX86_64::VisitVecSumReduce(HVecSumReduce* instruction) {
103   LOG(FATAL) << "No SIMD for " << instruction->GetId();
104 }
105 
VisitVecSumReduce(HVecSumReduce * instruction)106 void InstructionCodeGeneratorX86_64::VisitVecSumReduce(HVecSumReduce* instruction) {
107   LOG(FATAL) << "No SIMD for " << instruction->GetId();
108 }
109 
110 // Helper to set up locations for vector unary operations.
CreateVecUnOpLocations(ArenaAllocator * arena,HVecUnaryOperation * instruction)111 static void CreateVecUnOpLocations(ArenaAllocator* arena, HVecUnaryOperation* instruction) {
112   LocationSummary* locations = new (arena) LocationSummary(instruction);
113   switch (instruction->GetPackedType()) {
114     case Primitive::kPrimBoolean:
115     case Primitive::kPrimByte:
116     case Primitive::kPrimChar:
117     case Primitive::kPrimShort:
118     case Primitive::kPrimInt:
119     case Primitive::kPrimLong:
120     case Primitive::kPrimFloat:
121     case Primitive::kPrimDouble:
122       locations->SetInAt(0, Location::RequiresFpuRegister());
123       locations->SetOut(Location::RequiresFpuRegister());
124       break;
125     default:
126       LOG(FATAL) << "Unsupported SIMD type";
127       UNREACHABLE();
128   }
129 }
130 
VisitVecCnv(HVecCnv * instruction)131 void LocationsBuilderX86_64::VisitVecCnv(HVecCnv* instruction) {
132   CreateVecUnOpLocations(GetGraph()->GetArena(), instruction);
133 }
134 
VisitVecCnv(HVecCnv * instruction)135 void InstructionCodeGeneratorX86_64::VisitVecCnv(HVecCnv* instruction) {
136   LocationSummary* locations = instruction->GetLocations();
137   XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>();
138   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
139   Primitive::Type from = instruction->GetInputType();
140   Primitive::Type to = instruction->GetResultType();
141   if (from == Primitive::kPrimInt && to == Primitive::kPrimFloat) {
142     DCHECK_EQ(4u, instruction->GetVectorLength());
143     __ cvtdq2ps(dst, src);
144   } else {
145     LOG(FATAL) << "Unsupported SIMD type";
146   }
147 }
148 
VisitVecNeg(HVecNeg * instruction)149 void LocationsBuilderX86_64::VisitVecNeg(HVecNeg* instruction) {
150   CreateVecUnOpLocations(GetGraph()->GetArena(), instruction);
151 }
152 
VisitVecNeg(HVecNeg * instruction)153 void InstructionCodeGeneratorX86_64::VisitVecNeg(HVecNeg* instruction) {
154   LocationSummary* locations = instruction->GetLocations();
155   XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>();
156   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
157   switch (instruction->GetPackedType()) {
158     case Primitive::kPrimByte:
159       DCHECK_EQ(16u, instruction->GetVectorLength());
160       __ pxor(dst, dst);
161       __ psubb(dst, src);
162       break;
163     case Primitive::kPrimChar:
164     case Primitive::kPrimShort:
165       DCHECK_EQ(8u, instruction->GetVectorLength());
166       __ pxor(dst, dst);
167       __ psubw(dst, src);
168       break;
169     case Primitive::kPrimInt:
170       DCHECK_EQ(4u, instruction->GetVectorLength());
171       __ pxor(dst, dst);
172       __ psubd(dst, src);
173       break;
174     case Primitive::kPrimLong:
175       DCHECK_EQ(2u, instruction->GetVectorLength());
176       __ pxor(dst, dst);
177       __ psubq(dst, src);
178       break;
179     case Primitive::kPrimFloat:
180       DCHECK_EQ(4u, instruction->GetVectorLength());
181       __ xorps(dst, dst);
182       __ subps(dst, src);
183       break;
184     case Primitive::kPrimDouble:
185       DCHECK_EQ(2u, instruction->GetVectorLength());
186       __ xorpd(dst, dst);
187       __ subpd(dst, src);
188       break;
189     default:
190       LOG(FATAL) << "Unsupported SIMD type";
191       UNREACHABLE();
192   }
193 }
194 
VisitVecAbs(HVecAbs * instruction)195 void LocationsBuilderX86_64::VisitVecAbs(HVecAbs* instruction) {
196   CreateVecUnOpLocations(GetGraph()->GetArena(), instruction);
197   if (instruction->GetPackedType() == Primitive::kPrimInt) {
198     instruction->GetLocations()->AddTemp(Location::RequiresFpuRegister());
199   }
200 }
201 
VisitVecAbs(HVecAbs * instruction)202 void InstructionCodeGeneratorX86_64::VisitVecAbs(HVecAbs* instruction) {
203   LocationSummary* locations = instruction->GetLocations();
204   XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>();
205   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
206   switch (instruction->GetPackedType()) {
207     case Primitive::kPrimInt: {
208       DCHECK_EQ(4u, instruction->GetVectorLength());
209       XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
210       __ movaps(dst, src);
211       __ pxor(tmp, tmp);
212       __ pcmpgtd(tmp, dst);
213       __ pxor(dst, tmp);
214       __ psubd(dst, tmp);
215       break;
216     }
217     case Primitive::kPrimFloat:
218       DCHECK_EQ(4u, instruction->GetVectorLength());
219       __ pcmpeqb(dst, dst);  // all ones
220       __ psrld(dst, Immediate(1));
221       __ andps(dst, src);
222       break;
223     case Primitive::kPrimDouble:
224       DCHECK_EQ(2u, instruction->GetVectorLength());
225       __ pcmpeqb(dst, dst);  // all ones
226       __ psrlq(dst, Immediate(1));
227       __ andpd(dst, src);
228       break;
229     default:
230       LOG(FATAL) << "Unsupported SIMD type";
231       UNREACHABLE();
232   }
233 }
234 
VisitVecNot(HVecNot * instruction)235 void LocationsBuilderX86_64::VisitVecNot(HVecNot* instruction) {
236   CreateVecUnOpLocations(GetGraph()->GetArena(), instruction);
237   // Boolean-not requires a temporary to construct the 16 x one.
238   if (instruction->GetPackedType() == Primitive::kPrimBoolean) {
239     instruction->GetLocations()->AddTemp(Location::RequiresFpuRegister());
240   }
241 }
242 
VisitVecNot(HVecNot * instruction)243 void InstructionCodeGeneratorX86_64::VisitVecNot(HVecNot* instruction) {
244   LocationSummary* locations = instruction->GetLocations();
245   XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>();
246   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
247   switch (instruction->GetPackedType()) {
248     case Primitive::kPrimBoolean: {  // special case boolean-not
249       DCHECK_EQ(16u, instruction->GetVectorLength());
250       XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
251       __ pxor(dst, dst);
252       __ pcmpeqb(tmp, tmp);  // all ones
253       __ psubb(dst, tmp);  // 16 x one
254       __ pxor(dst, src);
255       break;
256     }
257     case Primitive::kPrimByte:
258     case Primitive::kPrimChar:
259     case Primitive::kPrimShort:
260     case Primitive::kPrimInt:
261     case Primitive::kPrimLong:
262       DCHECK_LE(2u, instruction->GetVectorLength());
263       DCHECK_LE(instruction->GetVectorLength(), 16u);
264       __ pcmpeqb(dst, dst);  // all ones
265       __ pxor(dst, src);
266       break;
267     case Primitive::kPrimFloat:
268       DCHECK_EQ(4u, instruction->GetVectorLength());
269       __ pcmpeqb(dst, dst);  // all ones
270       __ xorps(dst, src);
271       break;
272     case Primitive::kPrimDouble:
273       DCHECK_EQ(2u, instruction->GetVectorLength());
274       __ pcmpeqb(dst, dst);  // all ones
275       __ xorpd(dst, src);
276       break;
277     default:
278       LOG(FATAL) << "Unsupported SIMD type";
279       UNREACHABLE();
280   }
281 }
282 
283 // Helper to set up locations for vector binary operations.
CreateVecBinOpLocations(ArenaAllocator * arena,HVecBinaryOperation * instruction)284 static void CreateVecBinOpLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) {
285   LocationSummary* locations = new (arena) LocationSummary(instruction);
286   switch (instruction->GetPackedType()) {
287     case Primitive::kPrimBoolean:
288     case Primitive::kPrimByte:
289     case Primitive::kPrimChar:
290     case Primitive::kPrimShort:
291     case Primitive::kPrimInt:
292     case Primitive::kPrimLong:
293     case Primitive::kPrimFloat:
294     case Primitive::kPrimDouble:
295       locations->SetInAt(0, Location::RequiresFpuRegister());
296       locations->SetInAt(1, Location::RequiresFpuRegister());
297       locations->SetOut(Location::SameAsFirstInput());
298       break;
299     default:
300       LOG(FATAL) << "Unsupported SIMD type";
301       UNREACHABLE();
302   }
303 }
304 
VisitVecAdd(HVecAdd * instruction)305 void LocationsBuilderX86_64::VisitVecAdd(HVecAdd* instruction) {
306   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
307 }
308 
VisitVecAdd(HVecAdd * instruction)309 void InstructionCodeGeneratorX86_64::VisitVecAdd(HVecAdd* instruction) {
310   LocationSummary* locations = instruction->GetLocations();
311   DCHECK(locations->InAt(0).Equals(locations->Out()));
312   XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>();
313   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
314   switch (instruction->GetPackedType()) {
315     case Primitive::kPrimByte:
316       DCHECK_EQ(16u, instruction->GetVectorLength());
317       __ paddb(dst, src);
318       break;
319     case Primitive::kPrimChar:
320     case Primitive::kPrimShort:
321       DCHECK_EQ(8u, instruction->GetVectorLength());
322       __ paddw(dst, src);
323       break;
324     case Primitive::kPrimInt:
325       DCHECK_EQ(4u, instruction->GetVectorLength());
326       __ paddd(dst, src);
327       break;
328     case Primitive::kPrimLong:
329       DCHECK_EQ(2u, instruction->GetVectorLength());
330       __ paddq(dst, src);
331       break;
332     case Primitive::kPrimFloat:
333       DCHECK_EQ(4u, instruction->GetVectorLength());
334       __ addps(dst, src);
335       break;
336     case Primitive::kPrimDouble:
337       DCHECK_EQ(2u, instruction->GetVectorLength());
338       __ addpd(dst, src);
339       break;
340     default:
341       LOG(FATAL) << "Unsupported SIMD type";
342       UNREACHABLE();
343   }
344 }
345 
VisitVecHalvingAdd(HVecHalvingAdd * instruction)346 void LocationsBuilderX86_64::VisitVecHalvingAdd(HVecHalvingAdd* instruction) {
347   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
348 }
349 
VisitVecHalvingAdd(HVecHalvingAdd * instruction)350 void InstructionCodeGeneratorX86_64::VisitVecHalvingAdd(HVecHalvingAdd* instruction) {
351   LocationSummary* locations = instruction->GetLocations();
352   DCHECK(locations->InAt(0).Equals(locations->Out()));
353   XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>();
354   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
355   switch (instruction->GetPackedType()) {
356     case Primitive::kPrimByte:
357       DCHECK_EQ(16u, instruction->GetVectorLength());
358      __ pavgb(dst, src);
359      return;
360     case Primitive::kPrimChar:
361     case Primitive::kPrimShort:
362       DCHECK_EQ(8u, instruction->GetVectorLength());
363       __ pavgw(dst, src);
364       return;
365     default:
366       LOG(FATAL) << "Unsupported SIMD type";
367       UNREACHABLE();
368   }
369 }
370 
VisitVecSub(HVecSub * instruction)371 void LocationsBuilderX86_64::VisitVecSub(HVecSub* instruction) {
372   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
373 }
374 
VisitVecSub(HVecSub * instruction)375 void InstructionCodeGeneratorX86_64::VisitVecSub(HVecSub* instruction) {
376   LocationSummary* locations = instruction->GetLocations();
377   DCHECK(locations->InAt(0).Equals(locations->Out()));
378   XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>();
379   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
380   switch (instruction->GetPackedType()) {
381     case Primitive::kPrimByte:
382       DCHECK_EQ(16u, instruction->GetVectorLength());
383       __ psubb(dst, src);
384       break;
385     case Primitive::kPrimChar:
386     case Primitive::kPrimShort:
387       DCHECK_EQ(8u, instruction->GetVectorLength());
388       __ psubw(dst, src);
389       break;
390     case Primitive::kPrimInt:
391       DCHECK_EQ(4u, instruction->GetVectorLength());
392       __ psubd(dst, src);
393       break;
394     case Primitive::kPrimLong:
395       DCHECK_EQ(2u, instruction->GetVectorLength());
396       __ psubq(dst, src);
397       break;
398     case Primitive::kPrimFloat:
399       DCHECK_EQ(4u, instruction->GetVectorLength());
400       __ subps(dst, src);
401       break;
402     case Primitive::kPrimDouble:
403       DCHECK_EQ(2u, instruction->GetVectorLength());
404       __ subpd(dst, src);
405       break;
406     default:
407       LOG(FATAL) << "Unsupported SIMD type";
408       UNREACHABLE();
409   }
410 }
411 
VisitVecMul(HVecMul * instruction)412 void LocationsBuilderX86_64::VisitVecMul(HVecMul* instruction) {
413   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
414 }
415 
VisitVecMul(HVecMul * instruction)416 void InstructionCodeGeneratorX86_64::VisitVecMul(HVecMul* instruction) {
417   LocationSummary* locations = instruction->GetLocations();
418   DCHECK(locations->InAt(0).Equals(locations->Out()));
419   XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>();
420   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
421   switch (instruction->GetPackedType()) {
422     case Primitive::kPrimChar:
423     case Primitive::kPrimShort:
424       DCHECK_EQ(8u, instruction->GetVectorLength());
425       __ pmullw(dst, src);
426       break;
427     case Primitive::kPrimInt:
428       DCHECK_EQ(4u, instruction->GetVectorLength());
429       __ pmulld(dst, src);
430       break;
431     case Primitive::kPrimFloat:
432       DCHECK_EQ(4u, instruction->GetVectorLength());
433       __ mulps(dst, src);
434       break;
435     case Primitive::kPrimDouble:
436       DCHECK_EQ(2u, instruction->GetVectorLength());
437       __ mulpd(dst, src);
438       break;
439     default:
440       LOG(FATAL) << "Unsupported SIMD type";
441       UNREACHABLE();
442   }
443 }
444 
VisitVecDiv(HVecDiv * instruction)445 void LocationsBuilderX86_64::VisitVecDiv(HVecDiv* instruction) {
446   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
447 }
448 
VisitVecDiv(HVecDiv * instruction)449 void InstructionCodeGeneratorX86_64::VisitVecDiv(HVecDiv* instruction) {
450   LocationSummary* locations = instruction->GetLocations();
451   DCHECK(locations->InAt(0).Equals(locations->Out()));
452   XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>();
453   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
454   switch (instruction->GetPackedType()) {
455     case Primitive::kPrimFloat:
456       DCHECK_EQ(4u, instruction->GetVectorLength());
457       __ divps(dst, src);
458       break;
459     case Primitive::kPrimDouble:
460       DCHECK_EQ(2u, instruction->GetVectorLength());
461       __ divpd(dst, src);
462       break;
463     default:
464       LOG(FATAL) << "Unsupported SIMD type";
465       UNREACHABLE();
466   }
467 }
468 
VisitVecMin(HVecMin * instruction)469 void LocationsBuilderX86_64::VisitVecMin(HVecMin* instruction) {
470   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
471 }
472 
VisitVecMin(HVecMin * instruction)473 void InstructionCodeGeneratorX86_64::VisitVecMin(HVecMin* instruction) {
474   LOG(FATAL) << "No SIMD for " << instruction->GetId();
475 }
476 
VisitVecMax(HVecMax * instruction)477 void LocationsBuilderX86_64::VisitVecMax(HVecMax* instruction) {
478   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
479 }
480 
VisitVecMax(HVecMax * instruction)481 void InstructionCodeGeneratorX86_64::VisitVecMax(HVecMax* instruction) {
482   LOG(FATAL) << "No SIMD for " << instruction->GetId();
483 }
484 
VisitVecAnd(HVecAnd * instruction)485 void LocationsBuilderX86_64::VisitVecAnd(HVecAnd* instruction) {
486   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
487 }
488 
VisitVecAnd(HVecAnd * instruction)489 void InstructionCodeGeneratorX86_64::VisitVecAnd(HVecAnd* instruction) {
490   LocationSummary* locations = instruction->GetLocations();
491   DCHECK(locations->InAt(0).Equals(locations->Out()));
492   XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>();
493   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
494   switch (instruction->GetPackedType()) {
495     case Primitive::kPrimBoolean:
496     case Primitive::kPrimByte:
497     case Primitive::kPrimChar:
498     case Primitive::kPrimShort:
499     case Primitive::kPrimInt:
500     case Primitive::kPrimLong:
501       DCHECK_LE(2u, instruction->GetVectorLength());
502       DCHECK_LE(instruction->GetVectorLength(), 16u);
503       __ pand(dst, src);
504       break;
505     case Primitive::kPrimFloat:
506       DCHECK_EQ(4u, instruction->GetVectorLength());
507       __ andps(dst, src);
508       break;
509     case Primitive::kPrimDouble:
510       DCHECK_EQ(2u, instruction->GetVectorLength());
511       __ andpd(dst, src);
512       break;
513     default:
514       LOG(FATAL) << "Unsupported SIMD type";
515       UNREACHABLE();
516   }
517 }
518 
VisitVecAndNot(HVecAndNot * instruction)519 void LocationsBuilderX86_64::VisitVecAndNot(HVecAndNot* instruction) {
520   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
521 }
522 
VisitVecAndNot(HVecAndNot * instruction)523 void InstructionCodeGeneratorX86_64::VisitVecAndNot(HVecAndNot* instruction) {
524   LocationSummary* locations = instruction->GetLocations();
525   DCHECK(locations->InAt(0).Equals(locations->Out()));
526   XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>();
527   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
528   switch (instruction->GetPackedType()) {
529     case Primitive::kPrimBoolean:
530     case Primitive::kPrimByte:
531     case Primitive::kPrimChar:
532     case Primitive::kPrimShort:
533     case Primitive::kPrimInt:
534     case Primitive::kPrimLong:
535       DCHECK_LE(2u, instruction->GetVectorLength());
536       DCHECK_LE(instruction->GetVectorLength(), 16u);
537       __ pandn(dst, src);
538       break;
539     case Primitive::kPrimFloat:
540       DCHECK_EQ(4u, instruction->GetVectorLength());
541       __ andnps(dst, src);
542       break;
543     case Primitive::kPrimDouble:
544       DCHECK_EQ(2u, instruction->GetVectorLength());
545       __ andnpd(dst, src);
546       break;
547     default:
548       LOG(FATAL) << "Unsupported SIMD type";
549       UNREACHABLE();
550   }
551 }
552 
VisitVecOr(HVecOr * instruction)553 void LocationsBuilderX86_64::VisitVecOr(HVecOr* instruction) {
554   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
555 }
556 
VisitVecOr(HVecOr * instruction)557 void InstructionCodeGeneratorX86_64::VisitVecOr(HVecOr* instruction) {
558   LocationSummary* locations = instruction->GetLocations();
559   DCHECK(locations->InAt(0).Equals(locations->Out()));
560   XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>();
561   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
562   switch (instruction->GetPackedType()) {
563     case Primitive::kPrimBoolean:
564     case Primitive::kPrimByte:
565     case Primitive::kPrimChar:
566     case Primitive::kPrimShort:
567     case Primitive::kPrimInt:
568     case Primitive::kPrimLong:
569       DCHECK_LE(2u, instruction->GetVectorLength());
570       DCHECK_LE(instruction->GetVectorLength(), 16u);
571       __ por(dst, src);
572       break;
573     case Primitive::kPrimFloat:
574       DCHECK_EQ(4u, instruction->GetVectorLength());
575       __ orps(dst, src);
576       break;
577     case Primitive::kPrimDouble:
578       DCHECK_EQ(2u, instruction->GetVectorLength());
579       __ orpd(dst, src);
580       break;
581     default:
582       LOG(FATAL) << "Unsupported SIMD type";
583       UNREACHABLE();
584   }
585 }
586 
VisitVecXor(HVecXor * instruction)587 void LocationsBuilderX86_64::VisitVecXor(HVecXor* instruction) {
588   CreateVecBinOpLocations(GetGraph()->GetArena(), instruction);
589 }
590 
VisitVecXor(HVecXor * instruction)591 void InstructionCodeGeneratorX86_64::VisitVecXor(HVecXor* instruction) {
592   LocationSummary* locations = instruction->GetLocations();
593   DCHECK(locations->InAt(0).Equals(locations->Out()));
594   XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>();
595   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
596   switch (instruction->GetPackedType()) {
597     case Primitive::kPrimBoolean:
598     case Primitive::kPrimByte:
599     case Primitive::kPrimChar:
600     case Primitive::kPrimShort:
601     case Primitive::kPrimInt:
602     case Primitive::kPrimLong:
603       DCHECK_LE(2u, instruction->GetVectorLength());
604       DCHECK_LE(instruction->GetVectorLength(), 16u);
605       __ pxor(dst, src);
606       break;
607     case Primitive::kPrimFloat:
608       DCHECK_EQ(4u, instruction->GetVectorLength());
609       __ xorps(dst, src);
610       break;
611     case Primitive::kPrimDouble:
612       DCHECK_EQ(2u, instruction->GetVectorLength());
613       __ xorpd(dst, src);
614       break;
615     default:
616       LOG(FATAL) << "Unsupported SIMD type";
617       UNREACHABLE();
618   }
619 }
620 
621 // Helper to set up locations for vector shift operations.
CreateVecShiftLocations(ArenaAllocator * arena,HVecBinaryOperation * instruction)622 static void CreateVecShiftLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) {
623   LocationSummary* locations = new (arena) LocationSummary(instruction);
624   switch (instruction->GetPackedType()) {
625     case Primitive::kPrimChar:
626     case Primitive::kPrimShort:
627     case Primitive::kPrimInt:
628     case Primitive::kPrimLong:
629       locations->SetInAt(0, Location::RequiresFpuRegister());
630       locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant()));
631       locations->SetOut(Location::SameAsFirstInput());
632       break;
633     default:
634       LOG(FATAL) << "Unsupported SIMD type";
635       UNREACHABLE();
636   }
637 }
638 
VisitVecShl(HVecShl * instruction)639 void LocationsBuilderX86_64::VisitVecShl(HVecShl* instruction) {
640   CreateVecShiftLocations(GetGraph()->GetArena(), instruction);
641 }
642 
VisitVecShl(HVecShl * instruction)643 void InstructionCodeGeneratorX86_64::VisitVecShl(HVecShl* instruction) {
644   LocationSummary* locations = instruction->GetLocations();
645   DCHECK(locations->InAt(0).Equals(locations->Out()));
646   int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
647   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
648   switch (instruction->GetPackedType()) {
649     case Primitive::kPrimChar:
650     case Primitive::kPrimShort:
651       DCHECK_EQ(8u, instruction->GetVectorLength());
652       __ psllw(dst, Immediate(static_cast<int8_t>(value)));
653       break;
654     case Primitive::kPrimInt:
655       DCHECK_EQ(4u, instruction->GetVectorLength());
656       __ pslld(dst, Immediate(static_cast<int8_t>(value)));
657       break;
658     case Primitive::kPrimLong:
659       DCHECK_EQ(2u, instruction->GetVectorLength());
660       __ psllq(dst, Immediate(static_cast<int8_t>(value)));
661       break;
662     default:
663       LOG(FATAL) << "Unsupported SIMD type";
664       UNREACHABLE();
665   }
666 }
667 
VisitVecShr(HVecShr * instruction)668 void LocationsBuilderX86_64::VisitVecShr(HVecShr* instruction) {
669   CreateVecShiftLocations(GetGraph()->GetArena(), instruction);
670 }
671 
VisitVecShr(HVecShr * instruction)672 void InstructionCodeGeneratorX86_64::VisitVecShr(HVecShr* instruction) {
673   LocationSummary* locations = instruction->GetLocations();
674   DCHECK(locations->InAt(0).Equals(locations->Out()));
675   int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
676   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
677   switch (instruction->GetPackedType()) {
678     case Primitive::kPrimChar:
679     case Primitive::kPrimShort:
680       DCHECK_EQ(8u, instruction->GetVectorLength());
681       __ psraw(dst, Immediate(static_cast<int8_t>(value)));
682       break;
683     case Primitive::kPrimInt:
684       DCHECK_EQ(4u, instruction->GetVectorLength());
685       __ psrad(dst, Immediate(static_cast<int8_t>(value)));
686       break;
687     default:
688       LOG(FATAL) << "Unsupported SIMD type";
689       UNREACHABLE();
690   }
691 }
692 
VisitVecUShr(HVecUShr * instruction)693 void LocationsBuilderX86_64::VisitVecUShr(HVecUShr* instruction) {
694   CreateVecShiftLocations(GetGraph()->GetArena(), instruction);
695 }
696 
VisitVecUShr(HVecUShr * instruction)697 void InstructionCodeGeneratorX86_64::VisitVecUShr(HVecUShr* instruction) {
698   LocationSummary* locations = instruction->GetLocations();
699   DCHECK(locations->InAt(0).Equals(locations->Out()));
700   int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
701   XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>();
702   switch (instruction->GetPackedType()) {
703     case Primitive::kPrimChar:
704     case Primitive::kPrimShort:
705       DCHECK_EQ(8u, instruction->GetVectorLength());
706       __ psrlw(dst, Immediate(static_cast<int8_t>(value)));
707       break;
708     case Primitive::kPrimInt:
709       DCHECK_EQ(4u, instruction->GetVectorLength());
710       __ psrld(dst, Immediate(static_cast<int8_t>(value)));
711       break;
712     case Primitive::kPrimLong:
713       DCHECK_EQ(2u, instruction->GetVectorLength());
714       __ psrlq(dst, Immediate(static_cast<int8_t>(value)));
715       break;
716     default:
717       LOG(FATAL) << "Unsupported SIMD type";
718       UNREACHABLE();
719   }
720 }
721 
VisitVecMultiplyAccumulate(HVecMultiplyAccumulate * instr)722 void LocationsBuilderX86_64::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instr) {
723   LOG(FATAL) << "No SIMD for " << instr->GetId();
724 }
725 
VisitVecMultiplyAccumulate(HVecMultiplyAccumulate * instr)726 void InstructionCodeGeneratorX86_64::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instr) {
727   LOG(FATAL) << "No SIMD for " << instr->GetId();
728 }
729 
730 // Helper to set up locations for vector memory operations.
CreateVecMemLocations(ArenaAllocator * arena,HVecMemoryOperation * instruction,bool is_load)731 static void CreateVecMemLocations(ArenaAllocator* arena,
732                                   HVecMemoryOperation* instruction,
733                                   bool is_load) {
734   LocationSummary* locations = new (arena) LocationSummary(instruction);
735   switch (instruction->GetPackedType()) {
736     case Primitive::kPrimBoolean:
737     case Primitive::kPrimByte:
738     case Primitive::kPrimChar:
739     case Primitive::kPrimShort:
740     case Primitive::kPrimInt:
741     case Primitive::kPrimLong:
742     case Primitive::kPrimFloat:
743     case Primitive::kPrimDouble:
744       locations->SetInAt(0, Location::RequiresRegister());
745       locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
746       if (is_load) {
747         locations->SetOut(Location::RequiresFpuRegister());
748       } else {
749         locations->SetInAt(2, Location::RequiresFpuRegister());
750       }
751       break;
752     default:
753       LOG(FATAL) << "Unsupported SIMD type";
754       UNREACHABLE();
755   }
756 }
757 
758 // Helper to set up registers and address for vector memory operations.
CreateVecMemRegisters(HVecMemoryOperation * instruction,Location * reg_loc,bool is_load)759 static Address CreateVecMemRegisters(HVecMemoryOperation* instruction,
760                                      Location* reg_loc,
761                                      bool is_load) {
762   LocationSummary* locations = instruction->GetLocations();
763   Location base = locations->InAt(0);
764   Location index = locations->InAt(1);
765   *reg_loc = is_load ? locations->Out() : locations->InAt(2);
766   size_t size = Primitive::ComponentSize(instruction->GetPackedType());
767   uint32_t offset = mirror::Array::DataOffset(size).Uint32Value();
768   ScaleFactor scale = TIMES_1;
769   switch (size) {
770     case 2: scale = TIMES_2; break;
771     case 4: scale = TIMES_4; break;
772     case 8: scale = TIMES_8; break;
773     default: break;
774   }
775   return CodeGeneratorX86_64::ArrayAddress(base.AsRegister<CpuRegister>(), index, scale, offset);
776 }
777 
VisitVecLoad(HVecLoad * instruction)778 void LocationsBuilderX86_64::VisitVecLoad(HVecLoad* instruction) {
779   CreateVecMemLocations(GetGraph()->GetArena(), instruction, /*is_load*/ true);
780 }
781 
VisitVecLoad(HVecLoad * instruction)782 void InstructionCodeGeneratorX86_64::VisitVecLoad(HVecLoad* instruction) {
783   Location reg_loc = Location::NoLocation();
784   Address address = CreateVecMemRegisters(instruction, &reg_loc, /*is_load*/ true);
785   XmmRegister reg = reg_loc.AsFpuRegister<XmmRegister>();
786   bool is_aligned16 = instruction->GetAlignment().IsAlignedAt(16);
787   switch (instruction->GetPackedType()) {
788     case Primitive::kPrimBoolean:
789     case Primitive::kPrimByte:
790     case Primitive::kPrimChar:
791     case Primitive::kPrimShort:
792     case Primitive::kPrimInt:
793     case Primitive::kPrimLong:
794       DCHECK_LE(2u, instruction->GetVectorLength());
795       DCHECK_LE(instruction->GetVectorLength(), 16u);
796       is_aligned16 ? __ movdqa(reg, address) : __ movdqu(reg, address);
797       break;
798     case Primitive::kPrimFloat:
799       DCHECK_EQ(4u, instruction->GetVectorLength());
800       is_aligned16 ? __ movaps(reg, address) : __ movups(reg, address);
801       break;
802     case Primitive::kPrimDouble:
803       DCHECK_EQ(2u, instruction->GetVectorLength());
804       is_aligned16 ? __ movapd(reg, address) : __ movupd(reg, address);
805       break;
806     default:
807       LOG(FATAL) << "Unsupported SIMD type";
808       UNREACHABLE();
809   }
810 }
811 
VisitVecStore(HVecStore * instruction)812 void LocationsBuilderX86_64::VisitVecStore(HVecStore* instruction) {
813   CreateVecMemLocations(GetGraph()->GetArena(), instruction, /*is_load*/ false);
814 }
815 
VisitVecStore(HVecStore * instruction)816 void InstructionCodeGeneratorX86_64::VisitVecStore(HVecStore* instruction) {
817   Location reg_loc = Location::NoLocation();
818   Address address = CreateVecMemRegisters(instruction, &reg_loc, /*is_load*/ false);
819   XmmRegister reg = reg_loc.AsFpuRegister<XmmRegister>();
820   bool is_aligned16 = instruction->GetAlignment().IsAlignedAt(16);
821   switch (instruction->GetPackedType()) {
822     case Primitive::kPrimBoolean:
823     case Primitive::kPrimByte:
824     case Primitive::kPrimChar:
825     case Primitive::kPrimShort:
826     case Primitive::kPrimInt:
827     case Primitive::kPrimLong:
828       DCHECK_LE(2u, instruction->GetVectorLength());
829       DCHECK_LE(instruction->GetVectorLength(), 16u);
830       is_aligned16 ? __ movdqa(address, reg) : __ movdqu(address, reg);
831       break;
832     case Primitive::kPrimFloat:
833       DCHECK_EQ(4u, instruction->GetVectorLength());
834       is_aligned16 ? __ movaps(address, reg) : __ movups(address, reg);
835       break;
836     case Primitive::kPrimDouble:
837       DCHECK_EQ(2u, instruction->GetVectorLength());
838       is_aligned16 ? __ movapd(address, reg) : __ movupd(address, reg);
839       break;
840     default:
841       LOG(FATAL) << "Unsupported SIMD type";
842       UNREACHABLE();
843   }
844 }
845 
846 #undef __
847 
848 }  // namespace x86_64
849 }  // namespace art
850