Lines Matching refs:__
78 #define __ down_cast<X86Assembler*>(codegen->GetAssembler())-> // NOLINT macro
112 __ Bind(GetEntryLabel()); in EmitNativeCode()
124 __ xorl(temp1, temp1); in EmitNativeCode()
126 __ Bind(&loop); in EmitNativeCode()
131 __ movl(temp2, Address(src, temp1, ScaleFactor::TIMES_4, adjusted_offset)); in EmitNativeCode()
133 __ leal(temp2, Address(src_pos.AsRegister<Register>(), temp1, ScaleFactor::TIMES_1, 0)); in EmitNativeCode()
134 __ movl(temp2, Address(src, temp2, ScaleFactor::TIMES_4, offset)); in EmitNativeCode()
136 __ MaybeUnpoisonHeapReference(temp2); in EmitNativeCode()
150 __ MaybePoisonHeapReference(temp2); in EmitNativeCode()
155 __ movl(Address(dest, temp1, ScaleFactor::TIMES_4, adjusted_offset), temp2); in EmitNativeCode()
157 __ leal(temp3, Address(dest_pos.AsRegister<Register>(), temp1, ScaleFactor::TIMES_1, 0)); in EmitNativeCode()
158 __ movl(Address(dest, temp3, ScaleFactor::TIMES_4, offset), temp2); in EmitNativeCode()
161 __ addl(temp1, Immediate(1)); in EmitNativeCode()
164 __ j(kNotEqual, &loop); in EmitNativeCode()
165 __ jmp(GetExitLabel()); in EmitNativeCode()
174 #undef __
176 #define __ assembler-> macro
207 __ movsd(temp, input.AsFpuRegister<XmmRegister>()); in MoveFPToInt()
208 __ movd(output.AsRegisterPairLow<Register>(), temp); in MoveFPToInt()
209 __ psrlq(temp, Immediate(32)); in MoveFPToInt()
210 __ movd(output.AsRegisterPairHigh<Register>(), temp); in MoveFPToInt()
212 __ movd(output.AsRegister<Register>(), input.AsFpuRegister<XmmRegister>()); in MoveFPToInt()
223 __ movd(temp1, input.AsRegisterPairLow<Register>()); in MoveIntToFP()
224 __ movd(temp2, input.AsRegisterPairHigh<Register>()); in MoveIntToFP()
225 __ punpckldq(temp1, temp2); in MoveIntToFP()
226 __ movsd(output.AsFpuRegister<XmmRegister>(), temp1); in MoveIntToFP()
228 __ movd(output.AsFpuRegister<XmmRegister>(), input.AsRegister<Register>()); in MoveIntToFP()
292 __ bswapl(out); in GenReverseBytes()
293 __ sarl(out, Immediate(16)); in GenReverseBytes()
296 __ bswapl(out); in GenReverseBytes()
327 __ movl(output_lo, input_hi); in VisitLongReverseBytes()
328 __ movl(output_hi, input_lo); in VisitLongReverseBytes()
329 __ bswapl(output_lo); in VisitLongReverseBytes()
330 __ bswapl(output_hi); in VisitLongReverseBytes()
379 __ movsd(temp, codegen->LiteralInt64Address( in MathAbsFP()
381 __ andpd(output.AsFpuRegister<XmmRegister>(), temp); in MathAbsFP()
383 __ movss(temp, codegen->LiteralInt32Address( in MathAbsFP()
385 __ andps(output.AsFpuRegister<XmmRegister>(), temp); in MathAbsFP()
390 __ subl(ESP, Immediate(8)); in MathAbsFP()
391 __ pushl(Immediate(0x7FFFFFFF)); in MathAbsFP()
392 __ pushl(Immediate(0xFFFFFFFF)); in MathAbsFP()
393 __ andpd(output.AsFpuRegister<XmmRegister>(), Address(ESP, 0)); in MathAbsFP()
395 __ subl(ESP, Immediate(12)); in MathAbsFP()
396 __ pushl(Immediate(0x7FFFFFFF)); in MathAbsFP()
397 __ andps(output.AsFpuRegister<XmmRegister>(), Address(ESP, 0)); in MathAbsFP()
399 __ addl(ESP, Immediate(16)); in MathAbsFP()
436 __ cdq(); in GenAbsInteger()
439 __ xorl(EAX, EDX); in GenAbsInteger()
442 __ subl(EAX, EDX); in GenAbsInteger()
466 __ movl(temp, input_hi); in GenAbsLong()
467 __ sarl(temp, Immediate(31)); in GenAbsLong()
470 __ movl(output_lo, temp); in GenAbsLong()
471 __ movl(output_hi, temp); in GenAbsLong()
474 __ xorl(output_lo, input_lo); in GenAbsLong()
475 __ xorl(output_hi, input_hi); in GenAbsLong()
478 __ subl(output_lo, temp); in GenAbsLong()
479 __ sbbl(output_hi, temp); in GenAbsLong()
536 __ ucomisd(out, op2); in GenMinMaxFP()
538 __ ucomiss(out, op2); in GenMinMaxFP()
541 __ j(Condition::kParityEven, &nan); in GenMinMaxFP()
543 __ j(is_min ? Condition::kAbove : Condition::kBelow, &op2_label); in GenMinMaxFP()
544 __ j(is_min ? Condition::kBelow : Condition::kAbove, &done); in GenMinMaxFP()
549 __ orpd(out, op2); in GenMinMaxFP()
551 __ orps(out, op2); in GenMinMaxFP()
555 __ andpd(out, op2); in GenMinMaxFP()
557 __ andps(out, op2); in GenMinMaxFP()
560 __ jmp(&done); in GenMinMaxFP()
563 __ Bind(&nan); in GenMinMaxFP()
571 __ movsd(out, codegen->LiteralInt64Address(kDoubleNaN, method_address, constant_area)); in GenMinMaxFP()
573 __ movss(out, codegen->LiteralInt32Address(kFloatNaN, method_address, constant_area)); in GenMinMaxFP()
577 __ pushl(Immediate(kDoubleNaNHigh)); in GenMinMaxFP()
578 __ pushl(Immediate(kDoubleNaNLow)); in GenMinMaxFP()
579 __ movsd(out, Address(ESP, 0)); in GenMinMaxFP()
580 __ addl(ESP, Immediate(8)); in GenMinMaxFP()
582 __ pushl(Immediate(kFloatNaN)); in GenMinMaxFP()
583 __ movss(out, Address(ESP, 0)); in GenMinMaxFP()
584 __ addl(ESP, Immediate(4)); in GenMinMaxFP()
587 __ jmp(&done); in GenMinMaxFP()
590 __ Bind(&op2_label); in GenMinMaxFP()
592 __ movsd(out, op2); in GenMinMaxFP()
594 __ movss(out, op2); in GenMinMaxFP()
598 __ Bind(&done); in GenMinMaxFP()
694 __ movl(temp, output_lo); in GenMinMax()
695 __ subl(temp, op2_lo); in GenMinMax()
698 __ movl(temp, output_hi); in GenMinMax()
699 __ sbbl(temp, op2_hi); in GenMinMax()
703 __ cmovl(cond, output_lo, op2_lo); in GenMinMax()
704 __ cmovl(cond, output_hi, op2_hi); in GenMinMax()
715 __ cmpl(out, op2); in GenMinMax()
717 __ cmovl(cond, out, op2); in GenMinMax()
838 __ roundsd(out, in, Immediate(round_mode)); in GenSSE41FPToFPIntrinsic()
915 __ movss(t2, in); in VisitMathRoundFloat()
916 __ roundss(t1, in, Immediate(1)); in VisitMathRoundFloat()
917 __ subss(t2, t1); in VisitMathRoundFloat()
923 __ comiss(t2, codegen_->LiteralInt32Address(bit_cast<int32_t, float>(0.5f), in VisitMathRoundFloat()
926 __ j(kBelow, &skip_incr); in VisitMathRoundFloat()
927 __ addss(t1, codegen_->LiteralInt32Address(bit_cast<int32_t, float>(1.0f), in VisitMathRoundFloat()
930 __ Bind(&skip_incr); in VisitMathRoundFloat()
933 __ pushl(Immediate(bit_cast<int32_t, float>(0.5f))); in VisitMathRoundFloat()
934 __ pushl(Immediate(bit_cast<int32_t, float>(1.0f))); in VisitMathRoundFloat()
935 __ comiss(t2, Address(ESP, 4)); in VisitMathRoundFloat()
936 __ j(kBelow, &skip_incr); in VisitMathRoundFloat()
937 __ addss(t1, Address(ESP, 0)); in VisitMathRoundFloat()
938 __ Bind(&skip_incr); in VisitMathRoundFloat()
939 __ addl(ESP, Immediate(8)); in VisitMathRoundFloat()
945 __ movl(out, Immediate(kPrimIntMax)); in VisitMathRoundFloat()
946 __ cvtsi2ss(t2, out); in VisitMathRoundFloat()
947 __ comiss(t1, t2); in VisitMathRoundFloat()
948 __ j(kAboveEqual, &done); // clipped to max (already in out), does not jump on unordered in VisitMathRoundFloat()
949 __ movl(out, Immediate(0)); // does not change flags in VisitMathRoundFloat()
950 __ j(kUnordered, &done); // NaN mapped to 0 (just moved in out) in VisitMathRoundFloat()
951 __ cvttss2si(out, t1); in VisitMathRoundFloat()
952 __ Bind(&done); in VisitMathRoundFloat()
972 __ subl(ESP, Immediate(16)); in GenFPToFPCall()
973 __ cfi().AdjustCFAOffset(16); in GenFPToFPCall()
976 __ movsd(Address(ESP, 0), XMM0); in GenFPToFPCall()
980 __ movsd(Address(ESP, 8), XMM1); in GenFPToFPCall()
987 __ fstpl(Address(ESP, 0)); in GenFPToFPCall()
988 __ movsd(XMM0, Address(ESP, 0)); in GenFPToFPCall()
991 __ addl(ESP, Immediate(16)); in GenFPToFPCall()
992 __ cfi().AdjustCFAOffset(-16); in GenFPToFPCall()
1208 __ cmpl(Address(input, length_offset), in CheckPosition()
1211 __ cmpl(Address(input, length_offset), length.AsRegister<Register>()); in CheckPosition()
1213 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1217 __ movl(temp, Address(input, length_offset)); in CheckPosition()
1218 __ subl(temp, Immediate(pos_const)); in CheckPosition()
1219 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1223 __ cmpl(temp, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); in CheckPosition()
1225 __ cmpl(temp, length.AsRegister<Register>()); in CheckPosition()
1227 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1232 __ testl(pos_reg, pos_reg); in CheckPosition()
1233 __ j(kNotEqual, slow_path->GetEntryLabel()); in CheckPosition()
1237 __ testl(pos_reg, pos_reg); in CheckPosition()
1238 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1241 __ cmpl(Address(input, length_offset), pos_reg); in CheckPosition()
1242 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1245 __ movl(temp, Address(input, length_offset)); in CheckPosition()
1246 __ subl(temp, pos_reg); in CheckPosition()
1248 __ cmpl(temp, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); in CheckPosition()
1250 __ cmpl(temp, length.AsRegister<Register>()); in CheckPosition()
1252 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1278 __ cmpl(src, dest); in VisitSystemArrayCopyChar()
1279 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
1282 __ testl(src, src); in VisitSystemArrayCopyChar()
1283 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
1286 __ testl(dest, dest); in VisitSystemArrayCopyChar()
1287 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
1292 __ cmpl(length.AsRegister<Register>(), length.AsRegister<Register>()); in VisitSystemArrayCopyChar()
1293 __ j(kLess, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
1298 __ movl(count, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); in VisitSystemArrayCopyChar()
1300 __ movl(count, length.AsRegister<Register>()); in VisitSystemArrayCopyChar()
1318 __ leal(src_base, Address(src, char_size * srcPos_const + data_offset)); in VisitSystemArrayCopyChar()
1320 __ leal(src_base, Address(src, srcPos.AsRegister<Register>(), in VisitSystemArrayCopyChar()
1326 __ leal(dest_base, Address(dest, char_size * destPos_const + data_offset)); in VisitSystemArrayCopyChar()
1328 __ leal(dest_base, Address(dest, destPos.AsRegister<Register>(), in VisitSystemArrayCopyChar()
1333 __ rep_movsw(); in VisitSystemArrayCopyChar()
1335 __ Bind(slow_path->GetExitLabel()); in VisitSystemArrayCopyChar()
1357 __ testl(argument, argument); in VisitStringCompareTo()
1360 __ j(kEqual, slow_path->GetEntryLabel()); in VisitStringCompareTo()
1363 __ Bind(slow_path->GetExitLabel()); in VisitStringCompareTo()
1404 __ testl(arg, arg); in VisitStringEquals()
1405 __ j(kEqual, &return_false); in VisitStringEquals()
1413 __ movl(ecx, Address(str, class_offset)); in VisitStringEquals()
1414 __ cmpl(ecx, Address(arg, class_offset)); in VisitStringEquals()
1415 __ j(kNotEqual, &return_false); in VisitStringEquals()
1419 __ cmpl(str, arg); in VisitStringEquals()
1420 __ j(kEqual, &return_true); in VisitStringEquals()
1423 __ movl(ecx, Address(str, count_offset)); in VisitStringEquals()
1427 __ cmpl(ecx, Address(arg, count_offset)); in VisitStringEquals()
1428 __ j(kNotEqual, &return_false); in VisitStringEquals()
1432 __ jecxz(&return_true); in VisitStringEquals()
1438 __ shrl(ecx, Immediate(1)); in VisitStringEquals()
1439 __ j(kCarrySet, &string_uncompressed); in VisitStringEquals()
1441 __ addl(ecx, Immediate(1)); in VisitStringEquals()
1442 __ shrl(ecx, Immediate(1)); in VisitStringEquals()
1443 __ Bind(&string_uncompressed); in VisitStringEquals()
1446 __ leal(esi, Address(str, value_offset)); in VisitStringEquals()
1447 __ leal(edi, Address(arg, value_offset)); in VisitStringEquals()
1451 __ addl(ecx, Immediate(1)); in VisitStringEquals()
1452 __ shrl(ecx, Immediate(1)); in VisitStringEquals()
1460 __ repe_cmpsl(); in VisitStringEquals()
1462 __ j(kNotEqual, &return_false); in VisitStringEquals()
1466 __ Bind(&return_true); in VisitStringEquals()
1467 __ movl(esi, Immediate(1)); in VisitStringEquals()
1468 __ jmp(&end); in VisitStringEquals()
1471 __ Bind(&return_false); in VisitStringEquals()
1472 __ xorl(esi, esi); in VisitStringEquals()
1473 __ Bind(&end); in VisitStringEquals()
1540 __ jmp(slow_path->GetEntryLabel()); in GenerateStringIndexOf()
1541 __ Bind(slow_path->GetExitLabel()); in GenerateStringIndexOf()
1545 __ cmpl(search_value, Immediate(std::numeric_limits<uint16_t>::max())); in GenerateStringIndexOf()
1548 __ j(kAbove, slow_path->GetEntryLabel()); in GenerateStringIndexOf()
1558 __ movl(string_length, Address(string_obj, count_offset)); in GenerateStringIndexOf()
1565 __ testl(string_length, string_length); in GenerateStringIndexOf()
1566 __ j(kEqual, ¬_found_label); in GenerateStringIndexOf()
1570 __ movl(string_length_flagged, string_length); in GenerateStringIndexOf()
1572 __ shrl(string_length, Immediate(1)); in GenerateStringIndexOf()
1577 __ movl(counter, string_length); in GenerateStringIndexOf()
1580 __ addl(string_obj, Immediate(value_offset)); in GenerateStringIndexOf()
1585 __ cmpl(start_index, string_length); in GenerateStringIndexOf()
1586 __ j(kGreaterEqual, ¬_found_label); in GenerateStringIndexOf()
1589 __ xorl(counter, counter); in GenerateStringIndexOf()
1590 __ cmpl(start_index, Immediate(0)); in GenerateStringIndexOf()
1591 __ cmovl(kGreater, counter, start_index); in GenerateStringIndexOf()
1595 __ testl(string_length_flagged, Immediate(1)); in GenerateStringIndexOf()
1596 __ j(kNotZero, &offset_uncompressed_label); in GenerateStringIndexOf()
1598 __ leal(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_1, value_offset)); in GenerateStringIndexOf()
1599 __ jmp(&modify_counter); in GenerateStringIndexOf()
1602 __ Bind(&offset_uncompressed_label); in GenerateStringIndexOf()
1603 __ leal(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_2, value_offset)); in GenerateStringIndexOf()
1607 __ Bind(&modify_counter); in GenerateStringIndexOf()
1609 __ leal(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_2, value_offset)); in GenerateStringIndexOf()
1611 __ negl(counter); in GenerateStringIndexOf()
1612 __ leal(counter, Address(string_length, counter, ScaleFactor::TIMES_1, 0)); in GenerateStringIndexOf()
1618 __ testl(string_length_flagged, Immediate(1)); in GenerateStringIndexOf()
1619 __ j(kNotZero, &uncompressed_string_comparison); in GenerateStringIndexOf()
1622 __ cmpl(search_value, Immediate(127)); in GenerateStringIndexOf()
1623 __ j(kGreater, ¬_found_label); in GenerateStringIndexOf()
1625 __ repne_scasb(); in GenerateStringIndexOf()
1626 __ jmp(&comparison_done); in GenerateStringIndexOf()
1631 __ Bind(&uncompressed_string_comparison); in GenerateStringIndexOf()
1632 __ repne_scasw(); in GenerateStringIndexOf()
1633 __ Bind(&comparison_done); in GenerateStringIndexOf()
1635 __ repne_scasw(); in GenerateStringIndexOf()
1638 __ j(kNotEqual, ¬_found_label); in GenerateStringIndexOf()
1641 __ subl(string_length, counter); in GenerateStringIndexOf()
1642 __ leal(out, Address(string_length, -1)); in GenerateStringIndexOf()
1645 __ jmp(&done); in GenerateStringIndexOf()
1648 __ Bind(¬_found_label); in GenerateStringIndexOf()
1649 __ movl(out, Immediate(-1)); in GenerateStringIndexOf()
1652 __ Bind(&done); in GenerateStringIndexOf()
1654 __ Bind(slow_path->GetExitLabel()); in GenerateStringIndexOf()
1692 __ testl(byte_array, byte_array); in VisitStringNewStringFromBytes()
1695 __ j(kEqual, slow_path->GetEntryLabel()); in VisitStringNewStringFromBytes()
1699 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromBytes()
1738 __ testl(string_to_copy, string_to_copy); in VisitStringNewStringFromString()
1741 __ j(kEqual, slow_path->GetEntryLabel()); in VisitStringNewStringFromString()
1745 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromString()
1791 __ pushl(ECX); in VisitStringGetCharsNoCheck()
1793 __ cfi().AdjustCFAOffset(stack_adjust); in VisitStringGetCharsNoCheck()
1796 __ subl(ECX, Immediate(srcBegin_value)); in VisitStringGetCharsNoCheck()
1799 __ subl(ECX, srcBegin.AsRegister<Register>()); in VisitStringGetCharsNoCheck()
1808 __ pushl(EAX); in VisitStringGetCharsNoCheck()
1809 __ cfi().AdjustCFAOffset(stack_adjust); in VisitStringGetCharsNoCheck()
1812 __ testl(Address(obj, count_offset), Immediate(1)); in VisitStringGetCharsNoCheck()
1815 __ j(kNotZero, ©_uncompressed); in VisitStringGetCharsNoCheck()
1818 __ leal(ESI, CodeGeneratorX86::ArrayAddress(obj, srcBegin, TIMES_1, value_offset)); in VisitStringGetCharsNoCheck()
1821 __ leal(EDI, Address(dst, dstBegin, ScaleFactor::TIMES_2, data_offset)); in VisitStringGetCharsNoCheck()
1822 __ Bind(©_loop); in VisitStringGetCharsNoCheck()
1823 __ jecxz(&done); in VisitStringGetCharsNoCheck()
1826 __ movzxb(EAX, Address(ESI, 0)); in VisitStringGetCharsNoCheck()
1827 __ movw(Address(EDI, 0), EAX); in VisitStringGetCharsNoCheck()
1828 __ leal(EDI, Address(EDI, char_size)); in VisitStringGetCharsNoCheck()
1829 __ leal(ESI, Address(ESI, c_char_size)); in VisitStringGetCharsNoCheck()
1831 __ subl(ECX, Immediate(1)); in VisitStringGetCharsNoCheck()
1832 __ jmp(©_loop); in VisitStringGetCharsNoCheck()
1833 __ Bind(©_uncompressed); in VisitStringGetCharsNoCheck()
1838 __ leal(EDI, Address(dst, dstBegin, ScaleFactor::TIMES_2, data_offset)); in VisitStringGetCharsNoCheck()
1839 __ leal(ESI, CodeGeneratorX86::ArrayAddress(obj, srcBegin, TIMES_2, value_offset)); in VisitStringGetCharsNoCheck()
1840 __ rep_movsw(); in VisitStringGetCharsNoCheck()
1842 __ Bind(&done); in VisitStringGetCharsNoCheck()
1845 __ popl(EAX); in VisitStringGetCharsNoCheck()
1846 __ cfi().AdjustCFAOffset(-stack_adjust); in VisitStringGetCharsNoCheck()
1849 __ popl(ECX); in VisitStringGetCharsNoCheck()
1850 __ cfi().AdjustCFAOffset(-stack_adjust); in VisitStringGetCharsNoCheck()
1860 __ movsxb(out_loc.AsRegister<Register>(), Address(address, 0)); in GenPeek()
1863 __ movsxw(out_loc.AsRegister<Register>(), Address(address, 0)); in GenPeek()
1866 __ movl(out_loc.AsRegister<Register>(), Address(address, 0)); in GenPeek()
1869 __ movl(out_loc.AsRegisterPairLow<Register>(), Address(address, 0)); in GenPeek()
1870 __ movl(out_loc.AsRegisterPairHigh<Register>(), Address(address, 4)); in GenPeek()
1932 __ movb(Address(address, 0), in GenPoke()
1935 __ movb(Address(address, 0), value_loc.AsRegister<ByteRegister>()); in GenPoke()
1940 __ movw(Address(address, 0), in GenPoke()
1943 __ movw(Address(address, 0), value_loc.AsRegister<Register>()); in GenPoke()
1948 __ movl(Address(address, 0), in GenPoke()
1951 __ movl(Address(address, 0), value_loc.AsRegister<Register>()); in GenPoke()
1957 __ movl(Address(address, 0), Immediate(Low32Bits(value))); in GenPoke()
1958 __ movl(Address(address, 4), Immediate(High32Bits(value))); in GenPoke()
1960 __ movl(Address(address, 0), value_loc.AsRegisterPairLow<Register>()); in GenPoke()
1961 __ movl(Address(address, 4), value_loc.AsRegisterPairHigh<Register>()); in GenPoke()
2029 __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
2041 __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
2046 __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
2047 __ MaybeUnpoisonHeapReference(output); in GenUnsafeGet()
2058 __ movsd(temp, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
2059 __ movd(output_lo, temp); in GenUnsafeGet()
2060 __ psrlq(temp, Immediate(32)); in GenUnsafeGet()
2061 __ movd(output_hi, temp); in GenUnsafeGet()
2063 __ movl(output_lo, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
2064 __ movl(output_hi, Address(base, offset, ScaleFactor::TIMES_1, 4)); in GenUnsafeGet()
2223 __ movd(temp1, value_lo); in GenUnsafePut()
2224 __ movd(temp2, value_hi); in GenUnsafePut()
2225 __ punpckldq(temp1, temp2); in GenUnsafePut()
2226 __ movsd(Address(base, offset, ScaleFactor::TIMES_1, 0), temp1); in GenUnsafePut()
2228 __ movl(Address(base, offset, ScaleFactor::TIMES_1, 0), value_lo); in GenUnsafePut()
2229 __ movl(Address(base, offset, ScaleFactor::TIMES_1, 4), value_hi); in GenUnsafePut()
2233 __ movl(temp, value_loc.AsRegister<Register>()); in GenUnsafePut()
2234 __ PoisonHeapReference(temp); in GenUnsafePut()
2235 __ movl(Address(base, offset, ScaleFactor::TIMES_1, 0), temp); in GenUnsafePut()
2237 __ movl(Address(base, offset, ScaleFactor::TIMES_1, 0), value_loc.AsRegister<Register>()); in GenUnsafePut()
2387 __ movl(value, base); in GenCAS()
2401 __ PoisonHeapReference(expected); in GenCAS()
2402 __ PoisonHeapReference(value); in GenCAS()
2405 __ LockCmpxchgl(field_addr, value); in GenCAS()
2411 __ setb(kZero, out.AsRegister<Register>()); in GenCAS()
2412 __ movzxb(out.AsRegister<Register>(), out.AsRegister<ByteRegister>()); in GenCAS()
2424 __ UnpoisonHeapReference(value); in GenCAS()
2434 __ LockCmpxchgl(field_addr, locations->InAt(4).AsRegister<Register>()); in GenCAS()
2442 __ LockCmpxchg8b(field_addr); in GenCAS()
2451 __ setb(kZero, out.AsRegister<Register>()); in GenCAS()
2452 __ movzxb(out.AsRegister<Register>(), out.AsRegister<ByteRegister>()); in GenCAS()
2485 __ movl(temp, reg); in SwapBits()
2486 __ shrl(reg, imm_shift); in SwapBits()
2487 __ andl(temp, imm_mask); in SwapBits()
2488 __ andl(reg, imm_mask); in SwapBits()
2489 __ shll(temp, imm_shift); in SwapBits()
2490 __ orl(reg, temp); in SwapBits()
2509 __ bswapl(reg); in VisitIntegerReverse()
2535 __ movl(temp, reg_low); in VisitLongReverse()
2536 __ movl(reg_low, reg_high); in VisitLongReverse()
2537 __ movl(reg_high, temp); in VisitLongReverse()
2540 __ bswapl(reg_low); in VisitLongReverse()
2546 __ bswapl(reg_high); in VisitLongReverse()
2589 __ popcntl(out, src.AsRegister<Register>()); in GenBitCount()
2592 __ popcntl(out, Address(ESP, src.GetStackIndex())); in GenBitCount()
2598 __ popcntl(temp, src.AsRegisterPairLow<Register>()); in GenBitCount()
2599 __ popcntl(out, src.AsRegisterPairHigh<Register>()); in GenBitCount()
2602 __ popcntl(temp, Address(ESP, src.GetStackIndex())); in GenBitCount()
2603 __ popcntl(out, Address(ESP, src.GetHighStackIndex(kX86WordSize))); in GenBitCount()
2605 __ addl(out, temp); in GenBitCount()
2659 __ bsrl(out, src.AsRegister<Register>()); in GenLeadingZeros()
2662 __ bsrl(out, Address(ESP, src.GetStackIndex())); in GenLeadingZeros()
2667 __ j(kEqual, &all_zeroes); in GenLeadingZeros()
2670 __ xorl(out, Immediate(31)); in GenLeadingZeros()
2671 __ jmp(&done); in GenLeadingZeros()
2674 __ Bind(&all_zeroes); in GenLeadingZeros()
2675 __ movl(out, Immediate(32)); in GenLeadingZeros()
2677 __ Bind(&done); in GenLeadingZeros()
2688 __ testl(src_hi, src_hi); in GenLeadingZeros()
2689 __ j(kEqual, &handle_low); in GenLeadingZeros()
2692 __ bsrl(out, src_hi); in GenLeadingZeros()
2695 __ xorl(out, Immediate(31)); in GenLeadingZeros()
2696 __ jmp(&done); in GenLeadingZeros()
2699 __ Bind(&handle_low); in GenLeadingZeros()
2700 __ bsrl(out, src_lo); in GenLeadingZeros()
2701 __ j(kEqual, &all_zeroes); in GenLeadingZeros()
2704 __ xorl(out, Immediate(63)); in GenLeadingZeros()
2705 __ jmp(&done); in GenLeadingZeros()
2708 __ Bind(&all_zeroes); in GenLeadingZeros()
2709 __ movl(out, Immediate(64)); in GenLeadingZeros()
2711 __ Bind(&done); in GenLeadingZeros()
2764 __ bsfl(out, src.AsRegister<Register>()); in GenTrailingZeros()
2767 __ bsfl(out, Address(ESP, src.GetStackIndex())); in GenTrailingZeros()
2772 __ j(kNotEqual, &done); in GenTrailingZeros()
2775 __ movl(out, Immediate(32)); in GenTrailingZeros()
2777 __ Bind(&done); in GenTrailingZeros()
2788 __ bsfl(out, src_lo); in GenTrailingZeros()
2789 __ j(kNotEqual, &done); in GenTrailingZeros()
2792 __ bsfl(out, src_hi); in GenTrailingZeros()
2793 __ j(kEqual, &all_zeroes); in GenTrailingZeros()
2796 __ addl(out, Immediate(32)); in GenTrailingZeros()
2797 __ jmp(&done); in GenTrailingZeros()
2800 __ Bind(&all_zeroes); in GenTrailingZeros()
2801 __ movl(out, Immediate(64)); in GenTrailingZeros()
2803 __ Bind(&done); in GenTrailingZeros()
2855 __ movl(temp, Address(temp, ArtMethod::DeclaringClassOffset().Int32Value())); in VisitReferenceGetReferent()
2865 __ cmpw(Address(temp, disable_flag_offset), Immediate(0)); in VisitReferenceGetReferent()
2866 __ j(kNotEqual, slow_path->GetEntryLabel()); in VisitReferenceGetReferent()
2868 __ cmpb(Address(temp, disable_flag_offset), Immediate(0)); in VisitReferenceGetReferent()
2869 __ j(kNotEqual, slow_path->GetEntryLabel()); in VisitReferenceGetReferent()
2870 __ cmpb(Address(temp, slow_path_flag_offset), Immediate(0)); in VisitReferenceGetReferent()
2871 __ j(kNotEqual, slow_path->GetEntryLabel()); in VisitReferenceGetReferent()
2875 __ movl(out, Address(obj, mirror::Reference::ReferentOffset().Int32Value())); in VisitReferenceGetReferent()
2877 __ MaybeUnpoisonHeapReference(out); in VisitReferenceGetReferent()
2878 __ Bind(slow_path->GetExitLabel()); in VisitReferenceGetReferent()
2901 __ leal(base, Address(array, element_size * constant + data_offset)); in GenSystemArrayCopyBaseAddress()
2903 __ leal(base, Address(array, pos.AsRegister<Register>(), scale_factor, data_offset)); in GenSystemArrayCopyBaseAddress()
2922 __ leal(end, Address(base, element_size * constant)); in GenSystemArrayCopyEndAddress()
2924 __ leal(end, Address(base, copy_length.AsRegister<Register>(), scale_factor, 0)); in GenSystemArrayCopyEndAddress()
3001 __ cmpl(src, dest); in VisitSystemArrayCopy()
3002 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3006 __ cmpl(src, dest); in VisitSystemArrayCopy()
3007 __ j(kNotEqual, &conditions_on_positions_validated); in VisitSystemArrayCopy()
3009 __ cmpl(dest_pos.AsRegister<Register>(), Immediate(src_pos_constant)); in VisitSystemArrayCopy()
3010 __ j(kGreater, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3014 __ cmpl(src, dest); in VisitSystemArrayCopy()
3015 __ j(kNotEqual, &conditions_on_positions_validated); in VisitSystemArrayCopy()
3019 __ cmpl(src_pos.AsRegister<Register>(), Immediate(dest_pos_constant)); in VisitSystemArrayCopy()
3020 __ j(kLess, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3022 __ cmpl(src_pos.AsRegister<Register>(), dest_pos.AsRegister<Register>()); in VisitSystemArrayCopy()
3023 __ j(kLess, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3027 __ Bind(&conditions_on_positions_validated); in VisitSystemArrayCopy()
3031 __ testl(src, src); in VisitSystemArrayCopy()
3032 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3037 __ testl(dest, dest); in VisitSystemArrayCopy()
3038 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3044 __ movl(temp3, Address(ESP, length.GetStackIndex())); in VisitSystemArrayCopy()
3053 __ testl(length.AsRegister<Register>(), length.AsRegister<Register>()); in VisitSystemArrayCopy()
3054 __ j(kLess, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3090 __ testl(temp1, temp1); in VisitSystemArrayCopy()
3091 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3096 __ movl(temp1, Address(src, class_offset)); in VisitSystemArrayCopy()
3097 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
3100 __ movl(temp1, Address(temp1, component_offset)); in VisitSystemArrayCopy()
3101 __ testl(temp1, temp1); in VisitSystemArrayCopy()
3102 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3103 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
3105 __ cmpw(Address(temp1, primitive_offset), Immediate(Primitive::kPrimNot)); in VisitSystemArrayCopy()
3106 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3135 __ testl(temp2, temp2); in VisitSystemArrayCopy()
3136 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3139 __ cmpw(Address(temp2, primitive_offset), Immediate(Primitive::kPrimNot)); in VisitSystemArrayCopy()
3140 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3149 __ cmpl(temp1, temp2); in VisitSystemArrayCopy()
3153 __ j(kEqual, &do_copy); in VisitSystemArrayCopy()
3161 __ cmpl(Address(temp1, super_offset), Immediate(0)); in VisitSystemArrayCopy()
3162 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3163 __ Bind(&do_copy); in VisitSystemArrayCopy()
3165 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3171 __ movl(temp1, Address(dest, class_offset)); in VisitSystemArrayCopy()
3173 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
3176 __ movl(temp2, Address(temp1, component_offset)); in VisitSystemArrayCopy()
3177 __ testl(temp2, temp2); in VisitSystemArrayCopy()
3178 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3179 __ MaybeUnpoisonHeapReference(temp2); in VisitSystemArrayCopy()
3180 __ cmpw(Address(temp2, primitive_offset), Immediate(Primitive::kPrimNot)); in VisitSystemArrayCopy()
3181 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3184 __ PoisonHeapReference(temp1); in VisitSystemArrayCopy()
3188 __ cmpl(temp1, Address(src, class_offset)); in VisitSystemArrayCopy()
3192 __ j(kEqual, &do_copy); in VisitSystemArrayCopy()
3193 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
3195 __ movl(temp1, Address(temp1, component_offset)); in VisitSystemArrayCopy()
3196 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
3197 __ cmpl(Address(temp1, super_offset), Immediate(0)); in VisitSystemArrayCopy()
3198 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3199 __ Bind(&do_copy); in VisitSystemArrayCopy()
3201 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3214 __ testl(temp1, temp1); in VisitSystemArrayCopy()
3215 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3220 __ movl(temp1, Address(src, class_offset)); in VisitSystemArrayCopy()
3221 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
3223 __ movl(temp1, Address(temp1, component_offset)); in VisitSystemArrayCopy()
3224 __ testl(temp1, temp1); in VisitSystemArrayCopy()
3225 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3226 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
3228 __ cmpw(Address(temp1, primitive_offset), Immediate(Primitive::kPrimNot)); in VisitSystemArrayCopy()
3229 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3249 __ movl(temp3, Address(ESP, length.GetStackIndex())); in VisitSystemArrayCopy()
3278 __ cmpl(temp1, temp3); in VisitSystemArrayCopy()
3279 __ j(kEqual, &done); in VisitSystemArrayCopy()
3291 __ testb(Address(src, monitor_offset + gray_byte_position), Immediate(test_value)); in VisitSystemArrayCopy()
3303 __ j(kNotZero, read_barrier_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3310 __ Bind(&loop); in VisitSystemArrayCopy()
3311 __ pushl(Address(temp1, 0)); in VisitSystemArrayCopy()
3312 __ cfi().AdjustCFAOffset(4); in VisitSystemArrayCopy()
3313 __ popl(Address(temp2, 0)); in VisitSystemArrayCopy()
3314 __ cfi().AdjustCFAOffset(-4); in VisitSystemArrayCopy()
3315 __ addl(temp1, Immediate(element_size)); in VisitSystemArrayCopy()
3316 __ addl(temp2, Immediate(element_size)); in VisitSystemArrayCopy()
3317 __ cmpl(temp1, temp3); in VisitSystemArrayCopy()
3318 __ j(kNotEqual, &loop); in VisitSystemArrayCopy()
3320 __ Bind(read_barrier_slow_path->GetExitLabel()); in VisitSystemArrayCopy()
3321 __ Bind(&done); in VisitSystemArrayCopy()
3331 __ cmpl(temp1, temp3); in VisitSystemArrayCopy()
3332 __ j(kEqual, &done); in VisitSystemArrayCopy()
3333 __ Bind(&loop); in VisitSystemArrayCopy()
3334 __ pushl(Address(temp1, 0)); in VisitSystemArrayCopy()
3335 __ cfi().AdjustCFAOffset(4); in VisitSystemArrayCopy()
3336 __ popl(Address(temp2, 0)); in VisitSystemArrayCopy()
3337 __ cfi().AdjustCFAOffset(-4); in VisitSystemArrayCopy()
3338 __ addl(temp1, Immediate(element_size)); in VisitSystemArrayCopy()
3339 __ addl(temp2, Immediate(element_size)); in VisitSystemArrayCopy()
3340 __ cmpl(temp1, temp3); in VisitSystemArrayCopy()
3341 __ j(kNotEqual, &loop); in VisitSystemArrayCopy()
3342 __ Bind(&done); in VisitSystemArrayCopy()
3348 __ Bind(intrinsic_slow_path->GetExitLabel()); in VisitSystemArrayCopy()
3375 __ movl(out, Immediate(address)); in VisitIntegerValueOf()
3381 __ movl(calling_convention.GetRegisterAt(0), Immediate(address)); in VisitIntegerValueOf()
3384 __ movl(Address(out, info.value_offset), Immediate(value)); in VisitIntegerValueOf()
3389 __ leal(out, Address(in, -info.low)); in VisitIntegerValueOf()
3390 __ cmpl(out, Immediate(info.high - info.low + 1)); in VisitIntegerValueOf()
3392 __ j(kAboveEqual, &allocate); in VisitIntegerValueOf()
3396 __ movl(out, Address(out, TIMES_4, data_offset + address)); in VisitIntegerValueOf()
3397 __ MaybeUnpoisonHeapReference(out); in VisitIntegerValueOf()
3398 __ jmp(&done); in VisitIntegerValueOf()
3399 __ Bind(&allocate); in VisitIntegerValueOf()
3402 __ movl(calling_convention.GetRegisterAt(0), Immediate(address)); in VisitIntegerValueOf()
3405 __ movl(Address(out, info.value_offset), in); in VisitIntegerValueOf()
3406 __ Bind(&done); in VisitIntegerValueOf()
3436 #undef __