Lines Matching refs:__
71 #define __ down_cast<X86_64Assembler*>(codegen->GetAssembler())-> // NOLINT macro
98 __ Bind(GetEntryLabel()); in EmitNativeCode()
100 __ Bind(&loop); in EmitNativeCode()
101 __ movl(CpuRegister(TMP), Address(src_curr_addr, 0)); in EmitNativeCode()
102 __ MaybeUnpoisonHeapReference(CpuRegister(TMP)); in EmitNativeCode()
112 __ MaybePoisonHeapReference(CpuRegister(TMP)); in EmitNativeCode()
113 __ movl(Address(dst_curr_addr, 0), CpuRegister(TMP)); in EmitNativeCode()
114 __ addl(src_curr_addr, Immediate(element_size)); in EmitNativeCode()
115 __ addl(dst_curr_addr, Immediate(element_size)); in EmitNativeCode()
116 __ cmpl(src_curr_addr, src_stop_addr); in EmitNativeCode()
117 __ j(kNotEqual, &loop); in EmitNativeCode()
118 __ jmp(GetExitLabel()); in EmitNativeCode()
127 #undef __
129 #define __ assembler-> macro
150 __ movd(output.AsRegister<CpuRegister>(), input.AsFpuRegister<XmmRegister>(), is64bit); in MoveFPToInt()
156 __ movd(output.AsFpuRegister<XmmRegister>(), input.AsRegister<CpuRegister>(), is64bit); in MoveIntToFP()
203 __ bswapl(out); in GenReverseBytes()
204 __ sarl(out, Immediate(16)); in GenReverseBytes()
207 __ bswapl(out); in GenReverseBytes()
210 __ bswapq(out); in GenReverseBytes()
269 __ movsd(xmm_temp, codegen->LiteralInt64Address(INT64_C(0x7FFFFFFFFFFFFFFF))); in MathAbsFP()
270 __ andpd(output.AsFpuRegister<XmmRegister>(), xmm_temp); in MathAbsFP()
272 __ movss(xmm_temp, codegen->LiteralInt32Address(INT32_C(0x7FFFFFFF))); in MathAbsFP()
273 __ andps(output.AsFpuRegister<XmmRegister>(), xmm_temp); in MathAbsFP()
309 __ movq(mask, out); in GenAbsInteger()
310 __ sarq(mask, Immediate(63)); in GenAbsInteger()
312 __ addq(out, mask); in GenAbsInteger()
313 __ xorq(out, mask); in GenAbsInteger()
316 __ movl(mask, out); in GenAbsInteger()
317 __ sarl(mask, Immediate(31)); in GenAbsInteger()
319 __ addl(out, mask); in GenAbsInteger()
320 __ xorl(out, mask); in GenAbsInteger()
377 __ ucomisd(out, op2); in GenMinMaxFP()
379 __ ucomiss(out, op2); in GenMinMaxFP()
382 __ j(Condition::kParityEven, &nan); in GenMinMaxFP()
384 __ j(is_min ? Condition::kAbove : Condition::kBelow, &op2_label); in GenMinMaxFP()
385 __ j(is_min ? Condition::kBelow : Condition::kAbove, &done); in GenMinMaxFP()
390 __ orpd(out, op2); in GenMinMaxFP()
392 __ orps(out, op2); in GenMinMaxFP()
396 __ andpd(out, op2); in GenMinMaxFP()
398 __ andps(out, op2); in GenMinMaxFP()
401 __ jmp(&done); in GenMinMaxFP()
404 __ Bind(&nan); in GenMinMaxFP()
406 __ movsd(out, codegen->LiteralInt64Address(INT64_C(0x7FF8000000000000))); in GenMinMaxFP()
408 __ movss(out, codegen->LiteralInt32Address(INT32_C(0x7FC00000))); in GenMinMaxFP()
410 __ jmp(&done); in GenMinMaxFP()
413 __ Bind(&op2_label); in GenMinMaxFP()
415 __ movsd(out, op2); in GenMinMaxFP()
417 __ movss(out, op2); in GenMinMaxFP()
421 __ Bind(&done); in GenMinMaxFP()
495 __ cmpq(out, op2); in GenMinMax()
497 __ cmpl(out, op2); in GenMinMax()
500 __ cmov(is_min ? Condition::kGreater : Condition::kLess, out, op2, is_long); in GenMinMax()
609 __ roundsd(out, in, Immediate(round_mode)); in GenSSE41FPToFPIntrinsic()
685 __ movss(t2, in); in VisitMathRoundFloat()
686 __ roundss(t1, in, Immediate(1)); in VisitMathRoundFloat()
687 __ subss(t2, t1); in VisitMathRoundFloat()
688 __ comiss(t2, codegen_->LiteralFloatAddress(0.5f)); in VisitMathRoundFloat()
689 __ j(kBelow, &skip_incr); in VisitMathRoundFloat()
690 __ addss(t1, codegen_->LiteralFloatAddress(1.0f)); in VisitMathRoundFloat()
691 __ Bind(&skip_incr); in VisitMathRoundFloat()
697 __ cvtsi2ss(t2, out); in VisitMathRoundFloat()
698 __ comiss(t1, t2); in VisitMathRoundFloat()
699 __ j(kAboveEqual, &done); // clipped to max (already in out), does not jump on unordered in VisitMathRoundFloat()
700 __ movl(out, Immediate(0)); // does not change flags in VisitMathRoundFloat()
701 __ j(kUnordered, &done); // NaN mapped to 0 (just moved in out) in VisitMathRoundFloat()
702 __ cvttss2si(out, t1); in VisitMathRoundFloat()
703 __ Bind(&done); in VisitMathRoundFloat()
729 __ movsd(t2, in); in VisitMathRoundDouble()
730 __ roundsd(t1, in, Immediate(1)); in VisitMathRoundDouble()
731 __ subsd(t2, t1); in VisitMathRoundDouble()
732 __ comisd(t2, codegen_->LiteralDoubleAddress(0.5)); in VisitMathRoundDouble()
733 __ j(kBelow, &skip_incr); in VisitMathRoundDouble()
734 __ addsd(t1, codegen_->LiteralDoubleAddress(1.0f)); in VisitMathRoundDouble()
735 __ Bind(&skip_incr); in VisitMathRoundDouble()
741 __ cvtsi2sd(t2, out, /* is64bit */ true); in VisitMathRoundDouble()
742 __ comisd(t1, t2); in VisitMathRoundDouble()
743 __ j(kAboveEqual, &done); // clipped to max (already in out), does not jump on unordered in VisitMathRoundDouble()
744 __ movl(out, Immediate(0)); // does not change flags, implicit zero extension to 64-bit in VisitMathRoundDouble()
745 __ j(kUnordered, &done); // NaN mapped to 0 (just moved in out) in VisitMathRoundDouble()
746 __ cvttsd2si(out, t1, /* is64bit */ true); in VisitMathRoundDouble()
747 __ Bind(&done); in VisitMathRoundDouble()
985 __ cmpl(Address(input, length_offset), in CheckPosition()
988 __ cmpl(Address(input, length_offset), length.AsRegister<CpuRegister>()); in CheckPosition()
990 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
994 __ movl(temp, Address(input, length_offset)); in CheckPosition()
995 __ subl(temp, Immediate(pos_const)); in CheckPosition()
996 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1000 __ cmpl(temp, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); in CheckPosition()
1002 __ cmpl(temp, length.AsRegister<CpuRegister>()); in CheckPosition()
1004 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1009 __ testl(pos_reg, pos_reg); in CheckPosition()
1010 __ j(kNotEqual, slow_path->GetEntryLabel()); in CheckPosition()
1014 __ testl(pos_reg, pos_reg); in CheckPosition()
1015 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1018 __ cmpl(Address(input, length_offset), pos_reg); in CheckPosition()
1019 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1022 __ movl(temp, Address(input, length_offset)); in CheckPosition()
1023 __ subl(temp, pos_reg); in CheckPosition()
1025 __ cmpl(temp, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); in CheckPosition()
1027 __ cmpl(temp, length.AsRegister<CpuRegister>()); in CheckPosition()
1029 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1055 __ cmpl(src, dest); in VisitSystemArrayCopyChar()
1056 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
1059 __ testl(src, src); in VisitSystemArrayCopyChar()
1060 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
1063 __ testl(dest, dest); in VisitSystemArrayCopyChar()
1064 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
1069 __ testl(length.AsRegister<CpuRegister>(), length.AsRegister<CpuRegister>()); in VisitSystemArrayCopyChar()
1070 __ j(kLess, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
1081 __ movl(count, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); in VisitSystemArrayCopyChar()
1083 __ movl(count, length.AsRegister<CpuRegister>()); in VisitSystemArrayCopyChar()
1095 __ leal(src_base, Address(src, char_size * src_pos_const + data_offset)); in VisitSystemArrayCopyChar()
1097 __ leal(src_base, Address(src, src_pos.AsRegister<CpuRegister>(), in VisitSystemArrayCopyChar()
1102 __ leal(dest_base, Address(dest, char_size * dest_pos_const + data_offset)); in VisitSystemArrayCopyChar()
1104 __ leal(dest_base, Address(dest, dest_pos.AsRegister<CpuRegister>(), in VisitSystemArrayCopyChar()
1109 __ rep_movsw(); in VisitSystemArrayCopyChar()
1111 __ Bind(slow_path->GetExitLabel()); in VisitSystemArrayCopyChar()
1146 __ leal(src_base, Address(src, element_size * constant + data_offset)); in GenSystemArrayCopyAddresses()
1148 __ leal(src_base, Address(src, src_pos.AsRegister<CpuRegister>(), scale_factor, data_offset)); in GenSystemArrayCopyAddresses()
1153 __ leal(dst_base, Address(dst, element_size * constant + data_offset)); in GenSystemArrayCopyAddresses()
1155 __ leal(dst_base, Address(dst, dst_pos.AsRegister<CpuRegister>(), scale_factor, data_offset)); in GenSystemArrayCopyAddresses()
1160 __ leal(src_end, Address(src_base, element_size * constant)); in GenSystemArrayCopyAddresses()
1162 __ leal(src_end, Address(src_base, copy_length.AsRegister<CpuRegister>(), scale_factor, 0)); in GenSystemArrayCopyAddresses()
1209 __ cmpl(src, dest); in VisitSystemArrayCopy()
1210 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1214 __ cmpl(src, dest); in VisitSystemArrayCopy()
1215 __ j(kNotEqual, &conditions_on_positions_validated); in VisitSystemArrayCopy()
1217 __ cmpl(dest_pos.AsRegister<CpuRegister>(), Immediate(src_pos_constant)); in VisitSystemArrayCopy()
1218 __ j(kGreater, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1222 __ cmpl(src, dest); in VisitSystemArrayCopy()
1223 __ j(kNotEqual, &conditions_on_positions_validated); in VisitSystemArrayCopy()
1227 __ cmpl(src_pos.AsRegister<CpuRegister>(), Immediate(dest_pos_constant)); in VisitSystemArrayCopy()
1228 __ j(kLess, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1230 __ cmpl(src_pos.AsRegister<CpuRegister>(), dest_pos.AsRegister<CpuRegister>()); in VisitSystemArrayCopy()
1231 __ j(kLess, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1235 __ Bind(&conditions_on_positions_validated); in VisitSystemArrayCopy()
1239 __ testl(src, src); in VisitSystemArrayCopy()
1240 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1245 __ testl(dest, dest); in VisitSystemArrayCopy()
1246 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1254 __ testl(length.AsRegister<CpuRegister>(), length.AsRegister<CpuRegister>()); in VisitSystemArrayCopy()
1255 __ j(kLess, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1300 __ movl(temp1, Address(dest, class_offset)); in VisitSystemArrayCopy()
1302 __ movl(temp2, Address(src, class_offset)); in VisitSystemArrayCopy()
1307 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
1308 __ MaybeUnpoisonHeapReference(temp2); in VisitSystemArrayCopy()
1319 __ testl(CpuRegister(TMP), CpuRegister(TMP)); in VisitSystemArrayCopy()
1320 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1325 __ movl(CpuRegister(TMP), Address(temp1, component_offset)); in VisitSystemArrayCopy()
1326 __ testl(CpuRegister(TMP), CpuRegister(TMP)); in VisitSystemArrayCopy()
1327 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1328 __ MaybeUnpoisonHeapReference(CpuRegister(TMP)); in VisitSystemArrayCopy()
1330 __ cmpw(Address(CpuRegister(TMP), primitive_offset), Immediate(Primitive::kPrimNot)); in VisitSystemArrayCopy()
1331 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1342 __ testl(CpuRegister(TMP), CpuRegister(TMP)); in VisitSystemArrayCopy()
1343 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1348 __ movl(CpuRegister(TMP), Address(temp2, component_offset)); in VisitSystemArrayCopy()
1349 __ testl(CpuRegister(TMP), CpuRegister(TMP)); in VisitSystemArrayCopy()
1350 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1351 __ MaybeUnpoisonHeapReference(CpuRegister(TMP)); in VisitSystemArrayCopy()
1353 __ cmpw(Address(CpuRegister(TMP), primitive_offset), Immediate(Primitive::kPrimNot)); in VisitSystemArrayCopy()
1354 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1357 __ cmpl(temp1, temp2); in VisitSystemArrayCopy()
1361 __ j(kEqual, &do_copy); in VisitSystemArrayCopy()
1370 __ cmpl(Address(temp1, super_offset), Immediate(0)); in VisitSystemArrayCopy()
1373 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
1376 __ movl(temp1, Address(temp1, component_offset)); in VisitSystemArrayCopy()
1377 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
1380 __ cmpl(Address(temp1, super_offset), Immediate(0)); in VisitSystemArrayCopy()
1382 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1383 __ Bind(&do_copy); in VisitSystemArrayCopy()
1385 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1397 __ testl(CpuRegister(TMP), CpuRegister(TMP)); in VisitSystemArrayCopy()
1398 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1401 __ movl(temp1, Address(src, class_offset)); in VisitSystemArrayCopy()
1402 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
1404 __ movl(CpuRegister(TMP), Address(temp1, component_offset)); in VisitSystemArrayCopy()
1406 __ testl(CpuRegister(TMP), CpuRegister(TMP)); in VisitSystemArrayCopy()
1407 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1408 __ MaybeUnpoisonHeapReference(CpuRegister(TMP)); in VisitSystemArrayCopy()
1410 __ cmpw(Address(CpuRegister(TMP), primitive_offset), Immediate(Primitive::kPrimNot)); in VisitSystemArrayCopy()
1411 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1446 __ cmpl(temp1, temp3); in VisitSystemArrayCopy()
1447 __ j(kEqual, &done); in VisitSystemArrayCopy()
1459 __ testb(Address(src, monitor_offset + gray_byte_position), Immediate(test_value)); in VisitSystemArrayCopy()
1471 __ j(kNotZero, read_barrier_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1476 __ Bind(&loop); in VisitSystemArrayCopy()
1477 __ movl(CpuRegister(TMP), Address(temp1, 0)); in VisitSystemArrayCopy()
1478 __ movl(Address(temp2, 0), CpuRegister(TMP)); in VisitSystemArrayCopy()
1479 __ addl(temp1, Immediate(element_size)); in VisitSystemArrayCopy()
1480 __ addl(temp2, Immediate(element_size)); in VisitSystemArrayCopy()
1481 __ cmpl(temp1, temp3); in VisitSystemArrayCopy()
1482 __ j(kNotEqual, &loop); in VisitSystemArrayCopy()
1484 __ Bind(read_barrier_slow_path->GetExitLabel()); in VisitSystemArrayCopy()
1485 __ Bind(&done); in VisitSystemArrayCopy()
1492 __ cmpl(temp1, temp3); in VisitSystemArrayCopy()
1493 __ j(kEqual, &done); in VisitSystemArrayCopy()
1494 __ Bind(&loop); in VisitSystemArrayCopy()
1495 __ movl(CpuRegister(TMP), Address(temp1, 0)); in VisitSystemArrayCopy()
1496 __ movl(Address(temp2, 0), CpuRegister(TMP)); in VisitSystemArrayCopy()
1497 __ addl(temp1, Immediate(element_size)); in VisitSystemArrayCopy()
1498 __ addl(temp2, Immediate(element_size)); in VisitSystemArrayCopy()
1499 __ cmpl(temp1, temp3); in VisitSystemArrayCopy()
1500 __ j(kNotEqual, &loop); in VisitSystemArrayCopy()
1501 __ Bind(&done); in VisitSystemArrayCopy()
1507 __ Bind(intrinsic_slow_path->GetExitLabel()); in VisitSystemArrayCopy()
1528 __ testl(argument, argument); in VisitStringCompareTo()
1531 __ j(kEqual, slow_path->GetEntryLabel()); in VisitStringCompareTo()
1534 __ Bind(slow_path->GetExitLabel()); in VisitStringCompareTo()
1575 __ testl(arg, arg); in VisitStringEquals()
1576 __ j(kEqual, &return_false); in VisitStringEquals()
1584 __ movl(rcx, Address(str, class_offset)); in VisitStringEquals()
1585 __ cmpl(rcx, Address(arg, class_offset)); in VisitStringEquals()
1586 __ j(kNotEqual, &return_false); in VisitStringEquals()
1590 __ cmpl(str, arg); in VisitStringEquals()
1591 __ j(kEqual, &return_true); in VisitStringEquals()
1594 __ movl(rcx, Address(str, count_offset)); in VisitStringEquals()
1598 __ cmpl(rcx, Address(arg, count_offset)); in VisitStringEquals()
1599 __ j(kNotEqual, &return_false); in VisitStringEquals()
1603 __ jrcxz(&return_true); in VisitStringEquals()
1609 __ shrl(rcx, Immediate(1)); in VisitStringEquals()
1610 __ j(kCarrySet, &string_uncompressed); in VisitStringEquals()
1613 __ addl(rcx, Immediate(1)); in VisitStringEquals()
1614 __ shrl(rcx, Immediate(1)); in VisitStringEquals()
1615 __ Bind(&string_uncompressed); in VisitStringEquals()
1618 __ leal(rsi, Address(str, value_offset)); in VisitStringEquals()
1619 __ leal(rdi, Address(arg, value_offset)); in VisitStringEquals()
1622 __ addl(rcx, Immediate(3)); in VisitStringEquals()
1623 __ shrl(rcx, Immediate(2)); in VisitStringEquals()
1631 __ repe_cmpsq(); in VisitStringEquals()
1633 __ j(kNotEqual, &return_false); in VisitStringEquals()
1637 __ Bind(&return_true); in VisitStringEquals()
1638 __ movl(rsi, Immediate(1)); in VisitStringEquals()
1639 __ jmp(&end); in VisitStringEquals()
1642 __ Bind(&return_false); in VisitStringEquals()
1643 __ xorl(rsi, rsi); in VisitStringEquals()
1644 __ Bind(&end); in VisitStringEquals()
1705 __ jmp(slow_path->GetEntryLabel()); in GenerateStringIndexOf()
1706 __ Bind(slow_path->GetExitLabel()); in GenerateStringIndexOf()
1710 __ cmpl(search_value, Immediate(std::numeric_limits<uint16_t>::max())); in GenerateStringIndexOf()
1713 __ j(kAbove, slow_path->GetEntryLabel()); in GenerateStringIndexOf()
1724 __ movl(string_length, Address(string_obj, count_offset)); in GenerateStringIndexOf()
1729 __ testl(string_length, string_length); in GenerateStringIndexOf()
1730 __ j(kEqual, ¬_found_label); in GenerateStringIndexOf()
1734 __ movl(CpuRegister(TMP), string_length); in GenerateStringIndexOf()
1736 __ shrl(string_length, Immediate(1)); in GenerateStringIndexOf()
1741 __ movl(counter, string_length); in GenerateStringIndexOf()
1743 __ addq(string_obj, Immediate(value_offset)); in GenerateStringIndexOf()
1748 __ cmpl(start_index, string_length); in GenerateStringIndexOf()
1749 __ j(kGreaterEqual, ¬_found_label); in GenerateStringIndexOf()
1752 __ xorl(counter, counter); in GenerateStringIndexOf()
1753 __ cmpl(start_index, Immediate(0)); in GenerateStringIndexOf()
1754 __ cmov(kGreater, counter, start_index, /* is64bit */ false); // 32-bit copy is enough. in GenerateStringIndexOf()
1758 __ testl(CpuRegister(TMP), Immediate(1)); in GenerateStringIndexOf()
1759 __ j(kNotZero, &offset_uncompressed_label); in GenerateStringIndexOf()
1760 __ leaq(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_1, value_offset)); in GenerateStringIndexOf()
1761 __ jmp(&modify_counter); in GenerateStringIndexOf()
1763 __ Bind(&offset_uncompressed_label); in GenerateStringIndexOf()
1764 __ leaq(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_2, value_offset)); in GenerateStringIndexOf()
1765 __ Bind(&modify_counter); in GenerateStringIndexOf()
1767 __ leaq(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_2, value_offset)); in GenerateStringIndexOf()
1770 __ negq(counter); // Needs to be 64-bit negation, as the address computation is 64-bit. in GenerateStringIndexOf()
1771 __ leaq(counter, Address(string_length, counter, ScaleFactor::TIMES_1, 0)); in GenerateStringIndexOf()
1777 __ testl(CpuRegister(TMP), Immediate(1)); in GenerateStringIndexOf()
1778 __ j(kNotZero, &uncompressed_string_comparison); in GenerateStringIndexOf()
1780 __ cmpl(search_value, Immediate(127)); in GenerateStringIndexOf()
1781 __ j(kGreater, ¬_found_label); in GenerateStringIndexOf()
1783 __ repne_scasb(); in GenerateStringIndexOf()
1784 __ jmp(&comparison_done); in GenerateStringIndexOf()
1788 __ Bind(&uncompressed_string_comparison); in GenerateStringIndexOf()
1789 __ repne_scasw(); in GenerateStringIndexOf()
1790 __ Bind(&comparison_done); in GenerateStringIndexOf()
1792 __ repne_scasw(); in GenerateStringIndexOf()
1795 __ j(kNotEqual, ¬_found_label); in GenerateStringIndexOf()
1798 __ subl(string_length, counter); in GenerateStringIndexOf()
1799 __ leal(out, Address(string_length, -1)); in GenerateStringIndexOf()
1802 __ jmp(&done); in GenerateStringIndexOf()
1805 __ Bind(¬_found_label); in GenerateStringIndexOf()
1806 __ movl(out, Immediate(-1)); in GenerateStringIndexOf()
1809 __ Bind(&done); in GenerateStringIndexOf()
1811 __ Bind(slow_path->GetExitLabel()); in GenerateStringIndexOf()
1849 __ testl(byte_array, byte_array); in VisitStringNewStringFromBytes()
1852 __ j(kEqual, slow_path->GetEntryLabel()); in VisitStringNewStringFromBytes()
1856 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromBytes()
1895 __ testl(string_to_copy, string_to_copy); in VisitStringNewStringFromString()
1898 __ j(kEqual, slow_path->GetEntryLabel()); in VisitStringNewStringFromString()
1902 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromString()
1947 __ movl(CpuRegister(RCX), srcEnd); in VisitStringGetCharsNoCheck()
1949 __ subl(CpuRegister(RCX), Immediate(srcBegin_value)); in VisitStringGetCharsNoCheck()
1952 __ subl(CpuRegister(RCX), srcBegin.AsRegister<CpuRegister>()); in VisitStringGetCharsNoCheck()
1961 __ testl(Address(obj, count_offset), Immediate(1)); in VisitStringGetCharsNoCheck()
1964 __ j(kNotZero, ©_uncompressed); in VisitStringGetCharsNoCheck()
1967 __ leaq(CpuRegister(RSI), in VisitStringGetCharsNoCheck()
1970 __ leaq(CpuRegister(RDI), Address(dst, dstBegin, ScaleFactor::TIMES_2, data_offset)); in VisitStringGetCharsNoCheck()
1972 __ Bind(©_loop); in VisitStringGetCharsNoCheck()
1973 __ jrcxz(&done); in VisitStringGetCharsNoCheck()
1976 __ movzxb(CpuRegister(TMP), Address(CpuRegister(RSI), 0)); in VisitStringGetCharsNoCheck()
1977 __ movw(Address(CpuRegister(RDI), 0), CpuRegister(TMP)); in VisitStringGetCharsNoCheck()
1978 __ leaq(CpuRegister(RDI), Address(CpuRegister(RDI), char_size)); in VisitStringGetCharsNoCheck()
1979 __ leaq(CpuRegister(RSI), Address(CpuRegister(RSI), c_char_size)); in VisitStringGetCharsNoCheck()
1981 __ subl(CpuRegister(RCX), Immediate(1)); in VisitStringGetCharsNoCheck()
1982 __ jmp(©_loop); in VisitStringGetCharsNoCheck()
1984 __ Bind(©_uncompressed); in VisitStringGetCharsNoCheck()
1987 __ leaq(CpuRegister(RSI), in VisitStringGetCharsNoCheck()
1990 __ leaq(CpuRegister(RDI), Address(dst, dstBegin, ScaleFactor::TIMES_2, data_offset)); in VisitStringGetCharsNoCheck()
1992 __ rep_movsw(); in VisitStringGetCharsNoCheck()
1994 __ Bind(&done); in VisitStringGetCharsNoCheck()
2004 __ movsxb(out, Address(address, 0)); in GenPeek()
2007 __ movsxw(out, Address(address, 0)); in GenPeek()
2010 __ movl(out, Address(address, 0)); in GenPeek()
2013 __ movq(out, Address(address, 0)); in GenPeek()
2069 __ movb(Address(address, 0), in GenPoke()
2072 __ movb(Address(address, 0), value.AsRegister<CpuRegister>()); in GenPoke()
2077 __ movw(Address(address, 0), in GenPoke()
2080 __ movw(Address(address, 0), value.AsRegister<CpuRegister>()); in GenPoke()
2085 __ movl(Address(address, 0), in GenPoke()
2088 __ movl(Address(address, 0), value.AsRegister<CpuRegister>()); in GenPoke()
2096 __ movq(Address(address, 0), Immediate(v_32)); in GenPoke()
2098 __ movq(Address(address, 0), value.AsRegister<CpuRegister>()); in GenPoke()
2167 __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
2177 __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
2182 __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
2183 __ MaybeUnpoisonHeapReference(output); in GenUnsafeGet()
2189 __ movq(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
2312 __ movq(Address(base, offset, ScaleFactor::TIMES_1, 0), value); in GenUnsafePut()
2315 __ movl(temp, value); in GenUnsafePut()
2316 __ PoisonHeapReference(temp); in GenUnsafePut()
2317 __ movl(Address(base, offset, ScaleFactor::TIMES_1, 0), temp); in GenUnsafePut()
2319 __ movl(Address(base, offset, ScaleFactor::TIMES_1, 0), value); in GenUnsafePut()
2459 __ movl(CpuRegister(value_reg), base); in GenCAS()
2473 __ PoisonHeapReference(expected); in GenCAS()
2474 __ PoisonHeapReference(CpuRegister(value_reg)); in GenCAS()
2477 __ LockCmpxchgl(field_addr, CpuRegister(value_reg)); in GenCAS()
2483 __ setcc(kZero, out); in GenCAS()
2484 __ movzxb(out, out); in GenCAS()
2496 __ UnpoisonHeapReference(CpuRegister(value_reg)); in GenCAS()
2501 __ UnpoisonHeapReference(expected); in GenCAS()
2505 __ LockCmpxchgl(Address(base, offset, TIMES_1, 0), value); in GenCAS()
2507 __ LockCmpxchgq(Address(base, offset, TIMES_1, 0), value); in GenCAS()
2516 __ setcc(kZero, out); in GenCAS()
2517 __ movzxb(out, out); in GenCAS()
2550 __ movl(temp, reg); in SwapBits()
2551 __ shrl(reg, imm_shift); in SwapBits()
2552 __ andl(temp, imm_mask); in SwapBits()
2553 __ andl(reg, imm_mask); in SwapBits()
2554 __ shll(temp, imm_shift); in SwapBits()
2555 __ orl(reg, temp); in SwapBits()
2574 __ bswapl(reg); in VisitIntegerReverse()
2593 __ movq(temp_mask, Immediate(mask)); in SwapBits64()
2594 __ movq(temp, reg); in SwapBits64()
2595 __ shrq(reg, imm_shift); in SwapBits64()
2596 __ andq(temp, temp_mask); in SwapBits64()
2597 __ andq(reg, temp_mask); in SwapBits64()
2598 __ shlq(temp, imm_shift); in SwapBits64()
2599 __ orq(reg, temp); in SwapBits64()
2619 __ bswapq(reg); in VisitLongReverse()
2659 __ popcntq(out, src.AsRegister<CpuRegister>()); in GenBitCount()
2661 __ popcntl(out, src.AsRegister<CpuRegister>()); in GenBitCount()
2665 __ popcntq(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenBitCount()
2668 __ popcntl(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenBitCount()
2710 __ xorl(out, out); // Clears upper bits too. in GenOneBit()
2735 __ bsrq(tmp, src.AsRegister<CpuRegister>()); in GenOneBit()
2737 __ bsrl(tmp, src.AsRegister<CpuRegister>()); in GenOneBit()
2741 __ bsrq(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); in GenOneBit()
2744 __ bsrl(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); in GenOneBit()
2748 __ j(kEqual, &is_zero); in GenOneBit()
2749 __ movl(out, Immediate(1)); // Clears upper bits too. in GenOneBit()
2751 __ shlq(out, tmp); in GenOneBit()
2753 __ shll(out, tmp); in GenOneBit()
2755 __ jmp(&done); in GenOneBit()
2756 __ Bind(&is_zero); in GenOneBit()
2757 __ xorl(out, out); // Clears upper bits too. in GenOneBit()
2758 __ Bind(&done); in GenOneBit()
2763 __ movq(tmp, src.AsRegister<CpuRegister>()); in GenOneBit()
2765 __ movl(tmp, src.AsRegister<CpuRegister>()); in GenOneBit()
2769 __ movq(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); in GenOneBit()
2772 __ movl(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); in GenOneBit()
2776 __ movq(out, tmp); in GenOneBit()
2777 __ negq(tmp); in GenOneBit()
2778 __ andq(out, tmp); in GenOneBit()
2780 __ movl(out, tmp); in GenOneBit()
2781 __ negl(tmp); in GenOneBit()
2782 __ andl(out, tmp); in GenOneBit()
2850 __ bsrq(out, src.AsRegister<CpuRegister>()); in GenLeadingZeros()
2852 __ bsrl(out, src.AsRegister<CpuRegister>()); in GenLeadingZeros()
2856 __ bsrq(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenLeadingZeros()
2859 __ bsrl(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenLeadingZeros()
2864 __ j(kEqual, &is_zero); in GenLeadingZeros()
2867 __ xorl(out, Immediate(zero_value_result - 1)); in GenLeadingZeros()
2868 __ jmp(&done); in GenLeadingZeros()
2871 __ Bind(&is_zero); in GenLeadingZeros()
2872 __ movl(out, Immediate(zero_value_result)); in GenLeadingZeros()
2874 __ Bind(&done); in GenLeadingZeros()
2924 __ bsfq(out, src.AsRegister<CpuRegister>()); in GenTrailingZeros()
2926 __ bsfl(out, src.AsRegister<CpuRegister>()); in GenTrailingZeros()
2930 __ bsfq(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenTrailingZeros()
2933 __ bsfl(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenTrailingZeros()
2938 __ j(kNotEqual, &done); in GenTrailingZeros()
2941 __ movl(out, Immediate(zero_value_result)); in GenTrailingZeros()
2943 __ Bind(&done); in GenTrailingZeros()
2995 __ movl(temp, Address(temp, ArtMethod::DeclaringClassOffset().Int32Value())); in VisitReferenceGetReferent()
3005 __ cmpw(Address(temp, disable_flag_offset), Immediate(0)); in VisitReferenceGetReferent()
3006 __ j(kNotEqual, slow_path->GetEntryLabel()); in VisitReferenceGetReferent()
3008 __ cmpb(Address(temp, disable_flag_offset), Immediate(0)); in VisitReferenceGetReferent()
3009 __ j(kNotEqual, slow_path->GetEntryLabel()); in VisitReferenceGetReferent()
3010 __ cmpb(Address(temp, slow_path_flag_offset), Immediate(0)); in VisitReferenceGetReferent()
3011 __ j(kNotEqual, slow_path->GetEntryLabel()); in VisitReferenceGetReferent()
3015 __ movl(out, Address(obj, mirror::Reference::ReferentOffset().Int32Value())); in VisitReferenceGetReferent()
3017 __ MaybeUnpoisonHeapReference(out); in VisitReferenceGetReferent()
3018 __ Bind(slow_path->GetExitLabel()); in VisitReferenceGetReferent()
3045 __ movl(out, Immediate(static_cast<int32_t>(address))); in VisitIntegerValueOf()
3052 __ movl(argument, Immediate(static_cast<int32_t>(address))); in VisitIntegerValueOf()
3055 __ movl(Address(out, info.value_offset), Immediate(value)); in VisitIntegerValueOf()
3060 __ leal(out, Address(in, -info.low)); in VisitIntegerValueOf()
3061 __ cmpl(out, Immediate(info.high - info.low + 1)); in VisitIntegerValueOf()
3063 __ j(kAboveEqual, &allocate); in VisitIntegerValueOf()
3068 __ movl(out, Address(out, TIMES_4, data_offset + address)); in VisitIntegerValueOf()
3071 __ movl(temp, Immediate(static_cast<int32_t>(data_offset + address))); in VisitIntegerValueOf()
3072 __ movl(out, Address(temp, out, TIMES_4, 0)); in VisitIntegerValueOf()
3074 __ MaybeUnpoisonHeapReference(out); in VisitIntegerValueOf()
3075 __ jmp(&done); in VisitIntegerValueOf()
3076 __ Bind(&allocate); in VisitIntegerValueOf()
3080 __ movl(argument, Immediate(static_cast<int32_t>(address))); in VisitIntegerValueOf()
3083 __ movl(Address(out, info.value_offset), in); in VisitIntegerValueOf()
3084 __ Bind(&done); in VisitIntegerValueOf()
3109 #undef __