477 inline mirror::Object* ConcurrentCopying::GetFwdPtr(mirror::Object* from_ref) {  in GetFwdPtr()  argument
478   DCHECK(region_space_->IsInFromSpace(from_ref));  in GetFwdPtr()
479   LockWord lw = from_ref->GetLockWord(false);  in GetFwdPtr()
1284 mirror::Object* ConcurrentCopying::Copy(mirror::Object* from_ref) {  in Copy()  argument
1285   DCHECK(region_space_->IsInFromSpace(from_ref));  in Copy()
1289   size_t obj_size = from_ref->SizeOf<kDefaultVerifyFlags, kWithoutReadBarrier>();  in Copy()
1338     memcpy(to_ref, from_ref, obj_size);  in Copy()
1391     bool success = from_ref->CasLockWordWeakSequentiallyConsistent(old_lock_word, new_lock_word);  in Copy()
1405       DCHECK(GetFwdPtr(from_ref) == to_ref);  in Copy()
1416 mirror::Object* ConcurrentCopying::IsMarked(mirror::Object* from_ref) {  in IsMarked()  argument
1417   DCHECK(from_ref != nullptr);  in IsMarked()
1418   space::RegionSpace::RegionType rtype = region_space_->GetRegionType(from_ref);  in IsMarked()
1421     return from_ref;  in IsMarked()
1425     to_ref = GetFwdPtr(from_ref);  in IsMarked()
1428         << "from_ref=" << from_ref << " to_ref=" << to_ref;  in IsMarked()
1430     if (region_space_bitmap_->Test(from_ref)) {  in IsMarked()
1431       to_ref = from_ref;  in IsMarked()
1437     if (immune_region_.ContainsObject(from_ref)) {  in IsMarked()
1439           cc_heap_bitmap_->GetContinuousSpaceBitmap(from_ref);  in IsMarked()
1443         DCHECK(heap_mark_bitmap_->GetContinuousSpaceBitmap(from_ref)->Test(from_ref))  in IsMarked()
1446       if (cc_bitmap->Test(from_ref)) {  in IsMarked()
1448         to_ref = from_ref;  in IsMarked()
1456           heap_mark_bitmap_->GetContinuousSpaceBitmap(from_ref);  in IsMarked()
1458           heap_mark_bitmap_->GetLargeObjectBitmap(from_ref);  in IsMarked()
1461       if (!is_los && mark_bitmap->Test(from_ref)) {  in IsMarked()
1463         to_ref = from_ref;  in IsMarked()
1464       } else if (is_los && los_bitmap->Test(from_ref)) {  in IsMarked()
1466         to_ref = from_ref;  in IsMarked()
1469         if (IsOnAllocStack(from_ref)) {  in IsMarked()
1471           to_ref = from_ref;  in IsMarked()
1488 mirror::Object* ConcurrentCopying::Mark(mirror::Object* from_ref) {  in Mark()  argument
1489   if (from_ref == nullptr) {  in Mark()
1492   DCHECK(from_ref != nullptr);  in Mark()
1506     return from_ref;  in Mark()
1509   space::RegionSpace::RegionType rtype = region_space_->GetRegionType(from_ref);  in Mark()
1512     return from_ref;  in Mark()
1516     to_ref = GetFwdPtr(from_ref);  in Mark()
1518       DCHECK(to_ref != ReadBarrier::GrayPtr()) << "from_ref=" << from_ref << " to_ref=" << to_ref;  in Mark()
1522       to_ref = Copy(from_ref);  in Mark()
1525         << "from_ref=" << from_ref << " to_ref=" << to_ref;  in Mark()
1529       from_ref->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), ReadBarrier::GrayPtr());  in Mark()
1531     if (region_space_bitmap_->AtomicTestAndSet(from_ref)) {  in Mark()
1533       to_ref = from_ref;  in Mark()
1536       to_ref = from_ref;  in Mark()
1544     DCHECK(!region_space_->HasAddress(from_ref)) << from_ref;  in Mark()
1545     if (immune_region_.ContainsObject(from_ref)) {  in Mark()
1547           cc_heap_bitmap_->GetContinuousSpaceBitmap(from_ref);  in Mark()
1551         DCHECK(heap_mark_bitmap_->GetContinuousSpaceBitmap(from_ref)->Test(from_ref))  in Mark()
1556         from_ref->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), ReadBarrier::GrayPtr());  in Mark()
1558       if (cc_bitmap->AtomicTestAndSet(from_ref)) {  in Mark()
1560         to_ref = from_ref;  in Mark()
1563         to_ref = from_ref;  in Mark()
1572           heap_mark_bitmap_->GetContinuousSpaceBitmap(from_ref);  in Mark()
1574           heap_mark_bitmap_->GetLargeObjectBitmap(from_ref);  in Mark()
1577       if (!is_los && mark_bitmap->Test(from_ref)) {  in Mark()
1579         to_ref = from_ref;  in Mark()
1584       } else if (is_los && los_bitmap->Test(from_ref)) {  in Mark()
1586         to_ref = from_ref;  in Mark()
1593         if (IsOnAllocStack(from_ref)) {  in Mark()
1595           to_ref = from_ref;  in Mark()
1609             from_ref->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), ReadBarrier::GrayPtr());  in Mark()
1611           if (!is_los && mark_bitmap->AtomicTestAndSet(from_ref)) {  in Mark()
1613             to_ref = from_ref;  in Mark()
1614           } else if (is_los && los_bitmap->AtomicTestAndSet(from_ref)) {  in Mark()
1616             to_ref = from_ref;  in Mark()
1619             to_ref = from_ref;  in Mark()
1644 mirror::Object* ConcurrentCopying::IsMarkedCallback(mirror::Object* from_ref, void* arg) {  in IsMarkedCallback()  argument
1645   return reinterpret_cast<ConcurrentCopying*>(arg)->IsMarked(from_ref);  in IsMarkedCallback()
1650   mirror::Object* from_ref = field->AsMirrorPtr();  in IsHeapReferenceMarkedCallback()  local
1651   mirror::Object* to_ref = reinterpret_cast<ConcurrentCopying*>(arg)->IsMarked(from_ref);  in IsHeapReferenceMarkedCallback()
1655   if (from_ref != to_ref) {  in IsHeapReferenceMarkedCallback()
1663 mirror::Object* ConcurrentCopying::MarkCallback(mirror::Object* from_ref, void* arg) {  in MarkCallback()  argument
1664   return reinterpret_cast<ConcurrentCopying*>(arg)->Mark(from_ref);  in MarkCallback()