1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "image_writer.h"
18 
19 #include <lz4.h>
20 #include <lz4hc.h>
21 #include <sys/stat.h>
22 #include <zlib.h>
23 
24 #include <memory>
25 #include <numeric>
26 #include <unordered_set>
27 #include <vector>
28 
29 #include "art_field-inl.h"
30 #include "art_method-inl.h"
31 #include "base/callee_save_type.h"
32 #include "base/enums.h"
33 #include "base/globals.h"
34 #include "base/logging.h"  // For VLOG.
35 #include "base/stl_util.h"
36 #include "base/unix_file/fd_file.h"
37 #include "class_linker-inl.h"
38 #include "class_root.h"
39 #include "compiled_method.h"
40 #include "dex/dex_file-inl.h"
41 #include "dex/dex_file_types.h"
42 #include "driver/compiler_options.h"
43 #include "elf/elf_utils.h"
44 #include "elf_file.h"
45 #include "entrypoints/entrypoint_utils-inl.h"
46 #include "gc/accounting/card_table-inl.h"
47 #include "gc/accounting/heap_bitmap.h"
48 #include "gc/accounting/space_bitmap-inl.h"
49 #include "gc/collector/concurrent_copying.h"
50 #include "gc/heap-visit-objects-inl.h"
51 #include "gc/heap.h"
52 #include "gc/space/large_object_space.h"
53 #include "gc/space/region_space.h"
54 #include "gc/space/space-inl.h"
55 #include "gc/verification.h"
56 #include "handle_scope-inl.h"
57 #include "image-inl.h"
58 #include "imt_conflict_table.h"
59 #include "intern_table-inl.h"
60 #include "jni/jni_internal.h"
61 #include "linear_alloc.h"
62 #include "lock_word.h"
63 #include "mirror/array-inl.h"
64 #include "mirror/class-inl.h"
65 #include "mirror/class_ext-inl.h"
66 #include "mirror/class_loader.h"
67 #include "mirror/dex_cache-inl.h"
68 #include "mirror/dex_cache.h"
69 #include "mirror/executable.h"
70 #include "mirror/method.h"
71 #include "mirror/object-inl.h"
72 #include "mirror/object-refvisitor-inl.h"
73 #include "mirror/object_array-alloc-inl.h"
74 #include "mirror/object_array-inl.h"
75 #include "mirror/string-inl.h"
76 #include "oat.h"
77 #include "oat_file.h"
78 #include "oat_file_manager.h"
79 #include "optimizing/intrinsic_objects.h"
80 #include "runtime.h"
81 #include "scoped_thread_state_change-inl.h"
82 #include "subtype_check.h"
83 #include "utils/dex_cache_arrays_layout-inl.h"
84 #include "well_known_classes.h"
85 
86 using ::art::mirror::Class;
87 using ::art::mirror::DexCache;
88 using ::art::mirror::Object;
89 using ::art::mirror::ObjectArray;
90 using ::art::mirror::String;
91 
92 namespace art {
93 namespace linker {
94 
MaybeCompressData(ArrayRef<const uint8_t> source,ImageHeader::StorageMode image_storage_mode,std::vector<uint8_t> * storage)95 static ArrayRef<const uint8_t> MaybeCompressData(ArrayRef<const uint8_t> source,
96                                                  ImageHeader::StorageMode image_storage_mode,
97                                                  /*out*/ std::vector<uint8_t>* storage) {
98   const uint64_t compress_start_time = NanoTime();
99 
100   switch (image_storage_mode) {
101     case ImageHeader::kStorageModeLZ4: {
102       storage->resize(LZ4_compressBound(source.size()));
103       size_t data_size = LZ4_compress_default(
104           reinterpret_cast<char*>(const_cast<uint8_t*>(source.data())),
105           reinterpret_cast<char*>(storage->data()),
106           source.size(),
107           storage->size());
108       storage->resize(data_size);
109       break;
110     }
111     case ImageHeader::kStorageModeLZ4HC: {
112       // Bound is same as non HC.
113       storage->resize(LZ4_compressBound(source.size()));
114       size_t data_size = LZ4_compress_HC(
115           reinterpret_cast<const char*>(const_cast<uint8_t*>(source.data())),
116           reinterpret_cast<char*>(storage->data()),
117           source.size(),
118           storage->size(),
119           LZ4HC_CLEVEL_MAX);
120       storage->resize(data_size);
121       break;
122     }
123     case ImageHeader::kStorageModeUncompressed: {
124       return source;
125     }
126     default: {
127       LOG(FATAL) << "Unsupported";
128       UNREACHABLE();
129     }
130   }
131 
132   DCHECK(image_storage_mode == ImageHeader::kStorageModeLZ4 ||
133          image_storage_mode == ImageHeader::kStorageModeLZ4HC);
134   VLOG(compiler) << "Compressed from " << source.size() << " to " << storage->size() << " in "
135                  << PrettyDuration(NanoTime() - compress_start_time);
136   if (kIsDebugBuild) {
137     std::vector<uint8_t> decompressed(source.size());
138     const size_t decompressed_size = LZ4_decompress_safe(
139         reinterpret_cast<char*>(storage->data()),
140         reinterpret_cast<char*>(decompressed.data()),
141         storage->size(),
142         decompressed.size());
143     CHECK_EQ(decompressed_size, decompressed.size());
144     CHECK_EQ(memcmp(source.data(), decompressed.data(), source.size()), 0) << image_storage_mode;
145   }
146   return ArrayRef<const uint8_t>(*storage);
147 }
148 
149 // Separate objects into multiple bins to optimize dirty memory use.
150 static constexpr bool kBinObjects = true;
151 
AllocateBootImageLiveObjects(Thread * self,Runtime * runtime)152 ObjPtr<mirror::ObjectArray<mirror::Object>> AllocateBootImageLiveObjects(
153     Thread* self, Runtime* runtime) REQUIRES_SHARED(Locks::mutator_lock_) {
154   ClassLinker* class_linker = runtime->GetClassLinker();
155   // The objects used for the Integer.valueOf() intrinsic must remain live even if references
156   // to them are removed using reflection. Image roots are not accessible through reflection,
157   // so the array we construct here shall keep them alive.
158   StackHandleScope<1> hs(self);
159   Handle<mirror::ObjectArray<mirror::Object>> integer_cache =
160       hs.NewHandle(IntrinsicObjects::LookupIntegerCache(self, class_linker));
161   size_t live_objects_size =
162       enum_cast<size_t>(ImageHeader::kIntrinsicObjectsStart) +
163       ((integer_cache != nullptr) ? (/* cache */ 1u + integer_cache->GetLength()) : 0u);
164   ObjPtr<mirror::ObjectArray<mirror::Object>> live_objects =
165       mirror::ObjectArray<mirror::Object>::Alloc(
166           self, GetClassRoot<mirror::ObjectArray<mirror::Object>>(class_linker), live_objects_size);
167   int32_t index = 0u;
168   auto set_entry = [&](ImageHeader::BootImageLiveObjects entry,
169                        ObjPtr<mirror::Object> value) REQUIRES_SHARED(Locks::mutator_lock_) {
170     DCHECK_EQ(index, enum_cast<int32_t>(entry));
171     live_objects->Set</*kTransacrionActive=*/ false>(index, value);
172     ++index;
173   };
174   set_entry(ImageHeader::kOomeWhenThrowingException,
175             runtime->GetPreAllocatedOutOfMemoryErrorWhenThrowingException());
176   set_entry(ImageHeader::kOomeWhenThrowingOome,
177             runtime->GetPreAllocatedOutOfMemoryErrorWhenThrowingOOME());
178   set_entry(ImageHeader::kOomeWhenHandlingStackOverflow,
179             runtime->GetPreAllocatedOutOfMemoryErrorWhenHandlingStackOverflow());
180   set_entry(ImageHeader::kNoClassDefFoundError, runtime->GetPreAllocatedNoClassDefFoundError());
181   set_entry(ImageHeader::kClearedJniWeakSentinel, runtime->GetSentinel().Read());
182 
183   DCHECK_EQ(index, enum_cast<int32_t>(ImageHeader::kIntrinsicObjectsStart));
184   if (integer_cache != nullptr) {
185     live_objects->Set(index++, integer_cache.Get());
186     for (int32_t i = 0, length = integer_cache->GetLength(); i != length; ++i) {
187       live_objects->Set(index++, integer_cache->Get(i));
188     }
189   }
190   CHECK_EQ(index, live_objects->GetLength());
191 
192   if (kIsDebugBuild && integer_cache != nullptr) {
193     CHECK_EQ(integer_cache.Get(), IntrinsicObjects::GetIntegerValueOfCache(live_objects));
194     for (int32_t i = 0, len = integer_cache->GetLength(); i != len; ++i) {
195       CHECK_EQ(integer_cache->GetWithoutChecks(i),
196                IntrinsicObjects::GetIntegerValueOfObject(live_objects, i));
197     }
198   }
199   return live_objects;
200 }
201 
GetAppClassLoader() const202 ObjPtr<mirror::ClassLoader> ImageWriter::GetAppClassLoader() const
203     REQUIRES_SHARED(Locks::mutator_lock_) {
204   return compiler_options_.IsAppImage()
205       ? ObjPtr<mirror::ClassLoader>::DownCast(Thread::Current()->DecodeJObject(app_class_loader_))
206       : nullptr;
207 }
208 
IsImageDexCache(ObjPtr<mirror::DexCache> dex_cache) const209 bool ImageWriter::IsImageDexCache(ObjPtr<mirror::DexCache> dex_cache) const {
210   // For boot image, we keep all dex caches.
211   if (compiler_options_.IsBootImage()) {
212     return true;
213   }
214   // Dex caches already in the boot image do not belong to the image being written.
215   if (IsInBootImage(dex_cache.Ptr())) {
216     return false;
217   }
218   // Dex caches for the boot class path components that are not part of the boot image
219   // cannot be garbage collected in PrepareImageAddressSpace() but we do not want to
220   // include them in the app image.
221   if (!ContainsElement(compiler_options_.GetDexFilesForOatFile(), dex_cache->GetDexFile())) {
222     return false;
223   }
224   return true;
225 }
226 
ClearDexFileCookies()227 static void ClearDexFileCookies() REQUIRES_SHARED(Locks::mutator_lock_) {
228   auto visitor = [](Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) {
229     DCHECK(obj != nullptr);
230     Class* klass = obj->GetClass();
231     if (klass == WellKnownClasses::ToClass(WellKnownClasses::dalvik_system_DexFile)) {
232       ArtField* field = jni::DecodeArtField(WellKnownClasses::dalvik_system_DexFile_cookie);
233       // Null out the cookie to enable determinism. b/34090128
234       field->SetObject</*kTransactionActive*/false>(obj, nullptr);
235     }
236   };
237   Runtime::Current()->GetHeap()->VisitObjects(visitor);
238 }
239 
PrepareImageAddressSpace(bool preload_dex_caches,TimingLogger * timings)240 bool ImageWriter::PrepareImageAddressSpace(bool preload_dex_caches, TimingLogger* timings) {
241   target_ptr_size_ = InstructionSetPointerSize(compiler_options_.GetInstructionSet());
242 
243   Thread* const self = Thread::Current();
244 
245   gc::Heap* const heap = Runtime::Current()->GetHeap();
246   {
247     ScopedObjectAccess soa(self);
248     {
249       TimingLogger::ScopedTiming t("PruneNonImageClasses", timings);
250       PruneNonImageClasses();  // Remove junk
251     }
252 
253     if (compiler_options_.IsAppImage()) {
254       TimingLogger::ScopedTiming t("ClearDexFileCookies", timings);
255       // Clear dex file cookies for app images to enable app image determinism. This is required
256       // since the cookie field contains long pointers to DexFiles which are not deterministic.
257       // b/34090128
258       ClearDexFileCookies();
259     }
260   }
261 
262   {
263     TimingLogger::ScopedTiming t("CollectGarbage", timings);
264     heap->CollectGarbage(/* clear_soft_references */ false);  // Remove garbage.
265   }
266 
267   if (kIsDebugBuild) {
268     ScopedObjectAccess soa(self);
269     CheckNonImageClassesRemoved();
270   }
271 
272   {
273     // All remaining weak interns are referenced. Promote them to strong interns. Whether a
274     // string was strongly or weakly interned, we shall make it strongly interned in the image.
275     TimingLogger::ScopedTiming t("PromoteInterns", timings);
276     ScopedObjectAccess soa(self);
277     Runtime::Current()->GetInternTable()->PromoteWeakToStrong();
278   }
279 
280   if (preload_dex_caches) {
281     TimingLogger::ScopedTiming t("PreloadDexCaches", timings);
282     // Preload deterministic contents to the dex cache arrays we're going to write.
283     ScopedObjectAccess soa(self);
284     ObjPtr<mirror::ClassLoader> class_loader = GetAppClassLoader();
285     std::vector<ObjPtr<mirror::DexCache>> dex_caches = FindDexCaches(self);
286     for (ObjPtr<mirror::DexCache> dex_cache : dex_caches) {
287       if (!IsImageDexCache(dex_cache)) {
288         continue;  // Boot image DexCache is not written to the app image.
289       }
290       PreloadDexCache(dex_cache, class_loader);
291     }
292   }
293 
294   {
295     TimingLogger::ScopedTiming t("CalculateNewObjectOffsets", timings);
296     ScopedObjectAccess soa(self);
297     CalculateNewObjectOffsets();
298   }
299 
300   // Obtain class count for debugging purposes
301   if (VLOG_IS_ON(compiler) && compiler_options_.IsAppImage()) {
302     ScopedObjectAccess soa(self);
303 
304     size_t app_image_class_count  = 0;
305 
306     for (ImageInfo& info : image_infos_) {
307       info.class_table_->Visit([&](ObjPtr<mirror::Class> klass)
308                                    REQUIRES_SHARED(Locks::mutator_lock_) {
309         if (!IsInBootImage(klass.Ptr())) {
310           ++app_image_class_count;
311         }
312 
313         // Indicate that we would like to continue visiting classes.
314         return true;
315       });
316     }
317 
318     VLOG(compiler) << "Dex2Oat:AppImage:classCount = " << app_image_class_count;
319   }
320 
321   // This needs to happen after CalculateNewObjectOffsets since it relies on intern_table_bytes_ and
322   // bin size sums being calculated.
323   TimingLogger::ScopedTiming t("AllocMemory", timings);
324   return AllocMemory();
325 }
326 
CopyMetadata()327 void ImageWriter::CopyMetadata() {
328   DCHECK(compiler_options_.IsAppImage());
329   CHECK_EQ(image_infos_.size(), 1u);
330 
331   const ImageInfo& image_info = image_infos_.back();
332   std::vector<ImageSection> image_sections = image_info.CreateImageSections().second;
333 
334   auto* sfo_section_base = reinterpret_cast<AppImageReferenceOffsetInfo*>(
335       image_info.image_.Begin() +
336       image_sections[ImageHeader::kSectionStringReferenceOffsets].Offset());
337 
338   std::copy(image_info.string_reference_offsets_.begin(),
339             image_info.string_reference_offsets_.end(),
340             sfo_section_base);
341 }
342 
IsInternedAppImageStringReference(ObjPtr<mirror::Object> referred_obj) const343 bool ImageWriter::IsInternedAppImageStringReference(ObjPtr<mirror::Object> referred_obj) const {
344   return referred_obj != nullptr &&
345          !IsInBootImage(referred_obj.Ptr()) &&
346          referred_obj->IsString() &&
347          referred_obj == Runtime::Current()->GetInternTable()->LookupStrong(
348              Thread::Current(), referred_obj->AsString());
349 }
350 
351 // Helper class that erases the image file if it isn't properly flushed and closed.
352 class ImageWriter::ImageFileGuard {
353  public:
354   ImageFileGuard() noexcept = default;
355   ImageFileGuard(ImageFileGuard&& other) noexcept = default;
356   ImageFileGuard& operator=(ImageFileGuard&& other) noexcept = default;
357 
~ImageFileGuard()358   ~ImageFileGuard() {
359     if (image_file_ != nullptr) {
360       // Failure, erase the image file.
361       image_file_->Erase();
362     }
363   }
364 
reset(File * image_file)365   void reset(File* image_file) {
366     image_file_.reset(image_file);
367   }
368 
operator ==(std::nullptr_t)369   bool operator==(std::nullptr_t) {
370     return image_file_ == nullptr;
371   }
372 
operator !=(std::nullptr_t)373   bool operator!=(std::nullptr_t) {
374     return image_file_ != nullptr;
375   }
376 
operator ->() const377   File* operator->() const {
378     return image_file_.get();
379   }
380 
WriteHeaderAndClose(const std::string & image_filename,const ImageHeader * image_header)381   bool WriteHeaderAndClose(const std::string& image_filename, const ImageHeader* image_header) {
382     // The header is uncompressed since it contains whether the image is compressed or not.
383     if (!image_file_->PwriteFully(image_header, sizeof(ImageHeader), 0)) {
384       PLOG(ERROR) << "Failed to write image file header " << image_filename;
385       return false;
386     }
387 
388     // FlushCloseOrErase() takes care of erasing, so the destructor does not need
389     // to do that whether the FlushCloseOrErase() succeeds or fails.
390     std::unique_ptr<File> image_file = std::move(image_file_);
391     if (image_file->FlushCloseOrErase() != 0) {
392       PLOG(ERROR) << "Failed to flush and close image file " << image_filename;
393       return false;
394     }
395 
396     return true;
397   }
398 
399  private:
400   std::unique_ptr<File> image_file_;
401 };
402 
Write(int image_fd,const std::vector<std::string> & image_filenames,size_t component_count)403 bool ImageWriter::Write(int image_fd,
404                         const std::vector<std::string>& image_filenames,
405                         size_t component_count) {
406   // If image_fd or oat_fd are not kInvalidFd then we may have empty strings in image_filenames or
407   // oat_filenames.
408   CHECK(!image_filenames.empty());
409   if (image_fd != kInvalidFd) {
410     CHECK_EQ(image_filenames.size(), 1u);
411   }
412   DCHECK(!oat_filenames_.empty());
413   CHECK_EQ(image_filenames.size(), oat_filenames_.size());
414 
415   Thread* const self = Thread::Current();
416   {
417     ScopedObjectAccess soa(self);
418     for (size_t i = 0; i < oat_filenames_.size(); ++i) {
419       CreateHeader(i, component_count);
420       CopyAndFixupNativeData(i);
421     }
422   }
423 
424   {
425     // TODO: heap validation can't handle these fix up passes.
426     ScopedObjectAccess soa(self);
427     Runtime::Current()->GetHeap()->DisableObjectValidation();
428     CopyAndFixupObjects();
429   }
430 
431   if (compiler_options_.IsAppImage()) {
432     CopyMetadata();
433   }
434 
435   // Primary image header shall be written last for two reasons. First, this ensures
436   // that we shall not end up with a valid primary image and invalid secondary image.
437   // Second, its checksum shall include the checksums of the secondary images (XORed).
438   // This way only the primary image checksum needs to be checked to determine whether
439   // any of the images or oat files are out of date. (Oat file checksums are included
440   // in the image checksum calculation.)
441   ImageHeader* primary_header = reinterpret_cast<ImageHeader*>(image_infos_[0].image_.Begin());
442   ImageFileGuard primary_image_file;
443   for (size_t i = 0; i < image_filenames.size(); ++i) {
444     const std::string& image_filename = image_filenames[i];
445     ImageInfo& image_info = GetImageInfo(i);
446     ImageFileGuard image_file;
447     if (image_fd != kInvalidFd) {
448       // Ignore image_filename, it is supplied only for better diagnostic.
449       image_file.reset(new File(image_fd, unix_file::kCheckSafeUsage));
450       // Empty the file in case it already exists.
451       if (image_file != nullptr) {
452         TEMP_FAILURE_RETRY(image_file->SetLength(0));
453         TEMP_FAILURE_RETRY(image_file->Flush());
454       }
455     } else {
456       image_file.reset(OS::CreateEmptyFile(image_filename.c_str()));
457     }
458 
459     if (image_file == nullptr) {
460       LOG(ERROR) << "Failed to open image file " << image_filename;
461       return false;
462     }
463 
464     // Make file world readable if we have created it, i.e. when not passed as file descriptor.
465     if (image_fd == -1 && !compiler_options_.IsAppImage() && fchmod(image_file->Fd(), 0644) != 0) {
466       PLOG(ERROR) << "Failed to make image file world readable: " << image_filename;
467       return false;
468     }
469 
470     // Image data size excludes the bitmap and the header.
471     ImageHeader* const image_header = reinterpret_cast<ImageHeader*>(image_info.image_.Begin());
472 
473     // Block sources (from the image).
474     const bool is_compressed = image_storage_mode_ != ImageHeader::kStorageModeUncompressed;
475     std::vector<std::pair<uint32_t, uint32_t>> block_sources;
476     std::vector<ImageHeader::Block> blocks;
477 
478     // Add a set of solid blocks such that no block is larger than the maximum size. A solid block
479     // is a block that must be decompressed all at once.
480     auto add_blocks = [&](uint32_t offset, uint32_t size) {
481       while (size != 0u) {
482         const uint32_t cur_size = std::min(size, compiler_options_.MaxImageBlockSize());
483         block_sources.emplace_back(offset, cur_size);
484         offset += cur_size;
485         size -= cur_size;
486       }
487     };
488 
489     add_blocks(sizeof(ImageHeader), image_header->GetImageSize() - sizeof(ImageHeader));
490 
491     // Checksum of compressed image data and header.
492     uint32_t image_checksum = adler32(0L, Z_NULL, 0);
493     image_checksum = adler32(image_checksum,
494                              reinterpret_cast<const uint8_t*>(image_header),
495                              sizeof(ImageHeader));
496     // Copy and compress blocks.
497     size_t out_offset = sizeof(ImageHeader);
498     for (const std::pair<uint32_t, uint32_t> block : block_sources) {
499       ArrayRef<const uint8_t> raw_image_data(image_info.image_.Begin() + block.first,
500                                              block.second);
501       std::vector<uint8_t> compressed_data;
502       ArrayRef<const uint8_t> image_data =
503           MaybeCompressData(raw_image_data, image_storage_mode_, &compressed_data);
504 
505       if (!is_compressed) {
506         // For uncompressed, preserve alignment since the image will be directly mapped.
507         out_offset = block.first;
508       }
509 
510       // Fill in the compressed location of the block.
511       blocks.emplace_back(ImageHeader::Block(
512           image_storage_mode_,
513           /*data_offset=*/ out_offset,
514           /*data_size=*/ image_data.size(),
515           /*image_offset=*/ block.first,
516           /*image_size=*/ block.second));
517 
518       // Write out the image + fields + methods.
519       if (!image_file->PwriteFully(image_data.data(), image_data.size(), out_offset)) {
520         PLOG(ERROR) << "Failed to write image file data " << image_filename;
521         image_file->Erase();
522         return false;
523       }
524       out_offset += image_data.size();
525       image_checksum = adler32(image_checksum, image_data.data(), image_data.size());
526     }
527 
528     // Write the block metadata directly after the image sections.
529     // Note: This is not part of the mapped image and is not preserved after decompressing, it's
530     // only used for image loading. For this reason, only write it out for compressed images.
531     if (is_compressed) {
532       // Align up since the compressed data is not necessarily aligned.
533       out_offset = RoundUp(out_offset, alignof(ImageHeader::Block));
534       CHECK(!blocks.empty());
535       const size_t blocks_bytes = blocks.size() * sizeof(blocks[0]);
536       if (!image_file->PwriteFully(&blocks[0], blocks_bytes, out_offset)) {
537         PLOG(ERROR) << "Failed to write image blocks " << image_filename;
538         image_file->Erase();
539         return false;
540       }
541       image_header->blocks_offset_ = out_offset;
542       image_header->blocks_count_ = blocks.size();
543       out_offset += blocks_bytes;
544     }
545 
546     // Data size includes everything except the bitmap.
547     image_header->data_size_ = out_offset - sizeof(ImageHeader);
548 
549     // Update and write the bitmap section. Note that the bitmap section is relative to the
550     // possibly compressed image.
551     ImageSection& bitmap_section = image_header->GetImageSection(ImageHeader::kSectionImageBitmap);
552     // Align up since data size may be unaligned if the image is compressed.
553     out_offset = RoundUp(out_offset, kPageSize);
554     bitmap_section = ImageSection(out_offset, bitmap_section.Size());
555 
556     if (!image_file->PwriteFully(image_info.image_bitmap_.Begin(),
557                                  bitmap_section.Size(),
558                                  bitmap_section.Offset())) {
559       PLOG(ERROR) << "Failed to write image file bitmap " << image_filename;
560       return false;
561     }
562 
563     int err = image_file->Flush();
564     if (err < 0) {
565       PLOG(ERROR) << "Failed to flush image file " << image_filename << " with result " << err;
566       return false;
567     }
568 
569     // Calculate the image checksum of the remaining data.
570     image_checksum = adler32(image_checksum,
571                              reinterpret_cast<const uint8_t*>(image_info.image_bitmap_.Begin()),
572                              bitmap_section.Size());
573     image_header->SetImageChecksum(image_checksum);
574 
575     if (VLOG_IS_ON(compiler)) {
576       const size_t separately_written_section_size = bitmap_section.Size();
577       const size_t total_uncompressed_size = image_info.image_size_ +
578           separately_written_section_size;
579       const size_t total_compressed_size = out_offset + separately_written_section_size;
580 
581       VLOG(compiler) << "Dex2Oat:uncompressedImageSize = " << total_uncompressed_size;
582       if (total_uncompressed_size != total_compressed_size) {
583         VLOG(compiler) << "Dex2Oat:compressedImageSize = " << total_compressed_size;
584       }
585     }
586 
587     CHECK_EQ(bitmap_section.End(), static_cast<size_t>(image_file->GetLength()))
588         << "Bitmap should be at the end of the file";
589 
590     // Write header last in case the compiler gets killed in the middle of image writing.
591     // We do not want to have a corrupted image with a valid header.
592     // Delay the writing of the primary image header until after writing secondary images.
593     if (i == 0u) {
594       primary_image_file = std::move(image_file);
595     } else {
596       if (!image_file.WriteHeaderAndClose(image_filename, image_header)) {
597         return false;
598       }
599       // Update the primary image checksum with the secondary image checksum.
600       primary_header->SetImageChecksum(primary_header->GetImageChecksum() ^ image_checksum);
601     }
602   }
603   DCHECK(primary_image_file != nullptr);
604   if (!primary_image_file.WriteHeaderAndClose(image_filenames[0], primary_header)) {
605     return false;
606   }
607 
608   return true;
609 }
610 
GetImageOffset(mirror::Object * object,size_t oat_index) const611 size_t ImageWriter::GetImageOffset(mirror::Object* object, size_t oat_index) const {
612   BinSlot bin_slot = GetImageBinSlot(object, oat_index);
613   const ImageInfo& image_info = GetImageInfo(oat_index);
614   size_t offset = image_info.GetBinSlotOffset(bin_slot.GetBin()) + bin_slot.GetOffset();
615   DCHECK_LT(offset, image_info.image_end_);
616   return offset;
617 }
618 
SetImageBinSlot(mirror::Object * object,BinSlot bin_slot)619 void ImageWriter::SetImageBinSlot(mirror::Object* object, BinSlot bin_slot) {
620   DCHECK(object != nullptr);
621   DCHECK(!IsImageBinSlotAssigned(object));
622 
623   // Before we stomp over the lock word, save the hash code for later.
624   LockWord lw(object->GetLockWord(false));
625   switch (lw.GetState()) {
626     case LockWord::kFatLocked:
627       FALLTHROUGH_INTENDED;
628     case LockWord::kThinLocked: {
629       std::ostringstream oss;
630       bool thin = (lw.GetState() == LockWord::kThinLocked);
631       oss << (thin ? "Thin" : "Fat")
632           << " locked object " << object << "(" << object->PrettyTypeOf()
633           << ") found during object copy";
634       if (thin) {
635         oss << ". Lock owner:" << lw.ThinLockOwner();
636       }
637       LOG(FATAL) << oss.str();
638       UNREACHABLE();
639     }
640     case LockWord::kUnlocked:
641       // No hash, don't need to save it.
642       break;
643     case LockWord::kHashCode:
644       DCHECK(saved_hashcode_map_.find(object) == saved_hashcode_map_.end());
645       saved_hashcode_map_.emplace(object, lw.GetHashCode());
646       break;
647     default:
648       LOG(FATAL) << "Unreachable.";
649       UNREACHABLE();
650   }
651   object->SetLockWord(LockWord::FromForwardingAddress(bin_slot.Uint32Value()),
652                       /*as_volatile=*/ false);
653   DCHECK_EQ(object->GetLockWord(false).ReadBarrierState(), 0u);
654   DCHECK(IsImageBinSlotAssigned(object));
655 }
656 
PrepareDexCacheArraySlots()657 void ImageWriter::PrepareDexCacheArraySlots() {
658   // Prepare dex cache array starts based on the ordering specified in the CompilerOptions.
659   // Set the slot size early to avoid DCHECK() failures in IsImageBinSlotAssigned()
660   // when AssignImageBinSlot() assigns their indexes out or order.
661   for (const DexFile* dex_file : compiler_options_.GetDexFilesForOatFile()) {
662     auto it = dex_file_oat_index_map_.find(dex_file);
663     DCHECK(it != dex_file_oat_index_map_.end()) << dex_file->GetLocation();
664     ImageInfo& image_info = GetImageInfo(it->second);
665     image_info.dex_cache_array_starts_.Put(
666         dex_file, image_info.GetBinSlotSize(Bin::kDexCacheArray));
667     DexCacheArraysLayout layout(target_ptr_size_, dex_file);
668     image_info.IncrementBinSlotSize(Bin::kDexCacheArray, layout.Size());
669   }
670 
671   ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
672   Thread* const self = Thread::Current();
673   ReaderMutexLock mu(self, *Locks::dex_lock_);
674   for (const ClassLinker::DexCacheData& data : class_linker->GetDexCachesData()) {
675     ObjPtr<mirror::DexCache> dex_cache =
676         ObjPtr<mirror::DexCache>::DownCast(self->DecodeJObject(data.weak_root));
677     if (dex_cache == nullptr || !IsImageDexCache(dex_cache)) {
678       continue;
679     }
680     const DexFile* dex_file = dex_cache->GetDexFile();
681     CHECK(dex_file_oat_index_map_.find(dex_file) != dex_file_oat_index_map_.end())
682         << "Dex cache should have been pruned " << dex_file->GetLocation()
683         << "; possibly in class path";
684     DexCacheArraysLayout layout(target_ptr_size_, dex_file);
685     // Empty dex files will not have a "valid" DexCacheArraysLayout.
686     if (dex_file->NumTypeIds() + dex_file->NumStringIds() + dex_file->NumMethodIds() +
687         dex_file->NumFieldIds() + dex_file->NumProtoIds() + dex_file->NumCallSiteIds() != 0) {
688       DCHECK(layout.Valid());
689     }
690     size_t oat_index = GetOatIndexForDexFile(dex_file);
691     ImageInfo& image_info = GetImageInfo(oat_index);
692     uint32_t start = image_info.dex_cache_array_starts_.Get(dex_file);
693     DCHECK_EQ(dex_file->NumTypeIds() != 0u, dex_cache->GetResolvedTypes() != nullptr);
694     AddDexCacheArrayRelocation(dex_cache->GetResolvedTypes(),
695                                start + layout.TypesOffset(),
696                                oat_index);
697     DCHECK_EQ(dex_file->NumMethodIds() != 0u, dex_cache->GetResolvedMethods() != nullptr);
698     AddDexCacheArrayRelocation(dex_cache->GetResolvedMethods(),
699                                start + layout.MethodsOffset(),
700                                oat_index);
701     DCHECK_EQ(dex_file->NumFieldIds() != 0u, dex_cache->GetResolvedFields() != nullptr);
702     AddDexCacheArrayRelocation(dex_cache->GetResolvedFields(),
703                                start + layout.FieldsOffset(),
704                                oat_index);
705     DCHECK_EQ(dex_file->NumStringIds() != 0u, dex_cache->GetStrings() != nullptr);
706     AddDexCacheArrayRelocation(dex_cache->GetStrings(), start + layout.StringsOffset(), oat_index);
707 
708     AddDexCacheArrayRelocation(dex_cache->GetResolvedMethodTypes(),
709                                start + layout.MethodTypesOffset(),
710                                oat_index);
711     AddDexCacheArrayRelocation(dex_cache->GetResolvedCallSites(),
712                                 start + layout.CallSitesOffset(),
713                                 oat_index);
714 
715     // Preresolved strings aren't part of the special layout.
716     GcRoot<mirror::String>* preresolved_strings = dex_cache->GetPreResolvedStrings();
717     if (preresolved_strings != nullptr) {
718       DCHECK(!IsInBootImage(preresolved_strings));
719       // Add the array to the metadata section.
720       const size_t count = dex_cache->NumPreResolvedStrings();
721       auto bin = BinTypeForNativeRelocationType(NativeObjectRelocationType::kGcRootPointer);
722       for (size_t i = 0; i < count; ++i) {
723         native_object_relocations_.emplace(&preresolved_strings[i],
724             NativeObjectRelocation { oat_index,
725                                      image_info.GetBinSlotSize(bin),
726                                      NativeObjectRelocationType::kGcRootPointer });
727         image_info.IncrementBinSlotSize(bin, sizeof(GcRoot<mirror::Object>));
728       }
729     }
730   }
731 }
732 
AddDexCacheArrayRelocation(void * array,size_t offset,size_t oat_index)733 void ImageWriter::AddDexCacheArrayRelocation(void* array,
734                                              size_t offset,
735                                              size_t oat_index) {
736   if (array != nullptr) {
737     DCHECK(!IsInBootImage(array));
738     native_object_relocations_.emplace(array,
739         NativeObjectRelocation { oat_index, offset, NativeObjectRelocationType::kDexCacheArray });
740   }
741 }
742 
AddMethodPointerArray(ObjPtr<mirror::PointerArray> arr)743 void ImageWriter::AddMethodPointerArray(ObjPtr<mirror::PointerArray> arr) {
744   DCHECK(arr != nullptr);
745   if (kIsDebugBuild) {
746     for (size_t i = 0, len = arr->GetLength(); i < len; i++) {
747       ArtMethod* method = arr->GetElementPtrSize<ArtMethod*>(i, target_ptr_size_);
748       if (method != nullptr && !method->IsRuntimeMethod()) {
749         ObjPtr<mirror::Class> klass = method->GetDeclaringClass();
750         CHECK(klass == nullptr || KeepClass(klass))
751             << Class::PrettyClass(klass) << " should be a kept class";
752       }
753     }
754   }
755   // kBinArtMethodClean picked arbitrarily, just required to differentiate between ArtFields and
756   // ArtMethods.
757   pointer_arrays_.emplace(arr.Ptr(), Bin::kArtMethodClean);
758 }
759 
AssignImageBinSlot(mirror::Object * object,size_t oat_index)760 ImageWriter::Bin ImageWriter::AssignImageBinSlot(mirror::Object* object, size_t oat_index) {
761   DCHECK(object != nullptr);
762   size_t object_size = object->SizeOf();
763 
764   // The magic happens here. We segregate objects into different bins based
765   // on how likely they are to get dirty at runtime.
766   //
767   // Likely-to-dirty objects get packed together into the same bin so that
768   // at runtime their page dirtiness ratio (how many dirty objects a page has) is
769   // maximized.
770   //
771   // This means more pages will stay either clean or shared dirty (with zygote) and
772   // the app will use less of its own (private) memory.
773   Bin bin = Bin::kRegular;
774 
775   if (kBinObjects) {
776     //
777     // Changing the bin of an object is purely a memory-use tuning.
778     // It has no change on runtime correctness.
779     //
780     // Memory analysis has determined that the following types of objects get dirtied
781     // the most:
782     //
783     // * Dex cache arrays are stored in a special bin. The arrays for each dex cache have
784     //   a fixed layout which helps improve generated code (using PC-relative addressing),
785     //   so we pre-calculate their offsets separately in PrepareDexCacheArraySlots().
786     //   Since these arrays are huge, most pages do not overlap other objects and it's not
787     //   really important where they are for the clean/dirty separation. Due to their
788     //   special PC-relative addressing, we arbitrarily keep them at the end.
789     // * Class'es which are verified [their clinit runs only at runtime]
790     //   - classes in general [because their static fields get overwritten]
791     //   - initialized classes with all-final statics are unlikely to be ever dirty,
792     //     so bin them separately
793     // * Art Methods that are:
794     //   - native [their native entry point is not looked up until runtime]
795     //   - have declaring classes that aren't initialized
796     //            [their interpreter/quick entry points are trampolines until the class
797     //             becomes initialized]
798     //
799     // We also assume the following objects get dirtied either never or extremely rarely:
800     //  * Strings (they are immutable)
801     //  * Art methods that aren't native and have initialized declared classes
802     //
803     // We assume that "regular" bin objects are highly unlikely to become dirtied,
804     // so packing them together will not result in a noticeably tighter dirty-to-clean ratio.
805     //
806     if (object->IsClass()) {
807       bin = Bin::kClassVerified;
808       ObjPtr<mirror::Class> klass = object->AsClass();
809 
810       // Add non-embedded vtable to the pointer array table if there is one.
811       ObjPtr<mirror::PointerArray> vtable = klass->GetVTable();
812       if (vtable != nullptr) {
813         AddMethodPointerArray(vtable);
814       }
815       ObjPtr<mirror::IfTable> iftable = klass->GetIfTable();
816       if (iftable != nullptr) {
817         for (int32_t i = 0; i < klass->GetIfTableCount(); ++i) {
818           if (iftable->GetMethodArrayCount(i) > 0) {
819             AddMethodPointerArray(iftable->GetMethodArray(i));
820           }
821         }
822       }
823 
824       // Move known dirty objects into their own sections. This includes:
825       //   - classes with dirty static fields.
826       if (dirty_image_objects_ != nullptr &&
827           dirty_image_objects_->find(klass->PrettyDescriptor()) != dirty_image_objects_->end()) {
828         bin = Bin::kKnownDirty;
829       } else if (klass->GetStatus() == ClassStatus::kVisiblyInitialized) {
830         bin = Bin::kClassInitialized;
831 
832         // If the class's static fields are all final, put it into a separate bin
833         // since it's very likely it will stay clean.
834         uint32_t num_static_fields = klass->NumStaticFields();
835         if (num_static_fields == 0) {
836           bin = Bin::kClassInitializedFinalStatics;
837         } else {
838           // Maybe all the statics are final?
839           bool all_final = true;
840           for (uint32_t i = 0; i < num_static_fields; ++i) {
841             ArtField* field = klass->GetStaticField(i);
842             if (!field->IsFinal()) {
843               all_final = false;
844               break;
845             }
846           }
847 
848           if (all_final) {
849             bin = Bin::kClassInitializedFinalStatics;
850           }
851         }
852       }
853     } else if (object->GetClass<kVerifyNone>()->IsStringClass()) {
854       bin = Bin::kString;  // Strings are almost always immutable (except for object header).
855     } else if (object->GetClass<kVerifyNone>() == GetClassRoot<mirror::Object>()) {
856       // Instance of java lang object, probably a lock object. This means it will be dirty when we
857       // synchronize on it.
858       bin = Bin::kMiscDirty;
859     } else if (object->IsDexCache()) {
860       // Dex file field becomes dirty when the image is loaded.
861       bin = Bin::kMiscDirty;
862     }
863     // else bin = kBinRegular
864   }
865 
866   // Assign the oat index too.
867   DCHECK(oat_index_map_.find(object) == oat_index_map_.end());
868   oat_index_map_.emplace(object, oat_index);
869 
870   ImageInfo& image_info = GetImageInfo(oat_index);
871 
872   size_t offset_delta = RoundUp(object_size, kObjectAlignment);  // 64-bit alignment
873   // How many bytes the current bin is at (aligned).
874   size_t current_offset = image_info.GetBinSlotSize(bin);
875   // Move the current bin size up to accommodate the object we just assigned a bin slot.
876   image_info.IncrementBinSlotSize(bin, offset_delta);
877 
878   BinSlot new_bin_slot(bin, current_offset);
879   SetImageBinSlot(object, new_bin_slot);
880 
881   image_info.IncrementBinSlotCount(bin, 1u);
882 
883   // Grow the image closer to the end by the object we just assigned.
884   image_info.image_end_ += offset_delta;
885 
886   return bin;
887 }
888 
WillMethodBeDirty(ArtMethod * m) const889 bool ImageWriter::WillMethodBeDirty(ArtMethod* m) const {
890   if (m->IsNative()) {
891     return true;
892   }
893   ObjPtr<mirror::Class> declaring_class = m->GetDeclaringClass();
894   // Initialized is highly unlikely to dirty since there's no entry points to mutate.
895   return declaring_class == nullptr ||
896          declaring_class->GetStatus() != ClassStatus::kVisiblyInitialized;
897 }
898 
IsImageBinSlotAssigned(mirror::Object * object) const899 bool ImageWriter::IsImageBinSlotAssigned(mirror::Object* object) const {
900   DCHECK(object != nullptr);
901 
902   // We always stash the bin slot into a lockword, in the 'forwarding address' state.
903   // If it's in some other state, then we haven't yet assigned an image bin slot.
904   if (object->GetLockWord(false).GetState() != LockWord::kForwardingAddress) {
905     return false;
906   } else if (kIsDebugBuild) {
907     LockWord lock_word = object->GetLockWord(false);
908     size_t offset = lock_word.ForwardingAddress();
909     BinSlot bin_slot(offset);
910     size_t oat_index = GetOatIndex(object);
911     const ImageInfo& image_info = GetImageInfo(oat_index);
912     DCHECK_LT(bin_slot.GetOffset(), image_info.GetBinSlotSize(bin_slot.GetBin()))
913         << "bin slot offset should not exceed the size of that bin";
914   }
915   return true;
916 }
917 
GetImageBinSlot(mirror::Object * object,size_t oat_index) const918 ImageWriter::BinSlot ImageWriter::GetImageBinSlot(mirror::Object* object, size_t oat_index) const {
919   DCHECK(object != nullptr);
920   DCHECK(IsImageBinSlotAssigned(object));
921 
922   LockWord lock_word = object->GetLockWord(false);
923   size_t offset = lock_word.ForwardingAddress();  // TODO: ForwardingAddress should be uint32_t
924   DCHECK_LE(offset, std::numeric_limits<uint32_t>::max());
925 
926   BinSlot bin_slot(static_cast<uint32_t>(offset));
927   DCHECK_LT(bin_slot.GetOffset(), GetImageInfo(oat_index).GetBinSlotSize(bin_slot.GetBin()));
928 
929   return bin_slot;
930 }
931 
UpdateImageBinSlotOffset(mirror::Object * object,size_t oat_index,size_t new_offset)932 void ImageWriter::UpdateImageBinSlotOffset(mirror::Object* object,
933                                            size_t oat_index,
934                                            size_t new_offset) {
935   BinSlot old_bin_slot = GetImageBinSlot(object, oat_index);
936   DCHECK_LT(new_offset, GetImageInfo(oat_index).GetBinSlotSize(old_bin_slot.GetBin()));
937   BinSlot new_bin_slot(old_bin_slot.GetBin(), new_offset);
938   object->SetLockWord(LockWord::FromForwardingAddress(new_bin_slot.Uint32Value()),
939                       /*as_volatile=*/ false);
940   DCHECK_EQ(object->GetLockWord(false).ReadBarrierState(), 0u);
941   DCHECK(IsImageBinSlotAssigned(object));
942 }
943 
AllocMemory()944 bool ImageWriter::AllocMemory() {
945   for (ImageInfo& image_info : image_infos_) {
946     const size_t length = RoundUp(image_info.CreateImageSections().first, kPageSize);
947 
948     std::string error_msg;
949     image_info.image_ = MemMap::MapAnonymous("image writer image",
950                                              length,
951                                              PROT_READ | PROT_WRITE,
952                                              /*low_4gb=*/ false,
953                                              &error_msg);
954     if (UNLIKELY(!image_info.image_.IsValid())) {
955       LOG(ERROR) << "Failed to allocate memory for image file generation: " << error_msg;
956       return false;
957     }
958 
959     // Create the image bitmap, only needs to cover mirror object section which is up to image_end_.
960     CHECK_LE(image_info.image_end_, length);
961     image_info.image_bitmap_ = gc::accounting::ContinuousSpaceBitmap::Create(
962         "image bitmap", image_info.image_.Begin(), RoundUp(image_info.image_end_, kPageSize));
963     if (!image_info.image_bitmap_.IsValid()) {
964       LOG(ERROR) << "Failed to allocate memory for image bitmap";
965       return false;
966     }
967   }
968   return true;
969 }
970 
IsBootClassLoaderClass(ObjPtr<mirror::Class> klass)971 static bool IsBootClassLoaderClass(ObjPtr<mirror::Class> klass)
972     REQUIRES_SHARED(Locks::mutator_lock_) {
973   return klass->GetClassLoader() == nullptr;
974 }
975 
IsBootClassLoaderNonImageClass(mirror::Class * klass)976 bool ImageWriter::IsBootClassLoaderNonImageClass(mirror::Class* klass) {
977   return IsBootClassLoaderClass(klass) && !IsInBootImage(klass);
978 }
979 
980 // This visitor follows the references of an instance, recursively then prune this class
981 // if a type of any field is pruned.
982 class ImageWriter::PruneObjectReferenceVisitor {
983  public:
PruneObjectReferenceVisitor(ImageWriter * image_writer,bool * early_exit,std::unordered_set<mirror::Object * > * visited,bool * result)984   PruneObjectReferenceVisitor(ImageWriter* image_writer,
985                         bool* early_exit,
986                         std::unordered_set<mirror::Object*>* visited,
987                         bool* result)
988       : image_writer_(image_writer), early_exit_(early_exit), visited_(visited), result_(result) {}
989 
VisitRootIfNonNull(mirror::CompressedReference<mirror::Object> * root ATTRIBUTE_UNUSED) const990   ALWAYS_INLINE void VisitRootIfNonNull(
991       mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED) const
992       REQUIRES_SHARED(Locks::mutator_lock_) { }
993 
VisitRoot(mirror::CompressedReference<mirror::Object> * root ATTRIBUTE_UNUSED) const994   ALWAYS_INLINE void VisitRoot(
995       mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED) const
996       REQUIRES_SHARED(Locks::mutator_lock_) { }
997 
operator ()(ObjPtr<mirror::Object> obj,MemberOffset offset,bool is_static ATTRIBUTE_UNUSED) const998   ALWAYS_INLINE void operator() (ObjPtr<mirror::Object> obj,
999                                  MemberOffset offset,
1000                                  bool is_static ATTRIBUTE_UNUSED) const
1001       REQUIRES_SHARED(Locks::mutator_lock_) {
1002     mirror::Object* ref =
1003         obj->GetFieldObject<mirror::Object, kVerifyNone, kWithoutReadBarrier>(offset);
1004     if (ref == nullptr || visited_->find(ref) != visited_->end()) {
1005       return;
1006     }
1007 
1008     ObjPtr<mirror::ObjectArray<mirror::Class>> class_roots =
1009         Runtime::Current()->GetClassLinker()->GetClassRoots();
1010     ObjPtr<mirror::Class> klass = ref->IsClass() ? ref->AsClass() : ref->GetClass();
1011     if (klass == GetClassRoot<mirror::Method>(class_roots) ||
1012         klass == GetClassRoot<mirror::Constructor>(class_roots)) {
1013       // Prune all classes using reflection because the content they held will not be fixup.
1014       *result_ = true;
1015     }
1016 
1017     if (ref->IsClass()) {
1018       *result_ = *result_ ||
1019           image_writer_->PruneImageClassInternal(ref->AsClass(), early_exit_, visited_);
1020     } else {
1021       // Record the object visited in case of circular reference.
1022       visited_->emplace(ref);
1023       *result_ = *result_ ||
1024           image_writer_->PruneImageClassInternal(klass, early_exit_, visited_);
1025       ref->VisitReferences(*this, *this);
1026       // Clean up before exit for next call of this function.
1027       visited_->erase(ref);
1028     }
1029   }
1030 
operator ()(ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED,ObjPtr<mirror::Reference> ref) const1031   ALWAYS_INLINE void operator() (ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED,
1032                                  ObjPtr<mirror::Reference> ref) const
1033       REQUIRES_SHARED(Locks::mutator_lock_) {
1034     operator()(ref, mirror::Reference::ReferentOffset(), /* is_static */ false);
1035   }
1036 
1037  private:
1038   ImageWriter* image_writer_;
1039   bool* early_exit_;
1040   std::unordered_set<mirror::Object*>* visited_;
1041   bool* const result_;
1042 };
1043 
1044 
PruneImageClass(ObjPtr<mirror::Class> klass)1045 bool ImageWriter::PruneImageClass(ObjPtr<mirror::Class> klass) {
1046   bool early_exit = false;
1047   std::unordered_set<mirror::Object*> visited;
1048   return PruneImageClassInternal(klass, &early_exit, &visited);
1049 }
1050 
PruneImageClassInternal(ObjPtr<mirror::Class> klass,bool * early_exit,std::unordered_set<mirror::Object * > * visited)1051 bool ImageWriter::PruneImageClassInternal(
1052     ObjPtr<mirror::Class> klass,
1053     bool* early_exit,
1054     std::unordered_set<mirror::Object*>* visited) {
1055   DCHECK(early_exit != nullptr);
1056   DCHECK(visited != nullptr);
1057   DCHECK(compiler_options_.IsAppImage() || compiler_options_.IsBootImageExtension());
1058   if (klass == nullptr || IsInBootImage(klass.Ptr())) {
1059     return false;
1060   }
1061   auto found = prune_class_memo_.find(klass.Ptr());
1062   if (found != prune_class_memo_.end()) {
1063     // Already computed, return the found value.
1064     return found->second;
1065   }
1066   // Circular dependencies, return false but do not store the result in the memoization table.
1067   if (visited->find(klass.Ptr()) != visited->end()) {
1068     *early_exit = true;
1069     return false;
1070   }
1071   visited->emplace(klass.Ptr());
1072   bool result = IsBootClassLoaderClass(klass);
1073   std::string temp;
1074   // Prune if not an image class, this handles any broken sets of image classes such as having a
1075   // class in the set but not it's superclass.
1076   result = result || !compiler_options_.IsImageClass(klass->GetDescriptor(&temp));
1077   bool my_early_exit = false;  // Only for ourselves, ignore caller.
1078   // Remove classes that failed to verify since we don't want to have java.lang.VerifyError in the
1079   // app image.
1080   if (klass->IsErroneous()) {
1081     result = true;
1082   } else {
1083     ObjPtr<mirror::ClassExt> ext(klass->GetExtData());
1084     CHECK(ext.IsNull() || ext->GetVerifyError() == nullptr) << klass->PrettyClass();
1085   }
1086   if (!result) {
1087     // Check interfaces since these wont be visited through VisitReferences.)
1088     ObjPtr<mirror::IfTable> if_table = klass->GetIfTable();
1089     for (size_t i = 0, num_interfaces = klass->GetIfTableCount(); i < num_interfaces; ++i) {
1090       result = result || PruneImageClassInternal(if_table->GetInterface(i),
1091                                                  &my_early_exit,
1092                                                  visited);
1093     }
1094   }
1095   if (klass->IsObjectArrayClass()) {
1096     result = result || PruneImageClassInternal(klass->GetComponentType(),
1097                                                &my_early_exit,
1098                                                visited);
1099   }
1100   // Check static fields and their classes.
1101   if (klass->IsResolved() && klass->NumReferenceStaticFields() != 0) {
1102     size_t num_static_fields = klass->NumReferenceStaticFields();
1103     // Presumably GC can happen when we are cross compiling, it should not cause performance
1104     // problems to do pointer size logic.
1105     MemberOffset field_offset = klass->GetFirstReferenceStaticFieldOffset(
1106         Runtime::Current()->GetClassLinker()->GetImagePointerSize());
1107     for (size_t i = 0u; i < num_static_fields; ++i) {
1108       mirror::Object* ref = klass->GetFieldObject<mirror::Object>(field_offset);
1109       if (ref != nullptr) {
1110         if (ref->IsClass()) {
1111           result = result || PruneImageClassInternal(ref->AsClass(), &my_early_exit, visited);
1112         } else {
1113           mirror::Class* type = ref->GetClass();
1114           result = result || PruneImageClassInternal(type, &my_early_exit, visited);
1115           if (!result) {
1116             // For non-class case, also go through all the types mentioned by it's fields'
1117             // references recursively to decide whether to keep this class.
1118             bool tmp = false;
1119             PruneObjectReferenceVisitor visitor(this, &my_early_exit, visited, &tmp);
1120             ref->VisitReferences(visitor, visitor);
1121             result = result || tmp;
1122           }
1123         }
1124       }
1125       field_offset = MemberOffset(field_offset.Uint32Value() +
1126                                   sizeof(mirror::HeapReference<mirror::Object>));
1127     }
1128   }
1129   result = result || PruneImageClassInternal(klass->GetSuperClass(), &my_early_exit, visited);
1130   // Remove the class if the dex file is not in the set of dex files. This happens for classes that
1131   // are from uses-library if there is no profile. b/30688277
1132   ObjPtr<mirror::DexCache> dex_cache = klass->GetDexCache();
1133   if (dex_cache != nullptr) {
1134     result = result ||
1135         dex_file_oat_index_map_.find(dex_cache->GetDexFile()) == dex_file_oat_index_map_.end();
1136   }
1137   // Erase the element we stored earlier since we are exiting the function.
1138   auto it = visited->find(klass.Ptr());
1139   DCHECK(it != visited->end());
1140   visited->erase(it);
1141   // Only store result if it is true or none of the calls early exited due to circular
1142   // dependencies. If visited is empty then we are the root caller, in this case the cycle was in
1143   // a child call and we can remember the result.
1144   if (result == true || !my_early_exit || visited->empty()) {
1145     prune_class_memo_[klass.Ptr()] = result;
1146   }
1147   *early_exit |= my_early_exit;
1148   return result;
1149 }
1150 
KeepClass(ObjPtr<mirror::Class> klass)1151 bool ImageWriter::KeepClass(ObjPtr<mirror::Class> klass) {
1152   if (klass == nullptr) {
1153     return false;
1154   }
1155   if (IsInBootImage(klass.Ptr())) {
1156     // Already in boot image, return true.
1157     DCHECK(!compiler_options_.IsBootImage());
1158     return true;
1159   }
1160   std::string temp;
1161   if (!compiler_options_.IsImageClass(klass->GetDescriptor(&temp))) {
1162     return false;
1163   }
1164   if (compiler_options_.IsAppImage()) {
1165     // For app images, we need to prune classes that
1166     // are defined by the boot class path we're compiling against but not in
1167     // the boot image spaces since these may have already been loaded at
1168     // run time when this image is loaded. Keep classes in the boot image
1169     // spaces we're compiling against since we don't want to re-resolve these.
1170     return !PruneImageClass(klass);
1171   }
1172   return true;
1173 }
1174 
1175 class ImageWriter::PruneClassesVisitor : public ClassVisitor {
1176  public:
PruneClassesVisitor(ImageWriter * image_writer,ObjPtr<mirror::ClassLoader> class_loader)1177   PruneClassesVisitor(ImageWriter* image_writer, ObjPtr<mirror::ClassLoader> class_loader)
1178       : image_writer_(image_writer),
1179         class_loader_(class_loader),
1180         classes_to_prune_(),
1181         defined_class_count_(0u) { }
1182 
operator ()(ObjPtr<mirror::Class> klass)1183   bool operator()(ObjPtr<mirror::Class> klass) override REQUIRES_SHARED(Locks::mutator_lock_) {
1184     if (!image_writer_->KeepClass(klass.Ptr())) {
1185       classes_to_prune_.insert(klass.Ptr());
1186       if (klass->GetClassLoader() == class_loader_) {
1187         ++defined_class_count_;
1188       }
1189     }
1190     return true;
1191   }
1192 
Prune()1193   size_t Prune() REQUIRES_SHARED(Locks::mutator_lock_) {
1194     ClassTable* class_table =
1195         Runtime::Current()->GetClassLinker()->ClassTableForClassLoader(class_loader_);
1196     for (mirror::Class* klass : classes_to_prune_) {
1197       std::string storage;
1198       const char* descriptor = klass->GetDescriptor(&storage);
1199       bool result = class_table->Remove(descriptor);
1200       DCHECK(result);
1201       DCHECK(!class_table->Remove(descriptor)) << descriptor;
1202     }
1203     return defined_class_count_;
1204   }
1205 
1206  private:
1207   ImageWriter* const image_writer_;
1208   const ObjPtr<mirror::ClassLoader> class_loader_;
1209   std::unordered_set<mirror::Class*> classes_to_prune_;
1210   size_t defined_class_count_;
1211 };
1212 
1213 class ImageWriter::PruneClassLoaderClassesVisitor : public ClassLoaderVisitor {
1214  public:
PruneClassLoaderClassesVisitor(ImageWriter * image_writer)1215   explicit PruneClassLoaderClassesVisitor(ImageWriter* image_writer)
1216       : image_writer_(image_writer), removed_class_count_(0) {}
1217 
Visit(ObjPtr<mirror::ClassLoader> class_loader)1218   void Visit(ObjPtr<mirror::ClassLoader> class_loader) override
1219       REQUIRES_SHARED(Locks::mutator_lock_) {
1220     PruneClassesVisitor classes_visitor(image_writer_, class_loader);
1221     ClassTable* class_table =
1222         Runtime::Current()->GetClassLinker()->ClassTableForClassLoader(class_loader);
1223     class_table->Visit(classes_visitor);
1224     removed_class_count_ += classes_visitor.Prune();
1225   }
1226 
GetRemovedClassCount() const1227   size_t GetRemovedClassCount() const {
1228     return removed_class_count_;
1229   }
1230 
1231  private:
1232   ImageWriter* const image_writer_;
1233   size_t removed_class_count_;
1234 };
1235 
VisitClassLoaders(ClassLoaderVisitor * visitor)1236 void ImageWriter::VisitClassLoaders(ClassLoaderVisitor* visitor) {
1237   WriterMutexLock mu(Thread::Current(), *Locks::classlinker_classes_lock_);
1238   visitor->Visit(nullptr);  // Visit boot class loader.
1239   Runtime::Current()->GetClassLinker()->VisitClassLoaders(visitor);
1240 }
1241 
ClearDexCache(ObjPtr<mirror::DexCache> dex_cache)1242 void ImageWriter::ClearDexCache(ObjPtr<mirror::DexCache> dex_cache) {
1243   // Clear methods.
1244   mirror::MethodDexCacheType* resolved_methods = dex_cache->GetResolvedMethods();
1245   for (size_t slot_idx = 0, num = dex_cache->NumResolvedMethods(); slot_idx != num; ++slot_idx) {
1246     auto pair =
1247         mirror::DexCache::GetNativePairPtrSize(resolved_methods, slot_idx, target_ptr_size_);
1248     if (pair.object != nullptr) {
1249       dex_cache->ClearResolvedMethod(pair.index, target_ptr_size_);
1250     }
1251   }
1252   // Clear fields.
1253   mirror::FieldDexCacheType* resolved_fields = dex_cache->GetResolvedFields();
1254   for (size_t slot_idx = 0, num = dex_cache->NumResolvedFields(); slot_idx != num; ++slot_idx) {
1255     auto pair = mirror::DexCache::GetNativePairPtrSize(resolved_fields, slot_idx, target_ptr_size_);
1256     if (pair.object != nullptr) {
1257       dex_cache->ClearResolvedField(pair.index, target_ptr_size_);
1258     }
1259   }
1260   // Clear types.
1261   for (size_t slot_idx = 0, num = dex_cache->NumResolvedTypes(); slot_idx != num; ++slot_idx) {
1262     mirror::TypeDexCachePair pair =
1263         dex_cache->GetResolvedTypes()[slot_idx].load(std::memory_order_relaxed);
1264     if (!pair.object.IsNull()) {
1265       dex_cache->ClearResolvedType(dex::TypeIndex(pair.index));
1266     }
1267   }
1268   // Clear strings.
1269   for (size_t slot_idx = 0, num = dex_cache->NumStrings(); slot_idx != num; ++slot_idx) {
1270     mirror::StringDexCachePair pair =
1271         dex_cache->GetStrings()[slot_idx].load(std::memory_order_relaxed);
1272     if (!pair.object.IsNull()) {
1273       dex_cache->ClearString(dex::StringIndex(pair.index));
1274     }
1275   }
1276 }
1277 
PreloadDexCache(ObjPtr<mirror::DexCache> dex_cache,ObjPtr<mirror::ClassLoader> class_loader)1278 void ImageWriter::PreloadDexCache(ObjPtr<mirror::DexCache> dex_cache,
1279                                   ObjPtr<mirror::ClassLoader> class_loader) {
1280   // To ensure deterministic contents of the hash-based arrays, each slot shall contain
1281   // the candidate with the lowest index. As we're processing entries in increasing index
1282   // order, this means trying to look up the entry for the current index if the slot is
1283   // empty or if it contains a higher index.
1284 
1285   Runtime* runtime = Runtime::Current();
1286   ClassLinker* class_linker = runtime->GetClassLinker();
1287   const DexFile& dex_file = *dex_cache->GetDexFile();
1288   // Preload the methods array and make the contents deterministic.
1289   mirror::MethodDexCacheType* resolved_methods = dex_cache->GetResolvedMethods();
1290   dex::TypeIndex last_class_idx;  // Initialized to invalid index.
1291   ObjPtr<mirror::Class> last_class = nullptr;
1292   for (size_t i = 0, num = dex_cache->GetDexFile()->NumMethodIds(); i != num; ++i) {
1293     uint32_t slot_idx = dex_cache->MethodSlotIndex(i);
1294     auto pair =
1295         mirror::DexCache::GetNativePairPtrSize(resolved_methods, slot_idx, target_ptr_size_);
1296     uint32_t stored_index = pair.index;
1297     ArtMethod* method = pair.object;
1298     if (method != nullptr && i > stored_index) {
1299       continue;  // Already checked.
1300     }
1301     // Check if the referenced class is in the image. Note that we want to check the referenced
1302     // class rather than the declaring class to preserve the semantics, i.e. using a MethodId
1303     // results in resolving the referenced class and that can for example throw OOME.
1304     const dex::MethodId& method_id = dex_file.GetMethodId(i);
1305     if (method_id.class_idx_ != last_class_idx) {
1306       last_class_idx = method_id.class_idx_;
1307       last_class = class_linker->LookupResolvedType(last_class_idx, dex_cache, class_loader);
1308     }
1309     if (method == nullptr || i < stored_index) {
1310       if (last_class != nullptr) {
1311         // Try to resolve the method with the class linker, which will insert
1312         // it into the dex cache if successful.
1313         method = class_linker->FindResolvedMethod(last_class, dex_cache, class_loader, i);
1314         DCHECK(method == nullptr || dex_cache->GetResolvedMethod(i, target_ptr_size_) == method);
1315       }
1316     } else {
1317       DCHECK_EQ(i, stored_index);
1318       DCHECK(last_class != nullptr);
1319     }
1320   }
1321   // Preload the fields array and make the contents deterministic.
1322   mirror::FieldDexCacheType* resolved_fields = dex_cache->GetResolvedFields();
1323   last_class_idx = dex::TypeIndex();  // Initialized to invalid index.
1324   last_class = nullptr;
1325   for (size_t i = 0, end = dex_file.NumFieldIds(); i < end; ++i) {
1326     uint32_t slot_idx = dex_cache->FieldSlotIndex(i);
1327     auto pair = mirror::DexCache::GetNativePairPtrSize(resolved_fields, slot_idx, target_ptr_size_);
1328     uint32_t stored_index = pair.index;
1329     ArtField* field = pair.object;
1330     if (field != nullptr && i > stored_index) {
1331       continue;  // Already checked.
1332     }
1333     // Check if the referenced class is in the image. Note that we want to check the referenced
1334     // class rather than the declaring class to preserve the semantics, i.e. using a FieldId
1335     // results in resolving the referenced class and that can for example throw OOME.
1336     const dex::FieldId& field_id = dex_file.GetFieldId(i);
1337     if (field_id.class_idx_ != last_class_idx) {
1338       last_class_idx = field_id.class_idx_;
1339       last_class = class_linker->LookupResolvedType(last_class_idx, dex_cache, class_loader);
1340       if (last_class != nullptr && !KeepClass(last_class)) {
1341         last_class = nullptr;
1342       }
1343     }
1344     if (field == nullptr || i < stored_index) {
1345       if (last_class != nullptr) {
1346         // Try to resolve the field with the class linker, which will insert
1347         // it into the dex cache if successful.
1348         field = class_linker->FindResolvedFieldJLS(last_class, dex_cache, class_loader, i);
1349         DCHECK(field == nullptr || dex_cache->GetResolvedField(i, target_ptr_size_) == field);
1350       }
1351     } else {
1352       DCHECK_EQ(i, stored_index);
1353       DCHECK(last_class != nullptr);
1354     }
1355   }
1356   // Preload the types array and make the contents deterministic.
1357   // This is done after fields and methods as their lookup can touch the types array.
1358   for (size_t i = 0, end = dex_cache->GetDexFile()->NumTypeIds(); i < end; ++i) {
1359     dex::TypeIndex type_idx(i);
1360     uint32_t slot_idx = dex_cache->TypeSlotIndex(type_idx);
1361     mirror::TypeDexCachePair pair =
1362         dex_cache->GetResolvedTypes()[slot_idx].load(std::memory_order_relaxed);
1363     uint32_t stored_index = pair.index;
1364     ObjPtr<mirror::Class> klass = pair.object.Read();
1365     if (klass == nullptr || i < stored_index) {
1366       klass = class_linker->LookupResolvedType(type_idx, dex_cache, class_loader);
1367       DCHECK(klass == nullptr || dex_cache->GetResolvedType(type_idx) == klass);
1368     }
1369   }
1370   // Preload the strings array and make the contents deterministic.
1371   for (size_t i = 0, end = dex_cache->GetDexFile()->NumStringIds(); i < end; ++i) {
1372     dex::StringIndex string_idx(i);
1373     uint32_t slot_idx = dex_cache->StringSlotIndex(string_idx);
1374     mirror::StringDexCachePair pair =
1375         dex_cache->GetStrings()[slot_idx].load(std::memory_order_relaxed);
1376     uint32_t stored_index = pair.index;
1377     ObjPtr<mirror::String> string = pair.object.Read();
1378     if (string == nullptr || i < stored_index) {
1379       string = class_linker->LookupString(string_idx, dex_cache);
1380       DCHECK(string == nullptr || dex_cache->GetResolvedString(string_idx) == string);
1381     }
1382   }
1383 }
1384 
PruneNonImageClasses()1385 void ImageWriter::PruneNonImageClasses() {
1386   Runtime* runtime = Runtime::Current();
1387   ClassLinker* class_linker = runtime->GetClassLinker();
1388   Thread* self = Thread::Current();
1389   ScopedAssertNoThreadSuspension sa(__FUNCTION__);
1390 
1391   // Prune uses-library dex caches. Only prune the uses-library dex caches since we want to make
1392   // sure the other ones don't get unloaded before the OatWriter runs.
1393   class_linker->VisitClassTables(
1394       [&](ClassTable* table) REQUIRES_SHARED(Locks::mutator_lock_) {
1395     table->RemoveStrongRoots(
1396         [&](GcRoot<mirror::Object> root) REQUIRES_SHARED(Locks::mutator_lock_) {
1397       ObjPtr<mirror::Object> obj = root.Read();
1398       if (obj->IsDexCache()) {
1399         // Return true if the dex file is not one of the ones in the map.
1400         return dex_file_oat_index_map_.find(obj->AsDexCache()->GetDexFile()) ==
1401             dex_file_oat_index_map_.end();
1402       }
1403       // Return false to avoid removing.
1404       return false;
1405     });
1406   });
1407 
1408   // Remove the undesired classes from the class roots.
1409   {
1410     PruneClassLoaderClassesVisitor class_loader_visitor(this);
1411     VisitClassLoaders(&class_loader_visitor);
1412     VLOG(compiler) << "Pruned " << class_loader_visitor.GetRemovedClassCount() << " classes";
1413   }
1414 
1415   // Completely clear DexCaches. They shall be re-filled in PreloadDexCaches if requested.
1416   std::vector<ObjPtr<mirror::DexCache>> dex_caches = FindDexCaches(self);
1417   for (ObjPtr<mirror::DexCache> dex_cache : dex_caches) {
1418     ClearDexCache(dex_cache);
1419   }
1420 
1421   // Drop the array class cache in the ClassLinker, as these are roots holding those classes live.
1422   class_linker->DropFindArrayClassCache();
1423 
1424   // Clear to save RAM.
1425   prune_class_memo_.clear();
1426 }
1427 
FindDexCaches(Thread * self)1428 std::vector<ObjPtr<mirror::DexCache>> ImageWriter::FindDexCaches(Thread* self) {
1429   std::vector<ObjPtr<mirror::DexCache>> dex_caches;
1430   ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
1431   ReaderMutexLock mu2(self, *Locks::dex_lock_);
1432   dex_caches.reserve(class_linker->GetDexCachesData().size());
1433   for (const ClassLinker::DexCacheData& data : class_linker->GetDexCachesData()) {
1434     if (self->IsJWeakCleared(data.weak_root)) {
1435       continue;
1436     }
1437     dex_caches.push_back(self->DecodeJObject(data.weak_root)->AsDexCache());
1438   }
1439   return dex_caches;
1440 }
1441 
CheckNonImageClassesRemoved()1442 void ImageWriter::CheckNonImageClassesRemoved() {
1443   auto visitor = [&](Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) {
1444     if (obj->IsClass() && !IsInBootImage(obj)) {
1445       ObjPtr<Class> klass = obj->AsClass();
1446       if (!KeepClass(klass)) {
1447         DumpImageClasses();
1448         CHECK(KeepClass(klass))
1449             << Runtime::Current()->GetHeap()->GetVerification()->FirstPathFromRootSet(klass);
1450       }
1451     }
1452   };
1453   gc::Heap* heap = Runtime::Current()->GetHeap();
1454   heap->VisitObjects(visitor);
1455 }
1456 
DumpImageClasses()1457 void ImageWriter::DumpImageClasses() {
1458   for (const std::string& image_class : compiler_options_.GetImageClasses()) {
1459     LOG(INFO) << " " << image_class;
1460   }
1461 }
1462 
CollectDexCaches(Thread * self,size_t oat_index) const1463 ObjPtr<mirror::ObjectArray<mirror::Object>> ImageWriter::CollectDexCaches(Thread* self,
1464                                                                           size_t oat_index) const {
1465   std::unordered_set<const DexFile*> image_dex_files;
1466   for (auto& pair : dex_file_oat_index_map_) {
1467     const DexFile* image_dex_file = pair.first;
1468     size_t image_oat_index = pair.second;
1469     if (oat_index == image_oat_index) {
1470       image_dex_files.insert(image_dex_file);
1471     }
1472   }
1473 
1474   // build an Object[] of all the DexCaches used in the source_space_.
1475   // Since we can't hold the dex lock when allocating the dex_caches
1476   // ObjectArray, we lock the dex lock twice, first to get the number
1477   // of dex caches first and then lock it again to copy the dex
1478   // caches. We check that the number of dex caches does not change.
1479   ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
1480   size_t dex_cache_count = 0;
1481   {
1482     ReaderMutexLock mu(self, *Locks::dex_lock_);
1483     // Count number of dex caches not in the boot image.
1484     for (const ClassLinker::DexCacheData& data : class_linker->GetDexCachesData()) {
1485       ObjPtr<mirror::DexCache> dex_cache =
1486           ObjPtr<mirror::DexCache>::DownCast(self->DecodeJObject(data.weak_root));
1487       if (dex_cache == nullptr) {
1488         continue;
1489       }
1490       const DexFile* dex_file = dex_cache->GetDexFile();
1491       if (IsImageDexCache(dex_cache)) {
1492         dex_cache_count += image_dex_files.find(dex_file) != image_dex_files.end() ? 1u : 0u;
1493       }
1494     }
1495   }
1496   ObjPtr<ObjectArray<Object>> dex_caches = ObjectArray<Object>::Alloc(
1497       self, GetClassRoot<ObjectArray<Object>>(class_linker), dex_cache_count);
1498   CHECK(dex_caches != nullptr) << "Failed to allocate a dex cache array.";
1499   {
1500     ReaderMutexLock mu(self, *Locks::dex_lock_);
1501     size_t non_image_dex_caches = 0;
1502     // Re-count number of non image dex caches.
1503     for (const ClassLinker::DexCacheData& data : class_linker->GetDexCachesData()) {
1504       ObjPtr<mirror::DexCache> dex_cache =
1505           ObjPtr<mirror::DexCache>::DownCast(self->DecodeJObject(data.weak_root));
1506       if (dex_cache == nullptr) {
1507         continue;
1508       }
1509       const DexFile* dex_file = dex_cache->GetDexFile();
1510       if (IsImageDexCache(dex_cache)) {
1511         non_image_dex_caches += image_dex_files.find(dex_file) != image_dex_files.end() ? 1u : 0u;
1512       }
1513     }
1514     CHECK_EQ(dex_cache_count, non_image_dex_caches)
1515         << "The number of non-image dex caches changed.";
1516     size_t i = 0;
1517     for (const ClassLinker::DexCacheData& data : class_linker->GetDexCachesData()) {
1518       ObjPtr<mirror::DexCache> dex_cache =
1519           ObjPtr<mirror::DexCache>::DownCast(self->DecodeJObject(data.weak_root));
1520       if (dex_cache == nullptr) {
1521         continue;
1522       }
1523       const DexFile* dex_file = dex_cache->GetDexFile();
1524       if (IsImageDexCache(dex_cache) &&
1525           image_dex_files.find(dex_file) != image_dex_files.end()) {
1526         dex_caches->Set<false>(i, dex_cache.Ptr());
1527         ++i;
1528       }
1529     }
1530   }
1531   return dex_caches;
1532 }
1533 
CreateImageRoots(size_t oat_index,Handle<mirror::ObjectArray<mirror::Object>> boot_image_live_objects) const1534 ObjPtr<ObjectArray<Object>> ImageWriter::CreateImageRoots(
1535     size_t oat_index,
1536     Handle<mirror::ObjectArray<mirror::Object>> boot_image_live_objects) const {
1537   Runtime* runtime = Runtime::Current();
1538   ClassLinker* class_linker = runtime->GetClassLinker();
1539   Thread* self = Thread::Current();
1540   StackHandleScope<2> hs(self);
1541 
1542   Handle<ObjectArray<Object>> dex_caches(hs.NewHandle(CollectDexCaches(self, oat_index)));
1543 
1544   // build an Object[] of the roots needed to restore the runtime
1545   int32_t image_roots_size = ImageHeader::NumberOfImageRoots(compiler_options_.IsAppImage());
1546   Handle<ObjectArray<Object>> image_roots(hs.NewHandle(ObjectArray<Object>::Alloc(
1547       self, GetClassRoot<ObjectArray<Object>>(class_linker), image_roots_size)));
1548   image_roots->Set<false>(ImageHeader::kDexCaches, dex_caches.Get());
1549   image_roots->Set<false>(ImageHeader::kClassRoots, class_linker->GetClassRoots());
1550   if (!compiler_options_.IsAppImage()) {
1551     DCHECK(boot_image_live_objects != nullptr);
1552     image_roots->Set<false>(ImageHeader::kBootImageLiveObjects, boot_image_live_objects.Get());
1553   } else {
1554     DCHECK(boot_image_live_objects == nullptr);
1555     image_roots->Set<false>(ImageHeader::kAppImageClassLoader, GetAppClassLoader());
1556   }
1557   for (int32_t i = 0; i != image_roots_size; ++i) {
1558     CHECK(image_roots->Get(i) != nullptr);
1559   }
1560   return image_roots.Get();
1561 }
1562 
RecordNativeRelocations(ObjPtr<mirror::Object> obj,size_t oat_index)1563 void ImageWriter::RecordNativeRelocations(ObjPtr<mirror::Object> obj, size_t oat_index) {
1564   if (obj->IsString()) {
1565     ObjPtr<mirror::String> str = obj->AsString();
1566     InternTable* intern_table = Runtime::Current()->GetInternTable();
1567     Thread* const self = Thread::Current();
1568     if (intern_table->LookupStrong(self, str) == str) {
1569       DCHECK(std::none_of(image_infos_.begin(),
1570                           image_infos_.end(),
1571                           [=](ImageInfo& info) REQUIRES_SHARED(Locks::mutator_lock_) {
1572                             return info.intern_table_->LookupStrong(self, str) != nullptr;
1573                           }));
1574       ObjPtr<mirror::String> interned =
1575           GetImageInfo(oat_index).intern_table_->InternStrongImageString(str);
1576       DCHECK_EQ(interned, obj);
1577     }
1578   } else if (obj->IsDexCache()) {
1579     DCHECK_EQ(oat_index, GetOatIndexForDexFile(obj->AsDexCache()->GetDexFile()));
1580   } else if (obj->IsClass()) {
1581     // Visit and assign offsets for fields and field arrays.
1582     ObjPtr<mirror::Class> as_klass = obj->AsClass();
1583     DCHECK_EQ(oat_index, GetOatIndexForClass(as_klass));
1584     DCHECK(!as_klass->IsErroneous()) << as_klass->GetStatus();
1585     if (compiler_options_.IsAppImage()) {
1586       // Extra sanity, no boot loader classes should be left!
1587       CHECK(!IsBootClassLoaderClass(as_klass)) << as_klass->PrettyClass();
1588     }
1589     LengthPrefixedArray<ArtField>* fields[] = {
1590         as_klass->GetSFieldsPtr(), as_klass->GetIFieldsPtr(),
1591     };
1592     ImageInfo& image_info = GetImageInfo(oat_index);
1593     if (!compiler_options_.IsAppImage()) {
1594       // Note: Avoid locking to prevent lock order violations from root visiting;
1595       // image_info.class_table_ is only accessed from the image writer.
1596       image_info.class_table_->InsertWithoutLocks(as_klass);
1597     }
1598     for (LengthPrefixedArray<ArtField>* cur_fields : fields) {
1599       // Total array length including header.
1600       if (cur_fields != nullptr) {
1601         const size_t header_size = LengthPrefixedArray<ArtField>::ComputeSize(0);
1602         // Forward the entire array at once.
1603         auto it = native_object_relocations_.find(cur_fields);
1604         CHECK(it == native_object_relocations_.end()) << "Field array " << cur_fields
1605                                                 << " already forwarded";
1606         size_t offset = image_info.GetBinSlotSize(Bin::kArtField);
1607         DCHECK(!IsInBootImage(cur_fields));
1608         native_object_relocations_.emplace(
1609             cur_fields,
1610             NativeObjectRelocation {
1611                 oat_index, offset, NativeObjectRelocationType::kArtFieldArray
1612             });
1613         offset += header_size;
1614         // Forward individual fields so that we can quickly find where they belong.
1615         for (size_t i = 0, count = cur_fields->size(); i < count; ++i) {
1616           // Need to forward arrays separate of fields.
1617           ArtField* field = &cur_fields->At(i);
1618           auto it2 = native_object_relocations_.find(field);
1619           CHECK(it2 == native_object_relocations_.end()) << "Field at index=" << i
1620               << " already assigned " << field->PrettyField() << " static=" << field->IsStatic();
1621           DCHECK(!IsInBootImage(field));
1622           native_object_relocations_.emplace(
1623               field,
1624               NativeObjectRelocation { oat_index,
1625                                        offset,
1626                                        NativeObjectRelocationType::kArtField });
1627           offset += sizeof(ArtField);
1628         }
1629         image_info.IncrementBinSlotSize(
1630             Bin::kArtField, header_size + cur_fields->size() * sizeof(ArtField));
1631         DCHECK_EQ(offset, image_info.GetBinSlotSize(Bin::kArtField));
1632       }
1633     }
1634     // Visit and assign offsets for methods.
1635     size_t num_methods = as_klass->NumMethods();
1636     if (num_methods != 0) {
1637       bool any_dirty = false;
1638       for (auto& m : as_klass->GetMethods(target_ptr_size_)) {
1639         if (WillMethodBeDirty(&m)) {
1640           any_dirty = true;
1641           break;
1642         }
1643       }
1644       NativeObjectRelocationType type = any_dirty
1645           ? NativeObjectRelocationType::kArtMethodDirty
1646           : NativeObjectRelocationType::kArtMethodClean;
1647       Bin bin_type = BinTypeForNativeRelocationType(type);
1648       // Forward the entire array at once, but header first.
1649       const size_t method_alignment = ArtMethod::Alignment(target_ptr_size_);
1650       const size_t method_size = ArtMethod::Size(target_ptr_size_);
1651       const size_t header_size = LengthPrefixedArray<ArtMethod>::ComputeSize(0,
1652                                                                              method_size,
1653                                                                              method_alignment);
1654       LengthPrefixedArray<ArtMethod>* array = as_klass->GetMethodsPtr();
1655       auto it = native_object_relocations_.find(array);
1656       CHECK(it == native_object_relocations_.end())
1657           << "Method array " << array << " already forwarded";
1658       size_t offset = image_info.GetBinSlotSize(bin_type);
1659       DCHECK(!IsInBootImage(array));
1660       native_object_relocations_.emplace(array,
1661           NativeObjectRelocation {
1662               oat_index,
1663               offset,
1664               any_dirty ? NativeObjectRelocationType::kArtMethodArrayDirty
1665                         : NativeObjectRelocationType::kArtMethodArrayClean });
1666       image_info.IncrementBinSlotSize(bin_type, header_size);
1667       for (auto& m : as_klass->GetMethods(target_ptr_size_)) {
1668         AssignMethodOffset(&m, type, oat_index);
1669       }
1670       (any_dirty ? dirty_methods_ : clean_methods_) += num_methods;
1671     }
1672     // Assign offsets for all runtime methods in the IMT since these may hold conflict tables
1673     // live.
1674     if (as_klass->ShouldHaveImt()) {
1675       ImTable* imt = as_klass->GetImt(target_ptr_size_);
1676       if (TryAssignImTableOffset(imt, oat_index)) {
1677         // Since imt's can be shared only do this the first time to not double count imt method
1678         // fixups.
1679         for (size_t i = 0; i < ImTable::kSize; ++i) {
1680           ArtMethod* imt_method = imt->Get(i, target_ptr_size_);
1681           DCHECK(imt_method != nullptr);
1682           if (imt_method->IsRuntimeMethod() &&
1683               !IsInBootImage(imt_method) &&
1684               !NativeRelocationAssigned(imt_method)) {
1685             AssignMethodOffset(imt_method, NativeObjectRelocationType::kRuntimeMethod, oat_index);
1686           }
1687         }
1688       }
1689     }
1690   } else if (obj->IsClassLoader()) {
1691     // Register the class loader if it has a class table.
1692     // The fake boot class loader should not get registered.
1693     ObjPtr<mirror::ClassLoader> class_loader = obj->AsClassLoader();
1694     if (class_loader->GetClassTable() != nullptr) {
1695       DCHECK(compiler_options_.IsAppImage());
1696       if (class_loader == GetAppClassLoader()) {
1697         ImageInfo& image_info = GetImageInfo(oat_index);
1698         // Note: Avoid locking to prevent lock order violations from root visiting;
1699         // image_info.class_table_ table is only accessed from the image writer
1700         // and class_loader->GetClassTable() is iterated but not modified.
1701         image_info.class_table_->CopyWithoutLocks(*class_loader->GetClassTable());
1702       }
1703     }
1704   }
1705 }
1706 
NativeRelocationAssigned(void * ptr) const1707 bool ImageWriter::NativeRelocationAssigned(void* ptr) const {
1708   return native_object_relocations_.find(ptr) != native_object_relocations_.end();
1709 }
1710 
TryAssignImTableOffset(ImTable * imt,size_t oat_index)1711 bool ImageWriter::TryAssignImTableOffset(ImTable* imt, size_t oat_index) {
1712   // No offset, or already assigned.
1713   if (imt == nullptr || IsInBootImage(imt) || NativeRelocationAssigned(imt)) {
1714     return false;
1715   }
1716   // If the method is a conflict method we also want to assign the conflict table offset.
1717   ImageInfo& image_info = GetImageInfo(oat_index);
1718   const size_t size = ImTable::SizeInBytes(target_ptr_size_);
1719   native_object_relocations_.emplace(
1720       imt,
1721       NativeObjectRelocation {
1722           oat_index,
1723           image_info.GetBinSlotSize(Bin::kImTable),
1724           NativeObjectRelocationType::kIMTable});
1725   image_info.IncrementBinSlotSize(Bin::kImTable, size);
1726   return true;
1727 }
1728 
TryAssignConflictTableOffset(ImtConflictTable * table,size_t oat_index)1729 void ImageWriter::TryAssignConflictTableOffset(ImtConflictTable* table, size_t oat_index) {
1730   // No offset, or already assigned.
1731   if (table == nullptr || NativeRelocationAssigned(table)) {
1732     return;
1733   }
1734   CHECK(!IsInBootImage(table));
1735   // If the method is a conflict method we also want to assign the conflict table offset.
1736   ImageInfo& image_info = GetImageInfo(oat_index);
1737   const size_t size = table->ComputeSize(target_ptr_size_);
1738   native_object_relocations_.emplace(
1739       table,
1740       NativeObjectRelocation {
1741           oat_index,
1742           image_info.GetBinSlotSize(Bin::kIMTConflictTable),
1743           NativeObjectRelocationType::kIMTConflictTable});
1744   image_info.IncrementBinSlotSize(Bin::kIMTConflictTable, size);
1745 }
1746 
AssignMethodOffset(ArtMethod * method,NativeObjectRelocationType type,size_t oat_index)1747 void ImageWriter::AssignMethodOffset(ArtMethod* method,
1748                                      NativeObjectRelocationType type,
1749                                      size_t oat_index) {
1750   DCHECK(!IsInBootImage(method));
1751   CHECK(!NativeRelocationAssigned(method)) << "Method " << method << " already assigned "
1752       << ArtMethod::PrettyMethod(method);
1753   if (method->IsRuntimeMethod()) {
1754     TryAssignConflictTableOffset(method->GetImtConflictTable(target_ptr_size_), oat_index);
1755   }
1756   ImageInfo& image_info = GetImageInfo(oat_index);
1757   Bin bin_type = BinTypeForNativeRelocationType(type);
1758   size_t offset = image_info.GetBinSlotSize(bin_type);
1759   native_object_relocations_.emplace(method, NativeObjectRelocation { oat_index, offset, type });
1760   image_info.IncrementBinSlotSize(bin_type, ArtMethod::Size(target_ptr_size_));
1761 }
1762 
1763 class ImageWriter::LayoutHelper {
1764  public:
LayoutHelper(ImageWriter * image_writer)1765   explicit LayoutHelper(ImageWriter* image_writer)
1766       : image_writer_(image_writer) {
1767     bin_objects_.resize(image_writer_->image_infos_.size());
1768     for (auto& inner : bin_objects_) {
1769       inner.resize(enum_cast<size_t>(Bin::kMirrorCount));
1770     }
1771   }
1772 
1773   void ProcessDexFileObjects(Thread* self) REQUIRES_SHARED(Locks::mutator_lock_);
1774   void ProcessRoots(VariableSizedHandleScope* handles) REQUIRES_SHARED(Locks::mutator_lock_);
1775 
1776   void ProcessWorkQueue() REQUIRES_SHARED(Locks::mutator_lock_);
1777 
1778   void VerifyImageBinSlotsAssigned() REQUIRES_SHARED(Locks::mutator_lock_);
1779 
1780   void FinalizeBinSlotOffsets() REQUIRES_SHARED(Locks::mutator_lock_);
1781 
1782   /*
1783    * Collects the string reference info necessary for loading app images.
1784    *
1785    * Because AppImages may contain interned strings that must be deduplicated
1786    * with previously interned strings when loading the app image, we need to
1787    * visit references to these strings and update them to point to the correct
1788    * string. To speed up the visiting of references at load time we include
1789    * a list of offsets to string references in the AppImage.
1790    */
1791   void CollectStringReferenceInfo(Thread* self) REQUIRES_SHARED(Locks::mutator_lock_);
1792 
1793  private:
1794   class CollectClassesVisitor;
1795   class CollectRootsVisitor;
1796   class CollectStringReferenceVisitor;
1797   class VisitReferencesVisitor;
1798 
1799   using WorkQueue = std::deque<std::pair<ObjPtr<mirror::Object>, size_t>>;
1800 
1801   void VisitReferences(ObjPtr<mirror::Object> obj, size_t oat_index)
1802       REQUIRES_SHARED(Locks::mutator_lock_);
1803   bool TryAssignBinSlot(ObjPtr<mirror::Object> obj, size_t oat_index)
1804       REQUIRES_SHARED(Locks::mutator_lock_);
1805 
1806   ImageWriter* const image_writer_;
1807 
1808   // Work list of <object, oat_index> for objects. Everything in the queue must already be
1809   // assigned a bin slot.
1810   WorkQueue work_queue_;
1811 
1812   // Objects for individual bins. Indexed by `oat_index` and `bin`.
1813   // Cannot use ObjPtr<> because of invalidation in Heap::VisitObjects().
1814   dchecked_vector<dchecked_vector<dchecked_vector<mirror::Object*>>> bin_objects_;
1815 };
1816 
1817 class ImageWriter::LayoutHelper::CollectClassesVisitor : public ClassVisitor {
1818  public:
CollectClassesVisitor(ImageWriter * image_writer)1819   explicit CollectClassesVisitor(ImageWriter* image_writer)
1820       : image_writer_(image_writer),
1821         dex_files_(image_writer_->compiler_options_.GetDexFilesForOatFile()) {}
1822 
operator ()(ObjPtr<mirror::Class> klass)1823   bool operator()(ObjPtr<mirror::Class> klass) override REQUIRES_SHARED(Locks::mutator_lock_) {
1824     if (!image_writer_->IsInBootImage(klass.Ptr())) {
1825       ObjPtr<mirror::Class> component_type = klass;
1826       size_t dimension = 0u;
1827       while (component_type->IsArrayClass()) {
1828         ++dimension;
1829         component_type = component_type->GetComponentType();
1830       }
1831       DCHECK(!component_type->IsProxyClass());
1832       size_t dex_file_index;
1833       uint32_t class_def_index = 0u;
1834       if (UNLIKELY(component_type->IsPrimitive())) {
1835         DCHECK(image_writer_->compiler_options_.IsBootImage());
1836         dex_file_index = 0u;
1837         class_def_index = enum_cast<uint32_t>(component_type->GetPrimitiveType());
1838       } else {
1839         auto it = std::find(dex_files_.begin(), dex_files_.end(), &component_type->GetDexFile());
1840         DCHECK(it != dex_files_.end()) << klass->PrettyDescriptor();
1841         dex_file_index = std::distance(dex_files_.begin(), it) + 1u;  // 0 is for primitive types.
1842         class_def_index = component_type->GetDexClassDefIndex();
1843       }
1844       klasses_.push_back({klass, dex_file_index, class_def_index, dimension});
1845     }
1846     return true;
1847   }
1848 
SortAndReleaseClasses()1849   WorkQueue SortAndReleaseClasses()
1850       REQUIRES_SHARED(Locks::mutator_lock_) {
1851     std::sort(klasses_.begin(), klasses_.end());
1852 
1853     WorkQueue result;
1854     size_t last_dex_file_index = static_cast<size_t>(-1);
1855     size_t last_oat_index = static_cast<size_t>(-1);
1856     for (const ClassEntry& entry : klasses_) {
1857       if (last_dex_file_index != entry.dex_file_index) {
1858         if (UNLIKELY(entry.dex_file_index == 0u)) {
1859           last_oat_index = GetDefaultOatIndex();  // Primitive type.
1860         } else {
1861           uint32_t dex_file_index = entry.dex_file_index - 1u;  // 0 is for primitive types.
1862           last_oat_index = image_writer_->GetOatIndexForDexFile(dex_files_[dex_file_index]);
1863         }
1864         last_dex_file_index = entry.dex_file_index;
1865       }
1866       result.emplace_back(entry.klass, last_oat_index);
1867     }
1868     klasses_.clear();
1869     return result;
1870   }
1871 
1872  private:
1873   struct ClassEntry {
1874     ObjPtr<mirror::Class> klass;
1875     // We shall sort classes by dex file, class def index and array dimension.
1876     size_t dex_file_index;
1877     uint32_t class_def_index;
1878     size_t dimension;
1879 
operator <art::linker::ImageWriter::LayoutHelper::CollectClassesVisitor::ClassEntry1880     bool operator<(const ClassEntry& other) const {
1881       return std::tie(dex_file_index, class_def_index, dimension) <
1882              std::tie(other.dex_file_index, other.class_def_index, other.dimension);
1883     }
1884   };
1885 
1886   ImageWriter* const image_writer_;
1887   ArrayRef<const DexFile* const> dex_files_;
1888   std::deque<ClassEntry> klasses_;
1889 };
1890 
1891 class ImageWriter::LayoutHelper::CollectRootsVisitor {
1892  public:
1893   CollectRootsVisitor() = default;
1894 
ReleaseRoots()1895   std::vector<ObjPtr<mirror::Object>> ReleaseRoots() {
1896     std::vector<ObjPtr<mirror::Object>> roots;
1897     roots.swap(roots_);
1898     return roots;
1899   }
1900 
VisitRootIfNonNull(StackReference<mirror::Object> * ref)1901   void VisitRootIfNonNull(StackReference<mirror::Object>* ref) {
1902     if (!ref->IsNull()) {
1903       roots_.push_back(ref->AsMirrorPtr());
1904     }
1905   }
1906 
1907  private:
1908   std::vector<ObjPtr<mirror::Object>> roots_;
1909 };
1910 
1911 class ImageWriter::LayoutHelper::CollectStringReferenceVisitor {
1912  public:
CollectStringReferenceVisitor(const ImageWriter * image_writer,size_t oat_index,std::vector<AppImageReferenceOffsetInfo> * const string_reference_offsets,ObjPtr<mirror::Object> current_obj)1913   explicit CollectStringReferenceVisitor(
1914       const ImageWriter* image_writer,
1915       size_t oat_index,
1916       std::vector<AppImageReferenceOffsetInfo>* const string_reference_offsets,
1917       ObjPtr<mirror::Object> current_obj)
1918       : image_writer_(image_writer),
1919         oat_index_(oat_index),
1920         string_reference_offsets_(string_reference_offsets),
1921         current_obj_(current_obj) {}
1922 
VisitRootIfNonNull(mirror::CompressedReference<mirror::Object> * root) const1923   void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const
1924       REQUIRES_SHARED(Locks::mutator_lock_) {
1925     if (!root->IsNull()) {
1926       VisitRoot(root);
1927     }
1928   }
1929 
VisitRoot(mirror::CompressedReference<mirror::Object> * root) const1930   void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const
1931       REQUIRES_SHARED(Locks::mutator_lock_)  {
1932     // Only dex caches have native String roots. These are collected separately.
1933     DCHECK(current_obj_->IsDexCache() ||
1934            !image_writer_->IsInternedAppImageStringReference(root->AsMirrorPtr()))
1935         << mirror::Object::PrettyTypeOf(current_obj_);
1936   }
1937 
1938   // Collects info for managed fields that reference managed Strings.
operator ()(ObjPtr<mirror::Object> obj,MemberOffset member_offset,bool is_static ATTRIBUTE_UNUSED) const1939   void operator() (ObjPtr<mirror::Object> obj,
1940                    MemberOffset member_offset,
1941                    bool is_static ATTRIBUTE_UNUSED) const
1942       REQUIRES_SHARED(Locks::mutator_lock_) {
1943     ObjPtr<mirror::Object> referred_obj =
1944         obj->GetFieldObject<mirror::Object, kVerifyNone, kWithoutReadBarrier>(member_offset);
1945 
1946     if (image_writer_->IsInternedAppImageStringReference(referred_obj)) {
1947       size_t base_offset = image_writer_->GetImageOffset(current_obj_.Ptr(), oat_index_);
1948       string_reference_offsets_->emplace_back(base_offset, member_offset.Uint32Value());
1949     }
1950   }
1951 
1952   ALWAYS_INLINE
operator ()(ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED,ObjPtr<mirror::Reference> ref) const1953   void operator() (ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED,
1954                    ObjPtr<mirror::Reference> ref) const
1955       REQUIRES_SHARED(Locks::mutator_lock_) {
1956     operator()(ref, mirror::Reference::ReferentOffset(), /* is_static */ false);
1957   }
1958 
1959  private:
1960   const ImageWriter* const image_writer_;
1961   const size_t oat_index_;
1962   std::vector<AppImageReferenceOffsetInfo>* const string_reference_offsets_;
1963   const ObjPtr<mirror::Object> current_obj_;
1964 };
1965 
1966 class ImageWriter::LayoutHelper::VisitReferencesVisitor {
1967  public:
VisitReferencesVisitor(LayoutHelper * helper,size_t oat_index)1968   VisitReferencesVisitor(LayoutHelper* helper, size_t oat_index)
1969       : helper_(helper), oat_index_(oat_index) {}
1970 
1971   // Fix up separately since we also need to fix up method entrypoints.
VisitRootIfNonNull(mirror::CompressedReference<mirror::Object> * root) const1972   ALWAYS_INLINE void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const
1973       REQUIRES_SHARED(Locks::mutator_lock_) {
1974     if (!root->IsNull()) {
1975       VisitRoot(root);
1976     }
1977   }
1978 
VisitRoot(mirror::CompressedReference<mirror::Object> * root) const1979   ALWAYS_INLINE void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const
1980       REQUIRES_SHARED(Locks::mutator_lock_) {
1981     root->Assign(VisitReference(root->AsMirrorPtr()));
1982   }
1983 
operator ()(ObjPtr<mirror::Object> obj,MemberOffset offset,bool is_static ATTRIBUTE_UNUSED) const1984   ALWAYS_INLINE void operator() (ObjPtr<mirror::Object> obj,
1985                                  MemberOffset offset,
1986                                  bool is_static ATTRIBUTE_UNUSED) const
1987       REQUIRES_SHARED(Locks::mutator_lock_) {
1988     mirror::Object* ref =
1989         obj->GetFieldObject<mirror::Object, kVerifyNone, kWithoutReadBarrier>(offset);
1990     obj->SetFieldObject</*kTransactionActive*/false>(offset, VisitReference(ref));
1991   }
1992 
operator ()(ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED,ObjPtr<mirror::Reference> ref) const1993   ALWAYS_INLINE void operator() (ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED,
1994                                  ObjPtr<mirror::Reference> ref) const
1995       REQUIRES_SHARED(Locks::mutator_lock_) {
1996     operator()(ref, mirror::Reference::ReferentOffset(), /* is_static */ false);
1997   }
1998 
1999  private:
VisitReference(mirror::Object * ref) const2000   mirror::Object* VisitReference(mirror::Object* ref) const REQUIRES_SHARED(Locks::mutator_lock_) {
2001     if (helper_->TryAssignBinSlot(ref, oat_index_)) {
2002       // Remember how many objects we're adding at the front of the queue as we want
2003       // to reverse that range to process these references in the order of addition.
2004       helper_->work_queue_.emplace_front(ref, oat_index_);
2005     }
2006     if (ClassLinker::kAppImageMayContainStrings &&
2007         helper_->image_writer_->compiler_options_.IsAppImage() &&
2008         helper_->image_writer_->IsInternedAppImageStringReference(ref)) {
2009       helper_->image_writer_->image_infos_[oat_index_].num_string_references_ += 1u;
2010     }
2011     return ref;
2012   }
2013 
2014   LayoutHelper* const helper_;
2015   const size_t oat_index_;
2016 };
2017 
ProcessDexFileObjects(Thread * self)2018 void ImageWriter::LayoutHelper::ProcessDexFileObjects(Thread* self) {
2019   Runtime* runtime = Runtime::Current();
2020   ClassLinker* class_linker = runtime->GetClassLinker();
2021 
2022   // To ensure deterministic output, populate the work queue with objects in a pre-defined order.
2023   // Note: If we decide to implement a profile-guided layout, this is the place to do so.
2024 
2025   // Get initial work queue with the image classes and assign their bin slots.
2026   CollectClassesVisitor visitor(image_writer_);
2027   class_linker->VisitClasses(&visitor);
2028   DCHECK(work_queue_.empty());
2029   work_queue_ = visitor.SortAndReleaseClasses();
2030   for (const std::pair<ObjPtr<mirror::Object>, size_t>& entry : work_queue_) {
2031     DCHECK(entry.first->IsClass());
2032     bool assigned = TryAssignBinSlot(entry.first, entry.second);
2033     DCHECK(assigned);
2034   }
2035 
2036   // Assign bin slots to strings and dex caches.
2037   for (const DexFile* dex_file : image_writer_->compiler_options_.GetDexFilesForOatFile()) {
2038     auto it = image_writer_->dex_file_oat_index_map_.find(dex_file);
2039     DCHECK(it != image_writer_->dex_file_oat_index_map_.end()) << dex_file->GetLocation();
2040     const size_t oat_index = it->second;
2041     // Assign bin slots for strings defined in this dex file in StringId (lexicographical) order.
2042     InternTable* const intern_table = runtime->GetInternTable();
2043     for (size_t i = 0, count = dex_file->NumStringIds(); i < count; ++i) {
2044       uint32_t utf16_length;
2045       const char* utf8_data = dex_file->StringDataAndUtf16LengthByIdx(dex::StringIndex(i),
2046                                                                       &utf16_length);
2047       ObjPtr<mirror::String> string = intern_table->LookupStrong(self, utf16_length, utf8_data);
2048       if (string != nullptr && !image_writer_->IsInBootImage(string.Ptr())) {
2049         // Try to assign bin slot to this string but do not add it to the work list.
2050         // The only reference in a String is its class, processed above for the boot image.
2051         bool assigned = TryAssignBinSlot(string, oat_index);
2052         DCHECK(assigned ||
2053                // We could have seen the same string in an earlier dex file.
2054                dex_file != image_writer_->compiler_options_.GetDexFilesForOatFile().front());
2055       }
2056     }
2057     // Assign bin slot to this file's dex cache and add it to the end of the work queue.
2058     ObjPtr<mirror::DexCache> dex_cache = class_linker->FindDexCache(self, *dex_file);
2059     DCHECK(dex_cache != nullptr);
2060     bool assigned = TryAssignBinSlot(dex_cache, oat_index);
2061     DCHECK(assigned);
2062     work_queue_.emplace_back(dex_cache, oat_index);
2063   }
2064 
2065   // Since classes and dex caches have been assigned to their bins, when we process a class
2066   // we do not follow through the class references or dex caches, so we correctly process
2067   // only objects actually belonging to that class before taking a new class from the queue.
2068   // If multiple class statics reference the same object (directly or indirectly), the object
2069   // is treated as belonging to the first encountered referencing class.
2070   ProcessWorkQueue();
2071 }
2072 
ProcessRoots(VariableSizedHandleScope * handles)2073 void ImageWriter::LayoutHelper::ProcessRoots(VariableSizedHandleScope* handles) {
2074   // Assing bin slots to the image objects referenced by `handles`, add them to the work queue
2075   // and process the work queue. These objects are the image roots and boot image live objects
2076   // and they reference other objects needed for the image, for example the array of dex cache
2077   // references, or the pre-allocated exceptions for the boot image.
2078   DCHECK(work_queue_.empty());
2079   CollectRootsVisitor visitor;
2080   handles->VisitRoots(visitor);
2081   for (ObjPtr<mirror::Object> root : visitor.ReleaseRoots()) {
2082     if (TryAssignBinSlot(root, GetDefaultOatIndex())) {
2083       work_queue_.emplace_back(root, GetDefaultOatIndex());
2084     }
2085   }
2086   ProcessWorkQueue();
2087 }
2088 
ProcessWorkQueue()2089 void ImageWriter::LayoutHelper::ProcessWorkQueue() {
2090   while (!work_queue_.empty()) {
2091     std::pair<ObjPtr<mirror::Object>, size_t> pair = work_queue_.front();
2092     work_queue_.pop_front();
2093     VisitReferences(/*obj=*/ pair.first, /*oat_index=*/ pair.second);
2094   }
2095 }
2096 
VerifyImageBinSlotsAssigned()2097 void ImageWriter::LayoutHelper::VerifyImageBinSlotsAssigned() {
2098   std::vector<mirror::Object*> carveout;
2099   if (image_writer_->compiler_options_.IsAppImage()) {
2100     // Exclude boot class path dex caches that are not part of the boot image.
2101     // Also exclude their locations if they have not been visited through another path.
2102     ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
2103     Thread* self = Thread::Current();
2104     ReaderMutexLock mu(self, *Locks::dex_lock_);
2105     for (const ClassLinker::DexCacheData& data : class_linker->GetDexCachesData()) {
2106       ObjPtr<mirror::DexCache> dex_cache =
2107           ObjPtr<mirror::DexCache>::DownCast(self->DecodeJObject(data.weak_root));
2108       if (dex_cache == nullptr ||
2109           image_writer_->IsInBootImage(dex_cache.Ptr()) ||
2110           ContainsElement(image_writer_->compiler_options_.GetDexFilesForOatFile(),
2111                           dex_cache->GetDexFile())) {
2112         continue;
2113       }
2114       CHECK(!image_writer_->IsImageBinSlotAssigned(dex_cache.Ptr()));
2115       carveout.push_back(dex_cache.Ptr());
2116       ObjPtr<mirror::String> location = dex_cache->GetLocation();
2117       if (!image_writer_->IsImageBinSlotAssigned(location.Ptr())) {
2118         carveout.push_back(location.Ptr());
2119       }
2120     }
2121   }
2122 
2123   std::vector<mirror::Object*> missed_objects;
2124   auto ensure_bin_slots_assigned = [&](mirror::Object* obj)
2125       REQUIRES_SHARED(Locks::mutator_lock_) {
2126     if (!image_writer_->IsInBootImage(obj)) {
2127       if (!UNLIKELY(image_writer_->IsImageBinSlotAssigned(obj))) {
2128         // Ignore the `carveout` objects.
2129         if (ContainsElement(carveout, obj)) {
2130           return;
2131         }
2132         // Ignore finalizer references for the dalvik.system.DexFile objects referenced by
2133         // the app class loader.
2134         if (obj->IsFinalizerReferenceInstance()) {
2135           ArtField* ref_field =
2136               obj->GetClass()->FindInstanceField("referent", "Ljava/lang/Object;");
2137           CHECK(ref_field != nullptr);
2138           ObjPtr<mirror::Object> ref = ref_field->GetObject(obj);
2139           CHECK(ref != nullptr);
2140           CHECK(image_writer_->IsImageBinSlotAssigned(ref.Ptr()));
2141           ObjPtr<mirror::Class> klass = ref->GetClass();
2142           CHECK(klass == WellKnownClasses::ToClass(WellKnownClasses::dalvik_system_DexFile));
2143           // Note: The app class loader is used only for checking against the runtime
2144           // class loader, the dex file cookie is cleared and therefore we do not need
2145           // to run the finalizer even if we implement app image objects collection.
2146           ArtField* field = jni::DecodeArtField(WellKnownClasses::dalvik_system_DexFile_cookie);
2147           CHECK(field->GetObject(ref) == nullptr);
2148           return;
2149         }
2150         if (obj->IsString()) {
2151           // Ignore interned strings. These may come from reflection interning method names.
2152           // TODO: Make dex file strings weak interns and GC them before writing the image.
2153           Runtime* runtime = Runtime::Current();
2154           ObjPtr<mirror::String> interned =
2155               runtime->GetInternTable()->LookupStrong(Thread::Current(), obj->AsString());
2156           if (interned == obj) {
2157             return;
2158           }
2159         }
2160         missed_objects.push_back(obj);
2161       }
2162     }
2163   };
2164   Runtime::Current()->GetHeap()->VisitObjects(ensure_bin_slots_assigned);
2165   if (!missed_objects.empty()) {
2166     const gc::Verification* v = Runtime::Current()->GetHeap()->GetVerification();
2167     size_t num_missed_objects = missed_objects.size();
2168     size_t num_paths = std::min<size_t>(num_missed_objects, 5u);  // Do not flood the output.
2169     ArrayRef<mirror::Object*> missed_objects_head =
2170         ArrayRef<mirror::Object*>(missed_objects).SubArray(/*pos=*/ 0u, /*length=*/ num_paths);
2171     for (mirror::Object* obj : missed_objects_head) {
2172       LOG(ERROR) << "Image object without assigned bin slot: "
2173           << mirror::Object::PrettyTypeOf(obj) << " " << obj
2174           << " " << v->FirstPathFromRootSet(obj);
2175     }
2176     LOG(FATAL) << "Found " << num_missed_objects << " objects without assigned bin slots.";
2177   }
2178 }
2179 
FinalizeBinSlotOffsets()2180 void ImageWriter::LayoutHelper::FinalizeBinSlotOffsets() {
2181   // Calculate bin slot offsets and adjust for region padding if needed.
2182   const size_t region_size = image_writer_->region_size_;
2183   const size_t num_image_infos = image_writer_->image_infos_.size();
2184   for (size_t oat_index = 0; oat_index != num_image_infos; ++oat_index) {
2185     ImageInfo& image_info = image_writer_->image_infos_[oat_index];
2186     size_t bin_offset = image_writer_->image_objects_offset_begin_;
2187 
2188     for (size_t i = 0; i != kNumberOfBins; ++i) {
2189       Bin bin = enum_cast<Bin>(i);
2190       switch (bin) {
2191         case Bin::kArtMethodClean:
2192         case Bin::kArtMethodDirty: {
2193           bin_offset = RoundUp(bin_offset, ArtMethod::Alignment(image_writer_->target_ptr_size_));
2194           break;
2195         }
2196         case Bin::kDexCacheArray:
2197           bin_offset =
2198               RoundUp(bin_offset, DexCacheArraysLayout::Alignment(image_writer_->target_ptr_size_));
2199           break;
2200         case Bin::kImTable:
2201         case Bin::kIMTConflictTable: {
2202           bin_offset = RoundUp(bin_offset, static_cast<size_t>(image_writer_->target_ptr_size_));
2203           break;
2204         }
2205         default: {
2206           // Normal alignment.
2207         }
2208       }
2209       image_info.bin_slot_offsets_[i] = bin_offset;
2210 
2211       // If the bin is for mirror objects, we may need to add region padding and update offsets.
2212       if (i < enum_cast<size_t>(Bin::kMirrorCount) && region_size != 0u) {
2213         const size_t offset_after_header = bin_offset - sizeof(ImageHeader);
2214         size_t remaining_space =
2215             RoundUp(offset_after_header + 1u, region_size) - offset_after_header;
2216         // Exercise the loop below in debug builds to get coverage.
2217         if (kIsDebugBuild || remaining_space < image_info.bin_slot_sizes_[i]) {
2218           // The bin crosses a region boundary. Add padding if needed.
2219           size_t object_offset = 0u;
2220           size_t padding = 0u;
2221           for (mirror::Object* object : bin_objects_[oat_index][i]) {
2222             BinSlot bin_slot = image_writer_->GetImageBinSlot(object, oat_index);
2223             DCHECK_EQ(enum_cast<size_t>(bin_slot.GetBin()), i);
2224             DCHECK_EQ(bin_slot.GetOffset() + padding, object_offset);
2225             size_t object_size = RoundUp(object->SizeOf<kVerifyNone>(), kObjectAlignment);
2226 
2227             auto add_padding = [&](bool tail_region) {
2228               DCHECK_NE(remaining_space, 0u);
2229               DCHECK_LT(remaining_space, region_size);
2230               DCHECK_ALIGNED(remaining_space, kObjectAlignment);
2231               // TODO When copying to heap regions, leave the tail region padding zero-filled.
2232               if (!tail_region || true) {
2233                 image_info.padding_offsets_.push_back(bin_offset + object_offset);
2234               }
2235               image_info.bin_slot_sizes_[i] += remaining_space;
2236               padding += remaining_space;
2237               object_offset += remaining_space;
2238               remaining_space = region_size;
2239             };
2240             if (object_size > remaining_space) {
2241               // Padding needed if we're not at region boundary (with a multi-region object).
2242               if (remaining_space != region_size) {
2243                 // TODO: Instead of adding padding, we should consider reordering the bins
2244                 // or objects to reduce wasted space.
2245                 add_padding(/*tail_region=*/ false);
2246               }
2247               DCHECK_EQ(remaining_space, region_size);
2248               // For huge objects, adjust the remaining space to hold the object and some more.
2249               if (object_size > region_size) {
2250                 remaining_space = RoundUp(object_size + 1u, region_size);
2251               }
2252             } else if (remaining_space == object_size) {
2253               // Move to the next region, no padding needed.
2254               remaining_space += region_size;
2255             }
2256             DCHECK_GT(remaining_space, object_size);
2257             remaining_space -= object_size;
2258             image_writer_->UpdateImageBinSlotOffset(object, oat_index, object_offset);
2259             object_offset += object_size;
2260             // Add padding to the tail region of huge objects if not region-aligned.
2261             if (object_size > region_size && remaining_space != region_size) {
2262               DCHECK(!IsAlignedParam(object_size, region_size));
2263               add_padding(/*tail_region=*/ true);
2264             }
2265           }
2266           image_writer_->region_alignment_wasted_ += padding;
2267           image_info.image_end_ += padding;
2268         }
2269       }
2270       bin_offset += image_info.bin_slot_sizes_[i];
2271     }
2272     // NOTE: There may be additional padding between the bin slots and the intern table.
2273     DCHECK_EQ(
2274         image_info.image_end_,
2275         image_info.GetBinSizeSum(Bin::kMirrorCount) + image_writer_->image_objects_offset_begin_);
2276   }
2277 
2278   VLOG(image) << "Space wasted for region alignment " << image_writer_->region_alignment_wasted_;
2279 }
2280 
CollectStringReferenceInfo(Thread * self)2281 void ImageWriter::LayoutHelper::CollectStringReferenceInfo(Thread* self) {
2282   size_t managed_string_refs = 0u;
2283   size_t total_string_refs = 0u;
2284 
2285   const size_t num_image_infos = image_writer_->image_infos_.size();
2286   for (size_t oat_index = 0; oat_index != num_image_infos; ++oat_index) {
2287     ImageInfo& image_info = image_writer_->image_infos_[oat_index];
2288     DCHECK(image_info.string_reference_offsets_.empty());
2289     image_info.string_reference_offsets_.reserve(image_info.num_string_references_);
2290 
2291     for (size_t i = 0; i < enum_cast<size_t>(Bin::kMirrorCount); ++i) {
2292       for (mirror::Object* obj : bin_objects_[oat_index][i]) {
2293         CollectStringReferenceVisitor visitor(image_writer_,
2294                                               oat_index,
2295                                               &image_info.string_reference_offsets_,
2296                                               obj);
2297         /*
2298          * References to managed strings can occur either in the managed heap or in
2299          * native memory regions. Information about managed references is collected
2300          * by the CollectStringReferenceVisitor and directly added to the image info.
2301          *
2302          * Native references to managed strings can only occur through DexCache
2303          * objects. This is verified by the visitor in debug mode and the references
2304          * are collected separately below.
2305          */
2306         obj->VisitReferences</*kVisitNativeRoots=*/ kIsDebugBuild,
2307                              kVerifyNone,
2308                              kWithoutReadBarrier>(visitor, visitor);
2309       }
2310     }
2311 
2312     managed_string_refs += image_info.string_reference_offsets_.size();
2313 
2314     // Collect dex cache string arrays.
2315     for (const DexFile* dex_file : image_writer_->compiler_options_.GetDexFilesForOatFile()) {
2316       if (image_writer_->GetOatIndexForDexFile(dex_file) == oat_index) {
2317         ObjPtr<mirror::DexCache> dex_cache =
2318             Runtime::Current()->GetClassLinker()->FindDexCache(self, *dex_file);
2319         DCHECK(dex_cache != nullptr);
2320         size_t base_offset = image_writer_->GetImageOffset(dex_cache.Ptr(), oat_index);
2321 
2322         // Visit all string cache entries.
2323         mirror::StringDexCacheType* strings = dex_cache->GetStrings();
2324         const size_t num_strings = dex_cache->NumStrings();
2325         for (uint32_t index = 0; index != num_strings; ++index) {
2326           ObjPtr<mirror::String> referred_string = strings[index].load().object.Read();
2327           if (image_writer_->IsInternedAppImageStringReference(referred_string)) {
2328             image_info.string_reference_offsets_.emplace_back(
2329                 SetDexCacheStringNativeRefTag(base_offset), index);
2330           }
2331         }
2332 
2333         // Visit all pre-resolved string entries.
2334         GcRoot<mirror::String>* preresolved_strings = dex_cache->GetPreResolvedStrings();
2335         const size_t num_pre_resolved_strings = dex_cache->NumPreResolvedStrings();
2336         for (uint32_t index = 0; index != num_pre_resolved_strings; ++index) {
2337           ObjPtr<mirror::String> referred_string = preresolved_strings[index].Read();
2338           if (image_writer_->IsInternedAppImageStringReference(referred_string)) {
2339             image_info.string_reference_offsets_.emplace_back(
2340                 SetDexCachePreResolvedStringNativeRefTag(base_offset), index);
2341           }
2342         }
2343       }
2344     }
2345 
2346     total_string_refs += image_info.string_reference_offsets_.size();
2347 
2348     // Check that we collected the same number of string references as we saw in the previous pass.
2349     CHECK_EQ(image_info.string_reference_offsets_.size(), image_info.num_string_references_);
2350   }
2351 
2352   VLOG(compiler) << "Dex2Oat:AppImage:stringReferences = " << total_string_refs
2353       << " (managed: " << managed_string_refs
2354       << ", native: " << (total_string_refs - managed_string_refs) << ")";
2355 }
2356 
VisitReferences(ObjPtr<mirror::Object> obj,size_t oat_index)2357 void ImageWriter::LayoutHelper::VisitReferences(ObjPtr<mirror::Object> obj, size_t oat_index) {
2358   size_t old_work_queue_size = work_queue_.size();
2359   VisitReferencesVisitor visitor(this, oat_index);
2360   // Walk references and assign bin slots for them.
2361   obj->VisitReferences</*kVisitNativeRoots=*/ true, kVerifyNone, kWithoutReadBarrier>(
2362       visitor,
2363       visitor);
2364   // Put the added references in the queue in the order in which they were added.
2365   // The visitor just pushes them to the front as it visits them.
2366   DCHECK_LE(old_work_queue_size, work_queue_.size());
2367   size_t num_added = work_queue_.size() - old_work_queue_size;
2368   std::reverse(work_queue_.begin(), work_queue_.begin() + num_added);
2369 }
2370 
TryAssignBinSlot(ObjPtr<mirror::Object> obj,size_t oat_index)2371 bool ImageWriter::LayoutHelper::TryAssignBinSlot(ObjPtr<mirror::Object> obj, size_t oat_index) {
2372   if (obj == nullptr || image_writer_->IsInBootImage(obj.Ptr())) {
2373     // Object is null or already in the image, there is no work to do.
2374     return false;
2375   }
2376   bool assigned = false;
2377   if (!image_writer_->IsImageBinSlotAssigned(obj.Ptr())) {
2378     image_writer_->RecordNativeRelocations(obj, oat_index);
2379     Bin bin = image_writer_->AssignImageBinSlot(obj.Ptr(), oat_index);
2380     bin_objects_[oat_index][enum_cast<size_t>(bin)].push_back(obj.Ptr());
2381     assigned = true;
2382   }
2383   return assigned;
2384 }
2385 
GetBootImageLiveObjects()2386 static ObjPtr<ObjectArray<Object>> GetBootImageLiveObjects() REQUIRES_SHARED(Locks::mutator_lock_) {
2387   gc::Heap* heap = Runtime::Current()->GetHeap();
2388   DCHECK(!heap->GetBootImageSpaces().empty());
2389   const ImageHeader& primary_header = heap->GetBootImageSpaces().front()->GetImageHeader();
2390   return ObjPtr<ObjectArray<Object>>::DownCast(
2391       primary_header.GetImageRoot<kWithReadBarrier>(ImageHeader::kBootImageLiveObjects));
2392 }
2393 
CalculateNewObjectOffsets()2394 void ImageWriter::CalculateNewObjectOffsets() {
2395   Thread* const self = Thread::Current();
2396   Runtime* const runtime = Runtime::Current();
2397   VariableSizedHandleScope handles(self);
2398   MutableHandle<ObjectArray<Object>> boot_image_live_objects = handles.NewHandle(
2399       compiler_options_.IsBootImage()
2400           ? AllocateBootImageLiveObjects(self, runtime)
2401           : (compiler_options_.IsBootImageExtension() ? GetBootImageLiveObjects() : nullptr));
2402   std::vector<Handle<ObjectArray<Object>>> image_roots;
2403   for (size_t i = 0, size = oat_filenames_.size(); i != size; ++i) {
2404     image_roots.push_back(handles.NewHandle(CreateImageRoots(i, boot_image_live_objects)));
2405   }
2406 
2407   gc::Heap* const heap = runtime->GetHeap();
2408 
2409   // Leave space for the header, but do not write it yet, we need to
2410   // know where image_roots is going to end up
2411   image_objects_offset_begin_ = RoundUp(sizeof(ImageHeader), kObjectAlignment);  // 64-bit-alignment
2412 
2413   // Write the image runtime methods.
2414   image_methods_[ImageHeader::kResolutionMethod] = runtime->GetResolutionMethod();
2415   image_methods_[ImageHeader::kImtConflictMethod] = runtime->GetImtConflictMethod();
2416   image_methods_[ImageHeader::kImtUnimplementedMethod] = runtime->GetImtUnimplementedMethod();
2417   image_methods_[ImageHeader::kSaveAllCalleeSavesMethod] =
2418       runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveAllCalleeSaves);
2419   image_methods_[ImageHeader::kSaveRefsOnlyMethod] =
2420       runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsOnly);
2421   image_methods_[ImageHeader::kSaveRefsAndArgsMethod] =
2422       runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsAndArgs);
2423   image_methods_[ImageHeader::kSaveEverythingMethod] =
2424       runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverything);
2425   image_methods_[ImageHeader::kSaveEverythingMethodForClinit] =
2426       runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForClinit);
2427   image_methods_[ImageHeader::kSaveEverythingMethodForSuspendCheck] =
2428       runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForSuspendCheck);
2429   // Visit image methods first to have the main runtime methods in the first image.
2430   for (auto* m : image_methods_) {
2431     CHECK(m != nullptr);
2432     CHECK(m->IsRuntimeMethod());
2433     DCHECK_EQ(!compiler_options_.IsBootImage(), IsInBootImage(m))
2434         << "Trampolines should be in boot image";
2435     if (!IsInBootImage(m)) {
2436       AssignMethodOffset(m, NativeObjectRelocationType::kRuntimeMethod, GetDefaultOatIndex());
2437     }
2438   }
2439 
2440   // Deflate monitors before we visit roots since deflating acquires the monitor lock. Acquiring
2441   // this lock while holding other locks may cause lock order violations.
2442   {
2443     auto deflate_monitor = [](mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) {
2444       Monitor::Deflate(Thread::Current(), obj);
2445     };
2446     heap->VisitObjects(deflate_monitor);
2447   }
2448 
2449   // From this point on, there shall be no GC anymore and no objects shall be allocated.
2450   // We can now assign a BitSlot to each object and store it in its lockword.
2451 
2452   LayoutHelper layout_helper(this);
2453   layout_helper.ProcessDexFileObjects(self);
2454   layout_helper.ProcessRoots(&handles);
2455 
2456   // Verify that all objects have assigned image bin slots.
2457   layout_helper.VerifyImageBinSlotsAssigned();
2458 
2459   // Calculate size of the dex cache arrays slot and prepare offsets.
2460   PrepareDexCacheArraySlots();
2461 
2462   // Calculate the sizes of the intern tables, class tables, and fixup tables.
2463   for (ImageInfo& image_info : image_infos_) {
2464     // Calculate how big the intern table will be after being serialized.
2465     InternTable* const intern_table = image_info.intern_table_.get();
2466     CHECK_EQ(intern_table->WeakSize(), 0u) << " should have strong interned all the strings";
2467     if (intern_table->StrongSize() != 0u) {
2468       image_info.intern_table_bytes_ = intern_table->WriteToMemory(nullptr);
2469     }
2470 
2471     // Calculate the size of the class table.
2472     ReaderMutexLock mu(self, *Locks::classlinker_classes_lock_);
2473     DCHECK_EQ(image_info.class_table_->NumReferencedZygoteClasses(), 0u);
2474     if (image_info.class_table_->NumReferencedNonZygoteClasses() != 0u) {
2475       image_info.class_table_bytes_ += image_info.class_table_->WriteToMemory(nullptr);
2476     }
2477   }
2478 
2479   // Finalize bin slot offsets. This may add padding for regions.
2480   layout_helper.FinalizeBinSlotOffsets();
2481 
2482   // Collect string reference info for app images.
2483   if (ClassLinker::kAppImageMayContainStrings && compiler_options_.IsAppImage()) {
2484     layout_helper.CollectStringReferenceInfo(self);
2485   }
2486 
2487   // Calculate image offsets.
2488   size_t image_offset = 0;
2489   for (ImageInfo& image_info : image_infos_) {
2490     image_info.image_begin_ = global_image_begin_ + image_offset;
2491     image_info.image_offset_ = image_offset;
2492     image_info.image_size_ = RoundUp(image_info.CreateImageSections().first, kPageSize);
2493     // There should be no gaps until the next image.
2494     image_offset += image_info.image_size_;
2495   }
2496 
2497   size_t i = 0;
2498   for (ImageInfo& image_info : image_infos_) {
2499     image_info.image_roots_address_ = PointerToLowMemUInt32(GetImageAddress(image_roots[i].Get()));
2500     i++;
2501   }
2502 
2503   // Update the native relocations by adding their bin sums.
2504   for (auto& pair : native_object_relocations_) {
2505     NativeObjectRelocation& relocation = pair.second;
2506     Bin bin_type = BinTypeForNativeRelocationType(relocation.type);
2507     ImageInfo& image_info = GetImageInfo(relocation.oat_index);
2508     relocation.offset += image_info.GetBinSlotOffset(bin_type);
2509   }
2510 
2511   // Remember the boot image live objects as raw pointer. No GC can happen anymore.
2512   boot_image_live_objects_ = boot_image_live_objects.Get();
2513 }
2514 
CreateImageSections() const2515 std::pair<size_t, std::vector<ImageSection>> ImageWriter::ImageInfo::CreateImageSections() const {
2516   std::vector<ImageSection> sections(ImageHeader::kSectionCount);
2517 
2518   // Do not round up any sections here that are represented by the bins since it
2519   // will break offsets.
2520 
2521   /*
2522    * Objects section
2523    */
2524   sections[ImageHeader::kSectionObjects] =
2525       ImageSection(0u, image_end_);
2526 
2527   /*
2528    * Field section
2529    */
2530   sections[ImageHeader::kSectionArtFields] =
2531       ImageSection(GetBinSlotOffset(Bin::kArtField), GetBinSlotSize(Bin::kArtField));
2532 
2533   /*
2534    * Method section
2535    */
2536   sections[ImageHeader::kSectionArtMethods] =
2537       ImageSection(GetBinSlotOffset(Bin::kArtMethodClean),
2538                    GetBinSlotSize(Bin::kArtMethodClean) +
2539                    GetBinSlotSize(Bin::kArtMethodDirty));
2540 
2541   /*
2542    * IMT section
2543    */
2544   sections[ImageHeader::kSectionImTables] =
2545       ImageSection(GetBinSlotOffset(Bin::kImTable), GetBinSlotSize(Bin::kImTable));
2546 
2547   /*
2548    * Conflict Tables section
2549    */
2550   sections[ImageHeader::kSectionIMTConflictTables] =
2551       ImageSection(GetBinSlotOffset(Bin::kIMTConflictTable), GetBinSlotSize(Bin::kIMTConflictTable));
2552 
2553   /*
2554    * Runtime Methods section
2555    */
2556   sections[ImageHeader::kSectionRuntimeMethods] =
2557       ImageSection(GetBinSlotOffset(Bin::kRuntimeMethod), GetBinSlotSize(Bin::kRuntimeMethod));
2558 
2559   /*
2560    * DexCache Arrays section.
2561    */
2562   const ImageSection& dex_cache_arrays_section =
2563       sections[ImageHeader::kSectionDexCacheArrays] =
2564           ImageSection(GetBinSlotOffset(Bin::kDexCacheArray),
2565                        GetBinSlotSize(Bin::kDexCacheArray));
2566 
2567   /*
2568    * Interned Strings section
2569    */
2570 
2571   // Round up to the alignment the string table expects. See HashSet::WriteToMemory.
2572   size_t cur_pos = RoundUp(dex_cache_arrays_section.End(), sizeof(uint64_t));
2573 
2574   const ImageSection& interned_strings_section =
2575       sections[ImageHeader::kSectionInternedStrings] =
2576           ImageSection(cur_pos, intern_table_bytes_);
2577 
2578   /*
2579    * Class Table section
2580    */
2581 
2582   // Obtain the new position and round it up to the appropriate alignment.
2583   cur_pos = RoundUp(interned_strings_section.End(), sizeof(uint64_t));
2584 
2585   const ImageSection& class_table_section =
2586       sections[ImageHeader::kSectionClassTable] =
2587           ImageSection(cur_pos, class_table_bytes_);
2588 
2589   /*
2590    * String Field Offsets section
2591    */
2592 
2593   // Round up to the alignment of the offsets we are going to store.
2594   cur_pos = RoundUp(class_table_section.End(), sizeof(uint32_t));
2595 
2596   // The size of string_reference_offsets_ can't be used here because it hasn't
2597   // been filled with AppImageReferenceOffsetInfo objects yet.  The
2598   // num_string_references_ value is calculated separately, before we can
2599   // compute the actual offsets.
2600   const ImageSection& string_reference_offsets =
2601       sections[ImageHeader::kSectionStringReferenceOffsets] =
2602           ImageSection(cur_pos, sizeof(string_reference_offsets_[0]) * num_string_references_);
2603 
2604   /*
2605    * Metadata section.
2606    */
2607 
2608   // Round up to the alignment of the offsets we are going to store.
2609   cur_pos = RoundUp(string_reference_offsets.End(),
2610                     mirror::DexCache::PreResolvedStringsAlignment());
2611 
2612   const ImageSection& metadata_section =
2613       sections[ImageHeader::kSectionMetadata] =
2614           ImageSection(cur_pos, GetBinSlotSize(Bin::kMetadata));
2615 
2616   // Return the number of bytes described by these sections, and the sections
2617   // themselves.
2618   return make_pair(metadata_section.End(), std::move(sections));
2619 }
2620 
CreateHeader(size_t oat_index,size_t component_count)2621 void ImageWriter::CreateHeader(size_t oat_index, size_t component_count) {
2622   ImageInfo& image_info = GetImageInfo(oat_index);
2623   const uint8_t* oat_file_begin = image_info.oat_file_begin_;
2624   const uint8_t* oat_file_end = oat_file_begin + image_info.oat_loaded_size_;
2625   const uint8_t* oat_data_end = image_info.oat_data_begin_ + image_info.oat_size_;
2626 
2627   uint32_t image_reservation_size = image_info.image_size_;
2628   DCHECK_ALIGNED(image_reservation_size, kPageSize);
2629   uint32_t current_component_count = 1u;
2630   if (compiler_options_.IsAppImage()) {
2631     DCHECK_EQ(oat_index, 0u);
2632     DCHECK_EQ(component_count, current_component_count);
2633   } else {
2634     DCHECK(image_infos_.size() == 1u || image_infos_.size() == component_count)
2635         << image_infos_.size() << " " << component_count;
2636     if (oat_index == 0u) {
2637       const ImageInfo& last_info = image_infos_.back();
2638       const uint8_t* end = last_info.oat_file_begin_ + last_info.oat_loaded_size_;
2639       DCHECK_ALIGNED(image_info.image_begin_, kPageSize);
2640       image_reservation_size =
2641           dchecked_integral_cast<uint32_t>(RoundUp(end - image_info.image_begin_, kPageSize));
2642       current_component_count = component_count;
2643     } else {
2644       image_reservation_size = 0u;
2645       current_component_count = 0u;
2646     }
2647   }
2648 
2649   // Compute boot image checksums for the primary component, leave as 0 otherwise.
2650   uint32_t boot_image_components = 0u;
2651   uint32_t boot_image_checksums = 0u;
2652   if (oat_index == 0u) {
2653     const std::vector<gc::space::ImageSpace*>& image_spaces =
2654         Runtime::Current()->GetHeap()->GetBootImageSpaces();
2655     DCHECK_EQ(image_spaces.empty(), compiler_options_.IsBootImage());
2656     for (size_t i = 0u, size = image_spaces.size(); i != size; ) {
2657       const ImageHeader& header = image_spaces[i]->GetImageHeader();
2658       boot_image_components += header.GetComponentCount();
2659       boot_image_checksums ^= header.GetImageChecksum();
2660       DCHECK_LE(header.GetImageSpaceCount(), size - i);
2661       i += header.GetImageSpaceCount();
2662     }
2663   }
2664 
2665   // Create the image sections.
2666   auto section_info_pair = image_info.CreateImageSections();
2667   const size_t image_end = section_info_pair.first;
2668   std::vector<ImageSection>& sections = section_info_pair.second;
2669 
2670   // Finally bitmap section.
2671   const size_t bitmap_bytes = image_info.image_bitmap_.Size();
2672   auto* bitmap_section = &sections[ImageHeader::kSectionImageBitmap];
2673   *bitmap_section = ImageSection(RoundUp(image_end, kPageSize), RoundUp(bitmap_bytes, kPageSize));
2674   if (VLOG_IS_ON(compiler)) {
2675     LOG(INFO) << "Creating header for " << oat_filenames_[oat_index];
2676     size_t idx = 0;
2677     for (const ImageSection& section : sections) {
2678       LOG(INFO) << static_cast<ImageHeader::ImageSections>(idx) << " " << section;
2679       ++idx;
2680     }
2681     LOG(INFO) << "Methods: clean=" << clean_methods_ << " dirty=" << dirty_methods_;
2682     LOG(INFO) << "Image roots address=" << std::hex << image_info.image_roots_address_ << std::dec;
2683     LOG(INFO) << "Image begin=" << std::hex << reinterpret_cast<uintptr_t>(global_image_begin_)
2684               << " Image offset=" << image_info.image_offset_ << std::dec;
2685     LOG(INFO) << "Oat file begin=" << std::hex << reinterpret_cast<uintptr_t>(oat_file_begin)
2686               << " Oat data begin=" << reinterpret_cast<uintptr_t>(image_info.oat_data_begin_)
2687               << " Oat data end=" << reinterpret_cast<uintptr_t>(oat_data_end)
2688               << " Oat file end=" << reinterpret_cast<uintptr_t>(oat_file_end);
2689   }
2690 
2691   // Create the header, leave 0 for data size since we will fill this in as we are writing the
2692   // image.
2693   new (image_info.image_.Begin()) ImageHeader(
2694       image_reservation_size,
2695       current_component_count,
2696       PointerToLowMemUInt32(image_info.image_begin_),
2697       image_end,
2698       sections.data(),
2699       image_info.image_roots_address_,
2700       image_info.oat_checksum_,
2701       PointerToLowMemUInt32(oat_file_begin),
2702       PointerToLowMemUInt32(image_info.oat_data_begin_),
2703       PointerToLowMemUInt32(oat_data_end),
2704       PointerToLowMemUInt32(oat_file_end),
2705       boot_image_begin_,
2706       boot_image_size_,
2707       boot_image_components,
2708       boot_image_checksums,
2709       static_cast<uint32_t>(target_ptr_size_));
2710 }
2711 
GetImageMethodAddress(ArtMethod * method)2712 ArtMethod* ImageWriter::GetImageMethodAddress(ArtMethod* method) {
2713   NativeObjectRelocation relocation = GetNativeRelocation(method);
2714   const ImageInfo& image_info = GetImageInfo(relocation.oat_index);
2715   CHECK_GE(relocation.offset, image_info.image_end_) << "ArtMethods should be after Objects";
2716   return reinterpret_cast<ArtMethod*>(image_info.image_begin_ + relocation.offset);
2717 }
2718 
GetIntrinsicReferenceAddress(uint32_t intrinsic_data)2719 const void* ImageWriter::GetIntrinsicReferenceAddress(uint32_t intrinsic_data) {
2720   DCHECK(compiler_options_.IsBootImage());
2721   switch (IntrinsicObjects::DecodePatchType(intrinsic_data)) {
2722     case IntrinsicObjects::PatchType::kIntegerValueOfArray: {
2723       const uint8_t* base_address =
2724           reinterpret_cast<const uint8_t*>(GetImageAddress(boot_image_live_objects_));
2725       MemberOffset data_offset =
2726           IntrinsicObjects::GetIntegerValueOfArrayDataOffset(boot_image_live_objects_);
2727       return base_address + data_offset.Uint32Value();
2728     }
2729     case IntrinsicObjects::PatchType::kIntegerValueOfObject: {
2730       uint32_t index = IntrinsicObjects::DecodePatchIndex(intrinsic_data);
2731       ObjPtr<mirror::Object> value =
2732           IntrinsicObjects::GetIntegerValueOfObject(boot_image_live_objects_, index);
2733       return GetImageAddress(value.Ptr());
2734     }
2735   }
2736   LOG(FATAL) << "UNREACHABLE";
2737   UNREACHABLE();
2738 }
2739 
2740 
2741 class ImageWriter::FixupRootVisitor : public RootVisitor {
2742  public:
FixupRootVisitor(ImageWriter * image_writer)2743   explicit FixupRootVisitor(ImageWriter* image_writer) : image_writer_(image_writer) {
2744   }
2745 
VisitRoots(mirror::Object *** roots ATTRIBUTE_UNUSED,size_t count ATTRIBUTE_UNUSED,const RootInfo & info ATTRIBUTE_UNUSED)2746   void VisitRoots(mirror::Object*** roots ATTRIBUTE_UNUSED,
2747                   size_t count ATTRIBUTE_UNUSED,
2748                   const RootInfo& info ATTRIBUTE_UNUSED)
2749       override REQUIRES_SHARED(Locks::mutator_lock_) {
2750     LOG(FATAL) << "Unsupported";
2751   }
2752 
VisitRoots(mirror::CompressedReference<mirror::Object> ** roots,size_t count,const RootInfo & info ATTRIBUTE_UNUSED)2753   void VisitRoots(mirror::CompressedReference<mirror::Object>** roots,
2754                   size_t count,
2755                   const RootInfo& info ATTRIBUTE_UNUSED)
2756       override REQUIRES_SHARED(Locks::mutator_lock_) {
2757     for (size_t i = 0; i < count; ++i) {
2758       // Copy the reference. Since we do not have the address for recording the relocation,
2759       // it needs to be recorded explicitly by the user of FixupRootVisitor.
2760       ObjPtr<mirror::Object> old_ptr = roots[i]->AsMirrorPtr();
2761       roots[i]->Assign(image_writer_->GetImageAddress(old_ptr.Ptr()));
2762     }
2763   }
2764 
2765  private:
2766   ImageWriter* const image_writer_;
2767 };
2768 
CopyAndFixupImTable(ImTable * orig,ImTable * copy)2769 void ImageWriter::CopyAndFixupImTable(ImTable* orig, ImTable* copy) {
2770   for (size_t i = 0; i < ImTable::kSize; ++i) {
2771     ArtMethod* method = orig->Get(i, target_ptr_size_);
2772     void** address = reinterpret_cast<void**>(copy->AddressOfElement(i, target_ptr_size_));
2773     CopyAndFixupPointer(address, method);
2774     DCHECK_EQ(copy->Get(i, target_ptr_size_), NativeLocationInImage(method));
2775   }
2776 }
2777 
CopyAndFixupImtConflictTable(ImtConflictTable * orig,ImtConflictTable * copy)2778 void ImageWriter::CopyAndFixupImtConflictTable(ImtConflictTable* orig, ImtConflictTable* copy) {
2779   const size_t count = orig->NumEntries(target_ptr_size_);
2780   for (size_t i = 0; i < count; ++i) {
2781     ArtMethod* interface_method = orig->GetInterfaceMethod(i, target_ptr_size_);
2782     ArtMethod* implementation_method = orig->GetImplementationMethod(i, target_ptr_size_);
2783     CopyAndFixupPointer(copy->AddressOfInterfaceMethod(i, target_ptr_size_), interface_method);
2784     CopyAndFixupPointer(
2785         copy->AddressOfImplementationMethod(i, target_ptr_size_), implementation_method);
2786     DCHECK_EQ(copy->GetInterfaceMethod(i, target_ptr_size_),
2787               NativeLocationInImage(interface_method));
2788     DCHECK_EQ(copy->GetImplementationMethod(i, target_ptr_size_),
2789               NativeLocationInImage(implementation_method));
2790   }
2791 }
2792 
CopyAndFixupNativeData(size_t oat_index)2793 void ImageWriter::CopyAndFixupNativeData(size_t oat_index) {
2794   const ImageInfo& image_info = GetImageInfo(oat_index);
2795   // Copy ArtFields and methods to their locations and update the array for convenience.
2796   for (auto& pair : native_object_relocations_) {
2797     NativeObjectRelocation& relocation = pair.second;
2798     // Only work with fields and methods that are in the current oat file.
2799     if (relocation.oat_index != oat_index) {
2800       continue;
2801     }
2802     auto* dest = image_info.image_.Begin() + relocation.offset;
2803     DCHECK_GE(dest, image_info.image_.Begin() + image_info.image_end_);
2804     DCHECK(!IsInBootImage(pair.first));
2805     switch (relocation.type) {
2806       case NativeObjectRelocationType::kArtField: {
2807         memcpy(dest, pair.first, sizeof(ArtField));
2808         CopyAndFixupReference(
2809             reinterpret_cast<ArtField*>(dest)->GetDeclaringClassAddressWithoutBarrier(),
2810             reinterpret_cast<ArtField*>(pair.first)->GetDeclaringClass());
2811         break;
2812       }
2813       case NativeObjectRelocationType::kRuntimeMethod:
2814       case NativeObjectRelocationType::kArtMethodClean:
2815       case NativeObjectRelocationType::kArtMethodDirty: {
2816         CopyAndFixupMethod(reinterpret_cast<ArtMethod*>(pair.first),
2817                            reinterpret_cast<ArtMethod*>(dest),
2818                            oat_index);
2819         break;
2820       }
2821       // For arrays, copy just the header since the elements will get copied by their corresponding
2822       // relocations.
2823       case NativeObjectRelocationType::kArtFieldArray: {
2824         memcpy(dest, pair.first, LengthPrefixedArray<ArtField>::ComputeSize(0));
2825         break;
2826       }
2827       case NativeObjectRelocationType::kArtMethodArrayClean:
2828       case NativeObjectRelocationType::kArtMethodArrayDirty: {
2829         size_t size = ArtMethod::Size(target_ptr_size_);
2830         size_t alignment = ArtMethod::Alignment(target_ptr_size_);
2831         memcpy(dest, pair.first, LengthPrefixedArray<ArtMethod>::ComputeSize(0, size, alignment));
2832         // Clear padding to avoid non-deterministic data in the image.
2833         // Historical note: We also did that to placate Valgrind.
2834         reinterpret_cast<LengthPrefixedArray<ArtMethod>*>(dest)->ClearPadding(size, alignment);
2835         break;
2836       }
2837       case NativeObjectRelocationType::kDexCacheArray:
2838         // Nothing to copy here, everything is done in FixupDexCache().
2839         break;
2840       case NativeObjectRelocationType::kIMTable: {
2841         ImTable* orig_imt = reinterpret_cast<ImTable*>(pair.first);
2842         ImTable* dest_imt = reinterpret_cast<ImTable*>(dest);
2843         CopyAndFixupImTable(orig_imt, dest_imt);
2844         break;
2845       }
2846       case NativeObjectRelocationType::kIMTConflictTable: {
2847         auto* orig_table = reinterpret_cast<ImtConflictTable*>(pair.first);
2848         CopyAndFixupImtConflictTable(
2849             orig_table,
2850             new(dest)ImtConflictTable(orig_table->NumEntries(target_ptr_size_), target_ptr_size_));
2851         break;
2852       }
2853       case NativeObjectRelocationType::kGcRootPointer: {
2854         auto* orig_pointer = reinterpret_cast<GcRoot<mirror::Object>*>(pair.first);
2855         auto* dest_pointer = reinterpret_cast<GcRoot<mirror::Object>*>(dest);
2856         CopyAndFixupReference(dest_pointer->AddressWithoutBarrier(), orig_pointer->Read());
2857         break;
2858       }
2859     }
2860   }
2861   // Fixup the image method roots.
2862   auto* image_header = reinterpret_cast<ImageHeader*>(image_info.image_.Begin());
2863   for (size_t i = 0; i < ImageHeader::kImageMethodsCount; ++i) {
2864     ArtMethod* method = image_methods_[i];
2865     CHECK(method != nullptr);
2866     CopyAndFixupPointer(
2867         reinterpret_cast<void**>(&image_header->image_methods_[i]), method, PointerSize::k32);
2868   }
2869   FixupRootVisitor root_visitor(this);
2870 
2871   // Write the intern table into the image.
2872   if (image_info.intern_table_bytes_ > 0) {
2873     const ImageSection& intern_table_section = image_header->GetInternedStringsSection();
2874     InternTable* const intern_table = image_info.intern_table_.get();
2875     uint8_t* const intern_table_memory_ptr =
2876         image_info.image_.Begin() + intern_table_section.Offset();
2877     const size_t intern_table_bytes = intern_table->WriteToMemory(intern_table_memory_ptr);
2878     CHECK_EQ(intern_table_bytes, image_info.intern_table_bytes_);
2879     // Fixup the pointers in the newly written intern table to contain image addresses.
2880     InternTable temp_intern_table;
2881     // Note that we require that ReadFromMemory does not make an internal copy of the elements so
2882     // that the VisitRoots() will update the memory directly rather than the copies.
2883     // This also relies on visit roots not doing any verification which could fail after we update
2884     // the roots to be the image addresses.
2885     temp_intern_table.AddTableFromMemory(intern_table_memory_ptr,
2886                                          VoidFunctor(),
2887                                          /*is_boot_image=*/ false);
2888     CHECK_EQ(temp_intern_table.Size(), intern_table->Size());
2889     temp_intern_table.VisitRoots(&root_visitor, kVisitRootFlagAllRoots);
2890     // Record relocations. (The root visitor does not get to see the slot addresses.)
2891     MutexLock lock(Thread::Current(), *Locks::intern_table_lock_);
2892     DCHECK(!temp_intern_table.strong_interns_.tables_.empty());
2893     DCHECK(!temp_intern_table.strong_interns_.tables_[0].Empty());  // Inserted at the beginning.
2894   }
2895   // Write the class table(s) into the image. class_table_bytes_ may be 0 if there are multiple
2896   // class loaders. Writing multiple class tables into the image is currently unsupported.
2897   if (image_info.class_table_bytes_ > 0u) {
2898     const ImageSection& class_table_section = image_header->GetClassTableSection();
2899     uint8_t* const class_table_memory_ptr =
2900         image_info.image_.Begin() + class_table_section.Offset();
2901     Thread* self = Thread::Current();
2902     ReaderMutexLock mu(self, *Locks::classlinker_classes_lock_);
2903 
2904     ClassTable* table = image_info.class_table_.get();
2905     CHECK(table != nullptr);
2906     const size_t class_table_bytes = table->WriteToMemory(class_table_memory_ptr);
2907     CHECK_EQ(class_table_bytes, image_info.class_table_bytes_);
2908     // Fixup the pointers in the newly written class table to contain image addresses. See
2909     // above comment for intern tables.
2910     ClassTable temp_class_table;
2911     temp_class_table.ReadFromMemory(class_table_memory_ptr);
2912     CHECK_EQ(temp_class_table.NumReferencedZygoteClasses(),
2913              table->NumReferencedNonZygoteClasses() + table->NumReferencedZygoteClasses());
2914     UnbufferedRootVisitor visitor(&root_visitor, RootInfo(kRootUnknown));
2915     temp_class_table.VisitRoots(visitor);
2916     // Record relocations. (The root visitor does not get to see the slot addresses.)
2917     // Note that the low bits in the slots contain bits of the descriptors' hash codes
2918     // but the relocation works fine for these "adjusted" references.
2919     ReaderMutexLock lock(self, temp_class_table.lock_);
2920     DCHECK(!temp_class_table.classes_.empty());
2921     DCHECK(!temp_class_table.classes_[0].empty());  // The ClassSet was inserted at the beginning.
2922   }
2923 }
2924 
FixupPointerArray(mirror::Object * dst,mirror::PointerArray * arr,Bin array_type)2925 void ImageWriter::FixupPointerArray(mirror::Object* dst,
2926                                     mirror::PointerArray* arr,
2927                                     Bin array_type) {
2928   CHECK(arr->IsIntArray() || arr->IsLongArray()) << arr->GetClass()->PrettyClass() << " " << arr;
2929   // Fixup int and long pointers for the ArtMethod or ArtField arrays.
2930   const size_t num_elements = arr->GetLength();
2931   CopyAndFixupReference(
2932       dst->GetFieldObjectReferenceAddr<kVerifyNone>(Class::ClassOffset()), arr->GetClass());
2933   auto* dest_array = down_cast<mirror::PointerArray*>(dst);
2934   for (size_t i = 0, count = num_elements; i < count; ++i) {
2935     void* elem = arr->GetElementPtrSize<void*>(i, target_ptr_size_);
2936     if (kIsDebugBuild && elem != nullptr && !IsInBootImage(elem)) {
2937       auto it = native_object_relocations_.find(elem);
2938       if (UNLIKELY(it == native_object_relocations_.end())) {
2939         if (it->second.IsArtMethodRelocation()) {
2940           auto* method = reinterpret_cast<ArtMethod*>(elem);
2941           LOG(FATAL) << "No relocation entry for ArtMethod " << method->PrettyMethod() << " @ "
2942                      << method << " idx=" << i << "/" << num_elements << " with declaring class "
2943                      << Class::PrettyClass(method->GetDeclaringClass());
2944         } else {
2945           CHECK_EQ(array_type, Bin::kArtField);
2946           auto* field = reinterpret_cast<ArtField*>(elem);
2947           LOG(FATAL) << "No relocation entry for ArtField " << field->PrettyField() << " @ "
2948               << field << " idx=" << i << "/" << num_elements << " with declaring class "
2949               << Class::PrettyClass(field->GetDeclaringClass());
2950         }
2951         UNREACHABLE();
2952       }
2953     }
2954     CopyAndFixupPointer(dest_array->ElementAddress(i, target_ptr_size_), elem);
2955   }
2956 }
2957 
CopyAndFixupObject(Object * obj)2958 void ImageWriter::CopyAndFixupObject(Object* obj) {
2959   if (!IsImageBinSlotAssigned(obj)) {
2960     return;
2961   }
2962   size_t oat_index = GetOatIndex(obj);
2963   size_t offset = GetImageOffset(obj, oat_index);
2964   ImageInfo& image_info = GetImageInfo(oat_index);
2965   auto* dst = reinterpret_cast<Object*>(image_info.image_.Begin() + offset);
2966   DCHECK_LT(offset, image_info.image_end_);
2967   const auto* src = reinterpret_cast<const uint8_t*>(obj);
2968 
2969   image_info.image_bitmap_.Set(dst);  // Mark the obj as live.
2970 
2971   const size_t n = obj->SizeOf();
2972 
2973   if (kIsDebugBuild && region_size_ != 0u) {
2974     const size_t offset_after_header = offset - sizeof(ImageHeader);
2975     const size_t next_region = RoundUp(offset_after_header, region_size_);
2976     if (offset_after_header != next_region) {
2977       // If the object is not on a region bondary, it must not be cross region.
2978       CHECK_LT(offset_after_header, next_region)
2979           << "offset_after_header=" << offset_after_header << " size=" << n;
2980       CHECK_LE(offset_after_header + n, next_region)
2981           << "offset_after_header=" << offset_after_header << " size=" << n;
2982     }
2983   }
2984   DCHECK_LE(offset + n, image_info.image_.Size());
2985   memcpy(dst, src, n);
2986 
2987   // Write in a hash code of objects which have inflated monitors or a hash code in their monitor
2988   // word.
2989   const auto it = saved_hashcode_map_.find(obj);
2990   dst->SetLockWord(it != saved_hashcode_map_.end() ?
2991       LockWord::FromHashCode(it->second, 0u) : LockWord::Default(), false);
2992   if (kUseBakerReadBarrier && gc::collector::ConcurrentCopying::kGrayDirtyImmuneObjects) {
2993     // Treat all of the objects in the image as marked to avoid unnecessary dirty pages. This is
2994     // safe since we mark all of the objects that may reference non immune objects as gray.
2995     CHECK(dst->AtomicSetMarkBit(0, 1));
2996   }
2997   FixupObject(obj, dst);
2998 }
2999 
3000 // Rewrite all the references in the copied object to point to their image address equivalent
3001 class ImageWriter::FixupVisitor {
3002  public:
FixupVisitor(ImageWriter * image_writer,Object * copy)3003   FixupVisitor(ImageWriter* image_writer, Object* copy)
3004       : image_writer_(image_writer), copy_(copy) {
3005   }
3006 
3007   // Ignore class roots since we don't have a way to map them to the destination. These are handled
3008   // with other logic.
VisitRootIfNonNull(mirror::CompressedReference<mirror::Object> * root ATTRIBUTE_UNUSED) const3009   void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED)
3010       const {}
VisitRoot(mirror::CompressedReference<mirror::Object> * root ATTRIBUTE_UNUSED) const3011   void VisitRoot(mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED) const {}
3012 
operator ()(ObjPtr<Object> obj,MemberOffset offset,bool is_static ATTRIBUTE_UNUSED) const3013   void operator()(ObjPtr<Object> obj, MemberOffset offset, bool is_static ATTRIBUTE_UNUSED) const
3014       REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(Locks::heap_bitmap_lock_) {
3015     ObjPtr<Object> ref = obj->GetFieldObject<Object, kVerifyNone>(offset);
3016     // Copy the reference and record the fixup if necessary.
3017     image_writer_->CopyAndFixupReference(
3018         copy_->GetFieldObjectReferenceAddr<kVerifyNone>(offset), ref);
3019   }
3020 
3021   // java.lang.ref.Reference visitor.
operator ()(ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED,ObjPtr<mirror::Reference> ref) const3022   void operator()(ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED,
3023                   ObjPtr<mirror::Reference> ref) const
3024       REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(Locks::heap_bitmap_lock_) {
3025     operator()(ref, mirror::Reference::ReferentOffset(), /* is_static */ false);
3026   }
3027 
3028  protected:
3029   ImageWriter* const image_writer_;
3030   mirror::Object* const copy_;
3031 };
3032 
CopyAndFixupObjects()3033 void ImageWriter::CopyAndFixupObjects() {
3034   auto visitor = [&](Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) {
3035     DCHECK(obj != nullptr);
3036     CopyAndFixupObject(obj);
3037   };
3038   Runtime::Current()->GetHeap()->VisitObjects(visitor);
3039   // Fill the padding objects since they are required for in order traversal of the image space.
3040   for (ImageInfo& image_info : image_infos_) {
3041     for (const size_t start_offset : image_info.padding_offsets_) {
3042       const size_t offset_after_header = start_offset - sizeof(ImageHeader);
3043       size_t remaining_space =
3044           RoundUp(offset_after_header + 1u, region_size_) - offset_after_header;
3045       DCHECK_NE(remaining_space, 0u);
3046       DCHECK_LT(remaining_space, region_size_);
3047       Object* dst = reinterpret_cast<Object*>(image_info.image_.Begin() + start_offset);
3048       ObjPtr<Class> object_class = GetClassRoot<mirror::Object, kWithoutReadBarrier>();
3049       DCHECK_ALIGNED_PARAM(remaining_space, object_class->GetObjectSize());
3050       Object* end = dst + remaining_space / object_class->GetObjectSize();
3051       Class* image_object_class = GetImageAddress(object_class.Ptr());
3052       while (dst != end) {
3053         dst->SetClass<kVerifyNone>(image_object_class);
3054         dst->SetLockWord<kVerifyNone>(LockWord::Default(), /*as_volatile=*/ false);
3055         image_info.image_bitmap_.Set(dst);  // Mark the obj as live.
3056         ++dst;
3057       }
3058     }
3059   }
3060   // We no longer need the hashcode map, values have already been copied to target objects.
3061   saved_hashcode_map_.clear();
3062 }
3063 
3064 class ImageWriter::FixupClassVisitor final : public FixupVisitor {
3065  public:
FixupClassVisitor(ImageWriter * image_writer,Object * copy)3066   FixupClassVisitor(ImageWriter* image_writer, Object* copy)
3067       : FixupVisitor(image_writer, copy) {}
3068 
operator ()(ObjPtr<Object> obj,MemberOffset offset,bool is_static ATTRIBUTE_UNUSED) const3069   void operator()(ObjPtr<Object> obj, MemberOffset offset, bool is_static ATTRIBUTE_UNUSED) const
3070       REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) {
3071     DCHECK(obj->IsClass());
3072     FixupVisitor::operator()(obj, offset, /*is_static*/false);
3073   }
3074 
operator ()(ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED,ObjPtr<mirror::Reference> ref ATTRIBUTE_UNUSED) const3075   void operator()(ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED,
3076                   ObjPtr<mirror::Reference> ref ATTRIBUTE_UNUSED) const
3077       REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(Locks::heap_bitmap_lock_) {
3078     LOG(FATAL) << "Reference not expected here.";
3079   }
3080 };
3081 
GetNativeRelocation(void * obj)3082 ImageWriter::NativeObjectRelocation ImageWriter::GetNativeRelocation(void* obj) {
3083   DCHECK(obj != nullptr);
3084   DCHECK(!IsInBootImage(obj));
3085   auto it = native_object_relocations_.find(obj);
3086   CHECK(it != native_object_relocations_.end()) << obj << " spaces "
3087       << Runtime::Current()->GetHeap()->DumpSpaces();
3088   return it->second;
3089 }
3090 
3091 template <typename T>
PrettyPrint(T * ptr)3092 std::string PrettyPrint(T* ptr) REQUIRES_SHARED(Locks::mutator_lock_) {
3093   std::ostringstream oss;
3094   oss << ptr;
3095   return oss.str();
3096 }
3097 
3098 template <>
PrettyPrint(ArtMethod * method)3099 std::string PrettyPrint(ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_) {
3100   return ArtMethod::PrettyMethod(method);
3101 }
3102 
3103 template <typename T>
NativeLocationInImage(T * obj)3104 T* ImageWriter::NativeLocationInImage(T* obj) {
3105   if (obj == nullptr || IsInBootImage(obj)) {
3106     return obj;
3107   } else {
3108     NativeObjectRelocation relocation = GetNativeRelocation(obj);
3109     const ImageInfo& image_info = GetImageInfo(relocation.oat_index);
3110     return reinterpret_cast<T*>(image_info.image_begin_ + relocation.offset);
3111   }
3112 }
3113 
3114 template <typename T>
NativeCopyLocation(T * obj)3115 T* ImageWriter::NativeCopyLocation(T* obj) {
3116   const NativeObjectRelocation relocation = GetNativeRelocation(obj);
3117   const ImageInfo& image_info = GetImageInfo(relocation.oat_index);
3118   return reinterpret_cast<T*>(image_info.image_.Begin() + relocation.offset);
3119 }
3120 
3121 class ImageWriter::NativeLocationVisitor {
3122  public:
NativeLocationVisitor(ImageWriter * image_writer)3123   explicit NativeLocationVisitor(ImageWriter* image_writer)
3124       : image_writer_(image_writer) {}
3125 
3126   template <typename T>
operator ()(T * ptr,void ** dest_addr) const3127   T* operator()(T* ptr, void** dest_addr) const REQUIRES_SHARED(Locks::mutator_lock_) {
3128     if (ptr != nullptr) {
3129       image_writer_->CopyAndFixupPointer(dest_addr, ptr);
3130     }
3131     // TODO: The caller shall overwrite the value stored by CopyAndFixupPointer()
3132     // with the value we return here. We should try to avoid the duplicate work.
3133     return image_writer_->NativeLocationInImage(ptr);
3134   }
3135 
3136  private:
3137   ImageWriter* const image_writer_;
3138 };
3139 
FixupClass(mirror::Class * orig,mirror::Class * copy)3140 void ImageWriter::FixupClass(mirror::Class* orig, mirror::Class* copy) {
3141   orig->FixupNativePointers(copy, target_ptr_size_, NativeLocationVisitor(this));
3142   FixupClassVisitor visitor(this, copy);
3143   ObjPtr<mirror::Object>(orig)->VisitReferences(visitor, visitor);
3144 
3145   if (kBitstringSubtypeCheckEnabled && !compiler_options_.IsBootImage()) {
3146     // When we call SubtypeCheck::EnsureInitialize, it Assigns new bitstring
3147     // values to the parent of that class.
3148     //
3149     // Every time this happens, the parent class has to mutate to increment
3150     // the "Next" value.
3151     //
3152     // If any of these parents are in the boot image, the changes [in the parents]
3153     // would be lost when the app image is reloaded.
3154     //
3155     // To prevent newly loaded classes (not in the app image) from being reassigned
3156     // the same bitstring value as an existing app image class, uninitialize
3157     // all the classes in the app image.
3158     //
3159     // On startup, the class linker will then re-initialize all the app
3160     // image bitstrings. See also ClassLinker::AddImageSpace.
3161     //
3162     // FIXME: Deal with boot image extensions.
3163     MutexLock subtype_check_lock(Thread::Current(), *Locks::subtype_check_lock_);
3164     // Lock every time to prevent a dcheck failure when we suspend with the lock held.
3165     SubtypeCheck<mirror::Class*>::ForceUninitialize(copy);
3166   }
3167 
3168   // Remove the clinitThreadId. This is required for image determinism.
3169   copy->SetClinitThreadId(static_cast<pid_t>(0));
3170   // We never emit kRetryVerificationAtRuntime, instead we mark the class as
3171   // resolved and the class will therefore be re-verified at runtime.
3172   if (orig->ShouldVerifyAtRuntime()) {
3173     copy->SetStatusInternal(ClassStatus::kResolved);
3174   }
3175 }
3176 
FixupObject(Object * orig,Object * copy)3177 void ImageWriter::FixupObject(Object* orig, Object* copy) {
3178   DCHECK(orig != nullptr);
3179   DCHECK(copy != nullptr);
3180   if (kUseBakerReadBarrier) {
3181     orig->AssertReadBarrierState();
3182   }
3183   if (orig->IsIntArray() || orig->IsLongArray()) {
3184     // Is this a native pointer array?
3185     auto it = pointer_arrays_.find(down_cast<mirror::PointerArray*>(orig));
3186     if (it != pointer_arrays_.end()) {
3187       // Should only need to fixup every pointer array exactly once.
3188       FixupPointerArray(copy, down_cast<mirror::PointerArray*>(orig), it->second);
3189       pointer_arrays_.erase(it);
3190       return;
3191     }
3192   }
3193   if (orig->IsClass()) {
3194     FixupClass(orig->AsClass<kVerifyNone>().Ptr(), down_cast<mirror::Class*>(copy));
3195   } else {
3196     ObjPtr<mirror::ObjectArray<mirror::Class>> class_roots =
3197         Runtime::Current()->GetClassLinker()->GetClassRoots();
3198     ObjPtr<mirror::Class> klass = orig->GetClass();
3199     if (klass == GetClassRoot<mirror::Method>(class_roots) ||
3200         klass == GetClassRoot<mirror::Constructor>(class_roots)) {
3201       // Need to go update the ArtMethod.
3202       auto* dest = down_cast<mirror::Executable*>(copy);
3203       auto* src = down_cast<mirror::Executable*>(orig);
3204       ArtMethod* src_method = src->GetArtMethod();
3205       CopyAndFixupPointer(dest, mirror::Executable::ArtMethodOffset(), src_method);
3206     } else if (klass == GetClassRoot<mirror::DexCache>(class_roots)) {
3207       FixupDexCache(down_cast<mirror::DexCache*>(orig), down_cast<mirror::DexCache*>(copy));
3208     } else if (klass->IsClassLoaderClass()) {
3209       mirror::ClassLoader* copy_loader = down_cast<mirror::ClassLoader*>(copy);
3210       // If src is a ClassLoader, set the class table to null so that it gets recreated by the
3211       // ClassLoader.
3212       copy_loader->SetClassTable(nullptr);
3213       // Also set allocator to null to be safe. The allocator is created when we create the class
3214       // table. We also never expect to unload things in the image since they are held live as
3215       // roots.
3216       copy_loader->SetAllocator(nullptr);
3217     }
3218     FixupVisitor visitor(this, copy);
3219     orig->VisitReferences(visitor, visitor);
3220   }
3221 }
3222 
3223 template <typename T>
FixupDexCacheArrayEntry(std::atomic<mirror::DexCachePair<T>> * orig_array,std::atomic<mirror::DexCachePair<T>> * new_array,uint32_t array_index)3224 void ImageWriter::FixupDexCacheArrayEntry(std::atomic<mirror::DexCachePair<T>>* orig_array,
3225                                           std::atomic<mirror::DexCachePair<T>>* new_array,
3226                                           uint32_t array_index) {
3227   static_assert(sizeof(std::atomic<mirror::DexCachePair<T>>) == sizeof(mirror::DexCachePair<T>),
3228                 "Size check for removing std::atomic<>.");
3229   mirror::DexCachePair<T>* orig_pair =
3230       reinterpret_cast<mirror::DexCachePair<T>*>(&orig_array[array_index]);
3231   mirror::DexCachePair<T>* new_pair =
3232       reinterpret_cast<mirror::DexCachePair<T>*>(&new_array[array_index]);
3233   CopyAndFixupReference(
3234       new_pair->object.AddressWithoutBarrier(), orig_pair->object.Read());
3235   new_pair->index = orig_pair->index;
3236 }
3237 
3238 template <typename T>
FixupDexCacheArrayEntry(std::atomic<mirror::NativeDexCachePair<T>> * orig_array,std::atomic<mirror::NativeDexCachePair<T>> * new_array,uint32_t array_index)3239 void ImageWriter::FixupDexCacheArrayEntry(std::atomic<mirror::NativeDexCachePair<T>>* orig_array,
3240                                           std::atomic<mirror::NativeDexCachePair<T>>* new_array,
3241                                           uint32_t array_index) {
3242   static_assert(
3243       sizeof(std::atomic<mirror::NativeDexCachePair<T>>) == sizeof(mirror::NativeDexCachePair<T>),
3244       "Size check for removing std::atomic<>.");
3245   if (target_ptr_size_ == PointerSize::k64) {
3246     DexCache::ConversionPair64* orig_pair =
3247         reinterpret_cast<DexCache::ConversionPair64*>(orig_array) + array_index;
3248     DexCache::ConversionPair64* new_pair =
3249         reinterpret_cast<DexCache::ConversionPair64*>(new_array) + array_index;
3250     *new_pair = *orig_pair;  // Copy original value and index.
3251     if (orig_pair->first != 0u) {
3252       CopyAndFixupPointer(
3253           reinterpret_cast<void**>(&new_pair->first), reinterpret_cast64<void*>(orig_pair->first));
3254     }
3255   } else {
3256     DexCache::ConversionPair32* orig_pair =
3257         reinterpret_cast<DexCache::ConversionPair32*>(orig_array) + array_index;
3258     DexCache::ConversionPair32* new_pair =
3259         reinterpret_cast<DexCache::ConversionPair32*>(new_array) + array_index;
3260     *new_pair = *orig_pair;  // Copy original value and index.
3261     if (orig_pair->first != 0u) {
3262       CopyAndFixupPointer(
3263           reinterpret_cast<void**>(&new_pair->first), reinterpret_cast32<void*>(orig_pair->first));
3264     }
3265   }
3266 }
3267 
FixupDexCacheArrayEntry(GcRoot<mirror::CallSite> * orig_array,GcRoot<mirror::CallSite> * new_array,uint32_t array_index)3268 void ImageWriter::FixupDexCacheArrayEntry(GcRoot<mirror::CallSite>* orig_array,
3269                                           GcRoot<mirror::CallSite>* new_array,
3270                                           uint32_t array_index) {
3271   CopyAndFixupReference(
3272       new_array[array_index].AddressWithoutBarrier(), orig_array[array_index].Read());
3273 }
3274 
3275 template <typename EntryType>
FixupDexCacheArray(DexCache * orig_dex_cache,DexCache * copy_dex_cache,MemberOffset array_offset,uint32_t size)3276 void ImageWriter::FixupDexCacheArray(DexCache* orig_dex_cache,
3277                                      DexCache* copy_dex_cache,
3278                                      MemberOffset array_offset,
3279                                      uint32_t size) {
3280   EntryType* orig_array = orig_dex_cache->GetFieldPtr64<EntryType*>(array_offset);
3281   DCHECK_EQ(orig_array != nullptr, size != 0u);
3282   if (orig_array != nullptr) {
3283     // Though the DexCache array fields are usually treated as native pointers, we clear
3284     // the top 32 bits for 32-bit targets.
3285     CopyAndFixupPointer(copy_dex_cache, array_offset, orig_array, PointerSize::k64);
3286     EntryType* new_array = NativeCopyLocation(orig_array);
3287     for (uint32_t i = 0; i != size; ++i) {
3288       FixupDexCacheArrayEntry(orig_array, new_array, i);
3289     }
3290   }
3291 }
3292 
FixupDexCache(DexCache * orig_dex_cache,DexCache * copy_dex_cache)3293 void ImageWriter::FixupDexCache(DexCache* orig_dex_cache, DexCache* copy_dex_cache) {
3294   FixupDexCacheArray<mirror::StringDexCacheType>(orig_dex_cache,
3295                                                  copy_dex_cache,
3296                                                  DexCache::StringsOffset(),
3297                                                  orig_dex_cache->NumStrings());
3298   FixupDexCacheArray<mirror::TypeDexCacheType>(orig_dex_cache,
3299                                                copy_dex_cache,
3300                                                DexCache::ResolvedTypesOffset(),
3301                                                orig_dex_cache->NumResolvedTypes());
3302   FixupDexCacheArray<mirror::MethodDexCacheType>(orig_dex_cache,
3303                                                  copy_dex_cache,
3304                                                  DexCache::ResolvedMethodsOffset(),
3305                                                  orig_dex_cache->NumResolvedMethods());
3306   FixupDexCacheArray<mirror::FieldDexCacheType>(orig_dex_cache,
3307                                                 copy_dex_cache,
3308                                                 DexCache::ResolvedFieldsOffset(),
3309                                                 orig_dex_cache->NumResolvedFields());
3310   FixupDexCacheArray<mirror::MethodTypeDexCacheType>(orig_dex_cache,
3311                                                      copy_dex_cache,
3312                                                      DexCache::ResolvedMethodTypesOffset(),
3313                                                      orig_dex_cache->NumResolvedMethodTypes());
3314   FixupDexCacheArray<GcRoot<mirror::CallSite>>(orig_dex_cache,
3315                                                copy_dex_cache,
3316                                                DexCache::ResolvedCallSitesOffset(),
3317                                                orig_dex_cache->NumResolvedCallSites());
3318   if (orig_dex_cache->GetPreResolvedStrings() != nullptr) {
3319     CopyAndFixupPointer(copy_dex_cache,
3320                         DexCache::PreResolvedStringsOffset(),
3321                         orig_dex_cache->GetPreResolvedStrings(),
3322                         PointerSize::k64);
3323   }
3324 
3325   // Remove the DexFile pointers. They will be fixed up when the runtime loads the oat file. Leaving
3326   // compiler pointers in here will make the output non-deterministic.
3327   copy_dex_cache->SetDexFile(nullptr);
3328 }
3329 
GetOatAddress(StubType type) const3330 const uint8_t* ImageWriter::GetOatAddress(StubType type) const {
3331   DCHECK_LE(type, StubType::kLast);
3332   // If we are compiling a boot image extension or app image,
3333   // we need to use the stubs of the primary boot image.
3334   if (!compiler_options_.IsBootImage()) {
3335     // Use the current image pointers.
3336     const std::vector<gc::space::ImageSpace*>& image_spaces =
3337         Runtime::Current()->GetHeap()->GetBootImageSpaces();
3338     DCHECK(!image_spaces.empty());
3339     const OatFile* oat_file = image_spaces[0]->GetOatFile();
3340     CHECK(oat_file != nullptr);
3341     const OatHeader& header = oat_file->GetOatHeader();
3342     switch (type) {
3343       // TODO: We could maybe clean this up if we stored them in an array in the oat header.
3344       case StubType::kQuickGenericJNITrampoline:
3345         return static_cast<const uint8_t*>(header.GetQuickGenericJniTrampoline());
3346       case StubType::kJNIDlsymLookupTrampoline:
3347         return static_cast<const uint8_t*>(header.GetJniDlsymLookupTrampoline());
3348       case StubType::kJNIDlsymLookupCriticalTrampoline:
3349         return static_cast<const uint8_t*>(header.GetJniDlsymLookupCriticalTrampoline());
3350       case StubType::kQuickIMTConflictTrampoline:
3351         return static_cast<const uint8_t*>(header.GetQuickImtConflictTrampoline());
3352       case StubType::kQuickResolutionTrampoline:
3353         return static_cast<const uint8_t*>(header.GetQuickResolutionTrampoline());
3354       case StubType::kQuickToInterpreterBridge:
3355         return static_cast<const uint8_t*>(header.GetQuickToInterpreterBridge());
3356       default:
3357         UNREACHABLE();
3358     }
3359   }
3360   const ImageInfo& primary_image_info = GetImageInfo(0);
3361   return GetOatAddressForOffset(primary_image_info.GetStubOffset(type), primary_image_info);
3362 }
3363 
GetQuickCode(ArtMethod * method,const ImageInfo & image_info)3364 const uint8_t* ImageWriter::GetQuickCode(ArtMethod* method, const ImageInfo& image_info) {
3365   DCHECK(!method->IsResolutionMethod()) << method->PrettyMethod();
3366   DCHECK_NE(method, Runtime::Current()->GetImtConflictMethod()) << method->PrettyMethod();
3367   DCHECK(!method->IsImtUnimplementedMethod()) << method->PrettyMethod();
3368   DCHECK(method->IsInvokable()) << method->PrettyMethod();
3369   DCHECK(!IsInBootImage(method)) << method->PrettyMethod();
3370 
3371   // Use original code if it exists. Otherwise, set the code pointer to the resolution
3372   // trampoline.
3373 
3374   // Quick entrypoint:
3375   const void* quick_oat_entry_point =
3376       method->GetEntryPointFromQuickCompiledCodePtrSize(target_ptr_size_);
3377   const uint8_t* quick_code;
3378 
3379   if (UNLIKELY(IsInBootImage(method->GetDeclaringClass().Ptr()))) {
3380     DCHECK(method->IsCopied());
3381     // If the code is not in the oat file corresponding to this image (e.g. default methods)
3382     quick_code = reinterpret_cast<const uint8_t*>(quick_oat_entry_point);
3383   } else {
3384     uint32_t quick_oat_code_offset = PointerToLowMemUInt32(quick_oat_entry_point);
3385     quick_code = GetOatAddressForOffset(quick_oat_code_offset, image_info);
3386   }
3387 
3388   if (quick_code == nullptr) {
3389     // If we don't have code, use generic jni / interpreter bridge.
3390     // Both perform class initialization check if needed.
3391     quick_code = method->IsNative()
3392         ? GetOatAddress(StubType::kQuickGenericJNITrampoline)
3393         : GetOatAddress(StubType::kQuickToInterpreterBridge);
3394   } else if (NeedsClinitCheckBeforeCall(method) &&
3395              !method->GetDeclaringClass()->IsVisiblyInitialized()) {
3396     // If we do have code but the method needs a class initialization check before calling
3397     // that code, install the resolution stub that will perform the check.
3398     quick_code = GetOatAddress(StubType::kQuickResolutionTrampoline);
3399   }
3400   return quick_code;
3401 }
3402 
CopyAndFixupMethod(ArtMethod * orig,ArtMethod * copy,size_t oat_index)3403 void ImageWriter::CopyAndFixupMethod(ArtMethod* orig,
3404                                      ArtMethod* copy,
3405                                      size_t oat_index) {
3406   if (orig->IsAbstract()) {
3407     // Ignore the single-implementation info for abstract method.
3408     // Do this on orig instead of copy, otherwise there is a crash due to methods
3409     // are copied before classes.
3410     // TODO: handle fixup of single-implementation method for abstract method.
3411     orig->SetHasSingleImplementation(false);
3412     orig->SetSingleImplementation(
3413         nullptr, Runtime::Current()->GetClassLinker()->GetImagePointerSize());
3414   }
3415 
3416   memcpy(copy, orig, ArtMethod::Size(target_ptr_size_));
3417 
3418   CopyAndFixupReference(
3419       copy->GetDeclaringClassAddressWithoutBarrier(), orig->GetDeclaringClassUnchecked());
3420 
3421   // OatWriter replaces the code_ with an offset value. Here we re-adjust to a pointer relative to
3422   // oat_begin_
3423 
3424   // The resolution method has a special trampoline to call.
3425   Runtime* runtime = Runtime::Current();
3426   const void* quick_code;
3427   if (orig->IsRuntimeMethod()) {
3428     ImtConflictTable* orig_table = orig->GetImtConflictTable(target_ptr_size_);
3429     if (orig_table != nullptr) {
3430       // Special IMT conflict method, normal IMT conflict method or unimplemented IMT method.
3431       quick_code = GetOatAddress(StubType::kQuickIMTConflictTrampoline);
3432       CopyAndFixupPointer(copy, ArtMethod::DataOffset(target_ptr_size_), orig_table);
3433     } else if (UNLIKELY(orig == runtime->GetResolutionMethod())) {
3434       quick_code = GetOatAddress(StubType::kQuickResolutionTrampoline);
3435     } else {
3436       bool found_one = false;
3437       for (size_t i = 0; i < static_cast<size_t>(CalleeSaveType::kLastCalleeSaveType); ++i) {
3438         auto idx = static_cast<CalleeSaveType>(i);
3439         if (runtime->HasCalleeSaveMethod(idx) && runtime->GetCalleeSaveMethod(idx) == orig) {
3440           found_one = true;
3441           break;
3442         }
3443       }
3444       CHECK(found_one) << "Expected to find callee save method but got " << orig->PrettyMethod();
3445       CHECK(copy->IsRuntimeMethod());
3446       CHECK(copy->GetEntryPointFromQuickCompiledCode() == nullptr);
3447       quick_code = nullptr;
3448     }
3449   } else {
3450     // We assume all methods have code. If they don't currently then we set them to the use the
3451     // resolution trampoline. Abstract methods never have code and so we need to make sure their
3452     // use results in an AbstractMethodError. We use the interpreter to achieve this.
3453     if (UNLIKELY(!orig->IsInvokable())) {
3454       quick_code = GetOatAddress(StubType::kQuickToInterpreterBridge);
3455     } else {
3456       const ImageInfo& image_info = image_infos_[oat_index];
3457       quick_code = GetQuickCode(orig, image_info);
3458 
3459       // JNI entrypoint:
3460       if (orig->IsNative()) {
3461         // The native method's pointer is set to a stub to lookup via dlsym.
3462         // Note this is not the code_ pointer, that is handled above.
3463         StubType stub_type = orig->IsCriticalNative() ? StubType::kJNIDlsymLookupCriticalTrampoline
3464                                                       : StubType::kJNIDlsymLookupTrampoline;
3465         copy->SetEntryPointFromJniPtrSize(GetOatAddress(stub_type), target_ptr_size_);
3466       } else {
3467         CHECK(copy->GetDataPtrSize(target_ptr_size_) == nullptr);
3468       }
3469     }
3470   }
3471   if (quick_code != nullptr) {
3472     copy->SetEntryPointFromQuickCompiledCodePtrSize(quick_code, target_ptr_size_);
3473   }
3474 }
3475 
GetBinSizeSum(Bin up_to) const3476 size_t ImageWriter::ImageInfo::GetBinSizeSum(Bin up_to) const {
3477   DCHECK_LE(static_cast<size_t>(up_to), kNumberOfBins);
3478   return std::accumulate(&bin_slot_sizes_[0],
3479                          &bin_slot_sizes_[0] + static_cast<size_t>(up_to),
3480                          /*init*/ static_cast<size_t>(0));
3481 }
3482 
BinSlot(uint32_t lockword)3483 ImageWriter::BinSlot::BinSlot(uint32_t lockword) : lockword_(lockword) {
3484   // These values may need to get updated if more bins are added to the enum Bin
3485   static_assert(kBinBits == 3, "wrong number of bin bits");
3486   static_assert(kBinShift == 27, "wrong number of shift");
3487   static_assert(sizeof(BinSlot) == sizeof(LockWord), "BinSlot/LockWord must have equal sizes");
3488 
3489   DCHECK_LT(GetBin(), Bin::kMirrorCount);
3490   DCHECK_ALIGNED(GetOffset(), kObjectAlignment);
3491 }
3492 
BinSlot(Bin bin,uint32_t index)3493 ImageWriter::BinSlot::BinSlot(Bin bin, uint32_t index)
3494     : BinSlot(index | (static_cast<uint32_t>(bin) << kBinShift)) {
3495   DCHECK_EQ(index, GetOffset());
3496 }
3497 
GetBin() const3498 ImageWriter::Bin ImageWriter::BinSlot::GetBin() const {
3499   return static_cast<Bin>((lockword_ & kBinMask) >> kBinShift);
3500 }
3501 
GetOffset() const3502 uint32_t ImageWriter::BinSlot::GetOffset() const {
3503   return lockword_ & ~kBinMask;
3504 }
3505 
BinTypeForNativeRelocationType(NativeObjectRelocationType type)3506 ImageWriter::Bin ImageWriter::BinTypeForNativeRelocationType(NativeObjectRelocationType type) {
3507   switch (type) {
3508     case NativeObjectRelocationType::kArtField:
3509     case NativeObjectRelocationType::kArtFieldArray:
3510       return Bin::kArtField;
3511     case NativeObjectRelocationType::kArtMethodClean:
3512     case NativeObjectRelocationType::kArtMethodArrayClean:
3513       return Bin::kArtMethodClean;
3514     case NativeObjectRelocationType::kArtMethodDirty:
3515     case NativeObjectRelocationType::kArtMethodArrayDirty:
3516       return Bin::kArtMethodDirty;
3517     case NativeObjectRelocationType::kDexCacheArray:
3518       return Bin::kDexCacheArray;
3519     case NativeObjectRelocationType::kRuntimeMethod:
3520       return Bin::kRuntimeMethod;
3521     case NativeObjectRelocationType::kIMTable:
3522       return Bin::kImTable;
3523     case NativeObjectRelocationType::kIMTConflictTable:
3524       return Bin::kIMTConflictTable;
3525     case NativeObjectRelocationType::kGcRootPointer:
3526       return Bin::kMetadata;
3527   }
3528   UNREACHABLE();
3529 }
3530 
GetOatIndex(mirror::Object * obj) const3531 size_t ImageWriter::GetOatIndex(mirror::Object* obj) const {
3532   if (!IsMultiImage()) {
3533     return GetDefaultOatIndex();
3534   }
3535   auto it = oat_index_map_.find(obj);
3536   DCHECK(it != oat_index_map_.end()) << obj;
3537   return it->second;
3538 }
3539 
GetOatIndexForDexFile(const DexFile * dex_file) const3540 size_t ImageWriter::GetOatIndexForDexFile(const DexFile* dex_file) const {
3541   if (!IsMultiImage()) {
3542     return GetDefaultOatIndex();
3543   }
3544   auto it = dex_file_oat_index_map_.find(dex_file);
3545   DCHECK(it != dex_file_oat_index_map_.end()) << dex_file->GetLocation();
3546   return it->second;
3547 }
3548 
GetOatIndexForClass(ObjPtr<mirror::Class> klass) const3549 size_t ImageWriter::GetOatIndexForClass(ObjPtr<mirror::Class> klass) const {
3550   while (klass->IsArrayClass()) {
3551     klass = klass->GetComponentType();
3552   }
3553   if (UNLIKELY(klass->IsPrimitive())) {
3554     DCHECK(klass->GetDexCache() == nullptr);
3555     return GetDefaultOatIndex();
3556   } else {
3557     DCHECK(klass->GetDexCache() != nullptr);
3558     return GetOatIndexForDexFile(&klass->GetDexFile());
3559   }
3560 }
3561 
UpdateOatFileLayout(size_t oat_index,size_t oat_loaded_size,size_t oat_data_offset,size_t oat_data_size)3562 void ImageWriter::UpdateOatFileLayout(size_t oat_index,
3563                                       size_t oat_loaded_size,
3564                                       size_t oat_data_offset,
3565                                       size_t oat_data_size) {
3566   DCHECK_GE(oat_loaded_size, oat_data_offset);
3567   DCHECK_GE(oat_loaded_size - oat_data_offset, oat_data_size);
3568 
3569   const uint8_t* images_end = image_infos_.back().image_begin_ + image_infos_.back().image_size_;
3570   DCHECK(images_end != nullptr);  // Image space must be ready.
3571   for (const ImageInfo& info : image_infos_) {
3572     DCHECK_LE(info.image_begin_ + info.image_size_, images_end);
3573   }
3574 
3575   ImageInfo& cur_image_info = GetImageInfo(oat_index);
3576   cur_image_info.oat_file_begin_ = images_end + cur_image_info.oat_offset_;
3577   cur_image_info.oat_loaded_size_ = oat_loaded_size;
3578   cur_image_info.oat_data_begin_ = cur_image_info.oat_file_begin_ + oat_data_offset;
3579   cur_image_info.oat_size_ = oat_data_size;
3580 
3581   if (compiler_options_.IsAppImage()) {
3582     CHECK_EQ(oat_filenames_.size(), 1u) << "App image should have no next image.";
3583     return;
3584   }
3585 
3586   // Update the oat_offset of the next image info.
3587   if (oat_index + 1u != oat_filenames_.size()) {
3588     // There is a following one.
3589     ImageInfo& next_image_info = GetImageInfo(oat_index + 1u);
3590     next_image_info.oat_offset_ = cur_image_info.oat_offset_ + oat_loaded_size;
3591   }
3592 }
3593 
UpdateOatFileHeader(size_t oat_index,const OatHeader & oat_header)3594 void ImageWriter::UpdateOatFileHeader(size_t oat_index, const OatHeader& oat_header) {
3595   ImageInfo& cur_image_info = GetImageInfo(oat_index);
3596   cur_image_info.oat_checksum_ = oat_header.GetChecksum();
3597 
3598   if (oat_index == GetDefaultOatIndex()) {
3599     // Primary oat file, read the trampolines.
3600     cur_image_info.SetStubOffset(StubType::kJNIDlsymLookupTrampoline,
3601                                  oat_header.GetJniDlsymLookupTrampolineOffset());
3602     cur_image_info.SetStubOffset(StubType::kJNIDlsymLookupCriticalTrampoline,
3603                                  oat_header.GetJniDlsymLookupCriticalTrampolineOffset());
3604     cur_image_info.SetStubOffset(StubType::kQuickGenericJNITrampoline,
3605                                  oat_header.GetQuickGenericJniTrampolineOffset());
3606     cur_image_info.SetStubOffset(StubType::kQuickIMTConflictTrampoline,
3607                                  oat_header.GetQuickImtConflictTrampolineOffset());
3608     cur_image_info.SetStubOffset(StubType::kQuickResolutionTrampoline,
3609                                  oat_header.GetQuickResolutionTrampolineOffset());
3610     cur_image_info.SetStubOffset(StubType::kQuickToInterpreterBridge,
3611                                  oat_header.GetQuickToInterpreterBridgeOffset());
3612   }
3613 }
3614 
ImageWriter(const CompilerOptions & compiler_options,uintptr_t image_begin,ImageHeader::StorageMode image_storage_mode,const std::vector<std::string> & oat_filenames,const std::unordered_map<const DexFile *,size_t> & dex_file_oat_index_map,jobject class_loader,const HashSet<std::string> * dirty_image_objects)3615 ImageWriter::ImageWriter(
3616     const CompilerOptions& compiler_options,
3617     uintptr_t image_begin,
3618     ImageHeader::StorageMode image_storage_mode,
3619     const std::vector<std::string>& oat_filenames,
3620     const std::unordered_map<const DexFile*, size_t>& dex_file_oat_index_map,
3621     jobject class_loader,
3622     const HashSet<std::string>* dirty_image_objects)
3623     : compiler_options_(compiler_options),
3624       boot_image_begin_(Runtime::Current()->GetHeap()->GetBootImagesStartAddress()),
3625       boot_image_size_(Runtime::Current()->GetHeap()->GetBootImagesSize()),
3626       global_image_begin_(reinterpret_cast<uint8_t*>(image_begin)),
3627       image_objects_offset_begin_(0),
3628       target_ptr_size_(InstructionSetPointerSize(compiler_options.GetInstructionSet())),
3629       image_infos_(oat_filenames.size()),
3630       dirty_methods_(0u),
3631       clean_methods_(0u),
3632       app_class_loader_(class_loader),
3633       boot_image_live_objects_(nullptr),
3634       image_storage_mode_(image_storage_mode),
3635       oat_filenames_(oat_filenames),
3636       dex_file_oat_index_map_(dex_file_oat_index_map),
3637       dirty_image_objects_(dirty_image_objects) {
3638   DCHECK(compiler_options.IsBootImage() ||
3639          compiler_options.IsBootImageExtension() ||
3640          compiler_options.IsAppImage());
3641   DCHECK_EQ(compiler_options.IsBootImage(), boot_image_begin_ == 0u);
3642   DCHECK_EQ(compiler_options.IsBootImage(), boot_image_size_ == 0u);
3643   CHECK_NE(image_begin, 0U);
3644   std::fill_n(image_methods_, arraysize(image_methods_), nullptr);
3645   CHECK_EQ(compiler_options.IsBootImage(),
3646            Runtime::Current()->GetHeap()->GetBootImageSpaces().empty())
3647       << "Compiling a boot image should occur iff there are no boot image spaces loaded";
3648   if (compiler_options_.IsAppImage()) {
3649     // Make sure objects are not crossing region boundaries for app images.
3650     region_size_ = gc::space::RegionSpace::kRegionSize;
3651   }
3652 }
3653 
ImageInfo()3654 ImageWriter::ImageInfo::ImageInfo()
3655     : intern_table_(new InternTable),
3656       class_table_(new ClassTable) {}
3657 
3658 template <typename DestType>
CopyAndFixupReference(DestType * dest,ObjPtr<mirror::Object> src)3659 void ImageWriter::CopyAndFixupReference(DestType* dest, ObjPtr<mirror::Object> src) {
3660   static_assert(std::is_same<DestType, mirror::CompressedReference<mirror::Object>>::value ||
3661                     std::is_same<DestType, mirror::HeapReference<mirror::Object>>::value,
3662                 "DestType must be a Compressed-/HeapReference<Object>.");
3663   dest->Assign(GetImageAddress(src.Ptr()));
3664 }
3665 
CopyAndFixupPointer(void ** target,void * value,PointerSize pointer_size)3666 void ImageWriter::CopyAndFixupPointer(void** target, void* value, PointerSize pointer_size) {
3667   void* new_value = NativeLocationInImage(value);
3668   if (pointer_size == PointerSize::k32) {
3669     *reinterpret_cast<uint32_t*>(target) = reinterpret_cast32<uint32_t>(new_value);
3670   } else {
3671     *reinterpret_cast<uint64_t*>(target) = reinterpret_cast64<uint64_t>(new_value);
3672   }
3673   DCHECK(value != nullptr);
3674 }
3675 
CopyAndFixupPointer(void ** target,void * value)3676 void ImageWriter::CopyAndFixupPointer(void** target, void* value)
3677     REQUIRES_SHARED(Locks::mutator_lock_) {
3678   CopyAndFixupPointer(target, value, target_ptr_size_);
3679 }
3680 
CopyAndFixupPointer(void * object,MemberOffset offset,void * value,PointerSize pointer_size)3681 void ImageWriter::CopyAndFixupPointer(
3682     void* object, MemberOffset offset, void* value, PointerSize pointer_size) {
3683   void** target =
3684       reinterpret_cast<void**>(reinterpret_cast<uint8_t*>(object) + offset.Uint32Value());
3685   return CopyAndFixupPointer(target, value, pointer_size);
3686 }
3687 
CopyAndFixupPointer(void * object,MemberOffset offset,void * value)3688 void ImageWriter::CopyAndFixupPointer(void* object, MemberOffset offset, void* value) {
3689   return CopyAndFixupPointer(object, offset, value, target_ptr_size_);
3690 }
3691 
3692 }  // namespace linker
3693 }  // namespace art
3694