1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "image_space.h"
18 
19 #include <sys/statvfs.h>
20 #include <sys/types.h>
21 #include <unistd.h>
22 
23 #include <random>
24 
25 #include "android-base/stringprintf.h"
26 #include "android-base/strings.h"
27 #include "android-base/unique_fd.h"
28 
29 #include "arch/instruction_set.h"
30 #include "art_field-inl.h"
31 #include "art_method-inl.h"
32 #include "base/array_ref.h"
33 #include "base/bit_memory_region.h"
34 #include "base/callee_save_type.h"
35 #include "base/enums.h"
36 #include "base/file_utils.h"
37 #include "base/macros.h"
38 #include "base/memfd.h"
39 #include "base/os.h"
40 #include "base/scoped_flock.h"
41 #include "base/stl_util.h"
42 #include "base/string_view_cpp20.h"
43 #include "base/systrace.h"
44 #include "base/time_utils.h"
45 #include "base/utils.h"
46 #include "class_root.h"
47 #include "dex/art_dex_file_loader.h"
48 #include "dex/dex_file_loader.h"
49 #include "exec_utils.h"
50 #include "gc/accounting/space_bitmap-inl.h"
51 #include "gc/task_processor.h"
52 #include "image-inl.h"
53 #include "image_space_fs.h"
54 #include "intern_table-inl.h"
55 #include "mirror/class-inl.h"
56 #include "mirror/executable-inl.h"
57 #include "mirror/object-inl.h"
58 #include "mirror/object-refvisitor-inl.h"
59 #include "oat.h"
60 #include "oat_file.h"
61 #include "profile/profile_compilation_info.h"
62 #include "runtime.h"
63 #include "space-inl.h"
64 
65 namespace art {
66 namespace gc {
67 namespace space {
68 
69 using android::base::Join;
70 using android::base::StringAppendF;
71 using android::base::StringPrintf;
72 
73 // We do not allow the boot image and extensions to take more than 1GiB. They are
74 // supposed to be much smaller and allocating more that this would likely fail anyway.
75 static constexpr size_t kMaxTotalImageReservationSize = 1 * GB;
76 
77 Atomic<uint32_t> ImageSpace::bitmap_index_(0);
78 
ImageSpace(const std::string & image_filename,const char * image_location,const char * profile_file,MemMap && mem_map,accounting::ContinuousSpaceBitmap && live_bitmap,uint8_t * end)79 ImageSpace::ImageSpace(const std::string& image_filename,
80                        const char* image_location,
81                        const char* profile_file,
82                        MemMap&& mem_map,
83                        accounting::ContinuousSpaceBitmap&& live_bitmap,
84                        uint8_t* end)
85     : MemMapSpace(image_filename,
86                   std::move(mem_map),
87                   mem_map.Begin(),
88                   end,
89                   end,
90                   kGcRetentionPolicyNeverCollect),
91       live_bitmap_(std::move(live_bitmap)),
92       oat_file_non_owned_(nullptr),
93       image_location_(image_location),
94       profile_file_(profile_file) {
95   DCHECK(live_bitmap_.IsValid());
96 }
97 
ChooseRelocationOffsetDelta(int32_t min_delta,int32_t max_delta)98 static int32_t ChooseRelocationOffsetDelta(int32_t min_delta, int32_t max_delta) {
99   CHECK_ALIGNED(min_delta, kPageSize);
100   CHECK_ALIGNED(max_delta, kPageSize);
101   CHECK_LT(min_delta, max_delta);
102 
103   int32_t r = GetRandomNumber<int32_t>(min_delta, max_delta);
104   if (r % 2 == 0) {
105     r = RoundUp(r, kPageSize);
106   } else {
107     r = RoundDown(r, kPageSize);
108   }
109   CHECK_LE(min_delta, r);
110   CHECK_GE(max_delta, r);
111   CHECK_ALIGNED(r, kPageSize);
112   return r;
113 }
114 
ChooseRelocationOffsetDelta()115 static int32_t ChooseRelocationOffsetDelta() {
116   return ChooseRelocationOffsetDelta(ART_BASE_ADDRESS_MIN_DELTA, ART_BASE_ADDRESS_MAX_DELTA);
117 }
118 
GenerateImage(const std::string & image_filename,InstructionSet image_isa,std::string * error_msg)119 static bool GenerateImage(const std::string& image_filename,
120                           InstructionSet image_isa,
121                           std::string* error_msg) {
122   Runtime* runtime = Runtime::Current();
123   const std::vector<std::string>& boot_class_path = runtime->GetBootClassPath();
124   if (boot_class_path.empty()) {
125     *error_msg = "Failed to generate image because no boot class path specified";
126     return false;
127   }
128   // We should clean up so we are more likely to have room for the image.
129   if (Runtime::Current()->IsZygote()) {
130     LOG(INFO) << "Pruning dalvik-cache since we are generating an image and will need to recompile";
131     PruneDalvikCache(image_isa);
132   }
133 
134   std::vector<std::string> arg_vector;
135 
136   std::string dex2oat(Runtime::Current()->GetCompilerExecutable());
137   arg_vector.push_back(dex2oat);
138 
139   char* dex2oat_bcp = getenv("DEX2OATBOOTCLASSPATH");
140   std::vector<std::string> dex2oat_bcp_vector;
141   if (dex2oat_bcp != nullptr) {
142     arg_vector.push_back("--runtime-arg");
143     arg_vector.push_back(StringPrintf("-Xbootclasspath:%s", dex2oat_bcp));
144     Split(dex2oat_bcp, ':', &dex2oat_bcp_vector);
145   }
146 
147   std::string image_option_string("--image=");
148   image_option_string += image_filename;
149   arg_vector.push_back(image_option_string);
150 
151   if (!dex2oat_bcp_vector.empty()) {
152     for (size_t i = 0u; i < dex2oat_bcp_vector.size(); i++) {
153       arg_vector.push_back(std::string("--dex-file=") + dex2oat_bcp_vector[i]);
154       arg_vector.push_back(std::string("--dex-location=") + dex2oat_bcp_vector[i]);
155     }
156   } else {
157     const std::vector<std::string>& boot_class_path_locations =
158         runtime->GetBootClassPathLocations();
159     DCHECK_EQ(boot_class_path.size(), boot_class_path_locations.size());
160     for (size_t i = 0u; i < boot_class_path.size(); i++) {
161       arg_vector.push_back(std::string("--dex-file=") + boot_class_path[i]);
162       arg_vector.push_back(std::string("--dex-location=") + boot_class_path_locations[i]);
163     }
164   }
165 
166   std::string oat_file_option_string("--oat-file=");
167   oat_file_option_string += ImageHeader::GetOatLocationFromImageLocation(image_filename);
168   arg_vector.push_back(oat_file_option_string);
169 
170   // Note: we do not generate a fully debuggable boot image so we do not pass the
171   // compiler flag --debuggable here.
172 
173   Runtime::Current()->AddCurrentRuntimeFeaturesAsDex2OatArguments(&arg_vector);
174   CHECK_EQ(image_isa, kRuntimeISA)
175       << "We should always be generating an image for the current isa.";
176 
177   int32_t base_offset = ChooseRelocationOffsetDelta();
178   LOG(INFO) << "Using an offset of 0x" << std::hex << base_offset << " from default "
179             << "art base address of 0x" << std::hex << ART_BASE_ADDRESS;
180   arg_vector.push_back(StringPrintf("--base=0x%x", ART_BASE_ADDRESS + base_offset));
181 
182   if (!kIsTargetBuild) {
183     arg_vector.push_back("--host");
184   }
185 
186   // Check if there is a boot profile, and pass it to dex2oat.
187   if (OS::FileExists("/system/etc/boot-image.prof")) {
188     arg_vector.push_back("--profile-file=/system/etc/boot-image.prof");
189   } else {
190     // We will compile the boot image with compiler filter "speed" unless overridden below.
191     LOG(WARNING) << "Missing boot-image.prof file, /system/etc/boot-image.prof not found: "
192                  << strerror(errno);
193   }
194 
195   const std::vector<std::string>& compiler_options = Runtime::Current()->GetImageCompilerOptions();
196   for (size_t i = 0; i < compiler_options.size(); ++i) {
197     arg_vector.push_back(compiler_options[i].c_str());
198   }
199 
200   std::string command_line(Join(arg_vector, ' '));
201   LOG(INFO) << "GenerateImage: " << command_line;
202   return Exec(arg_vector, error_msg);
203 }
204 
FindImageFilenameImpl(const char * image_location,const InstructionSet image_isa,bool * has_system,std::string * system_filename,bool * dalvik_cache_exists,std::string * dalvik_cache,bool * is_global_cache,bool * has_cache,std::string * cache_filename)205 static bool FindImageFilenameImpl(const char* image_location,
206                                   const InstructionSet image_isa,
207                                   bool* has_system,
208                                   std::string* system_filename,
209                                   bool* dalvik_cache_exists,
210                                   std::string* dalvik_cache,
211                                   bool* is_global_cache,
212                                   bool* has_cache,
213                                   std::string* cache_filename) {
214   DCHECK(dalvik_cache != nullptr);
215 
216   *has_system = false;
217   *has_cache = false;
218   // image_location = /system/framework/boot.art
219   // system_image_location = /system/framework/<image_isa>/boot.art
220   std::string system_image_filename(GetSystemImageFilename(image_location, image_isa));
221   if (OS::FileExists(system_image_filename.c_str())) {
222     *system_filename = system_image_filename;
223     *has_system = true;
224   }
225 
226   bool have_android_data = false;
227   *dalvik_cache_exists = false;
228   GetDalvikCache(GetInstructionSetString(image_isa),
229                  /*create_if_absent=*/ true,
230                  dalvik_cache,
231                  &have_android_data,
232                  dalvik_cache_exists,
233                  is_global_cache);
234 
235   if (*dalvik_cache_exists) {
236     DCHECK(have_android_data);
237     // Always set output location even if it does not exist,
238     // so that the caller knows where to create the image.
239     //
240     // image_location = /system/framework/boot.art
241     // *image_filename = /data/dalvik-cache/<image_isa>/system@framework@boot.art
242     std::string error_msg;
243     if (!GetDalvikCacheFilename(image_location,
244                                 dalvik_cache->c_str(),
245                                 cache_filename,
246                                 &error_msg)) {
247       LOG(WARNING) << error_msg;
248       return *has_system;
249     }
250     *has_cache = OS::FileExists(cache_filename->c_str());
251   }
252   return *has_system || *has_cache;
253 }
254 
FindImageFilename(const char * image_location,const InstructionSet image_isa,std::string * system_filename,bool * has_system,std::string * cache_filename,bool * dalvik_cache_exists,bool * has_cache,bool * is_global_cache)255 bool ImageSpace::FindImageFilename(const char* image_location,
256                                    const InstructionSet image_isa,
257                                    std::string* system_filename,
258                                    bool* has_system,
259                                    std::string* cache_filename,
260                                    bool* dalvik_cache_exists,
261                                    bool* has_cache,
262                                    bool* is_global_cache) {
263   std::string dalvik_cache_unused;
264   return FindImageFilenameImpl(image_location,
265                                image_isa,
266                                has_system,
267                                system_filename,
268                                dalvik_cache_exists,
269                                &dalvik_cache_unused,
270                                is_global_cache,
271                                has_cache,
272                                cache_filename);
273 }
274 
ReadSpecificImageHeader(File * image_file,const char * file_description,ImageHeader * image_header,std::string * error_msg)275 static bool ReadSpecificImageHeader(File* image_file,
276                                     const char* file_description,
277                                     /*out*/ImageHeader* image_header,
278                                     /*out*/std::string* error_msg) {
279     if (!image_file->ReadFully(image_header, sizeof(ImageHeader))) {
280       *error_msg = StringPrintf("Unable to read image header from \"%s\"", file_description);
281       return false;
282     }
283     if (!image_header->IsValid()) {
284       *error_msg = StringPrintf("Image header from \"%s\" is invalid", file_description);
285       return false;
286     }
287     return true;
288 }
289 
ReadSpecificImageHeader(const char * filename,ImageHeader * image_header,std::string * error_msg)290 static bool ReadSpecificImageHeader(const char* filename,
291                                     /*out*/ImageHeader* image_header,
292                                     /*out*/std::string* error_msg) {
293   std::unique_ptr<File> image_file(OS::OpenFileForReading(filename));
294   if (image_file.get() == nullptr) {
295     *error_msg = StringPrintf("Unable to open file \"%s\" for reading image header", filename);
296     return false;
297   }
298   return ReadSpecificImageHeader(image_file.get(), filename, image_header, error_msg);
299 }
300 
ReadSpecificImageHeader(const char * filename,std::string * error_msg)301 static std::unique_ptr<ImageHeader> ReadSpecificImageHeader(const char* filename,
302                                                             std::string* error_msg) {
303   std::unique_ptr<ImageHeader> hdr(new ImageHeader);
304   if (!ReadSpecificImageHeader(filename, hdr.get(), error_msg)) {
305     return nullptr;
306   }
307   return hdr;
308 }
309 
CanWriteToDalvikCache(const InstructionSet isa)310 static bool CanWriteToDalvikCache(const InstructionSet isa) {
311   const std::string dalvik_cache = GetDalvikCache(GetInstructionSetString(isa));
312   if (access(dalvik_cache.c_str(), O_RDWR) == 0) {
313     return true;
314   } else if (errno != EACCES) {
315     PLOG(WARNING) << "CanWriteToDalvikCache returned error other than EACCES";
316   }
317   return false;
318 }
319 
ImageCreationAllowed(bool is_global_cache,const InstructionSet isa,bool is_zygote,std::string * error_msg)320 static bool ImageCreationAllowed(bool is_global_cache,
321                                  const InstructionSet isa,
322                                  bool is_zygote,
323                                  std::string* error_msg) {
324   // Anyone can write into a "local" cache.
325   if (!is_global_cache) {
326     return true;
327   }
328 
329   // Only the zygote running as root is allowed to create the global boot image.
330   // If the zygote is running as non-root (and cannot write to the dalvik-cache),
331   // then image creation is not allowed..
332   if (is_zygote) {
333     return CanWriteToDalvikCache(isa);
334   }
335 
336   *error_msg = "Only the zygote can create the global boot image.";
337   return false;
338 }
339 
VerifyImageAllocations()340 void ImageSpace::VerifyImageAllocations() {
341   uint8_t* current = Begin() + RoundUp(sizeof(ImageHeader), kObjectAlignment);
342   while (current < End()) {
343     CHECK_ALIGNED(current, kObjectAlignment);
344     auto* obj = reinterpret_cast<mirror::Object*>(current);
345     CHECK(obj->GetClass() != nullptr) << "Image object at address " << obj << " has null class";
346     CHECK(live_bitmap_.Test(obj)) << obj->PrettyTypeOf();
347     if (kUseBakerReadBarrier) {
348       obj->AssertReadBarrierState();
349     }
350     current += RoundUp(obj->SizeOf(), kObjectAlignment);
351   }
352 }
353 
354 // Helper class for relocating from one range of memory to another.
355 class RelocationRange {
356  public:
357   RelocationRange() = default;
358   RelocationRange(const RelocationRange&) = default;
RelocationRange(uintptr_t source,uintptr_t dest,uintptr_t length)359   RelocationRange(uintptr_t source, uintptr_t dest, uintptr_t length)
360       : source_(source),
361         dest_(dest),
362         length_(length) {}
363 
InSource(uintptr_t address) const364   bool InSource(uintptr_t address) const {
365     return address - source_ < length_;
366   }
367 
InDest(const void * dest) const368   bool InDest(const void* dest) const {
369     return InDest(reinterpret_cast<uintptr_t>(dest));
370   }
371 
InDest(uintptr_t address) const372   bool InDest(uintptr_t address) const {
373     return address - dest_ < length_;
374   }
375 
376   // Translate a source address to the destination space.
ToDest(uintptr_t address) const377   uintptr_t ToDest(uintptr_t address) const {
378     DCHECK(InSource(address));
379     return address + Delta();
380   }
381 
382   template <typename T>
ToDest(T * src) const383   T* ToDest(T* src) const {
384     return reinterpret_cast<T*>(ToDest(reinterpret_cast<uintptr_t>(src)));
385   }
386 
387   // Returns the delta between the dest from the source.
Delta() const388   uintptr_t Delta() const {
389     return dest_ - source_;
390   }
391 
Source() const392   uintptr_t Source() const {
393     return source_;
394   }
395 
Dest() const396   uintptr_t Dest() const {
397     return dest_;
398   }
399 
Length() const400   uintptr_t Length() const {
401     return length_;
402   }
403 
404  private:
405   const uintptr_t source_;
406   const uintptr_t dest_;
407   const uintptr_t length_;
408 };
409 
operator <<(std::ostream & os,const RelocationRange & reloc)410 std::ostream& operator<<(std::ostream& os, const RelocationRange& reloc) {
411   return os << "(" << reinterpret_cast<const void*>(reloc.Source()) << "-"
412             << reinterpret_cast<const void*>(reloc.Source() + reloc.Length()) << ")->("
413             << reinterpret_cast<const void*>(reloc.Dest()) << "-"
414             << reinterpret_cast<const void*>(reloc.Dest() + reloc.Length()) << ")";
415 }
416 
417 template <PointerSize kPointerSize, typename HeapVisitor, typename NativeVisitor>
418 class ImageSpace::PatchObjectVisitor final {
419  public:
PatchObjectVisitor(HeapVisitor heap_visitor,NativeVisitor native_visitor)420   explicit PatchObjectVisitor(HeapVisitor heap_visitor, NativeVisitor native_visitor)
421       : heap_visitor_(heap_visitor), native_visitor_(native_visitor) {}
422 
VisitClass(ObjPtr<mirror::Class> klass,ObjPtr<mirror::Class> class_class)423   void VisitClass(ObjPtr<mirror::Class> klass, ObjPtr<mirror::Class> class_class)
424       REQUIRES_SHARED(Locks::mutator_lock_) {
425     // A mirror::Class object consists of
426     //  - instance fields inherited from j.l.Object,
427     //  - instance fields inherited from j.l.Class,
428     //  - embedded tables (vtable, interface method table),
429     //  - static fields of the class itself.
430     // The reference fields are at the start of each field section (this is how the
431     // ClassLinker orders fields; except when that would create a gap between superclass
432     // fields and the first reference of the subclass due to alignment, it can be filled
433     // with smaller fields - but that's not the case for j.l.Object and j.l.Class).
434 
435     DCHECK_ALIGNED(klass.Ptr(), kObjectAlignment);
436     static_assert(IsAligned<kHeapReferenceSize>(kObjectAlignment), "Object alignment check.");
437     // First, patch the `klass->klass_`, known to be a reference to the j.l.Class.class.
438     // This should be the only reference field in j.l.Object and we assert that below.
439     DCHECK_EQ(class_class,
440               heap_visitor_(klass->GetClass<kVerifyNone, kWithoutReadBarrier>()));
441     klass->SetFieldObjectWithoutWriteBarrier<
442         /*kTransactionActive=*/ false,
443         /*kCheckTransaction=*/ true,
444         kVerifyNone>(mirror::Object::ClassOffset(), class_class);
445     // Then patch the reference instance fields described by j.l.Class.class.
446     // Use the sizeof(Object) to determine where these reference fields start;
447     // this is the same as `class_class->GetFirstReferenceInstanceFieldOffset()`
448     // after patching but the j.l.Class may not have been patched yet.
449     size_t num_reference_instance_fields = class_class->NumReferenceInstanceFields<kVerifyNone>();
450     DCHECK_NE(num_reference_instance_fields, 0u);
451     static_assert(IsAligned<kHeapReferenceSize>(sizeof(mirror::Object)), "Size alignment check.");
452     MemberOffset instance_field_offset(sizeof(mirror::Object));
453     for (size_t i = 0; i != num_reference_instance_fields; ++i) {
454       PatchReferenceField(klass, instance_field_offset);
455       static_assert(sizeof(mirror::HeapReference<mirror::Object>) == kHeapReferenceSize,
456                     "Heap reference sizes equality check.");
457       instance_field_offset =
458           MemberOffset(instance_field_offset.Uint32Value() + kHeapReferenceSize);
459     }
460     // Now that we have patched the `super_class_`, if this is the j.l.Class.class,
461     // we can get a reference to j.l.Object.class and assert that it has only one
462     // reference instance field (the `klass_` patched above).
463     if (kIsDebugBuild && klass == class_class) {
464       ObjPtr<mirror::Class> object_class =
465           klass->GetSuperClass<kVerifyNone, kWithoutReadBarrier>();
466       CHECK_EQ(object_class->NumReferenceInstanceFields<kVerifyNone>(), 1u);
467     }
468     // Then patch static fields.
469     size_t num_reference_static_fields = klass->NumReferenceStaticFields<kVerifyNone>();
470     if (num_reference_static_fields != 0u) {
471       MemberOffset static_field_offset =
472           klass->GetFirstReferenceStaticFieldOffset<kVerifyNone>(kPointerSize);
473       for (size_t i = 0; i != num_reference_static_fields; ++i) {
474         PatchReferenceField(klass, static_field_offset);
475         static_assert(sizeof(mirror::HeapReference<mirror::Object>) == kHeapReferenceSize,
476                       "Heap reference sizes equality check.");
477         static_field_offset =
478             MemberOffset(static_field_offset.Uint32Value() + kHeapReferenceSize);
479       }
480     }
481     // Then patch native pointers.
482     klass->FixupNativePointers<kVerifyNone>(klass.Ptr(), kPointerSize, *this);
483   }
484 
485   template <typename T>
operator ()(T * ptr,void ** dest_addr ATTRIBUTE_UNUSED) const486   T* operator()(T* ptr, void** dest_addr ATTRIBUTE_UNUSED) const {
487     return (ptr != nullptr) ? native_visitor_(ptr) : nullptr;
488   }
489 
VisitPointerArray(ObjPtr<mirror::PointerArray> pointer_array)490   void VisitPointerArray(ObjPtr<mirror::PointerArray> pointer_array)
491       REQUIRES_SHARED(Locks::mutator_lock_) {
492     // Fully patch the pointer array, including the `klass_` field.
493     PatchReferenceField</*kMayBeNull=*/ false>(pointer_array, mirror::Object::ClassOffset());
494 
495     int32_t length = pointer_array->GetLength<kVerifyNone>();
496     for (int32_t i = 0; i != length; ++i) {
497       ArtMethod** method_entry = reinterpret_cast<ArtMethod**>(
498           pointer_array->ElementAddress<kVerifyNone>(i, kPointerSize));
499       PatchNativePointer</*kMayBeNull=*/ false>(method_entry);
500     }
501   }
502 
VisitObject(mirror::Object * object)503   void VisitObject(mirror::Object* object) REQUIRES_SHARED(Locks::mutator_lock_) {
504     // Visit all reference fields.
505     object->VisitReferences</*kVisitNativeRoots=*/ false,
506                             kVerifyNone,
507                             kWithoutReadBarrier>(*this, *this);
508     // This function should not be called for classes.
509     DCHECK(!object->IsClass<kVerifyNone>());
510   }
511 
512   // Visitor for VisitReferences().
operator ()(ObjPtr<mirror::Object> object,MemberOffset field_offset,bool is_static) const513   ALWAYS_INLINE void operator()(ObjPtr<mirror::Object> object,
514                                 MemberOffset field_offset,
515                                 bool is_static)
516       const REQUIRES_SHARED(Locks::mutator_lock_) {
517     DCHECK(!is_static);
518     PatchReferenceField(object, field_offset);
519   }
520   // Visitor for VisitReferences(), java.lang.ref.Reference case.
operator ()(ObjPtr<mirror::Class> klass,ObjPtr<mirror::Reference> ref) const521   ALWAYS_INLINE void operator()(ObjPtr<mirror::Class> klass, ObjPtr<mirror::Reference> ref) const
522       REQUIRES_SHARED(Locks::mutator_lock_) {
523     DCHECK(klass->IsTypeOfReferenceClass());
524     this->operator()(ref, mirror::Reference::ReferentOffset(), /*is_static=*/ false);
525   }
526   // Ignore class native roots; not called from VisitReferences() for kVisitNativeRoots == false.
VisitRootIfNonNull(mirror::CompressedReference<mirror::Object> * root ATTRIBUTE_UNUSED) const527   void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED)
528       const {}
VisitRoot(mirror::CompressedReference<mirror::Object> * root ATTRIBUTE_UNUSED) const529   void VisitRoot(mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED) const {}
530 
VisitDexCacheArrays(ObjPtr<mirror::DexCache> dex_cache)531   void VisitDexCacheArrays(ObjPtr<mirror::DexCache> dex_cache)
532       REQUIRES_SHARED(Locks::mutator_lock_) {
533     ScopedTrace st("VisitDexCacheArrays");
534     FixupDexCacheArray<mirror::StringDexCacheType>(dex_cache,
535                                                    mirror::DexCache::StringsOffset(),
536                                                    dex_cache->NumStrings<kVerifyNone>());
537     FixupDexCacheArray<mirror::TypeDexCacheType>(dex_cache,
538                                                  mirror::DexCache::ResolvedTypesOffset(),
539                                                  dex_cache->NumResolvedTypes<kVerifyNone>());
540     FixupDexCacheArray<mirror::MethodDexCacheType>(dex_cache,
541                                                    mirror::DexCache::ResolvedMethodsOffset(),
542                                                    dex_cache->NumResolvedMethods<kVerifyNone>());
543     FixupDexCacheArray<mirror::FieldDexCacheType>(dex_cache,
544                                                   mirror::DexCache::ResolvedFieldsOffset(),
545                                                   dex_cache->NumResolvedFields<kVerifyNone>());
546     FixupDexCacheArray<mirror::MethodTypeDexCacheType>(
547         dex_cache,
548         mirror::DexCache::ResolvedMethodTypesOffset(),
549         dex_cache->NumResolvedMethodTypes<kVerifyNone>());
550     FixupDexCacheArray<GcRoot<mirror::CallSite>>(
551         dex_cache,
552         mirror::DexCache::ResolvedCallSitesOffset(),
553         dex_cache->NumResolvedCallSites<kVerifyNone>());
554     FixupDexCacheArray<GcRoot<mirror::String>>(
555         dex_cache,
556         mirror::DexCache::PreResolvedStringsOffset(),
557         dex_cache->NumPreResolvedStrings<kVerifyNone>());
558   }
559 
560   template <bool kMayBeNull = true, typename T>
PatchGcRoot(GcRoot<T> * root) const561   ALWAYS_INLINE void PatchGcRoot(/*inout*/GcRoot<T>* root) const
562       REQUIRES_SHARED(Locks::mutator_lock_) {
563     static_assert(sizeof(GcRoot<mirror::Class*>) == sizeof(uint32_t), "GcRoot size check");
564     T* old_value = root->template Read<kWithoutReadBarrier>();
565     DCHECK(kMayBeNull || old_value != nullptr);
566     if (!kMayBeNull || old_value != nullptr) {
567       *root = GcRoot<T>(heap_visitor_(old_value));
568     }
569   }
570 
571   template <bool kMayBeNull = true, typename T>
PatchNativePointer(T ** entry) const572   ALWAYS_INLINE void PatchNativePointer(/*inout*/T** entry) const {
573     if (kPointerSize == PointerSize::k64) {
574       uint64_t* raw_entry = reinterpret_cast<uint64_t*>(entry);
575       T* old_value = reinterpret_cast64<T*>(*raw_entry);
576       DCHECK(kMayBeNull || old_value != nullptr);
577       if (!kMayBeNull || old_value != nullptr) {
578         T* new_value = native_visitor_(old_value);
579         *raw_entry = reinterpret_cast64<uint64_t>(new_value);
580       }
581     } else {
582       uint32_t* raw_entry = reinterpret_cast<uint32_t*>(entry);
583       T* old_value = reinterpret_cast32<T*>(*raw_entry);
584       DCHECK(kMayBeNull || old_value != nullptr);
585       if (!kMayBeNull || old_value != nullptr) {
586         T* new_value = native_visitor_(old_value);
587         *raw_entry = reinterpret_cast32<uint32_t>(new_value);
588       }
589     }
590   }
591 
592   template <bool kMayBeNull = true>
PatchReferenceField(ObjPtr<mirror::Object> object,MemberOffset offset) const593   ALWAYS_INLINE void PatchReferenceField(ObjPtr<mirror::Object> object, MemberOffset offset) const
594       REQUIRES_SHARED(Locks::mutator_lock_) {
595     ObjPtr<mirror::Object> old_value =
596         object->GetFieldObject<mirror::Object, kVerifyNone, kWithoutReadBarrier>(offset);
597     DCHECK(kMayBeNull || old_value != nullptr);
598     if (!kMayBeNull || old_value != nullptr) {
599       ObjPtr<mirror::Object> new_value = heap_visitor_(old_value.Ptr());
600       object->SetFieldObjectWithoutWriteBarrier</*kTransactionActive=*/ false,
601                                                 /*kCheckTransaction=*/ true,
602                                                 kVerifyNone>(offset, new_value);
603     }
604   }
605 
606   template <typename T>
FixupDexCacheArrayEntry(std::atomic<mirror::DexCachePair<T>> * array,uint32_t index)607   void FixupDexCacheArrayEntry(std::atomic<mirror::DexCachePair<T>>* array, uint32_t index)
608       REQUIRES_SHARED(Locks::mutator_lock_) {
609     static_assert(sizeof(std::atomic<mirror::DexCachePair<T>>) == sizeof(mirror::DexCachePair<T>),
610                   "Size check for removing std::atomic<>.");
611     PatchGcRoot(&(reinterpret_cast<mirror::DexCachePair<T>*>(array)[index].object));
612   }
613 
614   template <typename T>
FixupDexCacheArrayEntry(std::atomic<mirror::NativeDexCachePair<T>> * array,uint32_t index)615   void FixupDexCacheArrayEntry(std::atomic<mirror::NativeDexCachePair<T>>* array, uint32_t index)
616       REQUIRES_SHARED(Locks::mutator_lock_) {
617     static_assert(sizeof(std::atomic<mirror::NativeDexCachePair<T>>) ==
618                       sizeof(mirror::NativeDexCachePair<T>),
619                   "Size check for removing std::atomic<>.");
620     mirror::NativeDexCachePair<T> pair =
621         mirror::DexCache::GetNativePairPtrSize(array, index, kPointerSize);
622     if (pair.object != nullptr) {
623       pair.object = native_visitor_(pair.object);
624       mirror::DexCache::SetNativePairPtrSize(array, index, pair, kPointerSize);
625     }
626   }
627 
FixupDexCacheArrayEntry(GcRoot<mirror::CallSite> * array,uint32_t index)628   void FixupDexCacheArrayEntry(GcRoot<mirror::CallSite>* array, uint32_t index)
629       REQUIRES_SHARED(Locks::mutator_lock_) {
630     PatchGcRoot(&array[index]);
631   }
632 
FixupDexCacheArrayEntry(GcRoot<mirror::String> * array,uint32_t index)633   void FixupDexCacheArrayEntry(GcRoot<mirror::String>* array, uint32_t index)
634       REQUIRES_SHARED(Locks::mutator_lock_) {
635     PatchGcRoot(&array[index]);
636   }
637 
638   template <typename EntryType>
FixupDexCacheArray(ObjPtr<mirror::DexCache> dex_cache,MemberOffset array_offset,uint32_t size)639   void FixupDexCacheArray(ObjPtr<mirror::DexCache> dex_cache,
640                           MemberOffset array_offset,
641                           uint32_t size) REQUIRES_SHARED(Locks::mutator_lock_) {
642     EntryType* old_array =
643         reinterpret_cast64<EntryType*>(dex_cache->GetField64<kVerifyNone>(array_offset));
644     DCHECK_EQ(old_array != nullptr, size != 0u);
645     if (old_array != nullptr) {
646       EntryType* new_array = native_visitor_(old_array);
647       dex_cache->SetField64<kVerifyNone>(array_offset, reinterpret_cast64<uint64_t>(new_array));
648       for (uint32_t i = 0; i != size; ++i) {
649         FixupDexCacheArrayEntry(new_array, i);
650       }
651     }
652   }
653 
654  private:
655   // Heap objects visitor.
656   HeapVisitor heap_visitor_;
657 
658   // Native objects visitor.
659   NativeVisitor native_visitor_;
660 };
661 
662 template <typename ReferenceVisitor>
663 class ImageSpace::ClassTableVisitor final {
664  public:
ClassTableVisitor(const ReferenceVisitor & reference_visitor)665   explicit ClassTableVisitor(const ReferenceVisitor& reference_visitor)
666       : reference_visitor_(reference_visitor) {}
667 
VisitRoot(mirror::CompressedReference<mirror::Object> * root) const668   void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const
669       REQUIRES_SHARED(Locks::mutator_lock_) {
670     DCHECK(root->AsMirrorPtr() != nullptr);
671     root->Assign(reference_visitor_(root->AsMirrorPtr()));
672   }
673 
674  private:
675   ReferenceVisitor reference_visitor_;
676 };
677 
678 class ImageSpace::RemapInternedStringsVisitor {
679  public:
RemapInternedStringsVisitor(const SafeMap<mirror::String *,mirror::String * > & intern_remap)680   explicit RemapInternedStringsVisitor(
681       const SafeMap<mirror::String*, mirror::String*>& intern_remap)
682       REQUIRES_SHARED(Locks::mutator_lock_)
683       : intern_remap_(intern_remap),
684         string_class_(GetStringClass()) {}
685 
686   // Visitor for VisitReferences().
operator ()(ObjPtr<mirror::Object> object,MemberOffset field_offset,bool is_static ATTRIBUTE_UNUSED) const687   ALWAYS_INLINE void operator()(ObjPtr<mirror::Object> object,
688                                 MemberOffset field_offset,
689                                 bool is_static ATTRIBUTE_UNUSED)
690       const REQUIRES_SHARED(Locks::mutator_lock_) {
691     ObjPtr<mirror::Object> old_value =
692         object->GetFieldObject<mirror::Object, kVerifyNone, kWithoutReadBarrier>(field_offset);
693     if (old_value != nullptr &&
694         old_value->GetClass<kVerifyNone, kWithoutReadBarrier>() == string_class_) {
695       auto it = intern_remap_.find(old_value->AsString().Ptr());
696       if (it != intern_remap_.end()) {
697         mirror::String* new_value = it->second;
698         object->SetFieldObjectWithoutWriteBarrier</*kTransactionActive=*/ false,
699                                                   /*kCheckTransaction=*/ true,
700                                                   kVerifyNone>(field_offset, new_value);
701       }
702     }
703   }
704   // Visitor for VisitReferences(), java.lang.ref.Reference case.
operator ()(ObjPtr<mirror::Class> klass,ObjPtr<mirror::Reference> ref) const705   ALWAYS_INLINE void operator()(ObjPtr<mirror::Class> klass, ObjPtr<mirror::Reference> ref) const
706       REQUIRES_SHARED(Locks::mutator_lock_) {
707     DCHECK(klass->IsTypeOfReferenceClass());
708     this->operator()(ref, mirror::Reference::ReferentOffset(), /*is_static=*/ false);
709   }
710   // Ignore class native roots; not called from VisitReferences() for kVisitNativeRoots == false.
VisitRootIfNonNull(mirror::CompressedReference<mirror::Object> * root ATTRIBUTE_UNUSED) const711   void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED)
712       const {}
VisitRoot(mirror::CompressedReference<mirror::Object> * root ATTRIBUTE_UNUSED) const713   void VisitRoot(mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED) const {}
714 
715  private:
GetStringClass()716   mirror::Class* GetStringClass() REQUIRES_SHARED(Locks::mutator_lock_) {
717     DCHECK(!intern_remap_.empty());
718     return intern_remap_.begin()->first->GetClass<kVerifyNone, kWithoutReadBarrier>();
719   }
720 
721   const SafeMap<mirror::String*, mirror::String*>& intern_remap_;
722   mirror::Class* const string_class_;
723 };
724 
725 // Helper class encapsulating loading, so we can access private ImageSpace members (this is a
726 // nested class), but not declare functions in the header.
727 class ImageSpace::Loader {
728  public:
InitAppImage(const char * image_filename,const char * image_location,const OatFile * oat_file,ArrayRef<ImageSpace * const> boot_image_spaces,std::string * error_msg)729   static std::unique_ptr<ImageSpace> InitAppImage(const char* image_filename,
730                                                   const char* image_location,
731                                                   const OatFile* oat_file,
732                                                   ArrayRef<ImageSpace* const> boot_image_spaces,
733                                                   /*out*/std::string* error_msg)
734       REQUIRES_SHARED(Locks::mutator_lock_) {
735     TimingLogger logger(__PRETTY_FUNCTION__, /*precise=*/ true, VLOG_IS_ON(image));
736 
737     std::unique_ptr<ImageSpace> space = Init(image_filename,
738                                              image_location,
739                                              &logger,
740                                              /*image_reservation=*/ nullptr,
741                                              error_msg);
742     if (space != nullptr) {
743       space->oat_file_non_owned_ = oat_file;
744       const ImageHeader& image_header = space->GetImageHeader();
745 
746       // Check the oat file checksum.
747       const uint32_t oat_checksum = oat_file->GetOatHeader().GetChecksum();
748       const uint32_t image_oat_checksum = image_header.GetOatChecksum();
749       if (oat_checksum != image_oat_checksum) {
750         *error_msg = StringPrintf("Oat checksum 0x%x does not match the image one 0x%x in image %s",
751                                   oat_checksum,
752                                   image_oat_checksum,
753                                   image_filename);
754         return nullptr;
755       }
756       size_t boot_image_space_dependencies;
757       if (!ValidateBootImageChecksum(image_filename,
758                                      image_header,
759                                      oat_file,
760                                      boot_image_spaces,
761                                      &boot_image_space_dependencies,
762                                      error_msg)) {
763         DCHECK(!error_msg->empty());
764         return nullptr;
765       }
766 
767       uint32_t expected_reservation_size = RoundUp(image_header.GetImageSize(), kPageSize);
768       if (!CheckImageReservationSize(*space, expected_reservation_size, error_msg) ||
769           !CheckImageComponentCount(*space, /*expected_component_count=*/ 1u, error_msg)) {
770         return nullptr;
771       }
772 
773       {
774         TimingLogger::ScopedTiming timing("RelocateImage", &logger);
775         const PointerSize pointer_size = image_header.GetPointerSize();
776         uint32_t boot_image_begin =
777             reinterpret_cast32<uint32_t>(boot_image_spaces.front()->Begin());
778         bool result;
779         if (pointer_size == PointerSize::k64) {
780           result = RelocateInPlace<PointerSize::k64>(boot_image_begin,
781                                                      space->GetMemMap()->Begin(),
782                                                      space->GetLiveBitmap(),
783                                                      oat_file,
784                                                      error_msg);
785         } else {
786           result = RelocateInPlace<PointerSize::k32>(boot_image_begin,
787                                                      space->GetMemMap()->Begin(),
788                                                      space->GetLiveBitmap(),
789                                                      oat_file,
790                                                      error_msg);
791         }
792         if (!result) {
793           return nullptr;
794         }
795       }
796 
797       DCHECK_LE(boot_image_space_dependencies, boot_image_spaces.size());
798       if (boot_image_space_dependencies != boot_image_spaces.size()) {
799         TimingLogger::ScopedTiming timing("DeduplicateInternedStrings", &logger);
800         // There shall be no duplicates with boot image spaces this app image depends on.
801         ArrayRef<ImageSpace* const> old_spaces =
802             boot_image_spaces.SubArray(/*pos=*/ boot_image_space_dependencies);
803         SafeMap<mirror::String*, mirror::String*> intern_remap;
804         RemoveInternTableDuplicates(old_spaces, space.get(), &intern_remap);
805         if (!intern_remap.empty()) {
806           RemapInternedStringDuplicates(intern_remap, space.get());
807         }
808       }
809 
810       const ImageHeader& primary_header = boot_image_spaces.front()->GetImageHeader();
811       static_assert(static_cast<size_t>(ImageHeader::kResolutionMethod) == 0u);
812       for (size_t i = 0u; i != static_cast<size_t>(ImageHeader::kImageMethodsCount); ++i) {
813         ImageHeader::ImageMethod method = static_cast<ImageHeader::ImageMethod>(i);
814         CHECK_EQ(primary_header.GetImageMethod(method), image_header.GetImageMethod(method))
815             << method;
816       }
817 
818       VLOG(image) << "ImageSpace::Loader::InitAppImage exiting " << *space.get();
819     }
820     if (VLOG_IS_ON(image)) {
821       logger.Dump(LOG_STREAM(INFO));
822     }
823     return space;
824   }
825 
Init(const char * image_filename,const char * image_location,TimingLogger * logger,MemMap * image_reservation,std::string * error_msg)826   static std::unique_ptr<ImageSpace> Init(const char* image_filename,
827                                           const char* image_location,
828                                           TimingLogger* logger,
829                                           /*inout*/MemMap* image_reservation,
830                                           /*out*/std::string* error_msg)
831       REQUIRES_SHARED(Locks::mutator_lock_) {
832     CHECK(image_filename != nullptr);
833     CHECK(image_location != nullptr);
834 
835     std::unique_ptr<File> file;
836     {
837       TimingLogger::ScopedTiming timing("OpenImageFile", logger);
838       file.reset(OS::OpenFileForReading(image_filename));
839       if (file == nullptr) {
840         *error_msg = StringPrintf("Failed to open '%s'", image_filename);
841         return nullptr;
842       }
843     }
844     return Init(file.get(),
845                 image_filename,
846                 image_location,
847                 /* profile_file=*/ "",
848                 /*allow_direct_mapping=*/ true,
849                 logger,
850                 image_reservation,
851                 error_msg);
852   }
853 
Init(File * file,const char * image_filename,const char * image_location,const char * profile_file,bool allow_direct_mapping,TimingLogger * logger,MemMap * image_reservation,std::string * error_msg)854   static std::unique_ptr<ImageSpace> Init(File* file,
855                                           const char* image_filename,
856                                           const char* image_location,
857                                           const char* profile_file,
858                                           bool allow_direct_mapping,
859                                           TimingLogger* logger,
860                                           /*inout*/MemMap* image_reservation,
861                                           /*out*/std::string* error_msg)
862       REQUIRES_SHARED(Locks::mutator_lock_) {
863     CHECK(image_filename != nullptr);
864     CHECK(image_location != nullptr);
865 
866     VLOG(image) << "ImageSpace::Init entering image_filename=" << image_filename;
867 
868     ImageHeader image_header;
869     {
870       TimingLogger::ScopedTiming timing("ReadImageHeader", logger);
871       bool success = file->PreadFully(&image_header, sizeof(image_header), /*offset=*/ 0u);
872       if (!success || !image_header.IsValid()) {
873         *error_msg = StringPrintf("Invalid image header in '%s'", image_filename);
874         return nullptr;
875       }
876     }
877     // Check that the file is larger or equal to the header size + data size.
878     const uint64_t image_file_size = static_cast<uint64_t>(file->GetLength());
879     if (image_file_size < sizeof(ImageHeader) + image_header.GetDataSize()) {
880       *error_msg = StringPrintf(
881           "Image file truncated: %" PRIu64 " vs. %" PRIu64 ".",
882            image_file_size,
883            static_cast<uint64_t>(sizeof(ImageHeader) + image_header.GetDataSize()));
884       return nullptr;
885     }
886 
887     if (VLOG_IS_ON(startup)) {
888       LOG(INFO) << "Dumping image sections";
889       for (size_t i = 0; i < ImageHeader::kSectionCount; ++i) {
890         const auto section_idx = static_cast<ImageHeader::ImageSections>(i);
891         auto& section = image_header.GetImageSection(section_idx);
892         LOG(INFO) << section_idx << " start="
893             << reinterpret_cast<void*>(image_header.GetImageBegin() + section.Offset()) << " "
894             << section;
895       }
896     }
897 
898     const auto& bitmap_section = image_header.GetImageBitmapSection();
899     // The location we want to map from is the first aligned page after the end of the stored
900     // (possibly compressed) data.
901     const size_t image_bitmap_offset =
902         RoundUp(sizeof(ImageHeader) + image_header.GetDataSize(), kPageSize);
903     const size_t end_of_bitmap = image_bitmap_offset + bitmap_section.Size();
904     if (end_of_bitmap != image_file_size) {
905       *error_msg = StringPrintf(
906           "Image file size does not equal end of bitmap: size=%" PRIu64 " vs. %zu.",
907           image_file_size,
908           end_of_bitmap);
909       return nullptr;
910     }
911 
912     // GetImageBegin is the preferred address to map the image. If we manage to map the
913     // image at the image begin, the amount of fixup work required is minimized.
914     // If it is pic we will retry with error_msg for the2 failure case. Pass a null error_msg to
915     // avoid reading proc maps for a mapping failure and slowing everything down.
916     // For the boot image, we have already reserved the memory and we load the image
917     // into the `image_reservation`.
918     MemMap map = LoadImageFile(
919         image_filename,
920         image_location,
921         image_header,
922         file->Fd(),
923         allow_direct_mapping,
924         logger,
925         image_reservation,
926         error_msg);
927     if (!map.IsValid()) {
928       DCHECK(!error_msg->empty());
929       return nullptr;
930     }
931     DCHECK_EQ(0, memcmp(&image_header, map.Begin(), sizeof(ImageHeader)));
932 
933     MemMap image_bitmap_map = MemMap::MapFile(bitmap_section.Size(),
934                                               PROT_READ,
935                                               MAP_PRIVATE,
936                                               file->Fd(),
937                                               image_bitmap_offset,
938                                               /*low_4gb=*/ false,
939                                               image_filename,
940                                               error_msg);
941     if (!image_bitmap_map.IsValid()) {
942       *error_msg = StringPrintf("Failed to map image bitmap: %s", error_msg->c_str());
943       return nullptr;
944     }
945     const uint32_t bitmap_index = ImageSpace::bitmap_index_.fetch_add(1);
946     std::string bitmap_name(StringPrintf("imagespace %s live-bitmap %u",
947                                          image_filename,
948                                          bitmap_index));
949     // Bitmap only needs to cover until the end of the mirror objects section.
950     const ImageSection& image_objects = image_header.GetObjectsSection();
951     // We only want the mirror object, not the ArtFields and ArtMethods.
952     uint8_t* const image_end = map.Begin() + image_objects.End();
953     accounting::ContinuousSpaceBitmap bitmap;
954     {
955       TimingLogger::ScopedTiming timing("CreateImageBitmap", logger);
956       bitmap = accounting::ContinuousSpaceBitmap::CreateFromMemMap(
957           bitmap_name,
958           std::move(image_bitmap_map),
959           reinterpret_cast<uint8_t*>(map.Begin()),
960           // Make sure the bitmap is aligned to card size instead of just bitmap word size.
961           RoundUp(image_objects.End(), gc::accounting::CardTable::kCardSize));
962       if (!bitmap.IsValid()) {
963         *error_msg = StringPrintf("Could not create bitmap '%s'", bitmap_name.c_str());
964         return nullptr;
965       }
966     }
967     // We only want the mirror object, not the ArtFields and ArtMethods.
968     std::unique_ptr<ImageSpace> space(new ImageSpace(image_filename,
969                                                      image_location,
970                                                      profile_file,
971                                                      std::move(map),
972                                                      std::move(bitmap),
973                                                      image_end));
974     return space;
975   }
976 
CheckImageComponentCount(const ImageSpace & space,uint32_t expected_component_count,std::string * error_msg)977   static bool CheckImageComponentCount(const ImageSpace& space,
978                                        uint32_t expected_component_count,
979                                        /*out*/std::string* error_msg) {
980     const ImageHeader& header = space.GetImageHeader();
981     if (header.GetComponentCount() != expected_component_count) {
982       *error_msg = StringPrintf("Unexpected component count in %s, received %u, expected %u",
983                                 space.GetImageFilename().c_str(),
984                                 header.GetComponentCount(),
985                                 expected_component_count);
986       return false;
987     }
988     return true;
989   }
990 
CheckImageReservationSize(const ImageSpace & space,uint32_t expected_reservation_size,std::string * error_msg)991   static bool CheckImageReservationSize(const ImageSpace& space,
992                                         uint32_t expected_reservation_size,
993                                         /*out*/std::string* error_msg) {
994     const ImageHeader& header = space.GetImageHeader();
995     if (header.GetImageReservationSize() != expected_reservation_size) {
996       *error_msg = StringPrintf("Unexpected reservation size in %s, received %u, expected %u",
997                                 space.GetImageFilename().c_str(),
998                                 header.GetImageReservationSize(),
999                                 expected_reservation_size);
1000       return false;
1001     }
1002     return true;
1003   }
1004 
1005   template <typename Container>
RemoveInternTableDuplicates(const Container & old_spaces,ImageSpace * new_space,SafeMap<mirror::String *,mirror::String * > * intern_remap)1006   static void RemoveInternTableDuplicates(
1007       const Container& old_spaces,
1008       /*inout*/ImageSpace* new_space,
1009       /*inout*/SafeMap<mirror::String*, mirror::String*>* intern_remap)
1010       REQUIRES_SHARED(Locks::mutator_lock_) {
1011     const ImageSection& new_interns = new_space->GetImageHeader().GetInternedStringsSection();
1012     if (new_interns.Size() != 0u) {
1013       const uint8_t* new_data = new_space->Begin() + new_interns.Offset();
1014       size_t new_read_count;
1015       InternTable::UnorderedSet new_set(new_data, /*make_copy_of_data=*/ false, &new_read_count);
1016       for (const auto& old_space : old_spaces) {
1017         const ImageSection& old_interns = old_space->GetImageHeader().GetInternedStringsSection();
1018         if (old_interns.Size() != 0u) {
1019           const uint8_t* old_data = old_space->Begin() + old_interns.Offset();
1020           size_t old_read_count;
1021           InternTable::UnorderedSet old_set(
1022               old_data, /*make_copy_of_data=*/ false, &old_read_count);
1023           RemoveDuplicates(old_set, &new_set, intern_remap);
1024         }
1025       }
1026     }
1027   }
1028 
RemapInternedStringDuplicates(const SafeMap<mirror::String *,mirror::String * > & intern_remap,ImageSpace * new_space)1029   static void RemapInternedStringDuplicates(
1030       const SafeMap<mirror::String*, mirror::String*>& intern_remap,
1031       ImageSpace* new_space) REQUIRES_SHARED(Locks::mutator_lock_) {
1032     RemapInternedStringsVisitor visitor(intern_remap);
1033     static_assert(IsAligned<kObjectAlignment>(sizeof(ImageHeader)), "Header alignment check");
1034     uint32_t objects_end = new_space->GetImageHeader().GetObjectsSection().Size();
1035     DCHECK_ALIGNED(objects_end, kObjectAlignment);
1036     for (uint32_t pos = sizeof(ImageHeader); pos != objects_end; ) {
1037       mirror::Object* object = reinterpret_cast<mirror::Object*>(new_space->Begin() + pos);
1038       object->VisitReferences</*kVisitNativeRoots=*/ false,
1039                               kVerifyNone,
1040                               kWithoutReadBarrier>(visitor, visitor);
1041       pos += RoundUp(object->SizeOf<kVerifyNone>(), kObjectAlignment);
1042     }
1043   }
1044 
1045  private:
1046   // Remove duplicates found in the `old_set` from the `new_set`.
1047   // Record the removed Strings for remapping. No read barriers are needed as the
1048   // tables are either just being loaded and not yet a part of the heap, or boot
1049   // image intern tables with non-moveable Strings used when loading an app image.
RemoveDuplicates(const InternTable::UnorderedSet & old_set,InternTable::UnorderedSet * new_set,SafeMap<mirror::String *,mirror::String * > * intern_remap)1050   static void RemoveDuplicates(const InternTable::UnorderedSet& old_set,
1051                                /*inout*/InternTable::UnorderedSet* new_set,
1052                                /*inout*/SafeMap<mirror::String*, mirror::String*>* intern_remap)
1053       REQUIRES_SHARED(Locks::mutator_lock_) {
1054     if (old_set.size() < new_set->size()) {
1055       for (const GcRoot<mirror::String>& old_s : old_set) {
1056         auto new_it = new_set->find(old_s);
1057         if (UNLIKELY(new_it != new_set->end())) {
1058           intern_remap->Put(new_it->Read<kWithoutReadBarrier>(), old_s.Read<kWithoutReadBarrier>());
1059           new_set->erase(new_it);
1060         }
1061       }
1062     } else {
1063       for (auto new_it = new_set->begin(), end = new_set->end(); new_it != end; ) {
1064         auto old_it = old_set.find(*new_it);
1065         if (UNLIKELY(old_it != old_set.end())) {
1066           intern_remap->Put(new_it->Read<kWithoutReadBarrier>(),
1067                             old_it->Read<kWithoutReadBarrier>());
1068           new_it = new_set->erase(new_it);
1069         } else {
1070           ++new_it;
1071         }
1072       }
1073     }
1074   }
1075 
ValidateBootImageChecksum(const char * image_filename,const ImageHeader & image_header,const OatFile * oat_file,ArrayRef<ImageSpace * const> boot_image_spaces,size_t * boot_image_space_dependencies,std::string * error_msg)1076   static bool ValidateBootImageChecksum(const char* image_filename,
1077                                         const ImageHeader& image_header,
1078                                         const OatFile* oat_file,
1079                                         ArrayRef<ImageSpace* const> boot_image_spaces,
1080                                         /*out*/size_t* boot_image_space_dependencies,
1081                                         /*out*/std::string* error_msg) {
1082     // Use the boot image component count to calculate the checksum from
1083     // the appropriate number of boot image chunks.
1084     uint32_t boot_image_component_count = image_header.GetBootImageComponentCount();
1085     size_t boot_image_spaces_size = boot_image_spaces.size();
1086     if (boot_image_component_count > boot_image_spaces_size) {
1087       *error_msg = StringPrintf("Too many boot image dependencies (%u > %zu) in image %s",
1088                                 boot_image_component_count,
1089                                 boot_image_spaces_size,
1090                                 image_filename);
1091       return false;
1092     }
1093     uint32_t checksum = 0u;
1094     size_t chunk_count = 0u;
1095     size_t space_pos = 0u;
1096     uint64_t boot_image_size = 0u;
1097     for (size_t component_count = 0u; component_count != boot_image_component_count; ) {
1098       const ImageHeader& current_header = boot_image_spaces[space_pos]->GetImageHeader();
1099       if (current_header.GetComponentCount() > boot_image_component_count - component_count) {
1100         *error_msg = StringPrintf("Boot image component count in %s ends in the middle of a chunk, "
1101                                       "%u is between %zu and %zu",
1102                                   image_filename,
1103                                   boot_image_component_count,
1104                                   component_count,
1105                                   component_count + current_header.GetComponentCount());
1106         return false;
1107       }
1108       component_count += current_header.GetComponentCount();
1109       checksum ^= current_header.GetImageChecksum();
1110       chunk_count += 1u;
1111       space_pos += current_header.GetImageSpaceCount();
1112       boot_image_size += current_header.GetImageReservationSize();
1113     }
1114     if (image_header.GetBootImageChecksum() != checksum) {
1115       *error_msg = StringPrintf("Boot image checksum mismatch (0x%08x != 0x%08x) in image %s",
1116                                 image_header.GetBootImageChecksum(),
1117                                 checksum,
1118                                 image_filename);
1119       return false;
1120     }
1121     if (image_header.GetBootImageSize() != boot_image_size) {
1122       *error_msg = StringPrintf("Boot image size mismatch (0x%08x != 0x%08" PRIx64 ") in image %s",
1123                                 image_header.GetBootImageSize(),
1124                                 boot_image_size,
1125                                 image_filename);
1126       return false;
1127     }
1128     // Oat checksums, if present, have already been validated, so we know that
1129     // they match the loaded image spaces. Therefore, we just verify that they
1130     // are consistent in the number of boot image chunks they list by looking
1131     // for the kImageChecksumPrefix at the start of each component.
1132     const char* oat_boot_class_path_checksums =
1133         oat_file->GetOatHeader().GetStoreValueByKey(OatHeader::kBootClassPathChecksumsKey);
1134     if (oat_boot_class_path_checksums != nullptr) {
1135       size_t oat_bcp_chunk_count = 0u;
1136       while (*oat_boot_class_path_checksums == kImageChecksumPrefix) {
1137         oat_bcp_chunk_count += 1u;
1138         // Find the start of the next component if any.
1139         const char* separator = strchr(oat_boot_class_path_checksums, ':');
1140         oat_boot_class_path_checksums = (separator != nullptr) ? separator + 1u : "";
1141       }
1142       if (oat_bcp_chunk_count != chunk_count) {
1143         *error_msg = StringPrintf("Boot image chunk count mismatch (%zu != %zu) in image %s",
1144                                   oat_bcp_chunk_count,
1145                                   chunk_count,
1146                                   image_filename);
1147         return false;
1148       }
1149     }
1150     *boot_image_space_dependencies = space_pos;
1151     return true;
1152   }
1153 
LoadImageFile(const char * image_filename,const char * image_location,const ImageHeader & image_header,int fd,bool allow_direct_mapping,TimingLogger * logger,MemMap * image_reservation,std::string * error_msg)1154   static MemMap LoadImageFile(const char* image_filename,
1155                               const char* image_location,
1156                               const ImageHeader& image_header,
1157                               int fd,
1158                               bool allow_direct_mapping,
1159                               TimingLogger* logger,
1160                               /*inout*/MemMap* image_reservation,
1161                               /*out*/std::string* error_msg)
1162         REQUIRES_SHARED(Locks::mutator_lock_) {
1163     TimingLogger::ScopedTiming timing("MapImageFile", logger);
1164     std::string temp_error_msg;
1165     const bool is_compressed = image_header.HasCompressedBlock();
1166     if (!is_compressed && allow_direct_mapping) {
1167       uint8_t* address = (image_reservation != nullptr) ? image_reservation->Begin() : nullptr;
1168       return MemMap::MapFileAtAddress(address,
1169                                       image_header.GetImageSize(),
1170                                       PROT_READ | PROT_WRITE,
1171                                       MAP_PRIVATE,
1172                                       fd,
1173                                       /*start=*/ 0,
1174                                       /*low_4gb=*/ true,
1175                                       image_filename,
1176                                       /*reuse=*/ false,
1177                                       image_reservation,
1178                                       error_msg);
1179     }
1180 
1181     // Reserve output and copy/decompress into it.
1182     MemMap map = MemMap::MapAnonymous(image_location,
1183                                       image_header.GetImageSize(),
1184                                       PROT_READ | PROT_WRITE,
1185                                       /*low_4gb=*/ true,
1186                                       image_reservation,
1187                                       error_msg);
1188     if (map.IsValid()) {
1189       const size_t stored_size = image_header.GetDataSize();
1190       MemMap temp_map = MemMap::MapFile(sizeof(ImageHeader) + stored_size,
1191                                         PROT_READ,
1192                                         MAP_PRIVATE,
1193                                         fd,
1194                                         /*start=*/ 0,
1195                                         /*low_4gb=*/ false,
1196                                         image_filename,
1197                                         error_msg);
1198       if (!temp_map.IsValid()) {
1199         DCHECK(error_msg == nullptr || !error_msg->empty());
1200         return MemMap::Invalid();
1201       }
1202 
1203       if (is_compressed) {
1204         memcpy(map.Begin(), &image_header, sizeof(ImageHeader));
1205 
1206         Runtime::ScopedThreadPoolUsage stpu;
1207         ThreadPool* const pool = stpu.GetThreadPool();
1208         const uint64_t start = NanoTime();
1209         Thread* const self = Thread::Current();
1210         static constexpr size_t kMinBlocks = 2u;
1211         const bool use_parallel = pool != nullptr && image_header.GetBlockCount() >= kMinBlocks;
1212         for (const ImageHeader::Block& block : image_header.GetBlocks(temp_map.Begin())) {
1213           auto function = [&](Thread*) {
1214             const uint64_t start2 = NanoTime();
1215             ScopedTrace trace("LZ4 decompress block");
1216             bool result = block.Decompress(/*out_ptr=*/map.Begin(),
1217                                            /*in_ptr=*/temp_map.Begin(),
1218                                            error_msg);
1219             if (!result && error_msg != nullptr) {
1220               *error_msg = "Failed to decompress image block " + *error_msg;
1221             }
1222             VLOG(image) << "Decompress block " << block.GetDataSize() << " -> "
1223                         << block.GetImageSize() << " in " << PrettyDuration(NanoTime() - start2);
1224           };
1225           if (use_parallel) {
1226             pool->AddTask(self, new FunctionTask(std::move(function)));
1227           } else {
1228             function(self);
1229           }
1230         }
1231         if (use_parallel) {
1232           ScopedTrace trace("Waiting for workers");
1233           // Go to native since we don't want to suspend while holding the mutator lock.
1234           ScopedThreadSuspension sts(Thread::Current(), kNative);
1235           pool->Wait(self, true, false);
1236         }
1237         const uint64_t time = NanoTime() - start;
1238         // Add one 1 ns to prevent possible divide by 0.
1239         VLOG(image) << "Decompressing image took " << PrettyDuration(time) << " ("
1240                     << PrettySize(static_cast<uint64_t>(map.Size()) * MsToNs(1000) / (time + 1))
1241                     << "/s)";
1242       } else {
1243         DCHECK(!allow_direct_mapping);
1244         // We do not allow direct mapping for boot image extensions compiled to a memfd.
1245         // This prevents wasting memory by kernel keeping the contents of the file alive
1246         // despite these contents being unreachable once the file descriptor is closed
1247         // and mmapped memory is copied for all existing mappings.
1248         //
1249         // Most pages would be copied during relocation while there is only one mapping.
1250         // We could use MAP_SHARED for relocation and then msync() and remap MAP_PRIVATE
1251         // as required for forking from zygote, but there would still be some pages
1252         // wasted anyway and we want to avoid that. (For example, static synchronized
1253         // methods use the class object for locking and thus modify its lockword.)
1254 
1255         // No other process should race to overwrite the extension in memfd.
1256         DCHECK_EQ(memcmp(temp_map.Begin(), &image_header, sizeof(ImageHeader)), 0);
1257         memcpy(map.Begin(), temp_map.Begin(), temp_map.Size());
1258       }
1259     }
1260 
1261     return map;
1262   }
1263 
1264   class EmptyRange {
1265    public:
InSource(uintptr_t) const1266     ALWAYS_INLINE bool InSource(uintptr_t) const { return false; }
InDest(uintptr_t) const1267     ALWAYS_INLINE bool InDest(uintptr_t) const { return false; }
ToDest(uintptr_t) const1268     ALWAYS_INLINE uintptr_t ToDest(uintptr_t) const { UNREACHABLE(); }
1269   };
1270 
1271   template <typename Range0, typename Range1 = EmptyRange, typename Range2 = EmptyRange>
1272   class ForwardAddress {
1273    public:
ForwardAddress(const Range0 & range0=Range0 (),const Range1 & range1=Range1 (),const Range2 & range2=Range2 ())1274     explicit ForwardAddress(const Range0& range0 = Range0(),
1275                             const Range1& range1 = Range1(),
1276                             const Range2& range2 = Range2())
1277         : range0_(range0), range1_(range1), range2_(range2) {}
1278 
1279     // Return the relocated address of a heap object.
1280     // Null checks must be performed in the caller (for performance reasons).
1281     template <typename T>
operator ()(T * src) const1282     ALWAYS_INLINE T* operator()(T* src) const {
1283       DCHECK(src != nullptr);
1284       const uintptr_t uint_src = reinterpret_cast<uintptr_t>(src);
1285       if (range2_.InSource(uint_src)) {
1286         return reinterpret_cast<T*>(range2_.ToDest(uint_src));
1287       }
1288       if (range1_.InSource(uint_src)) {
1289         return reinterpret_cast<T*>(range1_.ToDest(uint_src));
1290       }
1291       CHECK(range0_.InSource(uint_src))
1292           << reinterpret_cast<const void*>(src) << " not in "
1293           << reinterpret_cast<const void*>(range0_.Source()) << "-"
1294           << reinterpret_cast<const void*>(range0_.Source() + range0_.Length());
1295       return reinterpret_cast<T*>(range0_.ToDest(uint_src));
1296     }
1297 
1298    private:
1299     const Range0 range0_;
1300     const Range1 range1_;
1301     const Range2 range2_;
1302   };
1303 
1304   template <typename Forward>
1305   class FixupRootVisitor {
1306    public:
1307     template<typename... Args>
FixupRootVisitor(Args...args)1308     explicit FixupRootVisitor(Args... args) : forward_(args...) {}
1309 
VisitRootIfNonNull(mirror::CompressedReference<mirror::Object> * root) const1310     ALWAYS_INLINE void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const
1311         REQUIRES_SHARED(Locks::mutator_lock_) {
1312       if (!root->IsNull()) {
1313         VisitRoot(root);
1314       }
1315     }
1316 
VisitRoot(mirror::CompressedReference<mirror::Object> * root) const1317     ALWAYS_INLINE void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const
1318         REQUIRES_SHARED(Locks::mutator_lock_) {
1319       mirror::Object* ref = root->AsMirrorPtr();
1320       mirror::Object* new_ref = forward_(ref);
1321       if (ref != new_ref) {
1322         root->Assign(new_ref);
1323       }
1324     }
1325 
1326    private:
1327     Forward forward_;
1328   };
1329 
1330   template <typename Forward>
1331   class FixupObjectVisitor {
1332    public:
FixupObjectVisitor(gc::accounting::ContinuousSpaceBitmap * visited,const Forward & forward)1333     explicit FixupObjectVisitor(gc::accounting::ContinuousSpaceBitmap* visited,
1334                                 const Forward& forward)
1335         : visited_(visited), forward_(forward) {}
1336 
1337     // Fix up separately since we also need to fix up method entrypoints.
VisitRootIfNonNull(mirror::CompressedReference<mirror::Object> * root ATTRIBUTE_UNUSED) const1338     ALWAYS_INLINE void VisitRootIfNonNull(
1339         mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED) const {}
1340 
VisitRoot(mirror::CompressedReference<mirror::Object> * root ATTRIBUTE_UNUSED) const1341     ALWAYS_INLINE void VisitRoot(mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED)
1342         const {}
1343 
operator ()(ObjPtr<mirror::Object> obj,MemberOffset offset,bool is_static ATTRIBUTE_UNUSED) const1344     ALWAYS_INLINE void operator()(ObjPtr<mirror::Object> obj,
1345                                   MemberOffset offset,
1346                                   bool is_static ATTRIBUTE_UNUSED) const
1347         NO_THREAD_SAFETY_ANALYSIS {
1348       // Space is not yet added to the heap, don't do a read barrier.
1349       mirror::Object* ref = obj->GetFieldObject<mirror::Object, kVerifyNone, kWithoutReadBarrier>(
1350           offset);
1351       if (ref != nullptr) {
1352         // Use SetFieldObjectWithoutWriteBarrier to avoid card marking since we are writing to the
1353         // image.
1354         obj->SetFieldObjectWithoutWriteBarrier<false, true, kVerifyNone>(offset, forward_(ref));
1355       }
1356     }
1357 
1358     // java.lang.ref.Reference visitor.
operator ()(ObjPtr<mirror::Class> klass,ObjPtr<mirror::Reference> ref) const1359     ALWAYS_INLINE void operator()(ObjPtr<mirror::Class> klass, ObjPtr<mirror::Reference> ref) const
1360         REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(Locks::heap_bitmap_lock_) {
1361       DCHECK(klass->IsTypeOfReferenceClass());
1362       this->operator()(ref, mirror::Reference::ReferentOffset(), /*is_static=*/ false);
1363     }
1364 
operator ()(mirror::Object * obj) const1365     void operator()(mirror::Object* obj) const
1366         NO_THREAD_SAFETY_ANALYSIS {
1367       if (!visited_->Set(obj)) {
1368         // Not already visited.
1369         obj->VisitReferences</*visit native roots*/false, kVerifyNone, kWithoutReadBarrier>(
1370             *this,
1371             *this);
1372         CHECK(!obj->IsClass());
1373       }
1374     }
1375 
1376    private:
1377     gc::accounting::ContinuousSpaceBitmap* const visited_;
1378     Forward forward_;
1379   };
1380 
1381   // Relocate an image space mapped at target_base which possibly used to be at a different base
1382   // address. In place means modifying a single ImageSpace in place rather than relocating from
1383   // one ImageSpace to another.
1384   template <PointerSize kPointerSize>
RelocateInPlace(uint32_t boot_image_begin,uint8_t * target_base,accounting::ContinuousSpaceBitmap * bitmap,const OatFile * app_oat_file,std::string * error_msg)1385   static bool RelocateInPlace(uint32_t boot_image_begin,
1386                               uint8_t* target_base,
1387                               accounting::ContinuousSpaceBitmap* bitmap,
1388                               const OatFile* app_oat_file,
1389                               std::string* error_msg) {
1390     DCHECK(error_msg != nullptr);
1391     // Set up sections.
1392     ImageHeader* image_header = reinterpret_cast<ImageHeader*>(target_base);
1393     const uint32_t boot_image_size = image_header->GetBootImageSize();
1394     const ImageSection& objects_section = image_header->GetObjectsSection();
1395     // Where the app image objects are mapped to.
1396     uint8_t* objects_location = target_base + objects_section.Offset();
1397     TimingLogger logger(__FUNCTION__, true, false);
1398     RelocationRange boot_image(image_header->GetBootImageBegin(),
1399                                boot_image_begin,
1400                                boot_image_size);
1401     // Metadata is everything after the objects section, use exclusion to be safe.
1402     RelocationRange app_image_metadata(
1403         reinterpret_cast<uintptr_t>(image_header->GetImageBegin()) + objects_section.End(),
1404         reinterpret_cast<uintptr_t>(target_base) + objects_section.End(),
1405         image_header->GetImageSize() - objects_section.End());
1406     // App image heap objects, may be mapped in the heap.
1407     RelocationRange app_image_objects(
1408         reinterpret_cast<uintptr_t>(image_header->GetImageBegin()) + objects_section.Offset(),
1409         reinterpret_cast<uintptr_t>(objects_location),
1410         objects_section.Size());
1411     // Use the oat data section since this is where the OatFile::Begin is.
1412     RelocationRange app_oat(reinterpret_cast<uintptr_t>(image_header->GetOatDataBegin()),
1413                             // Not necessarily in low 4GB.
1414                             reinterpret_cast<uintptr_t>(app_oat_file->Begin()),
1415                             image_header->GetOatDataEnd() - image_header->GetOatDataBegin());
1416     VLOG(image) << "App image metadata " << app_image_metadata;
1417     VLOG(image) << "App image objects " << app_image_objects;
1418     VLOG(image) << "App oat " << app_oat;
1419     VLOG(image) << "Boot image " << boot_image;
1420     // True if we need to fixup any heap pointers.
1421     const bool fixup_image = boot_image.Delta() != 0 || app_image_metadata.Delta() != 0 ||
1422         app_image_objects.Delta() != 0;
1423     if (!fixup_image) {
1424       // Nothing to fix up.
1425       return true;
1426     }
1427     ScopedDebugDisallowReadBarriers sddrb(Thread::Current());
1428 
1429     using ForwardObject = ForwardAddress<RelocationRange, RelocationRange>;
1430     ForwardObject forward_object(boot_image, app_image_objects);
1431     ForwardObject forward_metadata(boot_image, app_image_metadata);
1432     using ForwardCode = ForwardAddress<RelocationRange, RelocationRange>;
1433     ForwardCode forward_code(boot_image, app_oat);
1434     PatchObjectVisitor<kPointerSize, ForwardObject, ForwardCode> patch_object_visitor(
1435         forward_object,
1436         forward_metadata);
1437     if (fixup_image) {
1438       // Two pass approach, fix up all classes first, then fix up non class-objects.
1439       // The visited bitmap is used to ensure that pointer arrays are not forwarded twice.
1440       gc::accounting::ContinuousSpaceBitmap visited_bitmap(
1441           gc::accounting::ContinuousSpaceBitmap::Create("Relocate bitmap",
1442                                                         target_base,
1443                                                         image_header->GetImageSize()));
1444       {
1445         TimingLogger::ScopedTiming timing("Fixup classes", &logger);
1446         ObjPtr<mirror::Class> class_class = [&]() NO_THREAD_SAFETY_ANALYSIS {
1447           ObjPtr<mirror::ObjectArray<mirror::Object>> image_roots = app_image_objects.ToDest(
1448               image_header->GetImageRoots<kWithoutReadBarrier>().Ptr());
1449           int32_t class_roots_index = enum_cast<int32_t>(ImageHeader::kClassRoots);
1450           DCHECK_LT(class_roots_index, image_roots->GetLength<kVerifyNone>());
1451           ObjPtr<mirror::ObjectArray<mirror::Class>> class_roots =
1452               ObjPtr<mirror::ObjectArray<mirror::Class>>::DownCast(boot_image.ToDest(
1453                   image_roots->GetWithoutChecks<kVerifyNone>(class_roots_index).Ptr()));
1454           return GetClassRoot<mirror::Class, kWithoutReadBarrier>(class_roots);
1455         }();
1456         const auto& class_table_section = image_header->GetClassTableSection();
1457         if (class_table_section.Size() > 0u) {
1458           ScopedObjectAccess soa(Thread::Current());
1459           ClassTableVisitor class_table_visitor(forward_object);
1460           size_t read_count = 0u;
1461           const uint8_t* data = target_base + class_table_section.Offset();
1462           // We avoid making a copy of the data since we want modifications to be propagated to the
1463           // memory map.
1464           ClassTable::ClassSet temp_set(data, /*make_copy_of_data=*/ false, &read_count);
1465           for (ClassTable::TableSlot& slot : temp_set) {
1466             slot.VisitRoot(class_table_visitor);
1467             ObjPtr<mirror::Class> klass = slot.Read<kWithoutReadBarrier>();
1468             if (!app_image_objects.InDest(klass.Ptr())) {
1469               continue;
1470             }
1471             const bool already_marked = visited_bitmap.Set(klass.Ptr());
1472             CHECK(!already_marked) << "App image class already visited";
1473             patch_object_visitor.VisitClass(klass, class_class);
1474             // Then patch the non-embedded vtable and iftable.
1475             ObjPtr<mirror::PointerArray> vtable =
1476                 klass->GetVTable<kVerifyNone, kWithoutReadBarrier>();
1477             if (vtable != nullptr &&
1478                 app_image_objects.InDest(vtable.Ptr()) &&
1479                 !visited_bitmap.Set(vtable.Ptr())) {
1480               patch_object_visitor.VisitPointerArray(vtable);
1481             }
1482             ObjPtr<mirror::IfTable> iftable = klass->GetIfTable<kVerifyNone, kWithoutReadBarrier>();
1483             if (iftable != nullptr && app_image_objects.InDest(iftable.Ptr())) {
1484               // Avoid processing the fields of iftable since we will process them later anyways
1485               // below.
1486               int32_t ifcount = klass->GetIfTableCount<kVerifyNone>();
1487               for (int32_t i = 0; i != ifcount; ++i) {
1488                 ObjPtr<mirror::PointerArray> unpatched_ifarray =
1489                     iftable->GetMethodArrayOrNull<kVerifyNone, kWithoutReadBarrier>(i);
1490                 if (unpatched_ifarray != nullptr) {
1491                   // The iftable has not been patched, so we need to explicitly adjust the pointer.
1492                   ObjPtr<mirror::PointerArray> ifarray = forward_object(unpatched_ifarray.Ptr());
1493                   if (app_image_objects.InDest(ifarray.Ptr()) &&
1494                       !visited_bitmap.Set(ifarray.Ptr())) {
1495                     patch_object_visitor.VisitPointerArray(ifarray);
1496                   }
1497                 }
1498               }
1499             }
1500           }
1501         }
1502       }
1503 
1504       // Fixup objects may read fields in the boot image, use the mutator lock here for sanity.
1505       // Though its probably not required.
1506       TimingLogger::ScopedTiming timing("Fixup objects", &logger);
1507       ScopedObjectAccess soa(Thread::Current());
1508       // Need to update the image to be at the target base.
1509       uintptr_t objects_begin = reinterpret_cast<uintptr_t>(target_base + objects_section.Offset());
1510       uintptr_t objects_end = reinterpret_cast<uintptr_t>(target_base + objects_section.End());
1511       FixupObjectVisitor<ForwardObject> fixup_object_visitor(&visited_bitmap, forward_object);
1512       bitmap->VisitMarkedRange(objects_begin, objects_end, fixup_object_visitor);
1513       // Fixup image roots.
1514       CHECK(app_image_objects.InSource(reinterpret_cast<uintptr_t>(
1515           image_header->GetImageRoots<kWithoutReadBarrier>().Ptr())));
1516       image_header->RelocateImageReferences(app_image_objects.Delta());
1517       image_header->RelocateBootImageReferences(boot_image.Delta());
1518       CHECK_EQ(image_header->GetImageBegin(), target_base);
1519       // Fix up dex cache DexFile pointers.
1520       ObjPtr<mirror::ObjectArray<mirror::DexCache>> dex_caches =
1521           image_header->GetImageRoot<kWithoutReadBarrier>(ImageHeader::kDexCaches)
1522               ->AsObjectArray<mirror::DexCache, kVerifyNone>();
1523       for (int32_t i = 0, count = dex_caches->GetLength(); i < count; ++i) {
1524         ObjPtr<mirror::DexCache> dex_cache = dex_caches->Get<kVerifyNone, kWithoutReadBarrier>(i);
1525         CHECK(dex_cache != nullptr);
1526         patch_object_visitor.VisitDexCacheArrays(dex_cache);
1527       }
1528     }
1529     {
1530       // Only touches objects in the app image, no need for mutator lock.
1531       TimingLogger::ScopedTiming timing("Fixup methods", &logger);
1532       image_header->VisitPackedArtMethods([&](ArtMethod& method) NO_THREAD_SAFETY_ANALYSIS {
1533         // TODO: Consider a separate visitor for runtime vs normal methods.
1534         if (UNLIKELY(method.IsRuntimeMethod())) {
1535           ImtConflictTable* table = method.GetImtConflictTable(kPointerSize);
1536           if (table != nullptr) {
1537             ImtConflictTable* new_table = forward_metadata(table);
1538             if (table != new_table) {
1539               method.SetImtConflictTable(new_table, kPointerSize);
1540             }
1541           }
1542           const void* old_code = method.GetEntryPointFromQuickCompiledCodePtrSize(kPointerSize);
1543           const void* new_code = forward_code(old_code);
1544           if (old_code != new_code) {
1545             method.SetEntryPointFromQuickCompiledCodePtrSize(new_code, kPointerSize);
1546           }
1547         } else {
1548           patch_object_visitor.PatchGcRoot(&method.DeclaringClassRoot());
1549           method.UpdateEntrypoints(forward_code, kPointerSize);
1550         }
1551       }, target_base, kPointerSize);
1552     }
1553     if (fixup_image) {
1554       {
1555         // Only touches objects in the app image, no need for mutator lock.
1556         TimingLogger::ScopedTiming timing("Fixup fields", &logger);
1557         image_header->VisitPackedArtFields([&](ArtField& field) NO_THREAD_SAFETY_ANALYSIS {
1558           patch_object_visitor.template PatchGcRoot</*kMayBeNull=*/ false>(
1559               &field.DeclaringClassRoot());
1560         }, target_base);
1561       }
1562       {
1563         TimingLogger::ScopedTiming timing("Fixup imt", &logger);
1564         image_header->VisitPackedImTables(forward_metadata, target_base, kPointerSize);
1565       }
1566       {
1567         TimingLogger::ScopedTiming timing("Fixup conflict tables", &logger);
1568         image_header->VisitPackedImtConflictTables(forward_metadata, target_base, kPointerSize);
1569       }
1570       // Fix up the intern table.
1571       const auto& intern_table_section = image_header->GetInternedStringsSection();
1572       if (intern_table_section.Size() > 0u) {
1573         TimingLogger::ScopedTiming timing("Fixup intern table", &logger);
1574         ScopedObjectAccess soa(Thread::Current());
1575         // Fixup the pointers in the newly written intern table to contain image addresses.
1576         InternTable temp_intern_table;
1577         // Note that we require that ReadFromMemory does not make an internal copy of the elements
1578         // so that the VisitRoots() will update the memory directly rather than the copies.
1579         temp_intern_table.AddTableFromMemory(target_base + intern_table_section.Offset(),
1580                                              [&](InternTable::UnorderedSet& strings)
1581             REQUIRES_SHARED(Locks::mutator_lock_) {
1582           for (GcRoot<mirror::String>& root : strings) {
1583             root = GcRoot<mirror::String>(forward_object(root.Read<kWithoutReadBarrier>()));
1584           }
1585         }, /*is_boot_image=*/ false);
1586       }
1587     }
1588     if (VLOG_IS_ON(image)) {
1589       logger.Dump(LOG_STREAM(INFO));
1590     }
1591     return true;
1592   }
1593 };
1594 
AppendImageChecksum(uint32_t component_count,uint32_t checksum,std::string * checksums)1595 static void AppendImageChecksum(uint32_t component_count,
1596                                 uint32_t checksum,
1597                                 /*inout*/std::string* checksums) {
1598   static_assert(ImageSpace::kImageChecksumPrefix == 'i', "Format prefix check.");
1599   StringAppendF(checksums, "i;%u/%08x", component_count, checksum);
1600 }
1601 
CheckAndRemoveImageChecksum(uint32_t component_count,uint32_t checksum,std::string_view * oat_checksums,std::string * error_msg)1602 static bool CheckAndRemoveImageChecksum(uint32_t component_count,
1603                                         uint32_t checksum,
1604                                         /*inout*/std::string_view* oat_checksums,
1605                                         /*out*/std::string* error_msg) {
1606   std::string image_checksum;
1607   AppendImageChecksum(component_count, checksum, &image_checksum);
1608   if (!StartsWith(*oat_checksums, image_checksum)) {
1609     *error_msg = StringPrintf("Image checksum mismatch, expected %s to start with %s",
1610                               std::string(*oat_checksums).c_str(),
1611                               image_checksum.c_str());
1612     return false;
1613   }
1614   oat_checksums->remove_prefix(image_checksum.size());
1615   return true;
1616 }
1617 
1618 // Helper class to find the primary boot image and boot image extensions
1619 // and determine the boot image layout.
1620 class ImageSpace::BootImageLayout {
1621  public:
1622   // Description of a "chunk" of the boot image, i.e. either primary boot image
1623   // or a boot image extension, used in conjunction with the boot class path to
1624   // load boot image components.
1625   struct ImageChunk {
1626     std::string base_location;
1627     std::string base_filename;
1628     std::string profile_file;
1629     size_t start_index;
1630     uint32_t component_count;
1631     uint32_t image_space_count;
1632     uint32_t reservation_size;
1633     uint32_t checksum;
1634     uint32_t boot_image_component_count;
1635     uint32_t boot_image_checksum;
1636     uint32_t boot_image_size;
1637 
1638     // The following file descriptors hold the memfd files for extensions compiled
1639     // in memory and described by the above fields. We want to use them to mmap()
1640     // the contents and then close them while treating the ImageChunk description
1641     // as immutable (const), so make these fields explicitly mutable.
1642     mutable android::base::unique_fd art_fd;
1643     mutable android::base::unique_fd vdex_fd;
1644     mutable android::base::unique_fd oat_fd;
1645   };
1646 
BootImageLayout(const std::string & image_location,ArrayRef<const std::string> boot_class_path,ArrayRef<const std::string> boot_class_path_locations)1647   BootImageLayout(const std::string& image_location,
1648                   ArrayRef<const std::string> boot_class_path,
1649                   ArrayRef<const std::string> boot_class_path_locations)
1650      : image_location_(image_location),
1651        boot_class_path_(boot_class_path),
1652        boot_class_path_locations_(boot_class_path_locations) {}
1653 
1654   std::string GetPrimaryImageLocation();
1655 
LoadFromSystem(InstructionSet image_isa,std::string * error_msg)1656   bool LoadFromSystem(InstructionSet image_isa, /*out*/std::string* error_msg) {
1657     return LoadOrValidateFromSystem(image_isa, /*oat_checksums=*/ nullptr, error_msg);
1658   }
1659 
ValidateFromSystem(InstructionSet image_isa,std::string_view * oat_checksums,std::string * error_msg)1660   bool ValidateFromSystem(InstructionSet image_isa,
1661                           /*inout*/std::string_view* oat_checksums,
1662                           /*out*/std::string* error_msg) {
1663     DCHECK(oat_checksums != nullptr);
1664     return LoadOrValidateFromSystem(image_isa, oat_checksums, error_msg);
1665   }
1666 
LoadFromDalvikCache(const std::string & dalvik_cache,std::string * error_msg)1667   bool LoadFromDalvikCache(const std::string& dalvik_cache, /*out*/std::string* error_msg) {
1668     return LoadOrValidateFromDalvikCache(dalvik_cache, /*oat_checksums=*/ nullptr, error_msg);
1669   }
1670 
ValidateFromDalvikCache(const std::string & dalvik_cache,std::string_view * oat_checksums,std::string * error_msg)1671   bool ValidateFromDalvikCache(const std::string& dalvik_cache,
1672                                /*inout*/std::string_view* oat_checksums,
1673                                /*out*/std::string* error_msg) {
1674     DCHECK(oat_checksums != nullptr);
1675     return LoadOrValidateFromDalvikCache(dalvik_cache, oat_checksums, error_msg);
1676   }
1677 
GetChunks() const1678   ArrayRef<const ImageChunk> GetChunks() const {
1679     return ArrayRef<const ImageChunk>(chunks_);
1680   }
1681 
GetBaseAddress() const1682   uint32_t GetBaseAddress() const {
1683     return base_address_;
1684   }
1685 
GetNextBcpIndex() const1686   size_t GetNextBcpIndex() const {
1687     return next_bcp_index_;
1688   }
1689 
GetTotalComponentCount() const1690   size_t GetTotalComponentCount() const {
1691     return total_component_count_;
1692   }
1693 
GetTotalReservationSize() const1694   size_t GetTotalReservationSize() const {
1695     return total_reservation_size_;
1696   }
1697 
1698  private:
1699   struct NamedComponentLocation {
1700     std::string base_location;
1701     size_t bcp_index;
1702     std::string profile_filename;
1703   };
1704 
ExpandLocationImpl(const std::string & location,size_t bcp_index,bool boot_image_extension)1705   std::string ExpandLocationImpl(const std::string& location,
1706                                  size_t bcp_index,
1707                                  bool boot_image_extension) {
1708     std::vector<std::string> expanded = ExpandMultiImageLocations(
1709         ArrayRef<const std::string>(boot_class_path_).SubArray(bcp_index, 1u),
1710         location,
1711         boot_image_extension);
1712     DCHECK_EQ(expanded.size(), 1u);
1713     return expanded[0];
1714   }
1715 
ExpandLocation(const std::string & location,size_t bcp_index)1716   std::string ExpandLocation(const std::string& location, size_t bcp_index) {
1717     if (bcp_index == 0u) {
1718       DCHECK_EQ(location, ExpandLocationImpl(location, bcp_index, /*boot_image_extension=*/ false));
1719       return location;
1720     } else {
1721       return ExpandLocationImpl(location, bcp_index, /*boot_image_extension=*/ true);
1722     }
1723   }
1724 
GetBcpComponentPath(size_t bcp_index)1725   std::string GetBcpComponentPath(size_t bcp_index) {
1726     DCHECK_LE(bcp_index, boot_class_path_.size());
1727     size_t bcp_slash_pos = boot_class_path_[bcp_index].rfind('/');
1728     DCHECK_NE(bcp_slash_pos, std::string::npos);
1729     return boot_class_path_[bcp_index].substr(0u, bcp_slash_pos + 1u);
1730   }
1731 
1732   bool VerifyImageLocation(const std::vector<std::string>& components,
1733                            /*out*/size_t* named_components_count,
1734                            /*out*/std::string* error_msg);
1735 
1736   bool MatchNamedComponents(
1737       ArrayRef<const std::string> named_components,
1738       /*out*/std::vector<NamedComponentLocation>* named_component_locations,
1739       /*out*/std::string* error_msg);
1740 
1741   bool ValidateBootImageChecksum(const char* file_description,
1742                                  const ImageHeader& header,
1743                                  /*out*/std::string* error_msg);
1744 
1745   bool ValidateHeader(const ImageHeader& header,
1746                       size_t bcp_index,
1747                       const char* file_description,
1748                       /*out*/std::string* error_msg);
1749 
1750   bool ReadHeader(const std::string& base_location,
1751                   const std::string& base_filename,
1752                   size_t bcp_index,
1753                   /*out*/std::string* error_msg);
1754 
1755   bool CompileExtension(const std::string& base_location,
1756                         const std::string& base_filename,
1757                         size_t bcp_index,
1758                         const std::string& profile_filename,
1759                         ArrayRef<std::string> dependencies,
1760                         /*out*/std::string* error_msg);
1761 
1762   bool CheckAndRemoveLastChunkChecksum(/*inout*/std::string_view* oat_checksums,
1763                                        /*out*/std::string* error_msg);
1764 
1765   template <typename FilenameFn>
1766   bool LoadOrValidate(FilenameFn&& filename_fn,
1767                       /*inout*/std::string_view* oat_checksums,
1768                       /*out*/std::string* error_msg);
1769 
1770   bool LoadOrValidateFromSystem(InstructionSet image_isa,
1771                                 /*inout*/std::string_view* oat_checksums,
1772                                 /*out*/std::string* error_msg);
1773 
1774   bool LoadOrValidateFromDalvikCache(const std::string& dalvik_cache,
1775                                      /*inout*/std::string_view* oat_checksums,
1776                                      /*out*/std::string* error_msg);
1777 
1778   const std::string& image_location_;
1779   ArrayRef<const std::string> boot_class_path_;
1780   ArrayRef<const std::string> boot_class_path_locations_;
1781 
1782   std::vector<ImageChunk> chunks_;
1783   uint32_t base_address_ = 0u;
1784   size_t next_bcp_index_ = 0u;
1785   size_t total_component_count_ = 0u;
1786   size_t total_reservation_size_ = 0u;
1787 };
1788 
GetPrimaryImageLocation()1789 std::string ImageSpace::BootImageLayout::GetPrimaryImageLocation() {
1790   size_t location_start = 0u;
1791   size_t location_end = image_location_.find(kComponentSeparator);
1792   while (location_end == location_start) {
1793     ++location_start;
1794     location_end = image_location_.find(location_start, kComponentSeparator);
1795   }
1796   std::string location = (location_end == std::string::npos)
1797       ? image_location_.substr(location_start)
1798       : image_location_.substr(location_start, location_end - location_start);
1799   if (location.find('/') == std::string::npos) {
1800     // No path, so use the path from the first boot class path component.
1801     size_t slash_pos = boot_class_path_.empty()
1802         ? std::string::npos
1803         : boot_class_path_[0].rfind('/');
1804     if (slash_pos == std::string::npos) {
1805       return std::string();
1806     }
1807     location.insert(0u, boot_class_path_[0].substr(0u, slash_pos + 1u));
1808   }
1809   return location;
1810 }
1811 
VerifyImageLocation(const std::vector<std::string> & components,size_t * named_components_count,std::string * error_msg)1812 bool ImageSpace::BootImageLayout::VerifyImageLocation(
1813     const std::vector<std::string>& components,
1814     /*out*/size_t* named_components_count,
1815     /*out*/std::string* error_msg) {
1816   DCHECK(named_components_count != nullptr);
1817 
1818   // Validate boot class path. Require a path and non-empty name in each component.
1819   for (const std::string& bcp_component : boot_class_path_) {
1820     size_t bcp_slash_pos = bcp_component.rfind('/');
1821     if (bcp_slash_pos == std::string::npos || bcp_slash_pos == bcp_component.size() - 1u) {
1822       *error_msg = StringPrintf("Invalid boot class path component: %s", bcp_component.c_str());
1823       return false;
1824     }
1825   }
1826 
1827   // Validate the format of image location components.
1828   size_t components_size = components.size();
1829   if (components_size == 0u) {
1830     *error_msg = "Empty image location.";
1831     return false;
1832   }
1833   size_t wildcards_start = components_size;  // No wildcards.
1834   for (size_t i = 0; i != components_size; ++i) {
1835     const std::string& component = components[i];
1836     DCHECK(!component.empty());  // Guaranteed by Split().
1837     const size_t profile_separator_pos = component.find(kProfileSeparator);
1838     size_t wildcard_pos = component.find('*');
1839     if (wildcard_pos == std::string::npos) {
1840       if (wildcards_start != components.size()) {
1841         *error_msg =
1842             StringPrintf("Image component without wildcard after component with wildcard: %s",
1843                          component.c_str());
1844         return false;
1845       }
1846       if (profile_separator_pos != std::string::npos) {
1847         if (component.find(kProfileSeparator, profile_separator_pos + 1u) != std::string::npos) {
1848           *error_msg = StringPrintf("Multiple profile delimiters in %s", component.c_str());
1849           return false;
1850         }
1851         if (profile_separator_pos == 0u || profile_separator_pos + 1u == component.size()) {
1852           *error_msg = StringPrintf("Missing component and/or profile name in %s",
1853                                     component.c_str());
1854           return false;
1855         }
1856         if (component.back() == '/') {
1857           *error_msg = StringPrintf("Profile name ends with path separator: %s",
1858                                     component.c_str());
1859           return false;
1860         }
1861       }
1862       size_t component_name_length =
1863           profile_separator_pos != std::string::npos ? profile_separator_pos : component.size();
1864       if (component[component_name_length - 1u] == '/') {
1865         *error_msg = StringPrintf("Image component ends with path separator: %s",
1866                                   component.c_str());
1867         return false;
1868       }
1869     } else {
1870       if (profile_separator_pos != std::string::npos) {
1871         *error_msg = StringPrintf("Unsupproted wildcard (*) and profile delimiter (!) in %s",
1872                                   component.c_str());
1873         return false;
1874       }
1875       if (wildcards_start == components_size) {
1876         wildcards_start = i;
1877       }
1878       // Wildcard must be the last character.
1879       if (wildcard_pos != component.size() - 1u) {
1880         *error_msg = StringPrintf("Unsupported wildcard (*) position in %s", component.c_str());
1881         return false;
1882       }
1883       // And it must be either plain wildcard or preceded by a path separator.
1884       if (component.size() != 1u && component[wildcard_pos - 1u] != '/') {
1885         *error_msg = StringPrintf("Non-plain wildcard (*) not preceded by path separator '/': %s",
1886                                   component.c_str());
1887         return false;
1888       }
1889       if (i == 0) {
1890         *error_msg = StringPrintf("Primary component contains wildcard (*): %s", component.c_str());
1891         return false;
1892       }
1893     }
1894   }
1895 
1896   *named_components_count = wildcards_start;
1897   return true;
1898 }
1899 
MatchNamedComponents(ArrayRef<const std::string> named_components,std::vector<NamedComponentLocation> * named_component_locations,std::string * error_msg)1900 bool ImageSpace::BootImageLayout::MatchNamedComponents(
1901     ArrayRef<const std::string> named_components,
1902     /*out*/std::vector<NamedComponentLocation>* named_component_locations,
1903     /*out*/std::string* error_msg) {
1904   DCHECK(!named_components.empty());
1905   DCHECK(named_component_locations->empty());
1906   named_component_locations->reserve(named_components.size());
1907   size_t bcp_component_count = boot_class_path_.size();
1908   size_t bcp_pos = 0;
1909   std::string base_name;
1910   for (size_t i = 0, size = named_components.size(); i != size; ++i) {
1911     std::string component = named_components[i];
1912     std::string profile_filename;  // Empty.
1913     const size_t profile_separator_pos = component.find(kProfileSeparator);
1914     if (profile_separator_pos != std::string::npos) {
1915       profile_filename = component.substr(profile_separator_pos + 1u);
1916       DCHECK(!profile_filename.empty());  // Checked by VerifyImageLocation()
1917       component.resize(profile_separator_pos);
1918       DCHECK(!component.empty());  // Checked by VerifyImageLocation()
1919     }
1920     size_t slash_pos = component.rfind('/');
1921     std::string base_location;
1922     if (i == 0u) {
1923       // The primary boot image name is taken as provided. It forms the base
1924       // for expanding the extension filenames.
1925       if (slash_pos != std::string::npos) {
1926         base_name = component.substr(slash_pos + 1u);
1927         base_location = component;
1928       } else {
1929         base_name = component;
1930         base_location = GetBcpComponentPath(0u) + component;
1931       }
1932     } else {
1933       std::string to_match;
1934       if (slash_pos != std::string::npos) {
1935         // If we have the full path, we just need to match the filename to the BCP component.
1936         base_location = component.substr(0u, slash_pos + 1u) + base_name;
1937         to_match = component;
1938       }
1939       while (true) {
1940         if (slash_pos == std::string::npos) {
1941           // If we do not have a full path, we need to update the path based on the BCP location.
1942           std::string path = GetBcpComponentPath(bcp_pos);
1943           to_match = path + component;
1944           base_location = path + base_name;
1945         }
1946         if (ExpandLocation(base_location, bcp_pos) == to_match) {
1947           break;
1948         }
1949         ++bcp_pos;
1950         if (bcp_pos == bcp_component_count) {
1951           *error_msg = StringPrintf("Image component %s does not match a boot class path component",
1952                                     component.c_str());
1953           return false;
1954         }
1955       }
1956     }
1957     if (!profile_filename.empty() && profile_filename.find('/') == std::string::npos) {
1958       profile_filename.insert(/*pos*/ 0u, GetBcpComponentPath(bcp_pos));
1959     }
1960     NamedComponentLocation location;
1961     location.base_location = base_location;
1962     location.bcp_index = bcp_pos;
1963     location.profile_filename = profile_filename;
1964     named_component_locations->push_back(location);
1965     ++bcp_pos;
1966   }
1967   return true;
1968 }
1969 
ValidateBootImageChecksum(const char * file_description,const ImageHeader & header,std::string * error_msg)1970 bool ImageSpace::BootImageLayout::ValidateBootImageChecksum(const char* file_description,
1971                                                             const ImageHeader& header,
1972                                                             /*out*/std::string* error_msg) {
1973   uint32_t boot_image_component_count = header.GetBootImageComponentCount();
1974   if (chunks_.empty() != (boot_image_component_count == 0u)) {
1975     *error_msg = StringPrintf("Unexpected boot image component count in %s: %u, %s",
1976                               file_description,
1977                               boot_image_component_count,
1978                               chunks_.empty() ? "should be 0" : "should not be 0");
1979     return false;
1980   }
1981   uint32_t component_count = 0u;
1982   uint32_t composite_checksum = 0u;
1983   uint64_t boot_image_size = 0u;
1984   for (const ImageChunk& chunk : chunks_) {
1985     if (component_count == boot_image_component_count) {
1986       break;  // Hit the component count.
1987     }
1988     if (chunk.start_index != component_count) {
1989       break;  // End of contiguous chunks, fail below; same as reaching end of `chunks_`.
1990     }
1991     if (chunk.component_count > boot_image_component_count - component_count) {
1992       *error_msg = StringPrintf("Boot image component count in %s ends in the middle of a chunk, "
1993                                     "%u is between %u and %u",
1994                                 file_description,
1995                                 boot_image_component_count,
1996                                 component_count,
1997                                 component_count + chunk.component_count);
1998       return false;
1999     }
2000     component_count += chunk.component_count;
2001     composite_checksum ^= chunk.checksum;
2002     boot_image_size += chunk.reservation_size;
2003   }
2004   DCHECK_LE(component_count, boot_image_component_count);
2005   if (component_count != boot_image_component_count) {
2006     *error_msg = StringPrintf("Missing boot image components for checksum in %s: %u > %u",
2007                               file_description,
2008                               boot_image_component_count,
2009                               component_count);
2010     return false;
2011   }
2012   if (composite_checksum != header.GetBootImageChecksum()) {
2013     *error_msg = StringPrintf("Boot image checksum mismatch in %s: 0x%08x != 0x%08x",
2014                               file_description,
2015                               header.GetBootImageChecksum(),
2016                               composite_checksum);
2017     return false;
2018   }
2019   if (boot_image_size != header.GetBootImageSize()) {
2020     *error_msg = StringPrintf("Boot image size mismatch in %s: 0x%08x != 0x%08" PRIx64,
2021                               file_description,
2022                               header.GetBootImageSize(),
2023                               boot_image_size);
2024     return false;
2025   }
2026   return true;
2027 }
2028 
ValidateHeader(const ImageHeader & header,size_t bcp_index,const char * file_description,std::string * error_msg)2029 bool ImageSpace::BootImageLayout::ValidateHeader(const ImageHeader& header,
2030                                                  size_t bcp_index,
2031                                                  const char* file_description,
2032                                                  /*out*/std::string* error_msg) {
2033   size_t bcp_component_count = boot_class_path_.size();
2034   DCHECK_LT(bcp_index, bcp_component_count);
2035   size_t allowed_component_count = bcp_component_count - bcp_index;
2036   DCHECK_LE(total_reservation_size_, kMaxTotalImageReservationSize);
2037   size_t allowed_reservation_size = kMaxTotalImageReservationSize - total_reservation_size_;
2038 
2039   if (header.GetComponentCount() == 0u ||
2040       header.GetComponentCount() > allowed_component_count) {
2041     *error_msg = StringPrintf("Unexpected component count in %s, received %u, "
2042                                   "expected non-zero and <= %zu",
2043                               file_description,
2044                               header.GetComponentCount(),
2045                               allowed_component_count);
2046     return false;
2047   }
2048   if (header.GetImageReservationSize() > allowed_reservation_size) {
2049     *error_msg = StringPrintf("Reservation size too big in %s: %u > %zu",
2050                               file_description,
2051                               header.GetImageReservationSize(),
2052                               allowed_reservation_size);
2053     return false;
2054   }
2055   if (!ValidateBootImageChecksum(file_description, header, error_msg)) {
2056     return false;
2057   }
2058 
2059   return true;
2060 }
2061 
ReadHeader(const std::string & base_location,const std::string & base_filename,size_t bcp_index,std::string * error_msg)2062 bool ImageSpace::BootImageLayout::ReadHeader(const std::string& base_location,
2063                                              const std::string& base_filename,
2064                                              size_t bcp_index,
2065                                              /*out*/std::string* error_msg) {
2066   DCHECK_LE(next_bcp_index_, bcp_index);
2067   DCHECK_LT(bcp_index, boot_class_path_.size());
2068 
2069   std::string actual_filename = ExpandLocation(base_filename, bcp_index);
2070   ImageHeader header;
2071   if (!ReadSpecificImageHeader(actual_filename.c_str(), &header, error_msg)) {
2072     return false;
2073   }
2074   const char* file_description = actual_filename.c_str();
2075   if (!ValidateHeader(header, bcp_index, file_description, error_msg)) {
2076     return false;
2077   }
2078 
2079   if (chunks_.empty()) {
2080     base_address_ = reinterpret_cast32<uint32_t>(header.GetImageBegin());
2081   }
2082   ImageChunk chunk;
2083   chunk.base_location = base_location;
2084   chunk.base_filename = base_filename;
2085   chunk.start_index = bcp_index;
2086   chunk.component_count = header.GetComponentCount();
2087   chunk.image_space_count = header.GetImageSpaceCount();
2088   chunk.reservation_size = header.GetImageReservationSize();
2089   chunk.checksum = header.GetImageChecksum();
2090   chunk.boot_image_component_count = header.GetBootImageComponentCount();
2091   chunk.boot_image_checksum = header.GetBootImageChecksum();
2092   chunk.boot_image_size = header.GetBootImageSize();
2093   chunks_.push_back(std::move(chunk));
2094   next_bcp_index_ = bcp_index + header.GetComponentCount();
2095   total_component_count_ += header.GetComponentCount();
2096   total_reservation_size_ += header.GetImageReservationSize();
2097   return true;
2098 }
2099 
CompileExtension(const std::string & base_location,const std::string & base_filename,size_t bcp_index,const std::string & profile_filename,ArrayRef<std::string> dependencies,std::string * error_msg)2100 bool ImageSpace::BootImageLayout::CompileExtension(const std::string& base_location,
2101                                                    const std::string& base_filename,
2102                                                    size_t bcp_index,
2103                                                    const std::string& profile_filename,
2104                                                    ArrayRef<std::string> dependencies,
2105                                                    /*out*/std::string* error_msg) {
2106   DCHECK_LE(total_component_count_, next_bcp_index_);
2107   DCHECK_LE(next_bcp_index_, bcp_index);
2108   size_t bcp_component_count = boot_class_path_.size();
2109   DCHECK_LT(bcp_index, bcp_component_count);
2110   DCHECK(!profile_filename.empty());
2111   if (total_component_count_ != bcp_index) {
2112     // We require all previous BCP components to have a boot image space (primary or extension).
2113     *error_msg = "Cannot compile extension because of missing dependencies.";
2114     return false;
2115   }
2116   Runtime* runtime = Runtime::Current();
2117   if (!runtime->IsImageDex2OatEnabled()) {
2118     *error_msg = "Cannot compile extension because dex2oat for image compilation is disabled.";
2119     return false;
2120   }
2121 
2122   // Check dependencies.
2123   DCHECK(!dependencies.empty());
2124   size_t dependency_component_count = 0;
2125   for (size_t i = 0, size = dependencies.size(); i != size; ++i) {
2126     if (chunks_.size() == i || chunks_[i].start_index != dependency_component_count) {
2127       *error_msg = StringPrintf("Missing extension dependency \"%s\"", dependencies[i].c_str());
2128       return false;
2129     }
2130     dependency_component_count += chunks_[i].component_count;
2131   }
2132 
2133   // Collect locations from the profile.
2134   std::set<std::string> dex_locations;
2135   {
2136     std::unique_ptr<File> profile_file(OS::OpenFileForReading(profile_filename.c_str()));
2137     if (profile_file == nullptr) {
2138       *error_msg = StringPrintf("Failed to open profile file \"%s\" for reading, error: %s",
2139                                 profile_filename.c_str(),
2140                                 strerror(errno));
2141       return false;
2142     }
2143 
2144     // TODO: Rewrite ProfileCompilationInfo to provide a better interface and
2145     // to store the dex locations in uncompressed section of the file.
2146     auto collect_fn = [&dex_locations](const std::string& dex_location,
2147                                        uint32_t checksum ATTRIBUTE_UNUSED) {
2148       dex_locations.insert(dex_location);  // Just collect locations.
2149       return false;                        // Do not read the profile data.
2150     };
2151     ProfileCompilationInfo info(/*for_boot_image=*/ true);
2152     if (!info.Load(profile_file->Fd(), /*merge_classes=*/ true, collect_fn)) {
2153       *error_msg = StringPrintf("Failed to scan profile from %s", profile_filename.c_str());
2154       return false;
2155     }
2156   }
2157 
2158   // Match boot class path components to locations from profile.
2159   // Note that the profile records only filenames without paths.
2160   size_t bcp_end = bcp_index;
2161   for (; bcp_end != bcp_component_count; ++bcp_end) {
2162     const std::string& bcp_component = boot_class_path_locations_[bcp_end];
2163     size_t slash_pos = bcp_component.rfind('/');
2164     DCHECK_NE(slash_pos, std::string::npos);
2165     std::string bcp_component_name = bcp_component.substr(slash_pos + 1u);
2166     if (dex_locations.count(bcp_component_name) == 0u) {
2167       break;  // Did not find the current location in dex file.
2168     }
2169   }
2170 
2171   if (bcp_end == bcp_index) {
2172     // No data for the first (requested) component.
2173     *error_msg = StringPrintf("The profile does not contain data for %s",
2174                               boot_class_path_locations_[bcp_index].c_str());
2175     return false;
2176   }
2177 
2178   // Create in-memory files.
2179   std::string art_filename = ExpandLocation(base_filename, bcp_index);
2180   std::string vdex_filename = ImageHeader::GetVdexLocationFromImageLocation(art_filename);
2181   std::string oat_filename = ImageHeader::GetOatLocationFromImageLocation(art_filename);
2182   android::base::unique_fd art_fd(memfd_create_compat(art_filename.c_str(), /*flags=*/ 0));
2183   android::base::unique_fd vdex_fd(memfd_create_compat(vdex_filename.c_str(), /*flags=*/ 0));
2184   android::base::unique_fd oat_fd(memfd_create_compat(oat_filename.c_str(), /*flags=*/ 0));
2185   if (art_fd.get() == -1 || vdex_fd.get() == -1 || oat_fd.get() == -1) {
2186     *error_msg = StringPrintf("Failed to create memfd handles for compiling extension for %s",
2187                               boot_class_path_locations_[bcp_index].c_str());
2188     return false;
2189   }
2190 
2191   // Construct the dex2oat command line.
2192   std::string dex2oat = runtime->GetCompilerExecutable();
2193   ArrayRef<const std::string> head_bcp =
2194       boot_class_path_.SubArray(/*pos=*/ 0u, /*length=*/ dependency_component_count);
2195   ArrayRef<const std::string> head_bcp_locations =
2196       boot_class_path_locations_.SubArray(/*pos=*/ 0u, /*length=*/ dependency_component_count);
2197   ArrayRef<const std::string> extension_bcp =
2198       boot_class_path_.SubArray(/*pos=*/ bcp_index, /*length=*/ bcp_end - bcp_index);
2199   ArrayRef<const std::string> extension_bcp_locations =
2200       boot_class_path_locations_.SubArray(/*pos=*/ bcp_index, /*length=*/ bcp_end - bcp_index);
2201   std::string boot_class_path = Join(head_bcp, ':') + ':' + Join(extension_bcp, ':');
2202   std::string boot_class_path_locations =
2203       Join(head_bcp_locations, ':') + ':' + Join(extension_bcp_locations, ':');
2204 
2205   std::vector<std::string> args;
2206   args.push_back(dex2oat);
2207   args.push_back("--runtime-arg");
2208   args.push_back("-Xbootclasspath:" + boot_class_path);
2209   args.push_back("--runtime-arg");
2210   args.push_back("-Xbootclasspath-locations:" + boot_class_path_locations);
2211   args.push_back("--boot-image=" + Join(dependencies, kComponentSeparator));
2212   for (size_t i = bcp_index; i != bcp_end; ++i) {
2213     args.push_back("--dex-file=" + boot_class_path_[i]);
2214     args.push_back("--dex-location=" + boot_class_path_locations_[i]);
2215   }
2216   args.push_back("--image-fd=" + std::to_string(art_fd.get()));
2217   args.push_back("--output-vdex-fd=" + std::to_string(vdex_fd.get()));
2218   args.push_back("--oat-fd=" + std::to_string(oat_fd.get()));
2219   args.push_back("--oat-location=" + ImageHeader::GetOatLocationFromImageLocation(base_filename));
2220   args.push_back("--single-image");
2221   args.push_back("--image-format=uncompressed");
2222 
2223   // We currently cannot guarantee that the boot class path has no verification failures.
2224   // And we do not want to compile anything, compilation should be done by JIT in zygote.
2225   args.push_back("--compiler-filter=verify");
2226 
2227   // Pass the profile.
2228   args.push_back("--profile-file=" + profile_filename);
2229 
2230   // Do not let the file descriptor numbers change the compilation output.
2231   args.push_back("--avoid-storing-invocation");
2232 
2233   runtime->AddCurrentRuntimeFeaturesAsDex2OatArguments(&args);
2234 
2235   if (!kIsTargetBuild) {
2236     args.push_back("--host");
2237   }
2238 
2239   // Image compiler options go last to allow overriding above args, such as --compiler-filter.
2240   for (const std::string& compiler_option : runtime->GetImageCompilerOptions()) {
2241     args.push_back(compiler_option);
2242   }
2243 
2244   // Compile the extension.
2245   VLOG(image) << "Compiling boot image extension for " << (bcp_end - bcp_index)
2246               << " components, starting from " << boot_class_path_locations_[bcp_index];
2247   if (!Exec(args, error_msg)) {
2248     return false;
2249   }
2250 
2251   // Read and validate the image header.
2252   ImageHeader header;
2253   {
2254     File image_file(art_fd.release(), /*check_usage=*/ false);
2255     if (!ReadSpecificImageHeader(&image_file, "compiled image file", &header, error_msg)) {
2256       return false;
2257     }
2258     art_fd.reset(image_file.Release());
2259   }
2260   const char* file_description = "compiled image file";
2261   if (!ValidateHeader(header, bcp_index, file_description, error_msg)) {
2262     return false;
2263   }
2264 
2265   DCHECK(!chunks_.empty());
2266   ImageChunk chunk;
2267   chunk.base_location = base_location;
2268   chunk.base_filename = base_filename;
2269   chunk.profile_file = profile_filename;
2270   chunk.start_index = bcp_index;
2271   chunk.component_count = header.GetComponentCount();
2272   chunk.image_space_count = header.GetImageSpaceCount();
2273   chunk.reservation_size = header.GetImageReservationSize();
2274   chunk.checksum = header.GetImageChecksum();
2275   chunk.boot_image_component_count = header.GetBootImageComponentCount();
2276   chunk.boot_image_checksum = header.GetBootImageChecksum();
2277   chunk.boot_image_size = header.GetBootImageSize();
2278   chunk.art_fd.reset(art_fd.release());
2279   chunk.vdex_fd.reset(vdex_fd.release());
2280   chunk.oat_fd.reset(oat_fd.release());
2281   chunks_.push_back(std::move(chunk));
2282   next_bcp_index_ = bcp_index + header.GetComponentCount();
2283   total_component_count_ += header.GetComponentCount();
2284   total_reservation_size_ += header.GetImageReservationSize();
2285   return true;
2286 }
2287 
CheckAndRemoveLastChunkChecksum(std::string_view * oat_checksums,std::string * error_msg)2288 bool ImageSpace::BootImageLayout::CheckAndRemoveLastChunkChecksum(
2289     /*inout*/std::string_view* oat_checksums,
2290     /*out*/std::string* error_msg) {
2291   DCHECK(oat_checksums != nullptr);
2292   DCHECK(!chunks_.empty());
2293   const ImageChunk& chunk = chunks_.back();
2294   size_t component_count = chunk.component_count;
2295   size_t checksum = chunk.checksum;
2296   if (!CheckAndRemoveImageChecksum(component_count, checksum, oat_checksums, error_msg)) {
2297     DCHECK(!error_msg->empty());
2298     return false;
2299   }
2300   if (oat_checksums->empty()) {
2301     if (next_bcp_index_ != boot_class_path_.size()) {
2302       *error_msg = StringPrintf("Checksum too short, missing %zu components.",
2303                                 boot_class_path_.size() - next_bcp_index_);
2304       return false;
2305     }
2306     return true;
2307   }
2308   if (!StartsWith(*oat_checksums, ":")) {
2309     *error_msg = StringPrintf("Missing ':' separator at start of %s",
2310                               std::string(*oat_checksums).c_str());
2311     return false;
2312   }
2313   oat_checksums->remove_prefix(1u);
2314   if (oat_checksums->empty()) {
2315     *error_msg = "Missing checksums after the ':' separator.";
2316     return false;
2317   }
2318   return true;
2319 }
2320 
2321 template <typename FilenameFn>
LoadOrValidate(FilenameFn && filename_fn,std::string_view * oat_checksums,std::string * error_msg)2322 bool ImageSpace::BootImageLayout::LoadOrValidate(FilenameFn&& filename_fn,
2323                                                  /*inout*/std::string_view* oat_checksums,
2324                                                  /*out*/std::string* error_msg) {
2325   DCHECK(GetChunks().empty());
2326   DCHECK_EQ(GetBaseAddress(), 0u);
2327   bool validate = (oat_checksums != nullptr);
2328   static_assert(ImageSpace::kImageChecksumPrefix == 'i', "Format prefix check.");
2329   DCHECK(!validate || StartsWith(*oat_checksums, "i"));
2330 
2331   std::vector<std::string> components;
2332   Split(image_location_, kComponentSeparator, &components);
2333   size_t named_components_count = 0u;
2334   if (!VerifyImageLocation(components, &named_components_count, error_msg)) {
2335     return false;
2336   }
2337 
2338   ArrayRef<const std::string> named_components =
2339       ArrayRef<const std::string>(components).SubArray(/*pos=*/ 0u, named_components_count);
2340 
2341   std::vector<NamedComponentLocation> named_component_locations;
2342   if (!MatchNamedComponents(named_components, &named_component_locations, error_msg)) {
2343     return false;
2344   }
2345 
2346   // Load the image headers of named components.
2347   DCHECK_EQ(named_component_locations.size(), named_components.size());
2348   const size_t bcp_component_count = boot_class_path_.size();
2349   size_t bcp_pos = 0u;
2350   ArrayRef<std::string> extension_dependencies;
2351   for (size_t i = 0, size = named_components.size(); i != size; ++i) {
2352     const std::string& base_location = named_component_locations[i].base_location;
2353     size_t bcp_index = named_component_locations[i].bcp_index;
2354     const std::string& profile_filename = named_component_locations[i].profile_filename;
2355     if (extension_dependencies.empty() && !profile_filename.empty()) {
2356       // Each extension is compiled against the same dependencies, namely the leading
2357       // named components that were specified without providing the profile filename.
2358       extension_dependencies =
2359           ArrayRef<std::string>(components).SubArray(/*pos=*/ 0, /*length=*/ i);
2360     }
2361     if (bcp_index < bcp_pos) {
2362       DCHECK_NE(i, 0u);
2363       LOG(ERROR) << "Named image component already covered by previous image: " << base_location;
2364       continue;
2365     }
2366     if (validate && bcp_index > bcp_pos) {
2367       *error_msg = StringPrintf("End of contiguous boot class path images, remaining checksum: %s",
2368                                 std::string(*oat_checksums).c_str());
2369       return false;
2370     }
2371     std::string local_error_msg;
2372     std::string* err_msg = (i == 0 || validate) ? error_msg : &local_error_msg;
2373     std::string base_filename;
2374     if (!filename_fn(base_location, &base_filename, err_msg) ||
2375         !ReadHeader(base_location, base_filename, bcp_index, err_msg)) {
2376       if (i == 0u || validate) {
2377         return false;
2378       }
2379       VLOG(image) << "Error reading named image component header for " << base_location
2380                   << ", error: " << local_error_msg;
2381       if (profile_filename.empty() ||
2382           !CompileExtension(base_location,
2383                             base_filename,
2384                             bcp_index,
2385                             profile_filename,
2386                             extension_dependencies,
2387                             &local_error_msg)) {
2388         if (!profile_filename.empty()) {
2389           VLOG(image) << "Error compiling extension for " << boot_class_path_[bcp_index]
2390                       << " error: " << local_error_msg;
2391         }
2392         bcp_pos = bcp_index + 1u;  // Skip at least this component.
2393         DCHECK_GT(bcp_pos, GetNextBcpIndex());
2394         continue;
2395       }
2396     }
2397     if (validate) {
2398       if (!CheckAndRemoveLastChunkChecksum(oat_checksums, error_msg)) {
2399         return false;
2400       }
2401       if (oat_checksums->empty() || !StartsWith(*oat_checksums, "i")) {
2402         return true;  // Let the caller deal with the dex file checksums if any.
2403       }
2404     }
2405     bcp_pos = GetNextBcpIndex();
2406   }
2407 
2408   // Look for remaining components if there are any wildcard specifications.
2409   ArrayRef<const std::string> search_paths =
2410       ArrayRef<const std::string>(components).SubArray(/*pos=*/ named_components_count);
2411   if (!search_paths.empty()) {
2412     const std::string& primary_base_location = named_component_locations[0].base_location;
2413     size_t base_slash_pos = primary_base_location.rfind('/');
2414     DCHECK_NE(base_slash_pos, std::string::npos);
2415     std::string base_name = primary_base_location.substr(base_slash_pos + 1u);
2416     DCHECK(!base_name.empty());
2417     while (bcp_pos != bcp_component_count) {
2418       const std::string& bcp_component =  boot_class_path_[bcp_pos];
2419       bool found = false;
2420       for (const std::string& path : search_paths) {
2421         std::string base_location;
2422         if (path.size() == 1u) {
2423           DCHECK_EQ(path, "*");
2424           size_t slash_pos = bcp_component.rfind('/');
2425           DCHECK_NE(slash_pos, std::string::npos);
2426           base_location = bcp_component.substr(0u, slash_pos + 1u) + base_name;
2427         } else {
2428           DCHECK(EndsWith(path, "/*"));
2429           base_location = path.substr(0u, path.size() - 1u) + base_name;
2430         }
2431         std::string err_msg;  // Ignored.
2432         std::string base_filename;
2433         if (filename_fn(base_location, &base_filename, &err_msg) &&
2434             ReadHeader(base_location, base_filename, bcp_pos, &err_msg)) {
2435           VLOG(image) << "Found image extension for " << ExpandLocation(base_location, bcp_pos);
2436           bcp_pos = GetNextBcpIndex();
2437           found = true;
2438           if (validate) {
2439             if (!CheckAndRemoveLastChunkChecksum(oat_checksums, error_msg)) {
2440               return false;
2441             }
2442             if (oat_checksums->empty() || !StartsWith(*oat_checksums, "i")) {
2443               return true;  // Let the caller deal with the dex file checksums if any.
2444             }
2445           }
2446           break;
2447         }
2448       }
2449       if (!found) {
2450         if (validate) {
2451           *error_msg = StringPrintf("Missing extension for %s, remaining checksum: %s",
2452                                     bcp_component.c_str(),
2453                                     std::string(*oat_checksums).c_str());
2454           return false;
2455         }
2456         ++bcp_pos;
2457       }
2458     }
2459   }
2460 
2461   return true;
2462 }
2463 
LoadOrValidateFromSystem(InstructionSet image_isa,std::string_view * oat_checksums,std::string * error_msg)2464 bool ImageSpace::BootImageLayout::LoadOrValidateFromSystem(InstructionSet image_isa,
2465                                                            /*inout*/std::string_view* oat_checksums,
2466                                                            /*out*/std::string* error_msg) {
2467   auto filename_fn = [image_isa](const std::string& location,
2468                                  /*out*/std::string* filename,
2469                                  /*out*/std::string* err_msg ATTRIBUTE_UNUSED) {
2470     *filename = GetSystemImageFilename(location.c_str(), image_isa);
2471     return true;
2472   };
2473   return LoadOrValidate(filename_fn, oat_checksums, error_msg);
2474 }
2475 
LoadOrValidateFromDalvikCache(const std::string & dalvik_cache,std::string_view * oat_checksums,std::string * error_msg)2476 bool ImageSpace::BootImageLayout::LoadOrValidateFromDalvikCache(
2477     const std::string& dalvik_cache,
2478     /*inout*/std::string_view* oat_checksums,
2479     /*out*/std::string* error_msg) {
2480   auto filename_fn = [&dalvik_cache](const std::string& location,
2481                                      /*out*/std::string* filename,
2482                                      /*out*/std::string* err_msg) {
2483     return GetDalvikCacheFilename(location.c_str(), dalvik_cache.c_str(), filename, err_msg);
2484   };
2485   return LoadOrValidate(filename_fn, oat_checksums, error_msg);
2486 }
2487 
2488 class ImageSpace::BootImageLoader {
2489  public:
BootImageLoader(const std::vector<std::string> & boot_class_path,const std::vector<std::string> & boot_class_path_locations,const std::string & image_location,InstructionSet image_isa,bool relocate,bool executable,bool is_zygote)2490   BootImageLoader(const std::vector<std::string>& boot_class_path,
2491                   const std::vector<std::string>& boot_class_path_locations,
2492                   const std::string& image_location,
2493                   InstructionSet image_isa,
2494                   bool relocate,
2495                   bool executable,
2496                   bool is_zygote)
2497       : boot_class_path_(boot_class_path),
2498         boot_class_path_locations_(boot_class_path_locations),
2499         image_location_(image_location),
2500         image_isa_(image_isa),
2501         relocate_(relocate),
2502         executable_(executable),
2503         is_zygote_(is_zygote),
2504         has_system_(false),
2505         has_cache_(false),
2506         is_global_cache_(true),
2507         dalvik_cache_exists_(false),
2508         dalvik_cache_(),
2509         cache_filename_() {
2510   }
2511 
IsZygote() const2512   bool IsZygote() const { return is_zygote_; }
2513 
FindImageFiles()2514   void FindImageFiles() {
2515     BootImageLayout layout(image_location_, boot_class_path_, boot_class_path_locations_);
2516     std::string image_location = layout.GetPrimaryImageLocation();
2517     std::string system_filename;
2518     bool found_image = FindImageFilenameImpl(image_location.c_str(),
2519                                              image_isa_,
2520                                              &has_system_,
2521                                              &system_filename,
2522                                              &dalvik_cache_exists_,
2523                                              &dalvik_cache_,
2524                                              &is_global_cache_,
2525                                              &has_cache_,
2526                                              &cache_filename_);
2527     DCHECK(!dalvik_cache_exists_ || !dalvik_cache_.empty());
2528     DCHECK_EQ(found_image, has_system_ || has_cache_);
2529   }
2530 
HasSystem() const2531   bool HasSystem() const { return has_system_; }
HasCache() const2532   bool HasCache() const { return has_cache_; }
2533 
DalvikCacheExists() const2534   bool DalvikCacheExists() const { return dalvik_cache_exists_; }
IsGlobalCache() const2535   bool IsGlobalCache() const { return is_global_cache_; }
2536 
GetDalvikCache() const2537   const std::string& GetDalvikCache() const {
2538     return dalvik_cache_;
2539   }
2540 
GetCacheFilename() const2541   const std::string& GetCacheFilename() const {
2542     return cache_filename_;
2543   }
2544 
2545   bool LoadFromSystem(bool validate_oat_file,
2546                       size_t extra_reservation_size,
2547                       /*out*/std::vector<std::unique_ptr<ImageSpace>>* boot_image_spaces,
2548                       /*out*/MemMap* extra_reservation,
2549                       /*out*/std::string* error_msg) REQUIRES_SHARED(Locks::mutator_lock_);
2550 
2551   bool LoadFromDalvikCache(
2552       bool validate_oat_file,
2553       size_t extra_reservation_size,
2554       /*out*/std::vector<std::unique_ptr<ImageSpace>>* boot_image_spaces,
2555       /*out*/MemMap* extra_reservation,
2556       /*out*/std::string* error_msg) REQUIRES_SHARED(Locks::mutator_lock_);
2557 
2558  private:
LoadImage(const BootImageLayout & layout,bool validate_oat_file,size_t extra_reservation_size,TimingLogger * logger,std::vector<std::unique_ptr<ImageSpace>> * boot_image_spaces,MemMap * extra_reservation,std::string * error_msg)2559   bool LoadImage(
2560       const BootImageLayout& layout,
2561       bool validate_oat_file,
2562       size_t extra_reservation_size,
2563       TimingLogger* logger,
2564       /*out*/std::vector<std::unique_ptr<ImageSpace>>* boot_image_spaces,
2565       /*out*/MemMap* extra_reservation,
2566       /*out*/std::string* error_msg) REQUIRES_SHARED(Locks::mutator_lock_) {
2567     ArrayRef<const BootImageLayout::ImageChunk> chunks = layout.GetChunks();
2568     DCHECK(!chunks.empty());
2569     const uint32_t base_address = layout.GetBaseAddress();
2570     const size_t image_component_count = layout.GetTotalComponentCount();
2571     const size_t image_reservation_size = layout.GetTotalReservationSize();
2572 
2573     DCHECK_LE(image_reservation_size, kMaxTotalImageReservationSize);
2574     static_assert(kMaxTotalImageReservationSize < std::numeric_limits<uint32_t>::max());
2575     if (extra_reservation_size > std::numeric_limits<uint32_t>::max() - image_reservation_size) {
2576       // Since the `image_reservation_size` is limited to kMaxTotalImageReservationSize,
2577       // the `extra_reservation_size` would have to be really excessive to fail this check.
2578       *error_msg = StringPrintf("Excessive extra reservation size: %zu", extra_reservation_size);
2579       return false;
2580     }
2581 
2582     // Reserve address space. If relocating, choose a random address for ALSR.
2583     uint8_t* addr = reinterpret_cast<uint8_t*>(
2584         relocate_ ? ART_BASE_ADDRESS + ChooseRelocationOffsetDelta() : base_address);
2585     MemMap image_reservation =
2586         ReserveBootImageMemory(addr, image_reservation_size + extra_reservation_size, error_msg);
2587     if (!image_reservation.IsValid()) {
2588       return false;
2589     }
2590 
2591     // Load components.
2592     std::vector<std::unique_ptr<ImageSpace>> spaces;
2593     spaces.reserve(image_component_count);
2594     size_t max_image_space_dependencies = 0u;
2595     for (size_t i = 0, num_chunks = chunks.size(); i != num_chunks; ++i) {
2596       const BootImageLayout::ImageChunk& chunk = chunks[i];
2597       std::string extension_error_msg;
2598       uint8_t* old_reservation_begin = image_reservation.Begin();
2599       size_t old_reservation_size = image_reservation.Size();
2600       DCHECK_LE(chunk.reservation_size, old_reservation_size);
2601       if (!LoadComponents(chunk,
2602                           validate_oat_file,
2603                           max_image_space_dependencies,
2604                           logger,
2605                           &spaces,
2606                           &image_reservation,
2607                           (i == 0) ? error_msg : &extension_error_msg)) {
2608         // Failed to load the chunk. If this is the primary boot image, report the error.
2609         if (i == 0) {
2610           return false;
2611         }
2612         // For extension, shrink the reservation (and remap if needed, see below).
2613         size_t new_reservation_size = old_reservation_size - chunk.reservation_size;
2614         if (new_reservation_size == 0u) {
2615           DCHECK_EQ(extra_reservation_size, 0u);
2616           DCHECK_EQ(i + 1u, num_chunks);
2617           image_reservation.Reset();
2618         } else if (old_reservation_begin != image_reservation.Begin()) {
2619           // Part of the image reservation has been used and then unmapped when
2620           // rollling back the partial boot image extension load. Try to remap
2621           // the image reservation. As this should be running single-threaded,
2622           // the address range should still be available to mmap().
2623           image_reservation.Reset();
2624           std::string remap_error_msg;
2625           image_reservation = ReserveBootImageMemory(old_reservation_begin,
2626                                                      new_reservation_size,
2627                                                      &remap_error_msg);
2628           if (!image_reservation.IsValid()) {
2629             *error_msg = StringPrintf("Failed to remap boot image reservation after failing "
2630                                           "to load boot image extension (%s: %s): %s",
2631                                       boot_class_path_locations_[chunk.start_index].c_str(),
2632                                       extension_error_msg.c_str(),
2633                                       remap_error_msg.c_str());
2634             return false;
2635           }
2636         } else {
2637           DCHECK_EQ(old_reservation_size, image_reservation.Size());
2638           image_reservation.SetSize(new_reservation_size);
2639         }
2640         LOG(ERROR) << "Failed to load boot image extension "
2641             << boot_class_path_locations_[chunk.start_index] << ": " << extension_error_msg;
2642       }
2643       // Update `max_image_space_dependencies` if all previous BCP components
2644       // were covered and loading the current chunk succeeded.
2645       if (max_image_space_dependencies == chunk.start_index &&
2646           spaces.size() == chunk.start_index + chunk.component_count) {
2647         max_image_space_dependencies = chunk.start_index + chunk.component_count;
2648       }
2649     }
2650 
2651     MemMap local_extra_reservation;
2652     if (!RemapExtraReservation(extra_reservation_size,
2653                                &image_reservation,
2654                                &local_extra_reservation,
2655                                error_msg)) {
2656       return false;
2657     }
2658 
2659     MaybeRelocateSpaces(spaces, logger);
2660     DeduplicateInternedStrings(ArrayRef<const std::unique_ptr<ImageSpace>>(spaces), logger);
2661     boot_image_spaces->swap(spaces);
2662     *extra_reservation = std::move(local_extra_reservation);
2663     return true;
2664   }
2665 
2666  private:
2667   class SimpleRelocateVisitor {
2668    public:
SimpleRelocateVisitor(uint32_t diff,uint32_t begin,uint32_t size)2669     SimpleRelocateVisitor(uint32_t diff, uint32_t begin, uint32_t size)
2670         : diff_(diff), begin_(begin), size_(size) {}
2671 
2672     // Adapter taking the same arguments as SplitRangeRelocateVisitor
2673     // to simplify constructing the various visitors in DoRelocateSpaces().
SimpleRelocateVisitor(uint32_t base_diff,uint32_t current_diff,uint32_t bound,uint32_t begin,uint32_t size)2674     SimpleRelocateVisitor(uint32_t base_diff,
2675                           uint32_t current_diff,
2676                           uint32_t bound,
2677                           uint32_t begin,
2678                           uint32_t size)
2679         : SimpleRelocateVisitor(base_diff, begin, size) {
2680       // Check arguments unused by this class.
2681       DCHECK_EQ(base_diff, current_diff);
2682       DCHECK_EQ(bound, begin);
2683     }
2684 
2685     template <typename T>
operator ()(T * src) const2686     ALWAYS_INLINE T* operator()(T* src) const {
2687       DCHECK(InSource(src));
2688       uint32_t raw_src = reinterpret_cast32<uint32_t>(src);
2689       return reinterpret_cast32<T*>(raw_src + diff_);
2690     }
2691 
2692     template <typename T>
InSource(T * ptr) const2693     ALWAYS_INLINE bool InSource(T* ptr) const {
2694       uint32_t raw_ptr = reinterpret_cast32<uint32_t>(ptr);
2695       return raw_ptr - begin_ < size_;
2696     }
2697 
2698     template <typename T>
InDest(T * ptr) const2699     ALWAYS_INLINE bool InDest(T* ptr) const {
2700       uint32_t raw_ptr = reinterpret_cast32<uint32_t>(ptr);
2701       uint32_t src_ptr = raw_ptr - diff_;
2702       return src_ptr - begin_ < size_;
2703     }
2704 
2705    private:
2706     const uint32_t diff_;
2707     const uint32_t begin_;
2708     const uint32_t size_;
2709   };
2710 
2711   class SplitRangeRelocateVisitor {
2712    public:
SplitRangeRelocateVisitor(uint32_t base_diff,uint32_t current_diff,uint32_t bound,uint32_t begin,uint32_t size)2713     SplitRangeRelocateVisitor(uint32_t base_diff,
2714                               uint32_t current_diff,
2715                               uint32_t bound,
2716                               uint32_t begin,
2717                               uint32_t size)
2718         : base_diff_(base_diff),
2719           current_diff_(current_diff),
2720           bound_(bound),
2721           begin_(begin),
2722           size_(size) {
2723       DCHECK_NE(begin_, bound_);
2724       // The bound separates the boot image range and the extension range.
2725       DCHECK_LT(bound_ - begin_, size_);
2726     }
2727 
2728     template <typename T>
operator ()(T * src) const2729     ALWAYS_INLINE T* operator()(T* src) const {
2730       DCHECK(InSource(src));
2731       uint32_t raw_src = reinterpret_cast32<uint32_t>(src);
2732       uint32_t diff = (raw_src < bound_) ? base_diff_ : current_diff_;
2733       return reinterpret_cast32<T*>(raw_src + diff);
2734     }
2735 
2736     template <typename T>
InSource(T * ptr) const2737     ALWAYS_INLINE bool InSource(T* ptr) const {
2738       uint32_t raw_ptr = reinterpret_cast32<uint32_t>(ptr);
2739       return raw_ptr - begin_ < size_;
2740     }
2741 
2742    private:
2743     const uint32_t base_diff_;
2744     const uint32_t current_diff_;
2745     const uint32_t bound_;
2746     const uint32_t begin_;
2747     const uint32_t size_;
2748   };
2749 
PointerAddress(ArtMethod * method,MemberOffset offset)2750   static void** PointerAddress(ArtMethod* method, MemberOffset offset) {
2751     return reinterpret_cast<void**>(reinterpret_cast<uint8_t*>(method) + offset.Uint32Value());
2752   }
2753 
2754   template <PointerSize kPointerSize>
DoRelocateSpaces(ArrayRef<const std::unique_ptr<ImageSpace>> & spaces,int64_t base_diff64)2755   static void DoRelocateSpaces(ArrayRef<const std::unique_ptr<ImageSpace>>& spaces,
2756                                int64_t base_diff64) REQUIRES_SHARED(Locks::mutator_lock_) {
2757     DCHECK(!spaces.empty());
2758     gc::accounting::ContinuousSpaceBitmap patched_objects(
2759         gc::accounting::ContinuousSpaceBitmap::Create(
2760             "Marked objects",
2761             spaces.front()->Begin(),
2762             spaces.back()->End() - spaces.front()->Begin()));
2763     const ImageHeader& base_header = spaces[0]->GetImageHeader();
2764     size_t base_image_space_count = base_header.GetImageSpaceCount();
2765     DCHECK_LE(base_image_space_count, spaces.size());
2766     DoRelocateSpaces<kPointerSize, /*kExtension=*/ false>(
2767         spaces.SubArray(/*pos=*/ 0u, base_image_space_count),
2768         base_diff64,
2769         &patched_objects);
2770 
2771     for (size_t i = base_image_space_count, size = spaces.size(); i != size; ) {
2772       const ImageHeader& ext_header = spaces[i]->GetImageHeader();
2773       size_t ext_image_space_count = ext_header.GetImageSpaceCount();
2774       DCHECK_LE(ext_image_space_count, size - i);
2775       DoRelocateSpaces<kPointerSize, /*kExtension=*/ true>(
2776           spaces.SubArray(/*pos=*/ i, ext_image_space_count),
2777           base_diff64,
2778           &patched_objects);
2779       i += ext_image_space_count;
2780     }
2781   }
2782 
2783   template <PointerSize kPointerSize, bool kExtension>
DoRelocateSpaces(ArrayRef<const std::unique_ptr<ImageSpace>> spaces,int64_t base_diff64,gc::accounting::ContinuousSpaceBitmap * patched_objects)2784   static void DoRelocateSpaces(ArrayRef<const std::unique_ptr<ImageSpace>> spaces,
2785                                int64_t base_diff64,
2786                                gc::accounting::ContinuousSpaceBitmap* patched_objects)
2787       REQUIRES_SHARED(Locks::mutator_lock_) {
2788     DCHECK(!spaces.empty());
2789     const ImageHeader& first_header = spaces.front()->GetImageHeader();
2790     uint32_t image_begin = reinterpret_cast32<uint32_t>(first_header.GetImageBegin());
2791     uint32_t image_size = first_header.GetImageReservationSize();
2792     DCHECK_NE(image_size, 0u);
2793     uint32_t source_begin = kExtension ? first_header.GetBootImageBegin() : image_begin;
2794     uint32_t source_size = kExtension ? first_header.GetBootImageSize() + image_size : image_size;
2795     if (kExtension) {
2796       DCHECK_EQ(first_header.GetBootImageBegin() + first_header.GetBootImageSize(), image_begin);
2797     }
2798     int64_t current_diff64 = kExtension
2799         ? static_cast<int64_t>(reinterpret_cast32<uint32_t>(spaces.front()->Begin())) -
2800               static_cast<int64_t>(image_begin)
2801         : base_diff64;
2802     if (base_diff64 == 0 && current_diff64 == 0) {
2803       return;
2804     }
2805     uint32_t base_diff = static_cast<uint32_t>(base_diff64);
2806     uint32_t current_diff = static_cast<uint32_t>(current_diff64);
2807 
2808     // For boot image the main visitor is a SimpleRelocateVisitor. For the boot image extension we
2809     // mostly use a SplitRelocationVisitor but some work can still use the SimpleRelocationVisitor.
2810     using MainRelocateVisitor = typename std::conditional<
2811         kExtension, SplitRangeRelocateVisitor, SimpleRelocateVisitor>::type;
2812     SimpleRelocateVisitor simple_relocate_visitor(current_diff, image_begin, image_size);
2813     MainRelocateVisitor main_relocate_visitor(
2814         base_diff, current_diff, /*bound=*/ image_begin, source_begin, source_size);
2815 
2816     using MainPatchRelocateVisitor =
2817         PatchObjectVisitor<kPointerSize, MainRelocateVisitor, MainRelocateVisitor>;
2818     using SimplePatchRelocateVisitor =
2819         PatchObjectVisitor<kPointerSize, SimpleRelocateVisitor, SimpleRelocateVisitor>;
2820     MainPatchRelocateVisitor main_patch_object_visitor(main_relocate_visitor,
2821                                                        main_relocate_visitor);
2822     SimplePatchRelocateVisitor simple_patch_object_visitor(simple_relocate_visitor,
2823                                                            simple_relocate_visitor);
2824 
2825     // Retrieve the Class.class, Method.class and Constructor.class needed in the loops below.
2826     ObjPtr<mirror::ObjectArray<mirror::Class>> class_roots;
2827     ObjPtr<mirror::Class> class_class;
2828     ObjPtr<mirror::Class> method_class;
2829     ObjPtr<mirror::Class> constructor_class;
2830     {
2831       ObjPtr<mirror::ObjectArray<mirror::Object>> image_roots =
2832           simple_relocate_visitor(first_header.GetImageRoots<kWithoutReadBarrier>().Ptr());
2833       DCHECK(!patched_objects->Test(image_roots.Ptr()));
2834 
2835       SimpleRelocateVisitor base_relocate_visitor(
2836           base_diff,
2837           source_begin,
2838           kExtension ? source_size - image_size : image_size);
2839       int32_t class_roots_index = enum_cast<int32_t>(ImageHeader::kClassRoots);
2840       DCHECK_LT(class_roots_index, image_roots->GetLength<kVerifyNone>());
2841       class_roots = ObjPtr<mirror::ObjectArray<mirror::Class>>::DownCast(base_relocate_visitor(
2842           image_roots->GetWithoutChecks<kVerifyNone>(class_roots_index).Ptr()));
2843       if (kExtension) {
2844         // Class roots must have been visited if we relocated the primary boot image.
2845         DCHECK(base_diff == 0 || patched_objects->Test(class_roots.Ptr()));
2846         class_class = GetClassRoot<mirror::Class, kWithoutReadBarrier>(class_roots);
2847         method_class = GetClassRoot<mirror::Method, kWithoutReadBarrier>(class_roots);
2848         constructor_class = GetClassRoot<mirror::Constructor, kWithoutReadBarrier>(class_roots);
2849       } else {
2850         DCHECK(!patched_objects->Test(class_roots.Ptr()));
2851         class_class = simple_relocate_visitor(
2852             GetClassRoot<mirror::Class, kWithoutReadBarrier>(class_roots).Ptr());
2853         method_class = simple_relocate_visitor(
2854             GetClassRoot<mirror::Method, kWithoutReadBarrier>(class_roots).Ptr());
2855         constructor_class = simple_relocate_visitor(
2856             GetClassRoot<mirror::Constructor, kWithoutReadBarrier>(class_roots).Ptr());
2857       }
2858     }
2859 
2860     for (const std::unique_ptr<ImageSpace>& space : spaces) {
2861       // First patch the image header.
2862       reinterpret_cast<ImageHeader*>(space->Begin())->RelocateImageReferences(current_diff64);
2863       reinterpret_cast<ImageHeader*>(space->Begin())->RelocateBootImageReferences(base_diff64);
2864 
2865       // Patch fields and methods.
2866       const ImageHeader& image_header = space->GetImageHeader();
2867       image_header.VisitPackedArtFields([&](ArtField& field) REQUIRES_SHARED(Locks::mutator_lock_) {
2868         // Fields always reference class in the current image.
2869         simple_patch_object_visitor.template PatchGcRoot</*kMayBeNull=*/ false>(
2870             &field.DeclaringClassRoot());
2871       }, space->Begin());
2872       image_header.VisitPackedArtMethods([&](ArtMethod& method)
2873           REQUIRES_SHARED(Locks::mutator_lock_) {
2874         main_patch_object_visitor.PatchGcRoot(&method.DeclaringClassRoot());
2875         void** data_address = PointerAddress(&method, ArtMethod::DataOffset(kPointerSize));
2876         main_patch_object_visitor.PatchNativePointer(data_address);
2877         void** entrypoint_address =
2878             PointerAddress(&method, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kPointerSize));
2879         main_patch_object_visitor.PatchNativePointer(entrypoint_address);
2880       }, space->Begin(), kPointerSize);
2881       auto method_table_visitor = [&](ArtMethod* method) {
2882         DCHECK(method != nullptr);
2883         return main_relocate_visitor(method);
2884       };
2885       image_header.VisitPackedImTables(method_table_visitor, space->Begin(), kPointerSize);
2886       image_header.VisitPackedImtConflictTables(method_table_visitor, space->Begin(), kPointerSize);
2887 
2888       // Patch the intern table.
2889       if (image_header.GetInternedStringsSection().Size() != 0u) {
2890         const uint8_t* data = space->Begin() + image_header.GetInternedStringsSection().Offset();
2891         size_t read_count;
2892         InternTable::UnorderedSet temp_set(data, /*make_copy_of_data=*/ false, &read_count);
2893         for (GcRoot<mirror::String>& slot : temp_set) {
2894           // The intern table contains only strings in the current image.
2895           simple_patch_object_visitor.template PatchGcRoot</*kMayBeNull=*/ false>(&slot);
2896         }
2897       }
2898 
2899       // Patch the class table and classes, so that we can traverse class hierarchy to
2900       // determine the types of other objects when we visit them later.
2901       if (image_header.GetClassTableSection().Size() != 0u) {
2902         uint8_t* data = space->Begin() + image_header.GetClassTableSection().Offset();
2903         size_t read_count;
2904         ClassTable::ClassSet temp_set(data, /*make_copy_of_data=*/ false, &read_count);
2905         DCHECK(!temp_set.empty());
2906         // The class table contains only classes in the current image.
2907         ClassTableVisitor class_table_visitor(simple_relocate_visitor);
2908         for (ClassTable::TableSlot& slot : temp_set) {
2909           slot.VisitRoot(class_table_visitor);
2910           ObjPtr<mirror::Class> klass = slot.Read<kWithoutReadBarrier>();
2911           DCHECK(klass != nullptr);
2912           DCHECK(!patched_objects->Test(klass.Ptr()));
2913           patched_objects->Set(klass.Ptr());
2914           main_patch_object_visitor.VisitClass(klass, class_class);
2915           // Then patch the non-embedded vtable and iftable.
2916           ObjPtr<mirror::PointerArray> vtable =
2917               klass->GetVTable<kVerifyNone, kWithoutReadBarrier>();
2918           if ((kExtension ? simple_relocate_visitor.InDest(vtable.Ptr()) : vtable != nullptr) &&
2919               !patched_objects->Set(vtable.Ptr())) {
2920             main_patch_object_visitor.VisitPointerArray(vtable);
2921           }
2922           ObjPtr<mirror::IfTable> iftable = klass->GetIfTable<kVerifyNone, kWithoutReadBarrier>();
2923           if (iftable != nullptr) {
2924             int32_t ifcount = klass->GetIfTableCount<kVerifyNone>();
2925             for (int32_t i = 0; i != ifcount; ++i) {
2926               ObjPtr<mirror::PointerArray> unpatched_ifarray =
2927                   iftable->GetMethodArrayOrNull<kVerifyNone, kWithoutReadBarrier>(i);
2928               if (kExtension ? simple_relocate_visitor.InSource(unpatched_ifarray.Ptr())
2929                              : unpatched_ifarray != nullptr) {
2930                 // The iftable has not been patched, so we need to explicitly adjust the pointer.
2931                 ObjPtr<mirror::PointerArray> ifarray =
2932                     simple_relocate_visitor(unpatched_ifarray.Ptr());
2933                 if (!patched_objects->Set(ifarray.Ptr())) {
2934                   main_patch_object_visitor.VisitPointerArray(ifarray);
2935                 }
2936               }
2937             }
2938           }
2939         }
2940       }
2941     }
2942 
2943     for (const std::unique_ptr<ImageSpace>& space : spaces) {
2944       const ImageHeader& image_header = space->GetImageHeader();
2945 
2946       static_assert(IsAligned<kObjectAlignment>(sizeof(ImageHeader)), "Header alignment check");
2947       uint32_t objects_end = image_header.GetObjectsSection().Size();
2948       DCHECK_ALIGNED(objects_end, kObjectAlignment);
2949       for (uint32_t pos = sizeof(ImageHeader); pos != objects_end; ) {
2950         mirror::Object* object = reinterpret_cast<mirror::Object*>(space->Begin() + pos);
2951         // Note: use Test() rather than Set() as this is the last time we're checking this object.
2952         if (!patched_objects->Test(object)) {
2953           // This is the last pass over objects, so we do not need to Set().
2954           main_patch_object_visitor.VisitObject(object);
2955           ObjPtr<mirror::Class> klass = object->GetClass<kVerifyNone, kWithoutReadBarrier>();
2956           if (klass->IsDexCacheClass<kVerifyNone>()) {
2957             // Patch dex cache array pointers and elements.
2958             ObjPtr<mirror::DexCache> dex_cache =
2959                 object->AsDexCache<kVerifyNone, kWithoutReadBarrier>();
2960             main_patch_object_visitor.VisitDexCacheArrays(dex_cache);
2961           } else if (klass == method_class || klass == constructor_class) {
2962             // Patch the ArtMethod* in the mirror::Executable subobject.
2963             ObjPtr<mirror::Executable> as_executable =
2964                 ObjPtr<mirror::Executable>::DownCast(object);
2965             ArtMethod* unpatched_method = as_executable->GetArtMethod<kVerifyNone>();
2966             ArtMethod* patched_method = main_relocate_visitor(unpatched_method);
2967             as_executable->SetArtMethod</*kTransactionActive=*/ false,
2968                                         /*kCheckTransaction=*/ true,
2969                                         kVerifyNone>(patched_method);
2970           }
2971         }
2972         pos += RoundUp(object->SizeOf<kVerifyNone>(), kObjectAlignment);
2973       }
2974     }
2975     if (kIsDebugBuild && !kExtension) {
2976       // We used just Test() instead of Set() above but we need to use Set()
2977       // for class roots to satisfy a DCHECK() for extensions.
2978       DCHECK(!patched_objects->Test(class_roots.Ptr()));
2979       patched_objects->Set(class_roots.Ptr());
2980     }
2981   }
2982 
MaybeRelocateSpaces(const std::vector<std::unique_ptr<ImageSpace>> & spaces,TimingLogger * logger)2983   void MaybeRelocateSpaces(const std::vector<std::unique_ptr<ImageSpace>>& spaces,
2984                            TimingLogger* logger)
2985       REQUIRES_SHARED(Locks::mutator_lock_) {
2986     TimingLogger::ScopedTiming timing("MaybeRelocateSpaces", logger);
2987     ImageSpace* first_space = spaces.front().get();
2988     const ImageHeader& first_space_header = first_space->GetImageHeader();
2989     int64_t base_diff64 =
2990         static_cast<int64_t>(reinterpret_cast32<uint32_t>(first_space->Begin())) -
2991         static_cast<int64_t>(reinterpret_cast32<uint32_t>(first_space_header.GetImageBegin()));
2992     if (!relocate_) {
2993       DCHECK_EQ(base_diff64, 0);
2994     }
2995 
2996     ArrayRef<const std::unique_ptr<ImageSpace>> spaces_ref(spaces);
2997     PointerSize pointer_size = first_space_header.GetPointerSize();
2998     if (pointer_size == PointerSize::k64) {
2999       DoRelocateSpaces<PointerSize::k64>(spaces_ref, base_diff64);
3000     } else {
3001       DoRelocateSpaces<PointerSize::k32>(spaces_ref, base_diff64);
3002     }
3003   }
3004 
DeduplicateInternedStrings(ArrayRef<const std::unique_ptr<ImageSpace>> spaces,TimingLogger * logger)3005   void DeduplicateInternedStrings(ArrayRef<const std::unique_ptr<ImageSpace>> spaces,
3006                                   TimingLogger* logger) REQUIRES_SHARED(Locks::mutator_lock_) {
3007     TimingLogger::ScopedTiming timing("DeduplicateInternedStrings", logger);
3008     DCHECK(!spaces.empty());
3009     size_t num_spaces = spaces.size();
3010     const ImageHeader& primary_header = spaces.front()->GetImageHeader();
3011     size_t primary_image_count = primary_header.GetImageSpaceCount();
3012     DCHECK_LE(primary_image_count, num_spaces);
3013     DCHECK_EQ(primary_image_count, primary_header.GetComponentCount());
3014     size_t component_count = primary_image_count;
3015     size_t space_pos = primary_image_count;
3016     while (space_pos != num_spaces) {
3017       const ImageHeader& current_header = spaces[space_pos]->GetImageHeader();
3018       size_t image_space_count = current_header.GetImageSpaceCount();
3019       DCHECK_LE(image_space_count, num_spaces - space_pos);
3020       size_t dependency_component_count = current_header.GetBootImageComponentCount();
3021       DCHECK_LE(dependency_component_count, component_count);
3022       if (dependency_component_count < component_count) {
3023         // There shall be no duplicate strings with the components that this space depends on.
3024         // Find the end of the dependencies, i.e. start of non-dependency images.
3025         size_t start_component_count = primary_image_count;
3026         size_t start_pos = primary_image_count;
3027         while (start_component_count != dependency_component_count) {
3028           const ImageHeader& dependency_header = spaces[start_pos]->GetImageHeader();
3029           DCHECK_LE(dependency_header.GetComponentCount(),
3030                     dependency_component_count - start_component_count);
3031           start_component_count += dependency_header.GetComponentCount();
3032           start_pos += dependency_header.GetImageSpaceCount();
3033         }
3034         // Remove duplicates from all intern tables belonging to the chunk.
3035         ArrayRef<const std::unique_ptr<ImageSpace>> old_spaces =
3036             spaces.SubArray(/*pos=*/ start_pos, space_pos - start_pos);
3037         SafeMap<mirror::String*, mirror::String*> intern_remap;
3038         for (size_t i = 0; i != image_space_count; ++i) {
3039           ImageSpace* new_space = spaces[space_pos + i].get();
3040           Loader::RemoveInternTableDuplicates(old_spaces, new_space, &intern_remap);
3041         }
3042         // Remap string for all spaces belonging to the chunk.
3043         if (!intern_remap.empty()) {
3044           for (size_t i = 0; i != image_space_count; ++i) {
3045             ImageSpace* new_space = spaces[space_pos + i].get();
3046             Loader::RemapInternedStringDuplicates(intern_remap, new_space);
3047           }
3048         }
3049       }
3050       component_count += current_header.GetComponentCount();
3051       space_pos += image_space_count;
3052     }
3053   }
3054 
Load(const std::string & image_location,const std::string & image_filename,const std::string & profile_file,android::base::unique_fd art_fd,TimingLogger * logger,MemMap * image_reservation,std::string * error_msg)3055   std::unique_ptr<ImageSpace> Load(const std::string& image_location,
3056                                    const std::string& image_filename,
3057                                    const std::string& profile_file,
3058                                    android::base::unique_fd art_fd,
3059                                    TimingLogger* logger,
3060                                    /*inout*/MemMap* image_reservation,
3061                                    /*out*/std::string* error_msg)
3062       REQUIRES_SHARED(Locks::mutator_lock_) {
3063     if (art_fd.get() != -1) {
3064       // No need to lock memfd for which we hold the only file descriptor
3065       // (see locking with ScopedFlock for normal files below).
3066       VLOG(startup) << "Using image file " << image_filename.c_str() << " for image location "
3067                     << image_location << " for compiled extension";
3068 
3069       File image_file(art_fd.release(), image_filename, /*check_usage=*/ false);
3070       std::unique_ptr<ImageSpace> result = Loader::Init(&image_file,
3071                                                         image_filename.c_str(),
3072                                                         image_location.c_str(),
3073                                                         profile_file.c_str(),
3074                                                         /*allow_direct_mapping=*/ false,
3075                                                         logger,
3076                                                         image_reservation,
3077                                                         error_msg);
3078       // Note: We're closing the image file descriptor here when we destroy
3079       // the `image_file` as we no longer need it.
3080       return result;
3081     }
3082 
3083     // Should this be a RDWR lock? This is only a defensive measure, as at
3084     // this point the image should exist.
3085     // However, only the zygote can write into the global dalvik-cache, so
3086     // restrict to zygote processes, or any process that isn't using
3087     // /data/dalvik-cache (which we assume to be allowed to write there).
3088     const bool rw_lock = is_zygote_ || !is_global_cache_;
3089 
3090     // Note that we must not use the file descriptor associated with
3091     // ScopedFlock::GetFile to Init the image file. We want the file
3092     // descriptor (and the associated exclusive lock) to be released when
3093     // we leave Create.
3094     ScopedFlock image = LockedFile::Open(image_filename.c_str(),
3095                                          /*flags=*/ rw_lock ? (O_CREAT | O_RDWR) : O_RDONLY,
3096                                          /*block=*/ true,
3097                                          error_msg);
3098 
3099     VLOG(startup) << "Using image file " << image_filename.c_str() << " for image location "
3100                   << image_location;
3101 
3102     // If we are in /system we can assume the image is good. We can also
3103     // assume this if we are using a relocated image (i.e. image checksum
3104     // matches) since this is only different by the offset. We need this to
3105     // make sure that host tests continue to work.
3106     // Since we are the boot image, pass null since we load the oat file from the boot image oat
3107     // file name.
3108     return Loader::Init(image_filename.c_str(),
3109                         image_location.c_str(),
3110                         logger,
3111                         image_reservation,
3112                         error_msg);
3113   }
3114 
OpenOatFile(ImageSpace * space,android::base::unique_fd vdex_fd,android::base::unique_fd oat_fd,ArrayRef<const std::string> dex_filenames,bool validate_oat_file,ArrayRef<const std::unique_ptr<ImageSpace>> dependencies,TimingLogger * logger,MemMap * image_reservation,std::string * error_msg)3115   bool OpenOatFile(ImageSpace* space,
3116                    android::base::unique_fd vdex_fd,
3117                    android::base::unique_fd oat_fd,
3118                    ArrayRef<const std::string> dex_filenames,
3119                    bool validate_oat_file,
3120                    ArrayRef<const std::unique_ptr<ImageSpace>> dependencies,
3121                    TimingLogger* logger,
3122                    /*inout*/MemMap* image_reservation,
3123                    /*out*/std::string* error_msg) {
3124     // VerifyImageAllocations() will be called later in Runtime::Init()
3125     // as some class roots like ArtMethod::java_lang_reflect_ArtMethod_
3126     // and ArtField::java_lang_reflect_ArtField_, which are used from
3127     // Object::SizeOf() which VerifyImageAllocations() calls, are not
3128     // set yet at this point.
3129     DCHECK(image_reservation != nullptr);
3130     std::unique_ptr<OatFile> oat_file;
3131     {
3132       TimingLogger::ScopedTiming timing("OpenOatFile", logger);
3133       std::string oat_filename =
3134           ImageHeader::GetOatLocationFromImageLocation(space->GetImageFilename());
3135       std::string oat_location =
3136           ImageHeader::GetOatLocationFromImageLocation(space->GetImageLocation());
3137 
3138       DCHECK_EQ(vdex_fd.get() != -1, oat_fd.get() != -1);
3139       if (vdex_fd.get() == -1) {
3140         oat_file.reset(OatFile::Open(/*zip_fd=*/ -1,
3141                                      oat_filename,
3142                                      oat_location,
3143                                      executable_,
3144                                      /*low_4gb=*/ false,
3145                                      dex_filenames,
3146                                      image_reservation,
3147                                      error_msg));
3148       } else {
3149         oat_file.reset(OatFile::Open(/*zip_fd=*/ -1,
3150                                      vdex_fd.get(),
3151                                      oat_fd.get(),
3152                                      oat_location,
3153                                      executable_,
3154                                      /*low_4gb=*/ false,
3155                                      dex_filenames,
3156                                      image_reservation,
3157                                      error_msg));
3158         // We no longer need the file descriptors and they will be closed by
3159         // the unique_fd destructor when we leave this function.
3160       }
3161 
3162       if (oat_file == nullptr) {
3163         *error_msg = StringPrintf("Failed to open oat file '%s' referenced from image %s: %s",
3164                                   oat_filename.c_str(),
3165                                   space->GetName(),
3166                                   error_msg->c_str());
3167         return false;
3168       }
3169       const ImageHeader& image_header = space->GetImageHeader();
3170       uint32_t oat_checksum = oat_file->GetOatHeader().GetChecksum();
3171       uint32_t image_oat_checksum = image_header.GetOatChecksum();
3172       if (oat_checksum != image_oat_checksum) {
3173         *error_msg = StringPrintf("Failed to match oat file checksum 0x%x to expected oat checksum"
3174                                   " 0x%x in image %s",
3175                                   oat_checksum,
3176                                   image_oat_checksum,
3177                                   space->GetName());
3178         return false;
3179       }
3180       const char* oat_boot_class_path =
3181           oat_file->GetOatHeader().GetStoreValueByKey(OatHeader::kBootClassPathKey);
3182       oat_boot_class_path = (oat_boot_class_path != nullptr) ? oat_boot_class_path : "";
3183       const char* oat_boot_class_path_checksums =
3184           oat_file->GetOatHeader().GetStoreValueByKey(OatHeader::kBootClassPathChecksumsKey);
3185       oat_boot_class_path_checksums =
3186           (oat_boot_class_path_checksums != nullptr) ? oat_boot_class_path_checksums : "";
3187       size_t component_count = image_header.GetComponentCount();
3188       if (component_count == 0u) {
3189         if (oat_boot_class_path[0] != 0 || oat_boot_class_path_checksums[0] != 0) {
3190           *error_msg = StringPrintf("Unexpected non-empty boot class path %s and/or checksums %s"
3191                                     " in image %s",
3192                                     oat_boot_class_path,
3193                                     oat_boot_class_path_checksums,
3194                                     space->GetName());
3195           return false;
3196         }
3197       } else if (dependencies.empty()) {
3198         std::string expected_boot_class_path = Join(ArrayRef<const std::string>(
3199               boot_class_path_locations_).SubArray(0u, component_count), ':');
3200         if (expected_boot_class_path != oat_boot_class_path) {
3201           *error_msg = StringPrintf("Failed to match oat boot class path %s to expected "
3202                                     "boot class path %s in image %s",
3203                                     oat_boot_class_path,
3204                                     expected_boot_class_path.c_str(),
3205                                     space->GetName());
3206           return false;
3207         }
3208       } else {
3209         std::string local_error_msg;
3210         if (!VerifyBootClassPathChecksums(
3211                  oat_boot_class_path_checksums,
3212                  oat_boot_class_path,
3213                  dependencies,
3214                  ArrayRef<const std::string>(boot_class_path_locations_),
3215                  ArrayRef<const std::string>(boot_class_path_),
3216                  &local_error_msg)) {
3217           *error_msg = StringPrintf("Failed to verify BCP %s with checksums %s in image %s: %s",
3218                                     oat_boot_class_path,
3219                                     oat_boot_class_path_checksums,
3220                                     space->GetName(),
3221                                     local_error_msg.c_str());
3222           return false;
3223         }
3224       }
3225       ptrdiff_t relocation_diff = space->Begin() - image_header.GetImageBegin();
3226       CHECK(image_header.GetOatDataBegin() != nullptr);
3227       uint8_t* oat_data_begin = image_header.GetOatDataBegin() + relocation_diff;
3228       if (oat_file->Begin() != oat_data_begin) {
3229         *error_msg = StringPrintf("Oat file '%s' referenced from image %s has unexpected begin"
3230                                       " %p v. %p",
3231                                   oat_filename.c_str(),
3232                                   space->GetName(),
3233                                   oat_file->Begin(),
3234                                   oat_data_begin);
3235         return false;
3236       }
3237     }
3238     if (validate_oat_file) {
3239       TimingLogger::ScopedTiming timing("ValidateOatFile", logger);
3240       if (!ImageSpace::ValidateOatFile(*oat_file, error_msg)) {
3241         DCHECK(!error_msg->empty());
3242         return false;
3243       }
3244     }
3245     space->oat_file_ = std::move(oat_file);
3246     space->oat_file_non_owned_ = space->oat_file_.get();
3247     return true;
3248   }
3249 
LoadComponents(const BootImageLayout::ImageChunk & chunk,bool validate_oat_file,size_t max_image_space_dependencies,TimingLogger * logger,std::vector<std::unique_ptr<ImageSpace>> * spaces,MemMap * image_reservation,std::string * error_msg)3250   bool LoadComponents(const BootImageLayout::ImageChunk& chunk,
3251                       bool validate_oat_file,
3252                       size_t max_image_space_dependencies,
3253                       TimingLogger* logger,
3254                       /*inout*/std::vector<std::unique_ptr<ImageSpace>>* spaces,
3255                       /*inout*/MemMap* image_reservation,
3256                       /*out*/std::string* error_msg)
3257       REQUIRES_SHARED(Locks::mutator_lock_) {
3258     // Make sure we destroy the spaces we created if we're returning an error.
3259     // Note that this can unmap part of the original `image_reservation`.
3260     class Guard {
3261      public:
3262       explicit Guard(std::vector<std::unique_ptr<ImageSpace>>* spaces_in)
3263           : spaces_(spaces_in), committed_(spaces_->size()) {}
3264       void Commit() {
3265         DCHECK_LT(committed_, spaces_->size());
3266         committed_ = spaces_->size();
3267       }
3268       ~Guard() {
3269         DCHECK_LE(committed_, spaces_->size());
3270         spaces_->resize(committed_);
3271       }
3272      private:
3273       std::vector<std::unique_ptr<ImageSpace>>* const spaces_;
3274       size_t committed_;
3275     };
3276     Guard guard(spaces);
3277 
3278     bool is_extension = (chunk.start_index != 0u);
3279     DCHECK_NE(spaces->empty(), is_extension);
3280     if (max_image_space_dependencies < chunk.boot_image_component_count) {
3281       DCHECK(is_extension);
3282       *error_msg = StringPrintf("Missing dependencies for extension component %s, %zu < %u",
3283                                 boot_class_path_locations_[chunk.start_index].c_str(),
3284                                 max_image_space_dependencies,
3285                                 chunk.boot_image_component_count);
3286       return false;
3287     }
3288     ArrayRef<const std::string> requested_bcp_locations =
3289         ArrayRef<const std::string>(boot_class_path_locations_).SubArray(
3290             chunk.start_index, chunk.image_space_count);
3291     std::vector<std::string> locations =
3292         ExpandMultiImageLocations(requested_bcp_locations, chunk.base_location, is_extension);
3293     std::vector<std::string> filenames =
3294         ExpandMultiImageLocations(requested_bcp_locations, chunk.base_filename, is_extension);
3295     DCHECK_EQ(locations.size(), filenames.size());
3296     for (size_t i = 0u, size = locations.size(); i != size; ++i) {
3297       spaces->push_back(Load(locations[i],
3298                              filenames[i],
3299                              chunk.profile_file,
3300                              std::move(chunk.art_fd),
3301                              logger,
3302                              image_reservation,
3303                              error_msg));
3304       const ImageSpace* space = spaces->back().get();
3305       if (space == nullptr) {
3306         return false;
3307       }
3308       uint32_t expected_component_count = (i == 0u) ? chunk.component_count : 0u;
3309       uint32_t expected_reservation_size = (i == 0u) ? chunk.reservation_size : 0u;
3310       if (!Loader::CheckImageReservationSize(*space, expected_reservation_size, error_msg) ||
3311           !Loader::CheckImageComponentCount(*space, expected_component_count, error_msg)) {
3312         return false;
3313       }
3314       const ImageHeader& header = space->GetImageHeader();
3315       if (i == 0 && (chunk.checksum != header.GetImageChecksum() ||
3316                      chunk.image_space_count != header.GetImageSpaceCount() ||
3317                      chunk.boot_image_component_count != header.GetBootImageComponentCount() ||
3318                      chunk.boot_image_checksum != header.GetBootImageChecksum() ||
3319                      chunk.boot_image_size != header.GetBootImageSize())) {
3320         *error_msg = StringPrintf("Image header modified since previously read from %s; "
3321                                       "checksum: 0x%08x -> 0x%08x,"
3322                                       "image_space_count: %u -> %u"
3323                                       "boot_image_component_count: %u -> %u, "
3324                                       "boot_image_checksum: 0x%08x -> 0x%08x"
3325                                       "boot_image_size: 0x%08x -> 0x%08x",
3326                                   space->GetImageFilename().c_str(),
3327                                   chunk.checksum,
3328                                   chunk.image_space_count,
3329                                   header.GetImageSpaceCount(),
3330                                   header.GetImageChecksum(),
3331                                   chunk.boot_image_component_count,
3332                                   header.GetBootImageComponentCount(),
3333                                   chunk.boot_image_checksum,
3334                                   header.GetBootImageChecksum(),
3335                                   chunk.boot_image_size,
3336                                   header.GetBootImageSize());
3337         return false;
3338       }
3339     }
3340     DCHECK_GE(max_image_space_dependencies, chunk.boot_image_component_count);
3341     ArrayRef<const std::unique_ptr<ImageSpace>> dependencies =
3342         ArrayRef<const std::unique_ptr<ImageSpace>>(*spaces).SubArray(
3343             /*pos=*/ 0u, chunk.boot_image_component_count);
3344     for (size_t i = 0u, size = locations.size(); i != size; ++i) {
3345       ImageSpace* space = (*spaces)[spaces->size() - chunk.image_space_count + i].get();
3346       size_t bcp_chunk_size = (chunk.image_space_count == 1u) ? chunk.component_count : 1u;
3347       if (!OpenOatFile(space,
3348                        std::move(chunk.vdex_fd),
3349                        std::move(chunk.oat_fd),
3350                        boot_class_path_.SubArray(/*pos=*/ chunk.start_index + i, bcp_chunk_size),
3351                        validate_oat_file,
3352                        dependencies,
3353                        logger,
3354                        image_reservation,
3355                        error_msg)) {
3356         return false;
3357       }
3358     }
3359 
3360     guard.Commit();
3361     return true;
3362   }
3363 
ReserveBootImageMemory(uint8_t * addr,uint32_t reservation_size,std::string * error_msg)3364   MemMap ReserveBootImageMemory(uint8_t* addr,
3365                                 uint32_t reservation_size,
3366                                 /*out*/std::string* error_msg) {
3367     DCHECK_ALIGNED(reservation_size, kPageSize);
3368     DCHECK_ALIGNED(addr, kPageSize);
3369     return MemMap::MapAnonymous("Boot image reservation",
3370                                 addr,
3371                                 reservation_size,
3372                                 PROT_NONE,
3373                                 /*low_4gb=*/ true,
3374                                 /*reuse=*/ false,
3375                                 /*reservation=*/ nullptr,
3376                                 error_msg);
3377   }
3378 
RemapExtraReservation(size_t extra_reservation_size,MemMap * image_reservation,MemMap * extra_reservation,std::string * error_msg)3379   bool RemapExtraReservation(size_t extra_reservation_size,
3380                              /*inout*/MemMap* image_reservation,
3381                              /*out*/MemMap* extra_reservation,
3382                              /*out*/std::string* error_msg) {
3383     DCHECK_ALIGNED(extra_reservation_size, kPageSize);
3384     DCHECK(!extra_reservation->IsValid());
3385     size_t expected_size = image_reservation->IsValid() ? image_reservation->Size() : 0u;
3386     if (extra_reservation_size != expected_size) {
3387       *error_msg = StringPrintf("Image reservation mismatch after loading boot image: %zu != %zu",
3388                                 extra_reservation_size,
3389                                 expected_size);
3390       return false;
3391     }
3392     if (extra_reservation_size != 0u) {
3393       DCHECK(image_reservation->IsValid());
3394       DCHECK_EQ(extra_reservation_size, image_reservation->Size());
3395       *extra_reservation = image_reservation->RemapAtEnd(image_reservation->Begin(),
3396                                                          "Boot image extra reservation",
3397                                                          PROT_NONE,
3398                                                          error_msg);
3399       if (!extra_reservation->IsValid()) {
3400         return false;
3401       }
3402     }
3403     DCHECK(!image_reservation->IsValid());
3404     return true;
3405   }
3406 
3407   const ArrayRef<const std::string> boot_class_path_;
3408   const ArrayRef<const std::string> boot_class_path_locations_;
3409   const std::string image_location_;
3410   const InstructionSet image_isa_;
3411   const bool relocate_;
3412   const bool executable_;
3413   const bool is_zygote_;
3414   bool has_system_;
3415   bool has_cache_;
3416   bool is_global_cache_;
3417   bool dalvik_cache_exists_;
3418   std::string dalvik_cache_;
3419   std::string cache_filename_;
3420 };
3421 
LoadFromSystem(bool validate_oat_file,size_t extra_reservation_size,std::vector<std::unique_ptr<ImageSpace>> * boot_image_spaces,MemMap * extra_reservation,std::string * error_msg)3422 bool ImageSpace::BootImageLoader::LoadFromSystem(
3423     bool validate_oat_file,
3424     size_t extra_reservation_size,
3425     /*out*/std::vector<std::unique_ptr<ImageSpace>>* boot_image_spaces,
3426     /*out*/MemMap* extra_reservation,
3427     /*out*/std::string* error_msg) {
3428   TimingLogger logger(__PRETTY_FUNCTION__, /*precise=*/ true, VLOG_IS_ON(image));
3429 
3430   BootImageLayout layout(image_location_, boot_class_path_, boot_class_path_locations_);
3431   if (!layout.LoadFromSystem(image_isa_, error_msg)) {
3432     return false;
3433   }
3434 
3435   if (!LoadImage(layout,
3436                  validate_oat_file,
3437                  extra_reservation_size,
3438                  &logger,
3439                  boot_image_spaces,
3440                  extra_reservation,
3441                  error_msg)) {
3442     return false;
3443   }
3444 
3445   if (VLOG_IS_ON(image)) {
3446     LOG(INFO) << "ImageSpace::BootImageLoader::LoadFromSystem exiting "
3447         << boot_image_spaces->front();
3448     logger.Dump(LOG_STREAM(INFO));
3449   }
3450   return true;
3451 }
3452 
LoadFromDalvikCache(bool validate_oat_file,size_t extra_reservation_size,std::vector<std::unique_ptr<ImageSpace>> * boot_image_spaces,MemMap * extra_reservation,std::string * error_msg)3453 bool ImageSpace::BootImageLoader::LoadFromDalvikCache(
3454     bool validate_oat_file,
3455     size_t extra_reservation_size,
3456     /*out*/std::vector<std::unique_ptr<ImageSpace>>* boot_image_spaces,
3457     /*out*/MemMap* extra_reservation,
3458     /*out*/std::string* error_msg) {
3459   TimingLogger logger(__PRETTY_FUNCTION__, /*precise=*/ true, VLOG_IS_ON(image));
3460   DCHECK(DalvikCacheExists());
3461 
3462   BootImageLayout layout(image_location_, boot_class_path_, boot_class_path_locations_);
3463   if (!layout.LoadFromDalvikCache(dalvik_cache_, error_msg)) {
3464     return false;
3465   }
3466   if (!LoadImage(layout,
3467                  validate_oat_file,
3468                  extra_reservation_size,
3469                  &logger,
3470                  boot_image_spaces,
3471                  extra_reservation,
3472                  error_msg)) {
3473     return false;
3474   }
3475 
3476   if (VLOG_IS_ON(image)) {
3477     LOG(INFO) << "ImageSpace::BootImageLoader::LoadFromDalvikCache exiting "
3478         << boot_image_spaces->front();
3479     logger.Dump(LOG_STREAM(INFO));
3480   }
3481   return true;
3482 }
3483 
IsBootClassPathOnDisk(InstructionSet image_isa)3484 bool ImageSpace::IsBootClassPathOnDisk(InstructionSet image_isa) {
3485   Runtime* runtime = Runtime::Current();
3486   BootImageLayout layout(runtime->GetImageLocation(),
3487                          ArrayRef<const std::string>(runtime->GetBootClassPath()),
3488                          ArrayRef<const std::string>(runtime->GetBootClassPathLocations()));
3489   const std::string image_location = layout.GetPrimaryImageLocation();
3490   ImageSpaceLoadingOrder order = runtime->GetImageSpaceLoadingOrder();
3491   std::unique_ptr<ImageHeader> image_header;
3492   std::string error_msg;
3493 
3494   std::string system_filename;
3495   bool has_system = false;
3496   std::string cache_filename;
3497   bool has_cache = false;
3498   bool dalvik_cache_exists = false;
3499   bool is_global_cache = false;
3500   if (FindImageFilename(image_location.c_str(),
3501                         image_isa,
3502                         &system_filename,
3503                         &has_system,
3504                         &cache_filename,
3505                         &dalvik_cache_exists,
3506                         &has_cache,
3507                         &is_global_cache)) {
3508     DCHECK(has_system || has_cache);
3509     const std::string& filename = (order == ImageSpaceLoadingOrder::kSystemFirst)
3510         ? (has_system ? system_filename : cache_filename)
3511         : (has_cache ? cache_filename : system_filename);
3512     image_header = ReadSpecificImageHeader(filename.c_str(), &error_msg);
3513   }
3514 
3515   return image_header != nullptr;
3516 }
3517 
3518 static constexpr uint64_t kLowSpaceValue = 50 * MB;
3519 static constexpr uint64_t kTmpFsSentinelValue = 384 * MB;
3520 
3521 // Read the free space of the cache partition and make a decision whether to keep the generated
3522 // image. This is to try to mitigate situations where the system might run out of space later.
CheckSpace(const std::string & cache_filename,std::string * error_msg)3523 static bool CheckSpace(const std::string& cache_filename, std::string* error_msg) {
3524   // Using statvfs vs statvfs64 because of b/18207376, and it is enough for all practical purposes.
3525   struct statvfs buf;
3526 
3527   int res = TEMP_FAILURE_RETRY(statvfs(cache_filename.c_str(), &buf));
3528   if (res != 0) {
3529     // Could not stat. Conservatively tell the system to delete the image.
3530     *error_msg = "Could not stat the filesystem, assuming low-memory situation.";
3531     return false;
3532   }
3533 
3534   uint64_t fs_overall_size = buf.f_bsize * static_cast<uint64_t>(buf.f_blocks);
3535   // Zygote is privileged, but other things are not. Use bavail.
3536   uint64_t fs_free_size = buf.f_bsize * static_cast<uint64_t>(buf.f_bavail);
3537 
3538   // Take the overall size as an indicator for a tmpfs, which is being used for the decryption
3539   // environment. We do not want to fail quickening the boot image there, as it is beneficial
3540   // for time-to-UI.
3541   if (fs_overall_size > kTmpFsSentinelValue) {
3542     if (fs_free_size < kLowSpaceValue) {
3543       *error_msg = StringPrintf("Low-memory situation: only %4.2f megabytes available, need at "
3544                                 "least %" PRIu64 ".",
3545                                 static_cast<double>(fs_free_size) / MB,
3546                                 kLowSpaceValue / MB);
3547       return false;
3548     }
3549   }
3550   return true;
3551 }
3552 
LoadBootImage(const std::vector<std::string> & boot_class_path,const std::vector<std::string> & boot_class_path_locations,const std::string & image_location,const InstructionSet image_isa,ImageSpaceLoadingOrder order,bool relocate,bool executable,bool is_zygote,size_t extra_reservation_size,std::vector<std::unique_ptr<ImageSpace>> * boot_image_spaces,MemMap * extra_reservation)3553 bool ImageSpace::LoadBootImage(
3554     const std::vector<std::string>& boot_class_path,
3555     const std::vector<std::string>& boot_class_path_locations,
3556     const std::string& image_location,
3557     const InstructionSet image_isa,
3558     ImageSpaceLoadingOrder order,
3559     bool relocate,
3560     bool executable,
3561     bool is_zygote,
3562     size_t extra_reservation_size,
3563     /*out*/std::vector<std::unique_ptr<ImageSpace>>* boot_image_spaces,
3564     /*out*/MemMap* extra_reservation) {
3565   ScopedTrace trace(__FUNCTION__);
3566 
3567   DCHECK(boot_image_spaces != nullptr);
3568   DCHECK(boot_image_spaces->empty());
3569   DCHECK_ALIGNED(extra_reservation_size, kPageSize);
3570   DCHECK(extra_reservation != nullptr);
3571   DCHECK_NE(image_isa, InstructionSet::kNone);
3572 
3573   if (image_location.empty()) {
3574     return false;
3575   }
3576 
3577   BootImageLoader loader(boot_class_path,
3578                          boot_class_path_locations,
3579                          image_location,
3580                          image_isa,
3581                          relocate,
3582                          executable,
3583                          is_zygote);
3584 
3585   // Step 0: Extra zygote work.
3586 
3587   loader.FindImageFiles();
3588 
3589   // Step 0.a: If we're the zygote, check for free space, and prune the cache preemptively,
3590   //           if necessary. While the runtime may be fine (it is pretty tolerant to
3591   //           out-of-disk-space situations), other parts of the platform are not.
3592   //
3593   //           The advantage of doing this proactively is that the later steps are simplified,
3594   //           i.e., we do not need to code retries.
3595   bool low_space = false;
3596   if (loader.IsZygote() && loader.DalvikCacheExists()) {
3597     // Extra checks for the zygote. These only apply when loading the first image, explained below.
3598     const std::string& dalvik_cache = loader.GetDalvikCache();
3599     DCHECK(!dalvik_cache.empty());
3600     std::string local_error_msg;
3601     bool check_space = CheckSpace(dalvik_cache, &local_error_msg);
3602     if (!check_space) {
3603       LOG(WARNING) << local_error_msg << " Preemptively pruning the dalvik cache.";
3604       PruneDalvikCache(image_isa);
3605 
3606       // Re-evaluate the image.
3607       loader.FindImageFiles();
3608 
3609       // Disable compilation/patching - we do not want to fill up the space again.
3610       low_space = true;
3611     }
3612   }
3613 
3614   // Collect all the errors.
3615   std::vector<std::string> error_msgs;
3616 
3617   auto try_load_from = [&](auto has_fn, auto load_fn, bool validate_oat_file) {
3618     if ((loader.*has_fn)()) {
3619       std::string local_error_msg;
3620       if ((loader.*load_fn)(validate_oat_file,
3621                             extra_reservation_size,
3622                             boot_image_spaces,
3623                             extra_reservation,
3624                             &local_error_msg)) {
3625         return true;
3626       }
3627       error_msgs.push_back(local_error_msg);
3628     }
3629     return false;
3630   };
3631 
3632   auto try_load_from_system = [&]() {
3633     // Validate the oat files if the loading order checks data first. Otherwise assume system
3634     // integrity.
3635     return try_load_from(&BootImageLoader::HasSystem,
3636                          &BootImageLoader::LoadFromSystem,
3637                          /*validate_oat_file=*/ order != ImageSpaceLoadingOrder::kSystemFirst);
3638   };
3639   auto try_load_from_cache = [&]() {
3640     // Always validate oat files from the dalvik cache.
3641     return try_load_from(&BootImageLoader::HasCache,
3642                          &BootImageLoader::LoadFromDalvikCache,
3643                          /*validate_oat_file=*/ true);
3644   };
3645 
3646   auto invoke_sequentially = [](auto first, auto second) {
3647     return first() || second();
3648   };
3649 
3650   // Step 1+2: Check system and cache images in the asked-for order.
3651   if (order == ImageSpaceLoadingOrder::kSystemFirst) {
3652     if (invoke_sequentially(try_load_from_system, try_load_from_cache)) {
3653       return true;
3654     }
3655   } else {
3656     if (invoke_sequentially(try_load_from_cache, try_load_from_system)) {
3657       return true;
3658     }
3659   }
3660 
3661   // Step 3: We do not have an existing image in /system,
3662   //         so generate an image into the dalvik cache.
3663   if (!loader.HasSystem() && loader.DalvikCacheExists()) {
3664     std::string local_error_msg;
3665     if (low_space || !Runtime::Current()->IsImageDex2OatEnabled()) {
3666       local_error_msg = "Image compilation disabled.";
3667     } else if (ImageCreationAllowed(loader.IsGlobalCache(),
3668                                     image_isa,
3669                                     is_zygote,
3670                                     &local_error_msg)) {
3671       bool compilation_success =
3672           GenerateImage(loader.GetCacheFilename(), image_isa, &local_error_msg);
3673       if (compilation_success) {
3674         if (loader.LoadFromDalvikCache(/*validate_oat_file=*/ false,
3675                                        extra_reservation_size,
3676                                        boot_image_spaces,
3677                                        extra_reservation,
3678                                        &local_error_msg)) {
3679           return true;
3680         }
3681       }
3682     }
3683     error_msgs.push_back(StringPrintf("Cannot compile image to %s: %s",
3684                                       loader.GetCacheFilename().c_str(),
3685                                       local_error_msg.c_str()));
3686   }
3687 
3688   // We failed. Prune the cache the free up space, create a compound error message
3689   // and return false.
3690   if (loader.DalvikCacheExists()) {
3691     PruneDalvikCache(image_isa);
3692   }
3693 
3694   std::ostringstream oss;
3695   bool first = true;
3696   for (const auto& msg : error_msgs) {
3697     if (!first) {
3698       oss << "\n    ";
3699     }
3700     oss << msg;
3701   }
3702 
3703   LOG(ERROR) << "Could not create image space with image file '" << image_location << "'. "
3704       << "Attempting to fall back to imageless running. Error was: " << oss.str();
3705 
3706   return false;
3707 }
3708 
~ImageSpace()3709 ImageSpace::~ImageSpace() {
3710   // Everything done by member destructors. Classes forward-declared in header are now defined.
3711 }
3712 
CreateFromAppImage(const char * image,const OatFile * oat_file,std::string * error_msg)3713 std::unique_ptr<ImageSpace> ImageSpace::CreateFromAppImage(const char* image,
3714                                                            const OatFile* oat_file,
3715                                                            std::string* error_msg) {
3716   // Note: The oat file has already been validated.
3717   const std::vector<ImageSpace*>& boot_image_spaces =
3718       Runtime::Current()->GetHeap()->GetBootImageSpaces();
3719   return CreateFromAppImage(image,
3720                             oat_file,
3721                             ArrayRef<ImageSpace* const>(boot_image_spaces),
3722                             error_msg);
3723 }
3724 
CreateFromAppImage(const char * image,const OatFile * oat_file,ArrayRef<ImageSpace * const> boot_image_spaces,std::string * error_msg)3725 std::unique_ptr<ImageSpace> ImageSpace::CreateFromAppImage(
3726     const char* image,
3727     const OatFile* oat_file,
3728     ArrayRef<ImageSpace* const> boot_image_spaces,
3729     std::string* error_msg) {
3730   return Loader::InitAppImage(image,
3731                               image,
3732                               oat_file,
3733                               boot_image_spaces,
3734                               error_msg);
3735 }
3736 
GetOatFile() const3737 const OatFile* ImageSpace::GetOatFile() const {
3738   return oat_file_non_owned_;
3739 }
3740 
ReleaseOatFile()3741 std::unique_ptr<const OatFile> ImageSpace::ReleaseOatFile() {
3742   CHECK(oat_file_ != nullptr);
3743   return std::move(oat_file_);
3744 }
3745 
Dump(std::ostream & os) const3746 void ImageSpace::Dump(std::ostream& os) const {
3747   os << GetType()
3748       << " begin=" << reinterpret_cast<void*>(Begin())
3749       << ",end=" << reinterpret_cast<void*>(End())
3750       << ",size=" << PrettySize(Size())
3751       << ",name=\"" << GetName() << "\"]";
3752 }
3753 
ValidateOatFile(const OatFile & oat_file,std::string * error_msg)3754 bool ImageSpace::ValidateOatFile(const OatFile& oat_file, std::string* error_msg) {
3755   const ArtDexFileLoader dex_file_loader;
3756   for (const OatDexFile* oat_dex_file : oat_file.GetOatDexFiles()) {
3757     const std::string& dex_file_location = oat_dex_file->GetDexFileLocation();
3758 
3759     // Skip multidex locations - These will be checked when we visit their
3760     // corresponding primary non-multidex location.
3761     if (DexFileLoader::IsMultiDexLocation(dex_file_location.c_str())) {
3762       continue;
3763     }
3764 
3765     std::vector<uint32_t> checksums;
3766     if (!dex_file_loader.GetMultiDexChecksums(dex_file_location.c_str(), &checksums, error_msg)) {
3767       *error_msg = StringPrintf("ValidateOatFile failed to get checksums of dex file '%s' "
3768                                 "referenced by oat file %s: %s",
3769                                 dex_file_location.c_str(),
3770                                 oat_file.GetLocation().c_str(),
3771                                 error_msg->c_str());
3772       return false;
3773     }
3774     CHECK(!checksums.empty());
3775     if (checksums[0] != oat_dex_file->GetDexFileLocationChecksum()) {
3776       *error_msg = StringPrintf("ValidateOatFile found checksum mismatch between oat file "
3777                                 "'%s' and dex file '%s' (0x%x != 0x%x)",
3778                                 oat_file.GetLocation().c_str(),
3779                                 dex_file_location.c_str(),
3780                                 oat_dex_file->GetDexFileLocationChecksum(),
3781                                 checksums[0]);
3782       return false;
3783     }
3784 
3785     // Verify checksums for any related multidex entries.
3786     for (size_t i = 1; i < checksums.size(); i++) {
3787       std::string multi_dex_location = DexFileLoader::GetMultiDexLocation(
3788           i,
3789           dex_file_location.c_str());
3790       const OatDexFile* multi_dex = oat_file.GetOatDexFile(multi_dex_location.c_str(),
3791                                                            nullptr,
3792                                                            error_msg);
3793       if (multi_dex == nullptr) {
3794         *error_msg = StringPrintf("ValidateOatFile oat file '%s' is missing entry '%s'",
3795                                   oat_file.GetLocation().c_str(),
3796                                   multi_dex_location.c_str());
3797         return false;
3798       }
3799 
3800       if (checksums[i] != multi_dex->GetDexFileLocationChecksum()) {
3801         *error_msg = StringPrintf("ValidateOatFile found checksum mismatch between oat file "
3802                                   "'%s' and dex file '%s' (0x%x != 0x%x)",
3803                                   oat_file.GetLocation().c_str(),
3804                                   multi_dex_location.c_str(),
3805                                   multi_dex->GetDexFileLocationChecksum(),
3806                                   checksums[i]);
3807         return false;
3808       }
3809     }
3810   }
3811   return true;
3812 }
3813 
GetBootClassPathChecksums(ArrayRef<ImageSpace * const> image_spaces,ArrayRef<const DexFile * const> boot_class_path)3814 std::string ImageSpace::GetBootClassPathChecksums(
3815     ArrayRef<ImageSpace* const> image_spaces,
3816     ArrayRef<const DexFile* const> boot_class_path) {
3817   DCHECK(!boot_class_path.empty());
3818   size_t bcp_pos = 0u;
3819   std::string boot_image_checksum;
3820 
3821   for (size_t image_pos = 0u, size = image_spaces.size(); image_pos != size; ) {
3822     const ImageSpace* main_space = image_spaces[image_pos];
3823     // Caller must make sure that the image spaces correspond to the head of the BCP.
3824     DCHECK_NE(main_space->oat_file_non_owned_->GetOatDexFiles().size(), 0u);
3825     DCHECK_EQ(main_space->oat_file_non_owned_->GetOatDexFiles()[0]->GetDexFileLocation(),
3826               boot_class_path[bcp_pos]->GetLocation());
3827     const ImageHeader& current_header = main_space->GetImageHeader();
3828     uint32_t image_space_count = current_header.GetImageSpaceCount();
3829     DCHECK_NE(image_space_count, 0u);
3830     DCHECK_LE(image_space_count, image_spaces.size() - image_pos);
3831     if (image_pos != 0u) {
3832       boot_image_checksum += ':';
3833     }
3834     uint32_t component_count = current_header.GetComponentCount();
3835     AppendImageChecksum(component_count, current_header.GetImageChecksum(), &boot_image_checksum);
3836     for (size_t space_index = 0; space_index != image_space_count; ++space_index) {
3837       const ImageSpace* space = image_spaces[image_pos + space_index];
3838       const OatFile* oat_file = space->oat_file_non_owned_;
3839       size_t num_dex_files = oat_file->GetOatDexFiles().size();
3840       if (kIsDebugBuild) {
3841         CHECK_NE(num_dex_files, 0u);
3842         CHECK_LE(oat_file->GetOatDexFiles().size(), boot_class_path.size() - bcp_pos);
3843         for (size_t i = 0; i != num_dex_files; ++i) {
3844           CHECK_EQ(oat_file->GetOatDexFiles()[i]->GetDexFileLocation(),
3845                    boot_class_path[bcp_pos + i]->GetLocation());
3846         }
3847       }
3848       bcp_pos += num_dex_files;
3849     }
3850     image_pos += image_space_count;
3851   }
3852 
3853   ArrayRef<const DexFile* const> boot_class_path_tail =
3854       ArrayRef<const DexFile* const>(boot_class_path).SubArray(bcp_pos);
3855   DCHECK(boot_class_path_tail.empty() ||
3856          !DexFileLoader::IsMultiDexLocation(boot_class_path_tail.front()->GetLocation().c_str()));
3857   for (const DexFile* dex_file : boot_class_path_tail) {
3858     if (!DexFileLoader::IsMultiDexLocation(dex_file->GetLocation().c_str())) {
3859       if (!boot_image_checksum.empty()) {
3860         boot_image_checksum += ':';
3861       }
3862       boot_image_checksum += kDexFileChecksumPrefix;
3863     }
3864     StringAppendF(&boot_image_checksum, "/%08x", dex_file->GetLocationChecksum());
3865   }
3866   return boot_image_checksum;
3867 }
3868 
CheckAndCountBCPComponents(std::string_view oat_boot_class_path,ArrayRef<const std::string> boot_class_path,std::string * error_msg)3869 static size_t CheckAndCountBCPComponents(std::string_view oat_boot_class_path,
3870                                          ArrayRef<const std::string> boot_class_path,
3871                                          /*out*/std::string* error_msg) {
3872   // Check that the oat BCP is a prefix of current BCP locations and count components.
3873   size_t component_count = 0u;
3874   std::string_view remaining_bcp(oat_boot_class_path);
3875   bool bcp_ok = false;
3876   for (const std::string& location : boot_class_path) {
3877     if (!StartsWith(remaining_bcp, location)) {
3878       break;
3879     }
3880     remaining_bcp.remove_prefix(location.size());
3881     ++component_count;
3882     if (remaining_bcp.empty()) {
3883       bcp_ok = true;
3884       break;
3885     }
3886     if (!StartsWith(remaining_bcp, ":")) {
3887       break;
3888     }
3889     remaining_bcp.remove_prefix(1u);
3890   }
3891   if (!bcp_ok) {
3892     *error_msg = StringPrintf("Oat boot class path (%s) is not a prefix of"
3893                               " runtime boot class path (%s)",
3894                               std::string(oat_boot_class_path).c_str(),
3895                               Join(boot_class_path, ':').c_str());
3896     return static_cast<size_t>(-1);
3897   }
3898   return component_count;
3899 }
3900 
VerifyBootClassPathChecksums(std::string_view oat_checksums,std::string_view oat_boot_class_path,const std::string & image_location,ArrayRef<const std::string> boot_class_path_locations,ArrayRef<const std::string> boot_class_path,InstructionSet image_isa,ImageSpaceLoadingOrder order,std::string * error_msg)3901 bool ImageSpace::VerifyBootClassPathChecksums(std::string_view oat_checksums,
3902                                               std::string_view oat_boot_class_path,
3903                                               const std::string& image_location,
3904                                               ArrayRef<const std::string> boot_class_path_locations,
3905                                               ArrayRef<const std::string> boot_class_path,
3906                                               InstructionSet image_isa,
3907                                               ImageSpaceLoadingOrder order,
3908                                               /*out*/std::string* error_msg) {
3909   if (oat_checksums.empty() || oat_boot_class_path.empty()) {
3910     *error_msg = oat_checksums.empty() ? "Empty checksums." : "Empty boot class path.";
3911     return false;
3912   }
3913 
3914   DCHECK_EQ(boot_class_path_locations.size(), boot_class_path.size());
3915   size_t bcp_size =
3916       CheckAndCountBCPComponents(oat_boot_class_path, boot_class_path_locations, error_msg);
3917   if (bcp_size == static_cast<size_t>(-1)) {
3918     DCHECK(!error_msg->empty());
3919     return false;
3920   }
3921 
3922   size_t bcp_pos = 0u;
3923   if (StartsWith(oat_checksums, "i")) {
3924     // Use only the matching part of the BCP for validation.
3925     BootImageLayout layout(image_location,
3926                            boot_class_path.SubArray(/*pos=*/ 0u, bcp_size),
3927                            boot_class_path_locations.SubArray(/*pos=*/ 0u, bcp_size));
3928     std::string primary_image_location = layout.GetPrimaryImageLocation();
3929     std::string system_filename;
3930     bool has_system = false;
3931     std::string cache_filename;
3932     bool has_cache = false;
3933     bool dalvik_cache_exists = false;
3934     bool is_global_cache = false;
3935     if (!FindImageFilename(primary_image_location.c_str(),
3936                            image_isa,
3937                            &system_filename,
3938                            &has_system,
3939                            &cache_filename,
3940                            &dalvik_cache_exists,
3941                            &has_cache,
3942                            &is_global_cache)) {
3943       *error_msg = StringPrintf("Unable to find image file for %s and %s",
3944                                 image_location.c_str(),
3945                                 GetInstructionSetString(image_isa));
3946       return false;
3947     }
3948 
3949     DCHECK(has_system || has_cache);
3950     bool use_system = (order == ImageSpaceLoadingOrder::kSystemFirst) ? has_system : !has_cache;
3951     bool image_checksums_ok = use_system
3952         ? layout.ValidateFromSystem(image_isa, &oat_checksums, error_msg)
3953         : layout.ValidateFromDalvikCache(cache_filename, &oat_checksums, error_msg);
3954     if (!image_checksums_ok) {
3955       return false;
3956     }
3957     bcp_pos = layout.GetNextBcpIndex();
3958   }
3959 
3960   for ( ; bcp_pos != bcp_size; ++bcp_pos) {
3961     static_assert(ImageSpace::kDexFileChecksumPrefix == 'd', "Format prefix check.");
3962     if (!StartsWith(oat_checksums, "d")) {
3963       *error_msg = StringPrintf("Missing dex checksums, expected %s to start with 'd'",
3964                                 std::string(oat_checksums).c_str());
3965       return false;
3966     }
3967     oat_checksums.remove_prefix(1u);
3968 
3969     const std::string& bcp_filename = boot_class_path[bcp_pos];
3970     std::vector<std::unique_ptr<const DexFile>> dex_files;
3971     const ArtDexFileLoader dex_file_loader;
3972     if (!dex_file_loader.Open(bcp_filename.c_str(),
3973                               bcp_filename,  // The location does not matter here.
3974                               /*verify=*/ false,
3975                               /*verify_checksum=*/ false,
3976                               error_msg,
3977                               &dex_files)) {
3978       return false;
3979     }
3980     DCHECK(!dex_files.empty());
3981     for (const std::unique_ptr<const DexFile>& dex_file : dex_files) {
3982       std::string dex_file_checksum = StringPrintf("/%08x", dex_file->GetLocationChecksum());
3983       if (!StartsWith(oat_checksums, dex_file_checksum)) {
3984         *error_msg = StringPrintf("Dex checksum mismatch, expected %s to start with %s",
3985                                   std::string(oat_checksums).c_str(),
3986                                   dex_file_checksum.c_str());
3987         return false;
3988       }
3989       oat_checksums.remove_prefix(dex_file_checksum.size());
3990     }
3991     if (bcp_pos + 1u != bcp_size) {
3992       if (!StartsWith(oat_checksums, ":")) {
3993         *error_msg = StringPrintf("Missing ':' separator at start of %s",
3994                                   std::string(oat_checksums).c_str());
3995         return false;
3996       }
3997       oat_checksums.remove_prefix(1u);
3998     }
3999   }
4000   if (!oat_checksums.empty()) {
4001     *error_msg = StringPrintf("Checksum too long, unexpected tail %s",
4002                               std::string(oat_checksums).c_str());
4003     return false;
4004   }
4005   return true;
4006 }
4007 
VerifyBootClassPathChecksums(std::string_view oat_checksums,std::string_view oat_boot_class_path,ArrayRef<const std::unique_ptr<ImageSpace>> image_spaces,ArrayRef<const std::string> boot_class_path_locations,ArrayRef<const std::string> boot_class_path,std::string * error_msg)4008 bool ImageSpace::VerifyBootClassPathChecksums(
4009     std::string_view oat_checksums,
4010     std::string_view oat_boot_class_path,
4011     ArrayRef<const std::unique_ptr<ImageSpace>> image_spaces,
4012     ArrayRef<const std::string> boot_class_path_locations,
4013     ArrayRef<const std::string> boot_class_path,
4014     /*out*/std::string* error_msg) {
4015   DCHECK_EQ(boot_class_path.size(), boot_class_path_locations.size());
4016   DCHECK_GE(boot_class_path_locations.size(), image_spaces.size());
4017   if (oat_checksums.empty() || oat_boot_class_path.empty()) {
4018     *error_msg = oat_checksums.empty() ? "Empty checksums." : "Empty boot class path.";
4019     return false;
4020   }
4021 
4022   size_t oat_bcp_size =
4023       CheckAndCountBCPComponents(oat_boot_class_path, boot_class_path_locations, error_msg);
4024   if (oat_bcp_size == static_cast<size_t>(-1)) {
4025     DCHECK(!error_msg->empty());
4026     return false;
4027   }
4028   const size_t num_image_spaces = image_spaces.size();
4029   if (num_image_spaces != oat_bcp_size) {
4030     *error_msg = StringPrintf("Image header records more dependencies (%zu) than BCP (%zu)",
4031                               num_image_spaces,
4032                               oat_bcp_size);
4033     return false;
4034   }
4035 
4036   // Verify image checksums.
4037   size_t bcp_pos = 0u;
4038   size_t image_pos = 0u;
4039   while (image_pos != num_image_spaces && StartsWith(oat_checksums, "i")) {
4040     // Verify the current image checksum.
4041     const ImageHeader& current_header = image_spaces[image_pos]->GetImageHeader();
4042     uint32_t image_space_count = current_header.GetImageSpaceCount();
4043     DCHECK_NE(image_space_count, 0u);
4044     DCHECK_LE(image_space_count, image_spaces.size() - image_pos);
4045     uint32_t component_count = current_header.GetComponentCount();
4046     uint32_t checksum = current_header.GetImageChecksum();
4047     if (!CheckAndRemoveImageChecksum(component_count, checksum, &oat_checksums, error_msg)) {
4048       DCHECK(!error_msg->empty());
4049       return false;
4050     }
4051 
4052     if (kIsDebugBuild) {
4053       for (size_t space_index = 0; space_index != image_space_count; ++space_index) {
4054         const OatFile* oat_file = image_spaces[image_pos + space_index]->oat_file_non_owned_;
4055         size_t num_dex_files = oat_file->GetOatDexFiles().size();
4056         CHECK_NE(num_dex_files, 0u);
4057         const std::string main_location = oat_file->GetOatDexFiles()[0]->GetDexFileLocation();
4058         CHECK_EQ(main_location, boot_class_path_locations[bcp_pos + space_index]);
4059         CHECK(!DexFileLoader::IsMultiDexLocation(main_location.c_str()));
4060         size_t num_base_locations = 1u;
4061         for (size_t i = 1u; i != num_dex_files; ++i) {
4062           if (DexFileLoader::IsMultiDexLocation(
4063                   oat_file->GetOatDexFiles()[i]->GetDexFileLocation().c_str())) {
4064             CHECK_EQ(image_space_count, 1u);  // We can find base locations only for --single-image.
4065             ++num_base_locations;
4066           }
4067         }
4068         if (image_space_count == 1u) {
4069           CHECK_EQ(num_base_locations, component_count);
4070         }
4071       }
4072     }
4073 
4074     image_pos += image_space_count;
4075     bcp_pos += component_count;
4076 
4077     if (!StartsWith(oat_checksums, ":")) {
4078       // Check that we've reached the end of checksums and BCP.
4079       if (!oat_checksums.empty()) {
4080          *error_msg = StringPrintf("Expected ':' separator or end of checksums, remaining %s.",
4081                                    std::string(oat_checksums).c_str());
4082          return false;
4083       }
4084       if (image_pos != oat_bcp_size) {
4085         *error_msg = StringPrintf("Component count mismatch between checksums (%zu) and BCP (%zu)",
4086                                   image_pos,
4087                                   oat_bcp_size);
4088         return false;
4089       }
4090       return true;
4091     }
4092     oat_checksums.remove_prefix(1u);
4093   }
4094 
4095   // We do not allow dependencies of extensions on dex files. That would require
4096   // interleaving the loading of the images with opening the other BCP dex files.
4097   return false;
4098 }
4099 
ExpandMultiImageLocations(ArrayRef<const std::string> dex_locations,const std::string & image_location,bool boot_image_extension)4100 std::vector<std::string> ImageSpace::ExpandMultiImageLocations(
4101     ArrayRef<const std::string> dex_locations,
4102     const std::string& image_location,
4103     bool boot_image_extension) {
4104   DCHECK(!dex_locations.empty());
4105 
4106   // Find the path.
4107   size_t last_slash = image_location.rfind('/');
4108   CHECK_NE(last_slash, std::string::npos);
4109 
4110   // We also need to honor path components that were encoded through '@'. Otherwise the loading
4111   // code won't be able to find the images.
4112   if (image_location.find('@', last_slash) != std::string::npos) {
4113     last_slash = image_location.rfind('@');
4114   }
4115 
4116   // Find the dot separating the primary image name from the extension.
4117   size_t last_dot = image_location.rfind('.');
4118   // Extract the extension and base (the path and primary image name).
4119   std::string extension;
4120   std::string base = image_location;
4121   if (last_dot != std::string::npos && last_dot > last_slash) {
4122     extension = image_location.substr(last_dot);  // Including the dot.
4123     base.resize(last_dot);
4124   }
4125   // For non-empty primary image name, add '-' to the `base`.
4126   if (last_slash + 1u != base.size()) {
4127     base += '-';
4128   }
4129 
4130   std::vector<std::string> locations;
4131   locations.reserve(dex_locations.size());
4132   size_t start_index = 0u;
4133   if (!boot_image_extension) {
4134     start_index = 1u;
4135     locations.push_back(image_location);
4136   }
4137 
4138   // Now create the other names. Use a counted loop to skip the first one if needed.
4139   for (size_t i = start_index; i < dex_locations.size(); ++i) {
4140     // Replace path with `base` (i.e. image path and prefix) and replace the original
4141     // extension (if any) with `extension`.
4142     std::string name = dex_locations[i];
4143     size_t last_dex_slash = name.rfind('/');
4144     if (last_dex_slash != std::string::npos) {
4145       name = name.substr(last_dex_slash + 1);
4146     }
4147     size_t last_dex_dot = name.rfind('.');
4148     if (last_dex_dot != std::string::npos) {
4149       name.resize(last_dex_dot);
4150     }
4151     locations.push_back(base + name + extension);
4152   }
4153   return locations;
4154 }
4155 
DumpSections(std::ostream & os) const4156 void ImageSpace::DumpSections(std::ostream& os) const {
4157   const uint8_t* base = Begin();
4158   const ImageHeader& header = GetImageHeader();
4159   for (size_t i = 0; i < ImageHeader::kSectionCount; ++i) {
4160     auto section_type = static_cast<ImageHeader::ImageSections>(i);
4161     const ImageSection& section = header.GetImageSection(section_type);
4162     os << section_type << " " << reinterpret_cast<const void*>(base + section.Offset())
4163        << "-" << reinterpret_cast<const void*>(base + section.End()) << "\n";
4164   }
4165 }
4166 
DisablePreResolvedStrings()4167 void ImageSpace::DisablePreResolvedStrings() {
4168   // Clear dex cache pointers.
4169   ObjPtr<mirror::ObjectArray<mirror::DexCache>> dex_caches =
4170       GetImageHeader().GetImageRoot(ImageHeader::kDexCaches)->AsObjectArray<mirror::DexCache>();
4171   for (size_t len = dex_caches->GetLength(), i = 0; i < len; ++i) {
4172     ObjPtr<mirror::DexCache> dex_cache = dex_caches->Get(i);
4173     dex_cache->ClearPreResolvedStrings();
4174   }
4175 }
4176 
ReleaseMetadata()4177 void ImageSpace::ReleaseMetadata() {
4178   const ImageSection& metadata = GetImageHeader().GetMetadataSection();
4179   VLOG(image) << "Releasing " << metadata.Size() << " image metadata bytes";
4180   // In the case where new app images may have been added around the checkpoint, ensure that we
4181   // don't madvise the cache for these.
4182   ObjPtr<mirror::ObjectArray<mirror::DexCache>> dex_caches =
4183       GetImageHeader().GetImageRoot(ImageHeader::kDexCaches)->AsObjectArray<mirror::DexCache>();
4184   bool have_startup_cache = false;
4185   for (size_t len = dex_caches->GetLength(), i = 0; i < len; ++i) {
4186     ObjPtr<mirror::DexCache> dex_cache = dex_caches->Get(i);
4187     if (dex_cache->NumPreResolvedStrings() != 0u) {
4188       have_startup_cache = true;
4189     }
4190   }
4191   // Only safe to do for images that have their preresolved strings caches disabled. This is because
4192   // uncompressed images madvise to the original unrelocated image contents.
4193   if (!have_startup_cache) {
4194     // Avoid using ZeroAndReleasePages since the zero fill might not be word atomic.
4195     uint8_t* const page_begin = AlignUp(Begin() + metadata.Offset(), kPageSize);
4196     uint8_t* const page_end = AlignDown(Begin() + metadata.End(), kPageSize);
4197     if (page_begin < page_end) {
4198       CHECK_NE(madvise(page_begin, page_end - page_begin, MADV_DONTNEED), -1) << "madvise failed";
4199     }
4200   }
4201 }
4202 
4203 }  // namespace space
4204 }  // namespace gc
4205 }  // namespace art
4206