1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef ART_COMPILER_OAT_WRITER_H_
18 #define ART_COMPILER_OAT_WRITER_H_
19 
20 #include <stdint.h>
21 #include <cstddef>
22 #include <memory>
23 
24 #include "base/array_ref.h"
25 #include "base/dchecked_vector.h"
26 #include "linker/relative_patcher.h"  // For linker::RelativePatcherTargetProvider.
27 #include "mem_map.h"
28 #include "method_reference.h"
29 #include "mirror/class.h"
30 #include "oat.h"
31 #include "os.h"
32 #include "safe_map.h"
33 #include "string_reference.h"
34 #include "utils/type_reference.h"
35 
36 namespace art {
37 
38 class BitVector;
39 class CompiledMethod;
40 class CompilerDriver;
41 class ImageWriter;
42 class ProfileCompilationInfo;
43 class OutputStream;
44 class TimingLogger;
45 class TypeLookupTable;
46 class VdexFile;
47 class ZipEntry;
48 
49 namespace debug {
50 struct MethodDebugInfo;
51 }  // namespace debug
52 
53 namespace linker {
54 class MultiOatRelativePatcher;
55 }  // namespace linker
56 
57 namespace verifier {
58   class VerifierDeps;
59 }  // namespace verifier
60 
61 // OatHeader         variable length with count of D OatDexFiles
62 //
63 // OatDexFile[0]     one variable sized OatDexFile with offsets to Dex and OatClasses
64 // OatDexFile[1]
65 // ...
66 // OatDexFile[D]
67 //
68 // TypeLookupTable[0] one descriptor to class def index hash table for each OatDexFile.
69 // TypeLookupTable[1]
70 // ...
71 // TypeLookupTable[D]
72 //
73 // ClassOffsets[0]   one table of OatClass offsets for each class def for each OatDexFile.
74 // ClassOffsets[1]
75 // ...
76 // ClassOffsets[D]
77 //
78 // OatClass[0]       one variable sized OatClass for each of C DexFile::ClassDefs
79 // OatClass[1]       contains OatClass entries with class status, offsets to code, etc.
80 // ...
81 // OatClass[C]
82 //
83 // GcMap             one variable sized blob with GC map.
84 // GcMap             GC maps are deduplicated.
85 // ...
86 // GcMap
87 //
88 // VmapTable         one variable sized VmapTable blob (quick compiler only).
89 // VmapTable         VmapTables are deduplicated.
90 // ...
91 // VmapTable
92 //
93 // MappingTable      one variable sized blob with MappingTable (quick compiler only).
94 // MappingTable      MappingTables are deduplicated.
95 // ...
96 // MappingTable
97 //
98 // padding           if necessary so that the following code will be page aligned
99 //
100 // OatMethodHeader   fixed size header for a CompiledMethod including the size of the MethodCode.
101 // MethodCode        one variable sized blob with the code of a CompiledMethod.
102 // OatMethodHeader   (OatMethodHeader, MethodCode) pairs are deduplicated.
103 // MethodCode
104 // ...
105 // OatMethodHeader
106 // MethodCode
107 //
108 class OatWriter {
109  public:
110   enum class CreateTypeLookupTable {
111     kCreate,
112     kDontCreate,
113     kDefault = kCreate
114   };
115 
116   OatWriter(bool compiling_boot_image, TimingLogger* timings, ProfileCompilationInfo* info);
117 
118   // To produce a valid oat file, the user must first add sources with any combination of
119   //   - AddDexFileSource(),
120   //   - AddZippedDexFilesSource(),
121   //   - AddRawDexFileSource(),
122   //   - AddVdexDexFilesSource().
123   // Then the user must call in order
124   //   - WriteAndOpenDexFiles()
125   //   - Initialize()
126   //   - WriteVerifierDeps()
127   //   - WriteQuickeningInfo()
128   //   - WriteChecksumsAndVdexHeader()
129   //   - PrepareLayout(),
130   //   - WriteRodata(),
131   //   - WriteCode(),
132   //   - WriteHeader().
133 
134   // Add dex file source(s) from a file, either a plain dex file or
135   // a zip file with one or more dex files.
136   bool AddDexFileSource(
137       const char* filename,
138       const char* location,
139       CreateTypeLookupTable create_type_lookup_table = CreateTypeLookupTable::kDefault);
140   // Add dex file source(s) from a zip file specified by a file handle.
141   bool AddZippedDexFilesSource(
142       File&& zip_fd,
143       const char* location,
144       CreateTypeLookupTable create_type_lookup_table = CreateTypeLookupTable::kDefault);
145   // Add dex file source from raw memory.
146   bool AddRawDexFileSource(
147       const ArrayRef<const uint8_t>& data,
148       const char* location,
149       uint32_t location_checksum,
150       CreateTypeLookupTable create_type_lookup_table = CreateTypeLookupTable::kDefault);
151   // Add dex file source(s) from a vdex file.
152   bool AddVdexDexFilesSource(
153       const VdexFile& vdex_file,
154       const char* location,
155       CreateTypeLookupTable create_type_lookup_table = CreateTypeLookupTable::kDefault);
156   dchecked_vector<const char*> GetSourceLocations() const;
157 
158   // Write raw dex files to the vdex file, mmap the file and open the dex files from it.
159   // Supporting data structures are written into the .rodata section of the oat file.
160   // The `verify` setting dictates whether the dex file verifier should check the dex files.
161   // This is generally the case, and should only be false for tests.
162   // If `update_input_vdex` is true, then this method won't actually write the dex files,
163   // and the compiler will just re-use the existing vdex file.
164   bool WriteAndOpenDexFiles(File* vdex_file,
165                             OutputStream* oat_rodata,
166                             InstructionSet instruction_set,
167                             const InstructionSetFeatures* instruction_set_features,
168                             SafeMap<std::string, std::string>* key_value_store,
169                             bool verify,
170                             bool update_input_vdex,
171                             /*out*/ std::unique_ptr<MemMap>* opened_dex_files_map,
172                             /*out*/ std::vector<std::unique_ptr<const DexFile>>* opened_dex_files);
173   bool WriteQuickeningInfo(OutputStream* vdex_out);
174   bool WriteVerifierDeps(OutputStream* vdex_out, verifier::VerifierDeps* verifier_deps);
175   bool WriteChecksumsAndVdexHeader(OutputStream* vdex_out);
176   // Initialize the writer with the given parameters.
Initialize(const CompilerDriver * compiler,ImageWriter * image_writer,const std::vector<const DexFile * > & dex_files)177   void Initialize(const CompilerDriver* compiler,
178                   ImageWriter* image_writer,
179                   const std::vector<const DexFile*>& dex_files) {
180     compiler_driver_ = compiler;
181     image_writer_ = image_writer;
182     dex_files_ = &dex_files;
183   }
184 
185   // Prepare layout of remaining data.
186   void PrepareLayout(linker::MultiOatRelativePatcher* relative_patcher);
187   // Write the rest of .rodata section (ClassOffsets[], OatClass[], maps).
188   bool WriteRodata(OutputStream* out);
189   // Write the code to the .text section.
190   bool WriteCode(OutputStream* out);
191   // Write the oat header. This finalizes the oat file.
192   bool WriteHeader(OutputStream* out,
193                    uint32_t image_file_location_oat_checksum,
194                    uintptr_t image_file_location_oat_begin,
195                    int32_t image_patch_delta);
196 
197   // Returns whether the oat file has an associated image.
HasImage()198   bool HasImage() const {
199     // Since the image is being created at the same time as the oat file,
200     // check if there's an image writer.
201     return image_writer_ != nullptr;
202   }
203 
HasBootImage()204   bool HasBootImage() const {
205     return compiling_boot_image_;
206   }
207 
GetOatHeader()208   const OatHeader& GetOatHeader() const {
209     return *oat_header_;
210   }
211 
GetOatSize()212   size_t GetOatSize() const {
213     return oat_size_;
214   }
215 
GetBssSize()216   size_t GetBssSize() const {
217     return bss_size_;
218   }
219 
GetBssRootsOffset()220   size_t GetBssRootsOffset() const {
221     return bss_roots_offset_;
222   }
223 
GetOatDataOffset()224   size_t GetOatDataOffset() const {
225     return oat_data_offset_;
226   }
227 
228   ~OatWriter();
229 
AddMethodDebugInfos(const std::vector<debug::MethodDebugInfo> & infos)230   void AddMethodDebugInfos(const std::vector<debug::MethodDebugInfo>& infos) {
231     method_info_.insert(method_info_.end(), infos.begin(), infos.end());
232   }
233 
GetMethodDebugInfo()234   ArrayRef<const debug::MethodDebugInfo> GetMethodDebugInfo() const {
235     return ArrayRef<const debug::MethodDebugInfo>(method_info_);
236   }
237 
GetCompilerDriver()238   const CompilerDriver* GetCompilerDriver() {
239     return compiler_driver_;
240   }
241 
242  private:
243   class DexFileSource;
244   class OatClass;
245   class OatDexFile;
246 
247   // The function VisitDexMethods() below iterates through all the methods in all
248   // the compiled dex files in order of their definitions. The method visitor
249   // classes provide individual bits of processing for each of the passes we need to
250   // first collect the data we want to write to the oat file and then, in later passes,
251   // to actually write it.
252   class DexMethodVisitor;
253   class OatDexMethodVisitor;
254   class InitOatClassesMethodVisitor;
255   class InitCodeMethodVisitor;
256   class InitMapMethodVisitor;
257   class InitMethodInfoVisitor;
258   class InitImageMethodVisitor;
259   class WriteCodeMethodVisitor;
260   class WriteMapMethodVisitor;
261   class WriteMethodInfoVisitor;
262   class WriteQuickeningInfoMethodVisitor;
263 
264   // Visit all the methods in all the compiled dex files in their definition order
265   // with a given DexMethodVisitor.
266   bool VisitDexMethods(DexMethodVisitor* visitor);
267 
268   // If `update_input_vdex` is true, then this method won't actually write the dex files,
269   // and the compiler will just re-use the existing vdex file.
270   bool WriteDexFiles(OutputStream* out, File* file, bool update_input_vdex);
271   bool WriteDexFile(OutputStream* out,
272                     File* file,
273                     OatDexFile* oat_dex_file,
274                     bool update_input_vdex);
275   bool SeekToDexFile(OutputStream* out, File* file, OatDexFile* oat_dex_file);
276   bool LayoutAndWriteDexFile(OutputStream* out, OatDexFile* oat_dex_file);
277   bool WriteDexFile(OutputStream* out,
278                     File* file,
279                     OatDexFile* oat_dex_file,
280                     ZipEntry* dex_file);
281   bool WriteDexFile(OutputStream* out,
282                     File* file,
283                     OatDexFile* oat_dex_file,
284                     File* dex_file);
285   bool WriteDexFile(OutputStream* out,
286                     OatDexFile* oat_dex_file,
287                     const uint8_t* dex_file,
288                     bool update_input_vdex);
289   bool OpenDexFiles(File* file,
290                     bool verify,
291                     /*out*/ std::unique_ptr<MemMap>* opened_dex_files_map,
292                     /*out*/ std::vector<std::unique_ptr<const DexFile>>* opened_dex_files);
293 
294   size_t InitOatHeader(InstructionSet instruction_set,
295                        const InstructionSetFeatures* instruction_set_features,
296                        uint32_t num_dex_files,
297                        SafeMap<std::string, std::string>* key_value_store);
298   size_t InitOatDexFiles(size_t offset);
299   size_t InitOatClasses(size_t offset);
300   size_t InitOatMaps(size_t offset);
301   size_t InitOatCode(size_t offset);
302   size_t InitOatCodeDexFiles(size_t offset);
303   void InitBssLayout(InstructionSet instruction_set);
304 
305   bool WriteClassOffsets(OutputStream* out);
306   bool WriteClasses(OutputStream* out);
307   size_t WriteMaps(OutputStream* out, const size_t file_offset, size_t relative_offset);
308   size_t WriteCode(OutputStream* out, const size_t file_offset, size_t relative_offset);
309   size_t WriteCodeDexFiles(OutputStream* out, const size_t file_offset, size_t relative_offset);
310 
311   bool RecordOatDataOffset(OutputStream* out);
312   bool ReadDexFileHeader(File* oat_file, OatDexFile* oat_dex_file);
313   bool ValidateDexFileHeader(const uint8_t* raw_header, const char* location);
314   bool WriteOatDexFiles(OutputStream* oat_rodata);
315   bool WriteTypeLookupTables(OutputStream* oat_rodata,
316                              const std::vector<std::unique_ptr<const DexFile>>& opened_dex_files);
317   bool WriteCodeAlignment(OutputStream* out, uint32_t aligned_code_delta);
318   void SetMultiOatRelativePatcherAdjustment();
319   void CloseSources();
320 
321   enum class WriteState {
322     kAddingDexFileSources,
323     kPrepareLayout,
324     kWriteRoData,
325     kWriteText,
326     kWriteHeader,
327     kDone
328   };
329 
330   WriteState write_state_;
331   TimingLogger* timings_;
332 
333   std::vector<std::unique_ptr<File>> raw_dex_files_;
334   std::vector<std::unique_ptr<ZipArchive>> zip_archives_;
335   std::vector<std::unique_ptr<ZipEntry>> zipped_dex_files_;
336 
337   // Using std::list<> which doesn't move elements around on push/emplace_back().
338   // We need this because we keep plain pointers to the strings' c_str().
339   std::list<std::string> zipped_dex_file_locations_;
340 
341   dchecked_vector<debug::MethodDebugInfo> method_info_;
342 
343   const CompilerDriver* compiler_driver_;
344   ImageWriter* image_writer_;
345   const bool compiling_boot_image_;
346 
347   // note OatFile does not take ownership of the DexFiles
348   const std::vector<const DexFile*>* dex_files_;
349 
350   // Size required for Vdex data structures.
351   size_t vdex_size_;
352 
353   // Offset of section holding Dex files inside Vdex.
354   size_t vdex_dex_files_offset_;
355 
356   // Offset of section holding VerifierDeps inside Vdex.
357   size_t vdex_verifier_deps_offset_;
358 
359   // Offset of section holding quickening info inside Vdex.
360   size_t vdex_quickening_info_offset_;
361 
362   // Size required for Oat data structures.
363   size_t oat_size_;
364 
365   // The start of the required .bss section.
366   size_t bss_start_;
367 
368   // The size of the required .bss section holding the DexCache data and GC roots.
369   size_t bss_size_;
370 
371   // The offset of the GC roots in .bss section.
372   size_t bss_roots_offset_;
373 
374   // Map for allocating Class entries in .bss. Indexed by TypeReference for the source
375   // type in the dex file with the "type value comparator" for deduplication. The value
376   // is the target offset for patching, starting at `bss_start_ + bss_roots_offset_`.
377   SafeMap<TypeReference, size_t, TypeReferenceValueComparator> bss_type_entries_;
378 
379   // Map for allocating String entries in .bss. Indexed by StringReference for the source
380   // string in the dex file with the "string value comparator" for deduplication. The value
381   // is the target offset for patching, starting at `bss_start_ + bss_roots_offset_`.
382   SafeMap<StringReference, size_t, StringReferenceValueComparator> bss_string_entries_;
383 
384   // Offsets of the dex cache arrays for each app dex file. For the
385   // boot image, this information is provided by the ImageWriter.
386   SafeMap<const DexFile*, size_t> dex_cache_arrays_offsets_;  // DexFiles not owned.
387 
388   // Offset of the oat data from the start of the mmapped region of the elf file.
389   size_t oat_data_offset_;
390 
391   // Fake OatDexFiles to hold type lookup tables for the compiler.
392   std::vector<std::unique_ptr<art::OatDexFile>> type_lookup_table_oat_dex_files_;
393 
394   // data to write
395   std::unique_ptr<OatHeader> oat_header_;
396   dchecked_vector<OatDexFile> oat_dex_files_;
397   dchecked_vector<OatClass> oat_classes_;
398   std::unique_ptr<const std::vector<uint8_t>> jni_dlsym_lookup_;
399   std::unique_ptr<const std::vector<uint8_t>> quick_generic_jni_trampoline_;
400   std::unique_ptr<const std::vector<uint8_t>> quick_imt_conflict_trampoline_;
401   std::unique_ptr<const std::vector<uint8_t>> quick_resolution_trampoline_;
402   std::unique_ptr<const std::vector<uint8_t>> quick_to_interpreter_bridge_;
403 
404   // output stats
405   uint32_t size_vdex_header_;
406   uint32_t size_vdex_checksums_;
407   uint32_t size_dex_file_alignment_;
408   uint32_t size_executable_offset_alignment_;
409   uint32_t size_oat_header_;
410   uint32_t size_oat_header_key_value_store_;
411   uint32_t size_dex_file_;
412   uint32_t size_verifier_deps_;
413   uint32_t size_verifier_deps_alignment_;
414   uint32_t size_quickening_info_;
415   uint32_t size_quickening_info_alignment_;
416   uint32_t size_interpreter_to_interpreter_bridge_;
417   uint32_t size_interpreter_to_compiled_code_bridge_;
418   uint32_t size_jni_dlsym_lookup_;
419   uint32_t size_quick_generic_jni_trampoline_;
420   uint32_t size_quick_imt_conflict_trampoline_;
421   uint32_t size_quick_resolution_trampoline_;
422   uint32_t size_quick_to_interpreter_bridge_;
423   uint32_t size_trampoline_alignment_;
424   uint32_t size_method_header_;
425   uint32_t size_code_;
426   uint32_t size_code_alignment_;
427   uint32_t size_relative_call_thunks_;
428   uint32_t size_misc_thunks_;
429   uint32_t size_vmap_table_;
430   uint32_t size_method_info_;
431   uint32_t size_oat_dex_file_location_size_;
432   uint32_t size_oat_dex_file_location_data_;
433   uint32_t size_oat_dex_file_location_checksum_;
434   uint32_t size_oat_dex_file_offset_;
435   uint32_t size_oat_dex_file_class_offsets_offset_;
436   uint32_t size_oat_dex_file_lookup_table_offset_;
437   uint32_t size_oat_lookup_table_alignment_;
438   uint32_t size_oat_lookup_table_;
439   uint32_t size_oat_class_offsets_alignment_;
440   uint32_t size_oat_class_offsets_;
441   uint32_t size_oat_class_type_;
442   uint32_t size_oat_class_status_;
443   uint32_t size_oat_class_method_bitmaps_;
444   uint32_t size_oat_class_method_offsets_;
445 
446   // The helper for processing relative patches is external so that we can patch across oat files.
447   linker::MultiOatRelativePatcher* relative_patcher_;
448 
449   // The locations of absolute patches relative to the start of the executable section.
450   dchecked_vector<uintptr_t> absolute_patch_locations_;
451 
452   // Profile info used to generate new layout of files.
453   ProfileCompilationInfo* profile_compilation_info_;
454 
455   DISALLOW_COPY_AND_ASSIGN(OatWriter);
456 };
457 
458 }  // namespace art
459 
460 #endif  // ART_COMPILER_OAT_WRITER_H_
461