1 //===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the GlobalModuleIndex class.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "ASTReaderInternals.h"
15 #include "clang/Basic/FileManager.h"
16 #include "clang/Lex/HeaderSearch.h"
17 #include "clang/Serialization/ASTBitCodes.h"
18 #include "clang/Serialization/GlobalModuleIndex.h"
19 #include "clang/Serialization/Module.h"
20 #include "llvm/ADT/DenseMap.h"
21 #include "llvm/ADT/MapVector.h"
22 #include "llvm/ADT/SmallString.h"
23 #include "llvm/ADT/StringExtras.h"
24 #include "llvm/Bitcode/BitstreamReader.h"
25 #include "llvm/Bitcode/BitstreamWriter.h"
26 #include "llvm/Support/FileSystem.h"
27 #include "llvm/Support/LockFileManager.h"
28 #include "llvm/Support/MemoryBuffer.h"
29 #include "llvm/Support/OnDiskHashTable.h"
30 #include "llvm/Support/Path.h"
31 #include <cstdio>
32 using namespace clang;
33 using namespace serialization;
34 
35 //----------------------------------------------------------------------------//
36 // Shared constants
37 //----------------------------------------------------------------------------//
38 namespace {
39   enum {
40     /// \brief The block containing the index.
41     GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
42   };
43 
44   /// \brief Describes the record types in the index.
45   enum IndexRecordTypes {
46     /// \brief Contains version information and potentially other metadata,
47     /// used to determine if we can read this global index file.
48     INDEX_METADATA,
49     /// \brief Describes a module, including its file name and dependencies.
50     MODULE,
51     /// \brief The index for identifiers.
52     IDENTIFIER_INDEX
53   };
54 }
55 
56 /// \brief The name of the global index file.
57 static const char * const IndexFileName = "modules.idx";
58 
59 /// \brief The global index file version.
60 static const unsigned CurrentVersion = 1;
61 
62 //----------------------------------------------------------------------------//
63 // Global module index reader.
64 //----------------------------------------------------------------------------//
65 
66 namespace {
67 
68 /// \brief Trait used to read the identifier index from the on-disk hash
69 /// table.
70 class IdentifierIndexReaderTrait {
71 public:
72   typedef StringRef external_key_type;
73   typedef StringRef internal_key_type;
74   typedef SmallVector<unsigned, 2> data_type;
75   typedef unsigned hash_value_type;
76   typedef unsigned offset_type;
77 
EqualKey(const internal_key_type & a,const internal_key_type & b)78   static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
79     return a == b;
80   }
81 
ComputeHash(const internal_key_type & a)82   static hash_value_type ComputeHash(const internal_key_type& a) {
83     return llvm::HashString(a);
84   }
85 
86   static std::pair<unsigned, unsigned>
ReadKeyDataLength(const unsigned char * & d)87   ReadKeyDataLength(const unsigned char*& d) {
88     using namespace llvm::support;
89     unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
90     unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
91     return std::make_pair(KeyLen, DataLen);
92   }
93 
94   static const internal_key_type&
GetInternalKey(const external_key_type & x)95   GetInternalKey(const external_key_type& x) { return x; }
96 
97   static const external_key_type&
GetExternalKey(const internal_key_type & x)98   GetExternalKey(const internal_key_type& x) { return x; }
99 
ReadKey(const unsigned char * d,unsigned n)100   static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
101     return StringRef((const char *)d, n);
102   }
103 
ReadData(const internal_key_type & k,const unsigned char * d,unsigned DataLen)104   static data_type ReadData(const internal_key_type& k,
105                             const unsigned char* d,
106                             unsigned DataLen) {
107     using namespace llvm::support;
108 
109     data_type Result;
110     while (DataLen > 0) {
111       unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
112       Result.push_back(ID);
113       DataLen -= 4;
114     }
115 
116     return Result;
117   }
118 };
119 
120 typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
121     IdentifierIndexTable;
122 
123 }
124 
GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer,llvm::BitstreamCursor Cursor)125 GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer,
126                                      llvm::BitstreamCursor Cursor)
127     : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(),
128       NumIdentifierLookupHits() {
129   // Read the global index.
130   bool InGlobalIndexBlock = false;
131   bool Done = false;
132   while (!Done) {
133     llvm::BitstreamEntry Entry = Cursor.advance();
134 
135     switch (Entry.Kind) {
136     case llvm::BitstreamEntry::Error:
137       return;
138 
139     case llvm::BitstreamEntry::EndBlock:
140       if (InGlobalIndexBlock) {
141         InGlobalIndexBlock = false;
142         Done = true;
143         continue;
144       }
145       return;
146 
147 
148     case llvm::BitstreamEntry::Record:
149       // Entries in the global index block are handled below.
150       if (InGlobalIndexBlock)
151         break;
152 
153       return;
154 
155     case llvm::BitstreamEntry::SubBlock:
156       if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
157         if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
158           return;
159 
160         InGlobalIndexBlock = true;
161       } else if (Cursor.SkipBlock()) {
162         return;
163       }
164       continue;
165     }
166 
167     SmallVector<uint64_t, 64> Record;
168     StringRef Blob;
169     switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) {
170     case INDEX_METADATA:
171       // Make sure that the version matches.
172       if (Record.size() < 1 || Record[0] != CurrentVersion)
173         return;
174       break;
175 
176     case MODULE: {
177       unsigned Idx = 0;
178       unsigned ID = Record[Idx++];
179 
180       // Make room for this module's information.
181       if (ID == Modules.size())
182         Modules.push_back(ModuleInfo());
183       else
184         Modules.resize(ID + 1);
185 
186       // Size/modification time for this module file at the time the
187       // global index was built.
188       Modules[ID].Size = Record[Idx++];
189       Modules[ID].ModTime = Record[Idx++];
190 
191       // File name.
192       unsigned NameLen = Record[Idx++];
193       Modules[ID].FileName.assign(Record.begin() + Idx,
194                                   Record.begin() + Idx + NameLen);
195       Idx += NameLen;
196 
197       // Dependencies
198       unsigned NumDeps = Record[Idx++];
199       Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
200                                       Record.begin() + Idx,
201                                       Record.begin() + Idx + NumDeps);
202       Idx += NumDeps;
203 
204       // Make sure we're at the end of the record.
205       assert(Idx == Record.size() && "More module info?");
206 
207       // Record this module as an unresolved module.
208       // FIXME: this doesn't work correctly for module names containing path
209       // separators.
210       StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
211       // Remove the -<hash of ModuleMapPath>
212       ModuleName = ModuleName.rsplit('-').first;
213       UnresolvedModules[ModuleName] = ID;
214       break;
215     }
216 
217     case IDENTIFIER_INDEX:
218       // Wire up the identifier index.
219       if (Record[0]) {
220         IdentifierIndex = IdentifierIndexTable::Create(
221             (const unsigned char *)Blob.data() + Record[0],
222             (const unsigned char *)Blob.data() + sizeof(uint32_t),
223             (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
224       }
225       break;
226     }
227   }
228 }
229 
~GlobalModuleIndex()230 GlobalModuleIndex::~GlobalModuleIndex() {
231   delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
232 }
233 
234 std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
readIndex(StringRef Path)235 GlobalModuleIndex::readIndex(StringRef Path) {
236   // Load the index file, if it's there.
237   llvm::SmallString<128> IndexPath;
238   IndexPath += Path;
239   llvm::sys::path::append(IndexPath, IndexFileName);
240 
241   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
242       llvm::MemoryBuffer::getFile(IndexPath.c_str());
243   if (!BufferOrErr)
244     return std::make_pair(nullptr, EC_NotFound);
245   std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
246 
247   /// \brief The bitstream reader from which we'll read the AST file.
248   llvm::BitstreamReader Reader((const unsigned char *)Buffer->getBufferStart(),
249                                (const unsigned char *)Buffer->getBufferEnd());
250 
251   /// \brief The main bitstream cursor for the main block.
252   llvm::BitstreamCursor Cursor(Reader);
253 
254   // Sniff for the signature.
255   if (Cursor.Read(8) != 'B' ||
256       Cursor.Read(8) != 'C' ||
257       Cursor.Read(8) != 'G' ||
258       Cursor.Read(8) != 'I') {
259     return std::make_pair(nullptr, EC_IOError);
260   }
261 
262   return std::make_pair(new GlobalModuleIndex(std::move(Buffer), Cursor),
263                         EC_None);
264 }
265 
266 void
getKnownModules(SmallVectorImpl<ModuleFile * > & ModuleFiles)267 GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) {
268   ModuleFiles.clear();
269   for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
270     if (ModuleFile *MF = Modules[I].File)
271       ModuleFiles.push_back(MF);
272   }
273 }
274 
getModuleDependencies(ModuleFile * File,SmallVectorImpl<ModuleFile * > & Dependencies)275 void GlobalModuleIndex::getModuleDependencies(
276        ModuleFile *File,
277        SmallVectorImpl<ModuleFile *> &Dependencies) {
278   // Look for information about this module file.
279   llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
280     = ModulesByFile.find(File);
281   if (Known == ModulesByFile.end())
282     return;
283 
284   // Record dependencies.
285   Dependencies.clear();
286   ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
287   for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
288     if (ModuleFile *MF = Modules[I].File)
289       Dependencies.push_back(MF);
290   }
291 }
292 
lookupIdentifier(StringRef Name,HitSet & Hits)293 bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
294   Hits.clear();
295 
296   // If there's no identifier index, there is nothing we can do.
297   if (!IdentifierIndex)
298     return false;
299 
300   // Look into the identifier index.
301   ++NumIdentifierLookups;
302   IdentifierIndexTable &Table
303     = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
304   IdentifierIndexTable::iterator Known = Table.find(Name);
305   if (Known == Table.end()) {
306     return true;
307   }
308 
309   SmallVector<unsigned, 2> ModuleIDs = *Known;
310   for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
311     if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
312       Hits.insert(MF);
313   }
314 
315   ++NumIdentifierLookupHits;
316   return true;
317 }
318 
loadedModuleFile(ModuleFile * File)319 bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
320   // Look for the module in the global module index based on the module name.
321   StringRef Name = File->ModuleName;
322   llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
323   if (Known == UnresolvedModules.end()) {
324     return true;
325   }
326 
327   // Rectify this module with the global module index.
328   ModuleInfo &Info = Modules[Known->second];
329 
330   //  If the size and modification time match what we expected, record this
331   // module file.
332   bool Failed = true;
333   if (File->File->getSize() == Info.Size &&
334       File->File->getModificationTime() == Info.ModTime) {
335     Info.File = File;
336     ModulesByFile[File] = Known->second;
337 
338     Failed = false;
339   }
340 
341   // One way or another, we have resolved this module file.
342   UnresolvedModules.erase(Known);
343   return Failed;
344 }
345 
printStats()346 void GlobalModuleIndex::printStats() {
347   std::fprintf(stderr, "*** Global Module Index Statistics:\n");
348   if (NumIdentifierLookups) {
349     fprintf(stderr, "  %u / %u identifier lookups succeeded (%f%%)\n",
350             NumIdentifierLookupHits, NumIdentifierLookups,
351             (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
352   }
353   std::fprintf(stderr, "\n");
354 }
355 
dump()356 void GlobalModuleIndex::dump() {
357   llvm::errs() << "*** Global Module Index Dump:\n";
358   llvm::errs() << "Module files:\n";
359   for (auto &MI : Modules) {
360     llvm::errs() << "** " << MI.FileName << "\n";
361     if (MI.File)
362       MI.File->dump();
363     else
364       llvm::errs() << "\n";
365   }
366   llvm::errs() << "\n";
367 }
368 
369 //----------------------------------------------------------------------------//
370 // Global module index writer.
371 //----------------------------------------------------------------------------//
372 
373 namespace {
374   /// \brief Provides information about a specific module file.
375   struct ModuleFileInfo {
376     /// \brief The numberic ID for this module file.
377     unsigned ID;
378 
379     /// \brief The set of modules on which this module depends. Each entry is
380     /// a module ID.
381     SmallVector<unsigned, 4> Dependencies;
382   };
383 
384   /// \brief Builder that generates the global module index file.
385   class GlobalModuleIndexBuilder {
386     FileManager &FileMgr;
387 
388     /// \brief Mapping from files to module file information.
389     typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
390 
391     /// \brief Information about each of the known module files.
392     ModuleFilesMap ModuleFiles;
393 
394     /// \brief Mapping from identifiers to the list of module file IDs that
395     /// consider this identifier to be interesting.
396     typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
397 
398     /// \brief A mapping from all interesting identifiers to the set of module
399     /// files in which those identifiers are considered interesting.
400     InterestingIdentifierMap InterestingIdentifiers;
401 
402     /// \brief Write the block-info block for the global module index file.
403     void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
404 
405     /// \brief Retrieve the module file information for the given file.
getModuleFileInfo(const FileEntry * File)406     ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
407       llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
408         = ModuleFiles.find(File);
409       if (Known != ModuleFiles.end())
410         return Known->second;
411 
412       unsigned NewID = ModuleFiles.size();
413       ModuleFileInfo &Info = ModuleFiles[File];
414       Info.ID = NewID;
415       return Info;
416     }
417 
418   public:
GlobalModuleIndexBuilder(FileManager & FileMgr)419     explicit GlobalModuleIndexBuilder(FileManager &FileMgr) : FileMgr(FileMgr){}
420 
421     /// \brief Load the contents of the given module file into the builder.
422     ///
423     /// \returns true if an error occurred, false otherwise.
424     bool loadModuleFile(const FileEntry *File);
425 
426     /// \brief Write the index to the given bitstream.
427     void writeIndex(llvm::BitstreamWriter &Stream);
428   };
429 }
430 
emitBlockID(unsigned ID,const char * Name,llvm::BitstreamWriter & Stream,SmallVectorImpl<uint64_t> & Record)431 static void emitBlockID(unsigned ID, const char *Name,
432                         llvm::BitstreamWriter &Stream,
433                         SmallVectorImpl<uint64_t> &Record) {
434   Record.clear();
435   Record.push_back(ID);
436   Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
437 
438   // Emit the block name if present.
439   if (!Name || Name[0] == 0) return;
440   Record.clear();
441   while (*Name)
442     Record.push_back(*Name++);
443   Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
444 }
445 
emitRecordID(unsigned ID,const char * Name,llvm::BitstreamWriter & Stream,SmallVectorImpl<uint64_t> & Record)446 static void emitRecordID(unsigned ID, const char *Name,
447                          llvm::BitstreamWriter &Stream,
448                          SmallVectorImpl<uint64_t> &Record) {
449   Record.clear();
450   Record.push_back(ID);
451   while (*Name)
452     Record.push_back(*Name++);
453   Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
454 }
455 
456 void
emitBlockInfoBlock(llvm::BitstreamWriter & Stream)457 GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
458   SmallVector<uint64_t, 64> Record;
459   Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3);
460 
461 #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
462 #define RECORD(X) emitRecordID(X, #X, Stream, Record)
463   BLOCK(GLOBAL_INDEX_BLOCK);
464   RECORD(INDEX_METADATA);
465   RECORD(MODULE);
466   RECORD(IDENTIFIER_INDEX);
467 #undef RECORD
468 #undef BLOCK
469 
470   Stream.ExitBlock();
471 }
472 
473 namespace {
474   class InterestingASTIdentifierLookupTrait
475     : public serialization::reader::ASTIdentifierLookupTraitBase {
476 
477   public:
478     /// \brief The identifier and whether it is "interesting".
479     typedef std::pair<StringRef, bool> data_type;
480 
ReadData(const internal_key_type & k,const unsigned char * d,unsigned DataLen)481     data_type ReadData(const internal_key_type& k,
482                        const unsigned char* d,
483                        unsigned DataLen) {
484       // The first bit indicates whether this identifier is interesting.
485       // That's all we care about.
486       using namespace llvm::support;
487       unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
488       bool IsInteresting = RawID & 0x01;
489       return std::make_pair(k, IsInteresting);
490     }
491   };
492 }
493 
loadModuleFile(const FileEntry * File)494 bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
495   // Open the module file.
496 
497   auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true);
498   if (!Buffer) {
499     return true;
500   }
501 
502   // Initialize the input stream
503   llvm::BitstreamReader InStreamFile;
504   InStreamFile.init((const unsigned char *)(*Buffer)->getBufferStart(),
505                     (const unsigned char *)(*Buffer)->getBufferEnd());
506   llvm::BitstreamCursor InStream(InStreamFile);
507 
508   // Sniff for the signature.
509   if (InStream.Read(8) != 'C' ||
510       InStream.Read(8) != 'P' ||
511       InStream.Read(8) != 'C' ||
512       InStream.Read(8) != 'H') {
513     return true;
514   }
515 
516   // Record this module file and assign it a unique ID (if it doesn't have
517   // one already).
518   unsigned ID = getModuleFileInfo(File).ID;
519 
520   // Search for the blocks and records we care about.
521   enum { Other, ControlBlock, ASTBlock } State = Other;
522   bool Done = false;
523   while (!Done) {
524     llvm::BitstreamEntry Entry = InStream.advance();
525     switch (Entry.Kind) {
526     case llvm::BitstreamEntry::Error:
527       Done = true;
528       continue;
529 
530     case llvm::BitstreamEntry::Record:
531       // In the 'other' state, just skip the record. We don't care.
532       if (State == Other) {
533         InStream.skipRecord(Entry.ID);
534         continue;
535       }
536 
537       // Handle potentially-interesting records below.
538       break;
539 
540     case llvm::BitstreamEntry::SubBlock:
541       if (Entry.ID == CONTROL_BLOCK_ID) {
542         if (InStream.EnterSubBlock(CONTROL_BLOCK_ID))
543           return true;
544 
545         // Found the control block.
546         State = ControlBlock;
547         continue;
548       }
549 
550       if (Entry.ID == AST_BLOCK_ID) {
551         if (InStream.EnterSubBlock(AST_BLOCK_ID))
552           return true;
553 
554         // Found the AST block.
555         State = ASTBlock;
556         continue;
557       }
558 
559       if (InStream.SkipBlock())
560         return true;
561 
562       continue;
563 
564     case llvm::BitstreamEntry::EndBlock:
565       State = Other;
566       continue;
567     }
568 
569     // Read the given record.
570     SmallVector<uint64_t, 64> Record;
571     StringRef Blob;
572     unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);
573 
574     // Handle module dependencies.
575     if (State == ControlBlock && Code == IMPORTS) {
576       // Load each of the imported PCH files.
577       unsigned Idx = 0, N = Record.size();
578       while (Idx < N) {
579         // Read information about the AST file.
580 
581         // Skip the imported kind
582         ++Idx;
583 
584         // Skip the import location
585         ++Idx;
586 
587         // Load stored size/modification time.
588         off_t StoredSize = (off_t)Record[Idx++];
589         time_t StoredModTime = (time_t)Record[Idx++];
590 
591         // Skip the stored signature.
592         // FIXME: we could read the signature out of the import and validate it.
593         Idx++;
594 
595         // Retrieve the imported file name.
596         unsigned Length = Record[Idx++];
597         SmallString<128> ImportedFile(Record.begin() + Idx,
598                                       Record.begin() + Idx + Length);
599         Idx += Length;
600 
601         // Find the imported module file.
602         const FileEntry *DependsOnFile
603           = FileMgr.getFile(ImportedFile, /*openFile=*/false,
604                             /*cacheFailure=*/false);
605         if (!DependsOnFile ||
606             (StoredSize != DependsOnFile->getSize()) ||
607             (StoredModTime != DependsOnFile->getModificationTime()))
608           return true;
609 
610         // Record the dependency.
611         unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
612         getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
613       }
614 
615       continue;
616     }
617 
618     // Handle the identifier table
619     if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
620       typedef llvm::OnDiskIterableChainedHashTable<
621           InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
622       std::unique_ptr<InterestingIdentifierTable> Table(
623           InterestingIdentifierTable::Create(
624               (const unsigned char *)Blob.data() + Record[0],
625               (const unsigned char *)Blob.data() + sizeof(uint32_t),
626               (const unsigned char *)Blob.data()));
627       for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
628                                                      DEnd = Table->data_end();
629            D != DEnd; ++D) {
630         std::pair<StringRef, bool> Ident = *D;
631         if (Ident.second)
632           InterestingIdentifiers[Ident.first].push_back(ID);
633         else
634           (void)InterestingIdentifiers[Ident.first];
635       }
636     }
637 
638     // We don't care about this record.
639   }
640 
641   return false;
642 }
643 
644 namespace {
645 
646 /// \brief Trait used to generate the identifier index as an on-disk hash
647 /// table.
648 class IdentifierIndexWriterTrait {
649 public:
650   typedef StringRef key_type;
651   typedef StringRef key_type_ref;
652   typedef SmallVector<unsigned, 2> data_type;
653   typedef const SmallVector<unsigned, 2> &data_type_ref;
654   typedef unsigned hash_value_type;
655   typedef unsigned offset_type;
656 
ComputeHash(key_type_ref Key)657   static hash_value_type ComputeHash(key_type_ref Key) {
658     return llvm::HashString(Key);
659   }
660 
661   std::pair<unsigned,unsigned>
EmitKeyDataLength(raw_ostream & Out,key_type_ref Key,data_type_ref Data)662   EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
663     using namespace llvm::support;
664     endian::Writer<little> LE(Out);
665     unsigned KeyLen = Key.size();
666     unsigned DataLen = Data.size() * 4;
667     LE.write<uint16_t>(KeyLen);
668     LE.write<uint16_t>(DataLen);
669     return std::make_pair(KeyLen, DataLen);
670   }
671 
EmitKey(raw_ostream & Out,key_type_ref Key,unsigned KeyLen)672   void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
673     Out.write(Key.data(), KeyLen);
674   }
675 
EmitData(raw_ostream & Out,key_type_ref Key,data_type_ref Data,unsigned DataLen)676   void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
677                 unsigned DataLen) {
678     using namespace llvm::support;
679     for (unsigned I = 0, N = Data.size(); I != N; ++I)
680       endian::Writer<little>(Out).write<uint32_t>(Data[I]);
681   }
682 };
683 
684 }
685 
writeIndex(llvm::BitstreamWriter & Stream)686 void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
687   using namespace llvm;
688 
689   // Emit the file header.
690   Stream.Emit((unsigned)'B', 8);
691   Stream.Emit((unsigned)'C', 8);
692   Stream.Emit((unsigned)'G', 8);
693   Stream.Emit((unsigned)'I', 8);
694 
695   // Write the block-info block, which describes the records in this bitcode
696   // file.
697   emitBlockInfoBlock(Stream);
698 
699   Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
700 
701   // Write the metadata.
702   SmallVector<uint64_t, 2> Record;
703   Record.push_back(CurrentVersion);
704   Stream.EmitRecord(INDEX_METADATA, Record);
705 
706   // Write the set of known module files.
707   for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
708                                 MEnd = ModuleFiles.end();
709        M != MEnd; ++M) {
710     Record.clear();
711     Record.push_back(M->second.ID);
712     Record.push_back(M->first->getSize());
713     Record.push_back(M->first->getModificationTime());
714 
715     // File name
716     StringRef Name(M->first->getName());
717     Record.push_back(Name.size());
718     Record.append(Name.begin(), Name.end());
719 
720     // Dependencies
721     Record.push_back(M->second.Dependencies.size());
722     Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
723     Stream.EmitRecord(MODULE, Record);
724   }
725 
726   // Write the identifier -> module file mapping.
727   {
728     llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
729     IdentifierIndexWriterTrait Trait;
730 
731     // Populate the hash table.
732     for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
733                                             IEnd = InterestingIdentifiers.end();
734          I != IEnd; ++I) {
735       Generator.insert(I->first(), I->second, Trait);
736     }
737 
738     // Create the on-disk hash table in a buffer.
739     SmallString<4096> IdentifierTable;
740     uint32_t BucketOffset;
741     {
742       using namespace llvm::support;
743       llvm::raw_svector_ostream Out(IdentifierTable);
744       // Make sure that no bucket is at offset 0
745       endian::Writer<little>(Out).write<uint32_t>(0);
746       BucketOffset = Generator.Emit(Out, Trait);
747     }
748 
749     // Create a blob abbreviation
750     BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
751     Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
752     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
753     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
754     unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev);
755 
756     // Write the identifier table
757     Record.clear();
758     Record.push_back(IDENTIFIER_INDEX);
759     Record.push_back(BucketOffset);
760     Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
761   }
762 
763   Stream.ExitBlock();
764 }
765 
766 GlobalModuleIndex::ErrorCode
writeIndex(FileManager & FileMgr,StringRef Path)767 GlobalModuleIndex::writeIndex(FileManager &FileMgr, StringRef Path) {
768   llvm::SmallString<128> IndexPath;
769   IndexPath += Path;
770   llvm::sys::path::append(IndexPath, IndexFileName);
771 
772   // Coordinate building the global index file with other processes that might
773   // try to do the same.
774   llvm::LockFileManager Locked(IndexPath);
775   switch (Locked) {
776   case llvm::LockFileManager::LFS_Error:
777     return EC_IOError;
778 
779   case llvm::LockFileManager::LFS_Owned:
780     // We're responsible for building the index ourselves. Do so below.
781     break;
782 
783   case llvm::LockFileManager::LFS_Shared:
784     // Someone else is responsible for building the index. We don't care
785     // when they finish, so we're done.
786     return EC_Building;
787   }
788 
789   // The module index builder.
790   GlobalModuleIndexBuilder Builder(FileMgr);
791 
792   // Load each of the module files.
793   std::error_code EC;
794   for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
795        D != DEnd && !EC;
796        D.increment(EC)) {
797     // If this isn't a module file, we don't care.
798     if (llvm::sys::path::extension(D->path()) != ".pcm") {
799       // ... unless it's a .pcm.lock file, which indicates that someone is
800       // in the process of rebuilding a module. They'll rebuild the index
801       // at the end of that translation unit, so we don't have to.
802       if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
803         return EC_Building;
804 
805       continue;
806     }
807 
808     // If we can't find the module file, skip it.
809     const FileEntry *ModuleFile = FileMgr.getFile(D->path());
810     if (!ModuleFile)
811       continue;
812 
813     // Load this module file.
814     if (Builder.loadModuleFile(ModuleFile))
815       return EC_IOError;
816   }
817 
818   // The output buffer, into which the global index will be written.
819   SmallVector<char, 16> OutputBuffer;
820   {
821     llvm::BitstreamWriter OutputStream(OutputBuffer);
822     Builder.writeIndex(OutputStream);
823   }
824 
825   // Write the global index file to a temporary file.
826   llvm::SmallString<128> IndexTmpPath;
827   int TmpFD;
828   if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
829                                       IndexTmpPath))
830     return EC_IOError;
831 
832   // Open the temporary global index file for output.
833   llvm::raw_fd_ostream Out(TmpFD, true);
834   if (Out.has_error())
835     return EC_IOError;
836 
837   // Write the index.
838   Out.write(OutputBuffer.data(), OutputBuffer.size());
839   Out.close();
840   if (Out.has_error())
841     return EC_IOError;
842 
843   // Remove the old index file. It isn't relevant any more.
844   llvm::sys::fs::remove(IndexPath);
845 
846   // Rename the newly-written index file to the proper name.
847   if (llvm::sys::fs::rename(IndexTmpPath, IndexPath)) {
848     // Rename failed; just remove the
849     llvm::sys::fs::remove(IndexTmpPath);
850     return EC_IOError;
851   }
852 
853   // We're done.
854   return EC_None;
855 }
856 
857 namespace {
858   class GlobalIndexIdentifierIterator : public IdentifierIterator {
859     /// \brief The current position within the identifier lookup table.
860     IdentifierIndexTable::key_iterator Current;
861 
862     /// \brief The end position within the identifier lookup table.
863     IdentifierIndexTable::key_iterator End;
864 
865   public:
GlobalIndexIdentifierIterator(IdentifierIndexTable & Idx)866     explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
867       Current = Idx.key_begin();
868       End = Idx.key_end();
869     }
870 
Next()871     StringRef Next() override {
872       if (Current == End)
873         return StringRef();
874 
875       StringRef Result = *Current;
876       ++Current;
877       return Result;
878     }
879   };
880 }
881 
createIdentifierIterator() const882 IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const {
883   IdentifierIndexTable &Table =
884     *static_cast<IdentifierIndexTable *>(IdentifierIndex);
885   return new GlobalIndexIdentifierIterator(Table);
886 }
887