1 //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the pruning of a directory based on least recently used.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/Support/CachePruning.h"
15 
16 #include "llvm/Support/Debug.h"
17 #include "llvm/Support/Errc.h"
18 #include "llvm/Support/Error.h"
19 #include "llvm/Support/FileSystem.h"
20 #include "llvm/Support/Path.h"
21 #include "llvm/Support/raw_ostream.h"
22 
23 #define DEBUG_TYPE "cache-pruning"
24 
25 #include <set>
26 #include <system_error>
27 
28 using namespace llvm;
29 
30 /// Write a new timestamp file with the given path. This is used for the pruning
31 /// interval option.
writeTimestampFile(StringRef TimestampFile)32 static void writeTimestampFile(StringRef TimestampFile) {
33   std::error_code EC;
34   raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
35 }
36 
parseDuration(StringRef Duration)37 static Expected<std::chrono::seconds> parseDuration(StringRef Duration) {
38   if (Duration.empty())
39     return make_error<StringError>("Duration must not be empty",
40                                    inconvertibleErrorCode());
41 
42   StringRef NumStr = Duration.slice(0, Duration.size()-1);
43   uint64_t Num;
44   if (NumStr.getAsInteger(0, Num))
45     return make_error<StringError>("'" + NumStr + "' not an integer",
46                                    inconvertibleErrorCode());
47 
48   switch (Duration.back()) {
49   case 's':
50     return std::chrono::seconds(Num);
51   case 'm':
52     return std::chrono::minutes(Num);
53   case 'h':
54     return std::chrono::hours(Num);
55   default:
56     return make_error<StringError>("'" + Duration +
57                                        "' must end with one of 's', 'm' or 'h'",
58                                    inconvertibleErrorCode());
59   }
60 }
61 
62 Expected<CachePruningPolicy>
parseCachePruningPolicy(StringRef PolicyStr)63 llvm::parseCachePruningPolicy(StringRef PolicyStr) {
64   CachePruningPolicy Policy;
65   std::pair<StringRef, StringRef> P = {"", PolicyStr};
66   while (!P.second.empty()) {
67     P = P.second.split(':');
68 
69     StringRef Key, Value;
70     std::tie(Key, Value) = P.first.split('=');
71     if (Key == "prune_interval") {
72       auto DurationOrErr = parseDuration(Value);
73       if (!DurationOrErr)
74         return DurationOrErr.takeError();
75       Policy.Interval = *DurationOrErr;
76     } else if (Key == "prune_after") {
77       auto DurationOrErr = parseDuration(Value);
78       if (!DurationOrErr)
79         return DurationOrErr.takeError();
80       Policy.Expiration = *DurationOrErr;
81     } else if (Key == "cache_size") {
82       if (Value.back() != '%')
83         return make_error<StringError>("'" + Value + "' must be a percentage",
84                                        inconvertibleErrorCode());
85       StringRef SizeStr = Value.drop_back();
86       uint64_t Size;
87       if (SizeStr.getAsInteger(0, Size))
88         return make_error<StringError>("'" + SizeStr + "' not an integer",
89                                        inconvertibleErrorCode());
90       if (Size > 100)
91         return make_error<StringError>("'" + SizeStr +
92                                            "' must be between 0 and 100",
93                                        inconvertibleErrorCode());
94       Policy.MaxSizePercentageOfAvailableSpace = Size;
95     } else if (Key == "cache_size_bytes") {
96       uint64_t Mult = 1;
97       switch (tolower(Value.back())) {
98       case 'k':
99         Mult = 1024;
100         Value = Value.drop_back();
101         break;
102       case 'm':
103         Mult = 1024 * 1024;
104         Value = Value.drop_back();
105         break;
106       case 'g':
107         Mult = 1024 * 1024 * 1024;
108         Value = Value.drop_back();
109         break;
110       }
111       uint64_t Size;
112       if (Value.getAsInteger(0, Size))
113         return make_error<StringError>("'" + Value + "' not an integer",
114                                        inconvertibleErrorCode());
115       Policy.MaxSizeBytes = Size * Mult;
116     } else if (Key == "cache_size_files") {
117       if (Value.getAsInteger(0, Policy.MaxSizeFiles))
118         return make_error<StringError>("'" + Value + "' not an integer",
119                                        inconvertibleErrorCode());
120     } else {
121       return make_error<StringError>("Unknown key: '" + Key + "'",
122                                      inconvertibleErrorCode());
123     }
124   }
125 
126   return Policy;
127 }
128 
129 /// Prune the cache of files that haven't been accessed in a long time.
pruneCache(StringRef Path,CachePruningPolicy Policy)130 bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
131   using namespace std::chrono;
132 
133   if (Path.empty())
134     return false;
135 
136   bool isPathDir;
137   if (sys::fs::is_directory(Path, isPathDir))
138     return false;
139 
140   if (!isPathDir)
141     return false;
142 
143   Policy.MaxSizePercentageOfAvailableSpace =
144       std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
145 
146   if (Policy.Expiration == seconds(0) &&
147       Policy.MaxSizePercentageOfAvailableSpace == 0 &&
148       Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
149     LLVM_DEBUG(dbgs() << "No pruning settings set, exit early\n");
150     // Nothing will be pruned, early exit
151     return false;
152   }
153 
154   // Try to stat() the timestamp file.
155   SmallString<128> TimestampFile(Path);
156   sys::path::append(TimestampFile, "llvmcache.timestamp");
157   sys::fs::file_status FileStatus;
158   const auto CurrentTime = system_clock::now();
159   if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {
160     if (EC == errc::no_such_file_or_directory) {
161       // If the timestamp file wasn't there, create one now.
162       writeTimestampFile(TimestampFile);
163     } else {
164       // Unknown error?
165       return false;
166     }
167   } else {
168     if (!Policy.Interval)
169       return false;
170     if (Policy.Interval != seconds(0)) {
171       // Check whether the time stamp is older than our pruning interval.
172       // If not, do nothing.
173       const auto TimeStampModTime = FileStatus.getLastModificationTime();
174       auto TimeStampAge = CurrentTime - TimeStampModTime;
175       if (TimeStampAge <= *Policy.Interval) {
176         LLVM_DEBUG(dbgs() << "Timestamp file too recent ("
177                           << duration_cast<seconds>(TimeStampAge).count()
178                           << "s old), do not prune.\n");
179         return false;
180       }
181     }
182     // Write a new timestamp file so that nobody else attempts to prune.
183     // There is a benign race condition here, if two processes happen to
184     // notice at the same time that the timestamp is out-of-date.
185     writeTimestampFile(TimestampFile);
186   }
187 
188   // Keep track of space. Needs to be kept ordered by size for determinism.
189   std::set<std::pair<uint64_t, std::string>> FileSizes;
190   uint64_t TotalSize = 0;
191 
192   // Walk the entire directory cache, looking for unused files.
193   std::error_code EC;
194   SmallString<128> CachePathNative;
195   sys::path::native(Path, CachePathNative);
196   // Walk all of the files within this directory.
197   for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
198        File != FileEnd && !EC; File.increment(EC)) {
199     // Ignore any files not beginning with the string "llvmcache-". This
200     // includes the timestamp file as well as any files created by the user.
201     // This acts as a safeguard against data loss if the user specifies the
202     // wrong directory as their cache directory.
203     if (!sys::path::filename(File->path()).startswith("llvmcache-"))
204       continue;
205 
206     // Look at this file. If we can't stat it, there's nothing interesting
207     // there.
208     ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();
209     if (!StatusOrErr) {
210       LLVM_DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
211       continue;
212     }
213 
214     // If the file hasn't been used recently enough, delete it
215     const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
216     auto FileAge = CurrentTime - FileAccessTime;
217     if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
218       LLVM_DEBUG(dbgs() << "Remove " << File->path() << " ("
219                         << duration_cast<seconds>(FileAge).count()
220                         << "s old)\n");
221       sys::fs::remove(File->path());
222       continue;
223     }
224 
225     // Leave it here for now, but add it to the list of size-based pruning.
226     TotalSize += StatusOrErr->getSize();
227     FileSizes.insert({StatusOrErr->getSize(), std::string(File->path())});
228   }
229 
230   auto FileAndSize = FileSizes.rbegin();
231   size_t NumFiles = FileSizes.size();
232 
233   auto RemoveCacheFile = [&]() {
234     // Remove the file.
235     sys::fs::remove(FileAndSize->second);
236     // Update size
237     TotalSize -= FileAndSize->first;
238     NumFiles--;
239     LLVM_DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
240                       << FileAndSize->first << "), new occupancy is "
241                       << TotalSize << "%\n");
242     ++FileAndSize;
243   };
244 
245   // Prune for number of files.
246   if (Policy.MaxSizeFiles)
247     while (NumFiles > Policy.MaxSizeFiles)
248       RemoveCacheFile();
249 
250   // Prune for size now if needed
251   if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
252     auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
253     if (!ErrOrSpaceInfo) {
254       report_fatal_error("Can't get available size");
255     }
256     sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
257     auto AvailableSpace = TotalSize + SpaceInfo.free;
258 
259     if (Policy.MaxSizePercentageOfAvailableSpace == 0)
260       Policy.MaxSizePercentageOfAvailableSpace = 100;
261     if (Policy.MaxSizeBytes == 0)
262       Policy.MaxSizeBytes = AvailableSpace;
263     auto TotalSizeTarget = std::min<uint64_t>(
264         AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
265         Policy.MaxSizeBytes);
266 
267     LLVM_DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
268                       << "% target is: "
269                       << Policy.MaxSizePercentageOfAvailableSpace << "%, "
270                       << Policy.MaxSizeBytes << " bytes\n");
271 
272     // Remove the oldest accessed files first, till we get below the threshold.
273     while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend())
274       RemoveCacheFile();
275   }
276   return true;
277 }
278