1 // Copyright 2016 Google Inc. All rights reserved
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "regen.h"
16
17 #include <sys/stat.h>
18
19 #include <algorithm>
20 #include <memory>
21 #include <mutex>
22 #include <vector>
23
24 #include "fileutil.h"
25 #include "find.h"
26 #include "io.h"
27 #include "log.h"
28 #include "ninja.h"
29 #include "stats.h"
30 #include "strutil.h"
31 #include "thread_pool.h"
32
33 namespace {
34
35 #define RETURN_TRUE do { \
36 if (g_flags.dump_kati_stamp) \
37 needs_regen_ = true; \
38 else \
39 return true; \
40 } while (0)
41
ShouldIgnoreDirty(StringPiece s)42 bool ShouldIgnoreDirty(StringPiece s) {
43 Pattern pat(g_flags.ignore_dirty_pattern);
44 Pattern nopat(g_flags.no_ignore_dirty_pattern);
45 return pat.Match(s) && !nopat.Match(s);
46 }
47
48 class StampChecker {
49 struct GlobResult {
50 string pat;
51 vector<string> result;
52 };
53
54 struct ShellResult {
55 string cmd;
56 string result;
57 vector<string> missing_dirs;
58 vector<string> read_dirs;
59 bool has_condition;
60 };
61
62 public:
StampChecker()63 StampChecker()
64 : needs_regen_(false) {
65 }
66
~StampChecker()67 ~StampChecker() {
68 for (GlobResult* gr : globs_) {
69 delete gr;
70 }
71 for (ShellResult* sr : commands_) {
72 delete sr;
73 }
74 }
75
NeedsRegen(double start_time,const string & orig_args)76 bool NeedsRegen(double start_time, const string& orig_args) {
77 if (IsMissingOutputs())
78 RETURN_TRUE;
79
80 if (CheckStep1(orig_args))
81 RETURN_TRUE;
82
83 if (CheckStep2())
84 RETURN_TRUE;
85
86 if (!needs_regen_) {
87 FILE* fp = fopen(GetNinjaStampFilename().c_str(), "rb+");
88 if (!fp)
89 return true;
90 ScopedFile sfp(fp);
91 if (fseek(fp, 0, SEEK_SET) < 0)
92 PERROR("fseek");
93 size_t r = fwrite(&start_time, sizeof(start_time), 1, fp);
94 CHECK(r == 1);
95 }
96 return needs_regen_;
97 }
98
99 private:
IsMissingOutputs()100 bool IsMissingOutputs() {
101 if (!Exists(GetNinjaFilename())) {
102 fprintf(stderr, "%s is missing, regenerating...\n",
103 GetNinjaFilename().c_str());
104 return true;
105 }
106 if (!Exists(GetNinjaShellScriptFilename())) {
107 fprintf(stderr, "%s is missing, regenerating...\n",
108 GetNinjaShellScriptFilename().c_str());
109 return true;
110 }
111 return false;
112 }
113
CheckStep1(const string & orig_args)114 bool CheckStep1(const string& orig_args) {
115 #define LOAD_INT(fp) ({ \
116 int v = LoadInt(fp); \
117 if (v < 0) { \
118 fprintf(stderr, "incomplete kati_stamp, regenerating...\n"); \
119 RETURN_TRUE; \
120 } \
121 v; \
122 })
123
124 #define LOAD_STRING(fp, s) ({ \
125 if (!LoadString(fp, s)) { \
126 fprintf(stderr, "incomplete kati_stamp, regenerating...\n"); \
127 RETURN_TRUE; \
128 } \
129 })
130
131 const string& stamp_filename = GetNinjaStampFilename();
132 FILE* fp = fopen(stamp_filename.c_str(), "rb");
133 if (!fp) {
134 if (g_flags.dump_kati_stamp)
135 printf("%s: %s\n", stamp_filename.c_str(), strerror(errno));
136 return true;
137 }
138 ScopedFile sfp(fp);
139
140 double gen_time;
141 size_t r = fread(&gen_time, sizeof(gen_time), 1, fp);
142 gen_time_ = gen_time;
143 if (r != 1) {
144 fprintf(stderr, "incomplete kati_stamp, regenerating...\n");
145 RETURN_TRUE;
146 }
147 if (g_flags.dump_kati_stamp)
148 printf("Generated time: %f\n", gen_time);
149
150 string s, s2;
151 int num_files = LOAD_INT(fp);
152 for (int i = 0; i < num_files; i++) {
153 LOAD_STRING(fp, &s);
154 double ts = GetTimestamp(s);
155 if (gen_time < ts) {
156 if (g_flags.regen_ignoring_kati_binary) {
157 string kati_binary;
158 GetExecutablePath(&kati_binary);
159 if (s == kati_binary) {
160 fprintf(stderr, "%s was modified, ignored.\n", s.c_str());
161 continue;
162 }
163 }
164 if (ShouldIgnoreDirty(s)) {
165 if (g_flags.dump_kati_stamp)
166 printf("file %s: ignored (%f)\n", s.c_str(), ts);
167 continue;
168 }
169 if (g_flags.dump_kati_stamp)
170 printf("file %s: dirty (%f)\n", s.c_str(), ts);
171 else
172 fprintf(stderr, "%s was modified, regenerating...\n", s.c_str());
173 RETURN_TRUE;
174 } else if (g_flags.dump_kati_stamp) {
175 printf("file %s: clean (%f)\n", s.c_str(), ts);
176 }
177 }
178
179 int num_undefineds = LOAD_INT(fp);
180 for (int i = 0; i < num_undefineds; i++) {
181 LOAD_STRING(fp, &s);
182 if (getenv(s.c_str())) {
183 if (g_flags.dump_kati_stamp) {
184 printf("env %s: dirty (unset => %s)\n", s.c_str(), getenv(s.c_str()));
185 } else {
186 fprintf(stderr, "Environment variable %s was set, regenerating...\n",
187 s.c_str());
188 }
189 RETURN_TRUE;
190 } else if (g_flags.dump_kati_stamp) {
191 printf("env %s: clean (unset)\n", s.c_str());
192 }
193 }
194
195 int num_envs = LOAD_INT(fp);
196 for (int i = 0; i < num_envs; i++) {
197 LOAD_STRING(fp, &s);
198 StringPiece val(getenv(s.c_str()));
199 LOAD_STRING(fp, &s2);
200 if (val != s2) {
201 if (g_flags.dump_kati_stamp) {
202 printf("env %s: dirty (%s => %.*s)\n",
203 s.c_str(), s2.c_str(), SPF(val));
204 } else {
205 fprintf(stderr, "Environment variable %s was modified (%s => %.*s), "
206 "regenerating...\n",
207 s.c_str(), s2.c_str(), SPF(val));
208 }
209 RETURN_TRUE;
210 } else if (g_flags.dump_kati_stamp) {
211 printf("env %s: clean (%.*s)\n", s.c_str(), SPF(val));
212 }
213 }
214
215 int num_globs = LOAD_INT(fp);
216 string pat;
217 for (int i = 0; i < num_globs; i++) {
218 GlobResult* gr = new GlobResult;
219 globs_.push_back(gr);
220
221 LOAD_STRING(fp, &gr->pat);
222 int num_files = LOAD_INT(fp);
223 gr->result.resize(num_files);
224 for (int j = 0; j < num_files; j++) {
225 LOAD_STRING(fp, &gr->result[j]);
226 }
227 }
228
229 int num_crs = LOAD_INT(fp);
230 for (int i = 0; i < num_crs; i++) {
231 ShellResult* sr = new ShellResult;
232 commands_.push_back(sr);
233 LOAD_STRING(fp, &sr->cmd);
234 LOAD_STRING(fp, &sr->result);
235 sr->has_condition = LOAD_INT(fp);
236 if (!sr->has_condition)
237 continue;
238
239 int num_missing_dirs = LOAD_INT(fp);
240 for (int j = 0; j < num_missing_dirs; j++) {
241 LOAD_STRING(fp, &s);
242 sr->missing_dirs.push_back(s);
243 }
244 int num_read_dirs = LOAD_INT(fp);
245 for (int j = 0; j < num_read_dirs; j++) {
246 LOAD_STRING(fp, &s);
247 sr->read_dirs.push_back(s);
248 }
249 }
250
251 LoadString(fp, &s);
252 if (orig_args != s) {
253 fprintf(stderr, "arguments changed, regenerating...\n");
254 RETURN_TRUE;
255 }
256
257 return needs_regen_;
258 }
259
CheckGlobResult(const GlobResult * gr,string * err)260 bool CheckGlobResult(const GlobResult* gr, string* err) {
261 COLLECT_STATS("glob time (regen)");
262 vector<string>* files;
263 Glob(gr->pat.c_str(), &files);
264 sort(files->begin(), files->end());
265 bool needs_regen = files->size() != gr->result.size();
266 for (size_t i = 0; i < gr->result.size(); i++) {
267 if (!needs_regen) {
268 if ((*files)[i] != gr->result[i]) {
269 needs_regen = true;
270 break;
271 }
272 }
273 }
274 if (needs_regen) {
275 if (ShouldIgnoreDirty(gr->pat)) {
276 if (g_flags.dump_kati_stamp) {
277 printf("wildcard %s: ignored\n", gr->pat.c_str());
278 }
279 return false;
280 }
281 if (g_flags.dump_kati_stamp) {
282 printf("wildcard %s: dirty\n", gr->pat.c_str());
283 } else {
284 *err = StringPrintf("wildcard(%s) was changed, regenerating...\n",
285 gr->pat.c_str());
286 }
287 } else if (g_flags.dump_kati_stamp) {
288 printf("wildcard %s: clean\n", gr->pat.c_str());
289 }
290 return needs_regen;
291 }
292
ShouldRunCommand(const ShellResult * sr)293 bool ShouldRunCommand(const ShellResult* sr) {
294 if (!sr->has_condition)
295 return true;
296
297 COLLECT_STATS("stat time (regen)");
298 for (const string& dir : sr->missing_dirs) {
299 if (Exists(dir))
300 return true;
301 }
302 for (const string& dir : sr->read_dirs) {
303 // We assume we rarely do a significant change for the top
304 // directory which affects the results of find command.
305 if (dir == "" || dir == "." || ShouldIgnoreDirty(dir))
306 continue;
307
308 struct stat st;
309 if (lstat(dir.c_str(), &st) != 0) {
310 return true;
311 }
312 double ts = GetTimestampFromStat(st);
313 if (gen_time_ < ts) {
314 return true;
315 }
316 if (S_ISLNK(st.st_mode)) {
317 ts = GetTimestamp(dir);
318 if (ts < 0 || gen_time_ < ts)
319 return true;
320 }
321 }
322 return false;
323 }
324
CheckShellResult(const ShellResult * sr,string * err)325 bool CheckShellResult(const ShellResult* sr, string* err) {
326 if (!ShouldRunCommand(sr)) {
327 if (g_flags.dump_kati_stamp)
328 printf("shell %s: clean (no rerun)\n", sr->cmd.c_str());
329 return false;
330 }
331
332 FindCommand fc;
333 if (fc.Parse(sr->cmd) &&
334 !fc.chdir.empty() && ShouldIgnoreDirty(fc.chdir)) {
335 if (g_flags.dump_kati_stamp)
336 printf("shell %s: ignored\n", sr->cmd.c_str());
337 return false;
338 }
339
340 COLLECT_STATS_WITH_SLOW_REPORT("shell time (regen)", sr->cmd.c_str());
341 string result;
342 RunCommand("/bin/sh", sr->cmd, RedirectStderr::DEV_NULL, &result);
343 FormatForCommandSubstitution(&result);
344 if (sr->result != result) {
345 if (g_flags.dump_kati_stamp) {
346 printf("shell %s: dirty\n", sr->cmd.c_str());
347 } else {
348 *err = StringPrintf("$(shell %s) was changed, regenerating...\n",
349 sr->cmd.c_str());
350 //*err += StringPrintf("%s => %s\n", expected.c_str(), result.c_str());
351 }
352 return true;
353 } else if (g_flags.dump_kati_stamp) {
354 printf("shell %s: clean (rerun)\n", sr->cmd.c_str());
355 }
356 return false;
357 }
358
CheckStep2()359 bool CheckStep2() {
360 unique_ptr<ThreadPool> tp(NewThreadPool(g_flags.num_jobs));
361
362 tp->Submit([this]() {
363 string err;
364 // TODO: Make glob cache thread safe and create a task for each glob.
365 for (GlobResult* gr : globs_) {
366 if (CheckGlobResult(gr, &err)) {
367 unique_lock<mutex> lock(mu_);
368 if (!needs_regen_) {
369 needs_regen_ = true;
370 msg_ = err;
371 }
372 break;
373 }
374 }
375 });
376
377 for (ShellResult* sr : commands_) {
378 tp->Submit([this, sr]() {
379 string err;
380 if (CheckShellResult(sr, &err)) {
381 unique_lock<mutex> lock(mu_);
382 if (!needs_regen_) {
383 needs_regen_ = true;
384 msg_ = err;
385 }
386 }
387 });
388 }
389
390 tp->Wait();
391 if (needs_regen_) {
392 fprintf(stderr, "%s", msg_.c_str());
393 }
394 return needs_regen_;
395 }
396
397 private:
398 double gen_time_;
399 vector<GlobResult*> globs_;
400 vector<ShellResult*> commands_;
401 mutex mu_;
402 bool needs_regen_;
403 string msg_;
404 };
405
406 } // namespace
407
NeedsRegen(double start_time,const string & orig_args)408 bool NeedsRegen(double start_time, const string& orig_args) {
409 return StampChecker().NeedsRegen(start_time, orig_args);
410 }
411