1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "src/traced/probes/ps/process_stats_data_source.h"
18 
19 #include <dirent.h>
20 
21 #include "perfetto/ext/base/file_utils.h"
22 #include "perfetto/ext/base/temp_file.h"
23 #include "perfetto/protozero/scattered_heap_buffer.h"
24 #include "perfetto/tracing/core/data_source_config.h"
25 #include "src/base/test/test_task_runner.h"
26 #include "src/traced/probes/common/cpu_freq_info_for_testing.h"
27 #include "src/tracing/core/trace_writer_for_testing.h"
28 #include "test/gtest_and_gmock.h"
29 
30 #include "protos/perfetto/config/process_stats/process_stats_config.gen.h"
31 #include "protos/perfetto/trace/ps/process_stats.gen.h"
32 #include "protos/perfetto/trace/ps/process_tree.gen.h"
33 
34 using ::perfetto::protos::gen::ProcessStatsConfig;
35 using ::testing::_;
36 using ::testing::ElementsAre;
37 using ::testing::ElementsAreArray;
38 using ::testing::Invoke;
39 using ::testing::Mock;
40 using ::testing::Return;
41 using ::testing::Truly;
42 
43 namespace perfetto {
44 namespace {
45 
46 class TestProcessStatsDataSource : public ProcessStatsDataSource {
47  public:
TestProcessStatsDataSource(base::TaskRunner * task_runner,TracingSessionID id,std::unique_ptr<TraceWriter> writer,const DataSourceConfig & config,std::unique_ptr<CpuFreqInfo> cpu_freq_info)48   TestProcessStatsDataSource(base::TaskRunner* task_runner,
49                              TracingSessionID id,
50                              std::unique_ptr<TraceWriter> writer,
51                              const DataSourceConfig& config,
52                              std::unique_ptr<CpuFreqInfo> cpu_freq_info)
53       : ProcessStatsDataSource(task_runner,
54                                id,
55                                std::move(writer),
56                                config,
57                                std::move(cpu_freq_info)) {}
58 
59   MOCK_METHOD0(OpenProcDir, base::ScopedDir());
60   MOCK_METHOD2(ReadProcPidFile, std::string(int32_t pid, const std::string&));
61   MOCK_METHOD1(OpenProcTaskDir, base::ScopedDir(int32_t pid));
62 };
63 
64 class ProcessStatsDataSourceTest : public ::testing::Test {
65  protected:
ProcessStatsDataSourceTest()66   ProcessStatsDataSourceTest() {}
67 
GetProcessStatsDataSource(const DataSourceConfig & cfg)68   std::unique_ptr<TestProcessStatsDataSource> GetProcessStatsDataSource(
69       const DataSourceConfig& cfg) {
70     auto writer =
71         std::unique_ptr<TraceWriterForTesting>(new TraceWriterForTesting());
72     writer_raw_ = writer.get();
73     return std::unique_ptr<TestProcessStatsDataSource>(
74         new TestProcessStatsDataSource(
75             &task_runner_, 0, std::move(writer), cfg,
76             cpu_freq_info_for_testing.GetInstance()));
77   }
78 
79   base::TestTaskRunner task_runner_;
80   TraceWriterForTesting* writer_raw_;
81   CpuFreqInfoForTesting cpu_freq_info_for_testing;
82 };
83 
TEST_F(ProcessStatsDataSourceTest,WriteOnceProcess)84 TEST_F(ProcessStatsDataSourceTest, WriteOnceProcess) {
85   auto data_source = GetProcessStatsDataSource(DataSourceConfig());
86   EXPECT_CALL(*data_source, ReadProcPidFile(42, "status"))
87       .WillOnce(Return(
88           "Name: foo\nTgid:\t42\nPid:   42\nPPid:  17\nUid:  43 44 45 56\n"));
89   EXPECT_CALL(*data_source, ReadProcPidFile(42, "cmdline"))
90       .WillOnce(Return(std::string("foo\0bar\0baz\0", 12)));
91 
92   data_source->OnPids({42});
93 
94   auto trace = writer_raw_->GetAllTracePackets();
95   ASSERT_EQ(trace.size(), 1u);
96   auto ps_tree = trace[0].process_tree();
97   ASSERT_EQ(ps_tree.processes_size(), 1);
98   auto first_process = ps_tree.processes()[0];
99   ASSERT_EQ(first_process.pid(), 42);
100   ASSERT_EQ(first_process.ppid(), 17);
101   ASSERT_EQ(first_process.uid(), 43);
102   ASSERT_THAT(first_process.cmdline(), ElementsAreArray({"foo", "bar", "baz"}));
103 }
104 
105 // Regression test for b/147438623.
TEST_F(ProcessStatsDataSourceTest,NonNulTerminatedCmdline)106 TEST_F(ProcessStatsDataSourceTest, NonNulTerminatedCmdline) {
107   auto data_source = GetProcessStatsDataSource(DataSourceConfig());
108   EXPECT_CALL(*data_source, ReadProcPidFile(42, "status"))
109       .WillOnce(Return(
110           "Name: foo\nTgid:\t42\nPid:   42\nPPid:  17\nUid:  43 44 45 56\n"));
111   EXPECT_CALL(*data_source, ReadProcPidFile(42, "cmdline"))
112       .WillOnce(Return(std::string("surfaceflinger", 14)));
113 
114   data_source->OnPids({42});
115 
116   auto trace = writer_raw_->GetAllTracePackets();
117   ASSERT_EQ(trace.size(), 1u);
118   auto ps_tree = trace[0].process_tree();
119   ASSERT_EQ(ps_tree.processes_size(), 1);
120   auto first_process = ps_tree.processes()[0];
121   ASSERT_THAT(first_process.cmdline(), ElementsAreArray({"surfaceflinger"}));
122 }
123 
TEST_F(ProcessStatsDataSourceTest,DontRescanCachedPIDsAndTIDs)124 TEST_F(ProcessStatsDataSourceTest, DontRescanCachedPIDsAndTIDs) {
125   // assertion helpers
126   auto expected_process = [](int pid) {
127     return [pid](protos::gen::ProcessTree::Process process) {
128       return process.pid() == pid && process.cmdline_size() > 0 &&
129              process.cmdline()[0] == "proc_" + std::to_string(pid);
130     };
131   };
132   auto expected_thread = [](int tid) {
133     return [tid](protos::gen::ProcessTree::Thread thread) {
134       return thread.tid() == tid && thread.tgid() == tid / 10 * 10 &&
135              thread.name() == "thread_" + std::to_string(tid);
136     };
137   };
138 
139   DataSourceConfig ds_config;
140   ProcessStatsConfig cfg;
141   cfg.set_record_thread_names(true);
142   ds_config.set_process_stats_config_raw(cfg.SerializeAsString());
143   auto data_source = GetProcessStatsDataSource(ds_config);
144   for (int p : {10, 11, 12, 20, 21, 22, 30, 31, 32}) {
145     EXPECT_CALL(*data_source, ReadProcPidFile(p, "status"))
146         .WillOnce(Invoke([](int32_t pid, const std::string&) {
147           int32_t tgid = (pid / 10) * 10;
148           return "Name: \tthread_" + std::to_string(pid) +
149                  "\nTgid:  " + std::to_string(tgid) +
150                  "\nPid:   " + std::to_string(pid) + "\nPPid:  1\n";
151         }));
152     if (p % 10 == 0) {
153       std::string proc_name = "proc_" + std::to_string(p);
154       proc_name.resize(proc_name.size() + 1);  // Add a trailing \0.
155       EXPECT_CALL(*data_source, ReadProcPidFile(p, "cmdline"))
156           .WillOnce(Return(proc_name));
157     }
158   }
159 
160   data_source->OnPids({10, 11, 12, 20, 21, 22, 10, 20, 11, 21});
161   data_source->OnPids({30});
162   data_source->OnPids({10, 30, 10, 31, 32});
163 
164   // check written contents
165   auto trace = writer_raw_->GetAllTracePackets();
166   EXPECT_EQ(trace.size(), 3u);
167 
168   // first packet - two unique processes, four threads
169   auto ps_tree = trace[0].process_tree();
170   EXPECT_THAT(ps_tree.processes(),
171               UnorderedElementsAre(Truly(expected_process(10)),
172                                    Truly(expected_process(20))));
173   EXPECT_THAT(ps_tree.threads(),
174               UnorderedElementsAre(
175                   Truly(expected_thread(11)), Truly(expected_thread(12)),
176                   Truly(expected_thread(21)), Truly(expected_thread(22))));
177 
178   // second packet - one new process
179   ps_tree = trace[1].process_tree();
180   EXPECT_THAT(ps_tree.processes(),
181               UnorderedElementsAre(Truly(expected_process(30))));
182   EXPECT_EQ(ps_tree.threads_size(), 0);
183 
184   // third packet - two threads that haven't been seen before
185   ps_tree = trace[2].process_tree();
186   EXPECT_EQ(ps_tree.processes_size(), 0);
187   EXPECT_THAT(ps_tree.threads(),
188               UnorderedElementsAre(Truly(expected_thread(31)),
189                                    Truly(expected_thread(32))));
190 }
191 
TEST_F(ProcessStatsDataSourceTest,IncrementalStateClear)192 TEST_F(ProcessStatsDataSourceTest, IncrementalStateClear) {
193   auto data_source = GetProcessStatsDataSource(DataSourceConfig());
194   EXPECT_CALL(*data_source, ReadProcPidFile(42, "status"))
195       .WillOnce(Return("Name: foo\nTgid:\t42\nPid:   42\nPPid:  17\n"));
196   EXPECT_CALL(*data_source, ReadProcPidFile(42, "cmdline"))
197       .WillOnce(Return(std::string("first_cmdline\0", 14)));
198 
199   data_source->OnPids({42});
200 
201   {
202     auto trace = writer_raw_->GetAllTracePackets();
203     ASSERT_EQ(trace.size(), 1u);
204     auto packet = trace[0];
205     // First packet in the trace has no previous state, so the clear marker is
206     // emitted.
207     ASSERT_TRUE(packet.incremental_state_cleared());
208 
209     auto ps_tree = packet.process_tree();
210     ASSERT_EQ(ps_tree.processes_size(), 1);
211     ASSERT_EQ(ps_tree.processes()[0].pid(), 42);
212     ASSERT_EQ(ps_tree.processes()[0].ppid(), 17);
213     ASSERT_THAT(ps_tree.processes()[0].cmdline(),
214                 ElementsAreArray({"first_cmdline"}));
215   }
216 
217   // Look up the same pid, which shouldn't be re-emitted.
218   Mock::VerifyAndClearExpectations(data_source.get());
219   EXPECT_CALL(*data_source, ReadProcPidFile(42, "status")).Times(0);
220   EXPECT_CALL(*data_source, ReadProcPidFile(42, "cmdline")).Times(0);
221 
222   data_source->OnPids({42});
223 
224   {
225     auto trace = writer_raw_->GetAllTracePackets();
226     ASSERT_EQ(trace.size(), 1u);
227   }
228 
229   // Invalidate incremental state, and look up the same pid again, which should
230   // re-emit the proc tree info.
231   Mock::VerifyAndClearExpectations(data_source.get());
232   EXPECT_CALL(*data_source, ReadProcPidFile(42, "status"))
233       .WillOnce(Return("Name: foo\nTgid:\t42\nPid:   42\nPPid:  18\n"));
234   EXPECT_CALL(*data_source, ReadProcPidFile(42, "cmdline"))
235       .WillOnce(Return(std::string("second_cmdline\0", 15)));
236 
237   data_source->ClearIncrementalState();
238   data_source->OnPids({42});
239 
240   {
241     // Second packet with new proc information.
242     auto trace = writer_raw_->GetAllTracePackets();
243     ASSERT_EQ(trace.size(), 2u);
244     auto packet = trace[1];
245     ASSERT_TRUE(packet.incremental_state_cleared());
246 
247     auto ps_tree = packet.process_tree();
248     ASSERT_EQ(ps_tree.processes_size(), 1);
249     ASSERT_EQ(ps_tree.processes()[0].pid(), 42);
250     ASSERT_EQ(ps_tree.processes()[0].ppid(), 18);
251     ASSERT_THAT(ps_tree.processes()[0].cmdline(),
252                 ElementsAreArray({"second_cmdline"}));
253   }
254 }
255 
TEST_F(ProcessStatsDataSourceTest,RenamePids)256 TEST_F(ProcessStatsDataSourceTest, RenamePids) {
257   // assertion helpers
258   auto expected_old_process = [](int pid) {
259     return [pid](protos::gen::ProcessTree::Process process) {
260       return process.pid() == pid && process.cmdline_size() > 0 &&
261              process.cmdline()[0] == "proc_" + std::to_string(pid);
262     };
263   };
264   auto expected_new_process = [](int pid) {
265     return [pid](protos::gen::ProcessTree::Process process) {
266       return process.pid() == pid && process.cmdline_size() > 0 &&
267              process.cmdline()[0] == "new_" + std::to_string(pid);
268     };
269   };
270 
271   DataSourceConfig config;
272   auto data_source = GetProcessStatsDataSource(config);
273   for (int p : {10, 20}) {
274     EXPECT_CALL(*data_source, ReadProcPidFile(p, "status"))
275         .WillRepeatedly(Invoke([](int32_t pid, const std::string&) {
276           return "Name: \tthread_" + std::to_string(pid) +
277                  "\nTgid:  " + std::to_string(pid) +
278                  "\nPid:   " + std::to_string(pid) + "\nPPid:  1\n";
279         }));
280 
281     std::string old_proc_name = "proc_" + std::to_string(p);
282     old_proc_name.resize(old_proc_name.size() + 1);  // Add a trailing \0.
283 
284     std::string new_proc_name = "new_" + std::to_string(p);
285     new_proc_name.resize(new_proc_name.size() + 1);  // Add a trailing \0.
286     EXPECT_CALL(*data_source, ReadProcPidFile(p, "cmdline"))
287         .WillOnce(Return(old_proc_name))
288         .WillOnce(Return(new_proc_name));
289   }
290 
291   data_source->OnPids({10, 20});
292   data_source->OnRenamePids({10});
293   data_source->OnPids({10, 20});
294   data_source->OnRenamePids({20});
295   data_source->OnPids({10, 20});
296 
297   // check written contents
298   auto trace = writer_raw_->GetAllTracePackets();
299   EXPECT_EQ(trace.size(), 3u);
300 
301   // first packet - two unique processes
302   auto ps_tree = trace[0].process_tree();
303   EXPECT_THAT(ps_tree.processes(),
304               UnorderedElementsAre(Truly(expected_old_process(10)),
305                                    Truly(expected_old_process(20))));
306   EXPECT_EQ(ps_tree.threads_size(), 0);
307 
308   // second packet - one new process
309   ps_tree = trace[1].process_tree();
310   EXPECT_THAT(ps_tree.processes(),
311               UnorderedElementsAre(Truly(expected_new_process(10))));
312   EXPECT_EQ(ps_tree.threads_size(), 0);
313 
314   // third packet - two threads that haven't been seen before
315   ps_tree = trace[2].process_tree();
316   EXPECT_THAT(ps_tree.processes(),
317               UnorderedElementsAre(Truly(expected_new_process(20))));
318   EXPECT_EQ(ps_tree.threads_size(), 0);
319 }
320 
TEST_F(ProcessStatsDataSourceTest,ProcessStats)321 TEST_F(ProcessStatsDataSourceTest, ProcessStats) {
322   DataSourceConfig ds_config;
323   ProcessStatsConfig cfg;
324   cfg.set_proc_stats_poll_ms(1);
325   cfg.add_quirks(ProcessStatsConfig::DISABLE_ON_DEMAND);
326   ds_config.set_process_stats_config_raw(cfg.SerializeAsString());
327   auto data_source = GetProcessStatsDataSource(ds_config);
328 
329   // Populate a fake /proc/ directory.
330   auto fake_proc = base::TempDir::Create();
331   const int kPids[] = {1, 2};
332   std::vector<std::string> dirs_to_delete;
333   for (int pid : kPids) {
334     char path[256];
335     sprintf(path, "%s/%d", fake_proc.path().c_str(), pid);
336     dirs_to_delete.push_back(path);
337     mkdir(path, 0755);
338   }
339 
340   auto checkpoint = task_runner_.CreateCheckpoint("all_done");
341 
342   EXPECT_CALL(*data_source, OpenProcDir()).WillRepeatedly(Invoke([&fake_proc] {
343     return base::ScopedDir(opendir(fake_proc.path().c_str()));
344   }));
345 
346   const int kNumIters = 4;
347   int iter = 0;
348   for (int pid : kPids) {
349     EXPECT_CALL(*data_source, ReadProcPidFile(pid, "status"))
350         .WillRepeatedly(Invoke([checkpoint, &iter](int32_t p,
351                                                    const std::string&) {
352           char ret[1024];
353           sprintf(ret, "Name:	pid_10\nVmSize:	 %d kB\nVmRSS:\t%d  kB\n",
354                   p * 100 + iter * 10 + 1, p * 100 + iter * 10 + 2);
355           return std::string(ret);
356         }));
357 
358     EXPECT_CALL(*data_source, ReadProcPidFile(pid, "oom_score_adj"))
359         .WillRepeatedly(Invoke(
360             [checkpoint, kPids, &iter](int32_t inner_pid, const std::string&) {
361               auto oom_score = inner_pid * 100 + iter * 10 + 3;
362               if (inner_pid == kPids[base::ArraySize(kPids) - 1]) {
363                 if (++iter == kNumIters)
364                   checkpoint();
365               }
366               return std::to_string(oom_score);
367             }));
368   }
369 
370   data_source->Start();
371   task_runner_.RunUntilCheckpoint("all_done");
372   data_source->Flush(1 /* FlushRequestId */, []() {});
373 
374   std::vector<protos::gen::ProcessStats::Process> processes;
375   auto trace = writer_raw_->GetAllTracePackets();
376   for (const auto& packet : trace) {
377     for (const auto& process : packet.process_stats().processes()) {
378       processes.push_back(process);
379     }
380   }
381   ASSERT_EQ(processes.size(), kNumIters * base::ArraySize(kPids));
382   iter = 0;
383   for (const auto& proc_counters : processes) {
384     int32_t pid = proc_counters.pid();
385     ASSERT_EQ(static_cast<int>(proc_counters.vm_size_kb()),
386               pid * 100 + iter * 10 + 1);
387     ASSERT_EQ(static_cast<int>(proc_counters.vm_rss_kb()),
388               pid * 100 + iter * 10 + 2);
389     ASSERT_EQ(static_cast<int>(proc_counters.oom_score_adj()),
390               pid * 100 + iter * 10 + 3);
391     if (pid == kPids[base::ArraySize(kPids) - 1])
392       iter++;
393   }
394 
395   // Cleanup |fake_proc|. TempDir checks that the directory is empty.
396   for (std::string& path : dirs_to_delete)
397     base::Rmdir(path);
398 }
399 
TEST_F(ProcessStatsDataSourceTest,CacheProcessStats)400 TEST_F(ProcessStatsDataSourceTest, CacheProcessStats) {
401   DataSourceConfig ds_config;
402   ProcessStatsConfig cfg;
403   cfg.set_proc_stats_poll_ms(105);
404   cfg.set_proc_stats_cache_ttl_ms(220);
405   cfg.add_quirks(ProcessStatsConfig::DISABLE_ON_DEMAND);
406   ds_config.set_process_stats_config_raw(cfg.SerializeAsString());
407   auto data_source = GetProcessStatsDataSource(ds_config);
408 
409   // Populate a fake /proc/ directory.
410   auto fake_proc = base::TempDir::Create();
411   const int kPid = 1;
412 
413   char path[256];
414   sprintf(path, "%s/%d", fake_proc.path().c_str(), kPid);
415   mkdir(path, 0755);
416 
417   auto checkpoint = task_runner_.CreateCheckpoint("all_done");
418 
419   EXPECT_CALL(*data_source, OpenProcDir()).WillRepeatedly(Invoke([&fake_proc] {
420     return base::ScopedDir(opendir(fake_proc.path().c_str()));
421   }));
422 
423   const int kNumIters = 4;
424   int iter = 0;
425   EXPECT_CALL(*data_source, ReadProcPidFile(kPid, "status"))
426       .WillRepeatedly(Invoke([checkpoint](int32_t p, const std::string&) {
427         char ret[1024];
428         sprintf(ret, "Name:	pid_10\nVmSize:	 %d kB\nVmRSS:\t%d  kB\n",
429                 p * 100 + 1, p * 100 + 2);
430         return std::string(ret);
431       }));
432 
433   EXPECT_CALL(*data_source, ReadProcPidFile(kPid, "oom_score_adj"))
434       .WillRepeatedly(
435           Invoke([checkpoint, &iter](int32_t inner_pid, const std::string&) {
436             if (++iter == kNumIters)
437               checkpoint();
438             return std::to_string(inner_pid * 100);
439           }));
440 
441   data_source->Start();
442   task_runner_.RunUntilCheckpoint("all_done");
443   data_source->Flush(1 /* FlushRequestId */, []() {});
444 
445   std::vector<protos::gen::ProcessStats::Process> processes;
446   auto trace = writer_raw_->GetAllTracePackets();
447   for (const auto& packet : trace) {
448     for (const auto& process : packet.process_stats().processes()) {
449       processes.push_back(process);
450     }
451   }
452   // We should get two counter events because:
453   // a) emissions happen at 0ms, 105ms, 210ms, 315ms
454   // b) clear events happen at 220ms, 440ms...
455   // Therefore, we should see the emissions at 0ms and 315ms.
456   ASSERT_EQ(processes.size(), 2u);
457   for (const auto& proc_counters : processes) {
458     ASSERT_EQ(proc_counters.pid(), kPid);
459     ASSERT_EQ(static_cast<int>(proc_counters.vm_size_kb()), kPid * 100 + 1);
460     ASSERT_EQ(static_cast<int>(proc_counters.vm_rss_kb()), kPid * 100 + 2);
461     ASSERT_EQ(static_cast<int>(proc_counters.oom_score_adj()), kPid * 100);
462   }
463 
464   // Cleanup |fake_proc|. TempDir checks that the directory is empty.
465   base::Rmdir(path);
466 }
467 
TEST_F(ProcessStatsDataSourceTest,ThreadTimeInState)468 TEST_F(ProcessStatsDataSourceTest, ThreadTimeInState) {
469   DataSourceConfig ds_config;
470   ProcessStatsConfig config;
471   // Do 2 ticks before cache clear.
472   config.set_proc_stats_poll_ms(100);
473   config.set_proc_stats_cache_ttl_ms(200);
474   config.add_quirks(ProcessStatsConfig::DISABLE_ON_DEMAND);
475   config.set_record_thread_time_in_state(true);
476   ds_config.set_process_stats_config_raw(config.SerializeAsString());
477   auto data_source = GetProcessStatsDataSource(ds_config);
478 
479   std::vector<std::string> dirs_to_delete;
480   auto make_proc_path = [&dirs_to_delete](base::TempDir& temp_dir, int pid) {
481     char path[256];
482     sprintf(path, "%s/%d", temp_dir.path().c_str(), pid);
483     dirs_to_delete.push_back(path);
484     mkdir(path, 0755);
485   };
486   // Populate a fake /proc/ directory.
487   auto fake_proc = base::TempDir::Create();
488   const int kPid = 1;
489   make_proc_path(fake_proc, kPid);
490   const int kIgnoredPid = 5;
491   make_proc_path(fake_proc, kIgnoredPid);
492 
493   // Populate a fake /proc/1/task directory.
494   auto fake_proc_task = base::TempDir::Create();
495   const int kTids[] = {1, 2};
496   for (int tid : kTids)
497     make_proc_path(fake_proc_task, tid);
498   // Populate a fake /proc/5/task directory.
499   auto fake_ignored_proc_task = base::TempDir::Create();
500   const int kIgnoredTid = 5;
501   make_proc_path(fake_ignored_proc_task, kIgnoredTid);
502 
503   auto checkpoint = task_runner_.CreateCheckpoint("all_done");
504 
505   EXPECT_CALL(*data_source, OpenProcDir()).WillRepeatedly(Invoke([&fake_proc] {
506     return base::ScopedDir(opendir(fake_proc.path().c_str()));
507   }));
508   EXPECT_CALL(*data_source, ReadProcPidFile(kPid, "status"))
509       .WillRepeatedly(
510           Return("Name:	pid_1\nVmSize:	 100 kB\nVmRSS:\t100  kB\n"));
511   EXPECT_CALL(*data_source, ReadProcPidFile(kPid, "oom_score_adj"))
512       .WillRepeatedly(Return("901"));
513   EXPECT_CALL(*data_source, ReadProcPidFile(kPid, "stat"))
514       .WillOnce(Return("1 (pid_1) S 1 1 0 0 -1 4210944 2197 2451 0 1 54 117 4"))
515       // ctime++
516       .WillOnce(Return("1 (pid_1) S 1 1 0 0 -1 4210944 2197 2451 0 1 55 117 4"))
517       // stime++
518       .WillOnce(Return("1 (pid_1) S 1 1 0 0 -1 4210944 2197 2451 0 1 55 118 4"))
519       // ctime++, stime++
520       .WillOnce(
521           Return("1 (pid_1) S 1 1 0 0 -1 4210944 2197 2451 0 1 56 119 4"));
522   EXPECT_CALL(*data_source, OpenProcTaskDir(kPid))
523       .WillRepeatedly(Invoke([&fake_proc_task](int32_t) {
524         return base::ScopedDir(opendir(fake_proc_task.path().c_str()));
525       }));
526   EXPECT_CALL(*data_source, ReadProcPidFile(kTids[0], "time_in_state"))
527       .Times(4)
528       .WillRepeatedly(Return("cpu0\n300000 1\n748800 1\ncpu1\n300000 5\n"));
529   EXPECT_CALL(*data_source, ReadProcPidFile(kTids[1], "status"))
530       .WillRepeatedly(Return("Name: tid_2"));
531   EXPECT_CALL(*data_source, ReadProcPidFile(kTids[1], "time_in_state"))
532       .WillOnce(
533           Return("cpu0\n300000 10\n748800 0\ncpu1\n300000 50\n652800 60\n"))
534       .WillOnce(Return("cpu0\n300000 20\n748800 0\n1324800 30\ncpu1\n300000 "
535                        "100\n652800 60\n"))
536       .WillOnce(Return("cpu0\n300000 200\n748800 0\n1324800 30\ncpu1\n300000 "
537                        "100\n652800 60\n"))
538       .WillOnce(Invoke([&checkpoint](int32_t, const std::string&) {
539         // Call checkpoint here to stop after the third tick.
540         checkpoint();
541         return "cpu0\n300000 300\n748800 0\n1324800 40\ncpu1\n300000 "
542                "200\n652800 70\n";
543       }));
544   EXPECT_CALL(*data_source, ReadProcPidFile(kIgnoredPid, "status"))
545       .WillRepeatedly(
546           Return("Name:	pid_5\nVmSize:	 100 kB\nVmRSS:\t100  kB\n"));
547   EXPECT_CALL(*data_source, ReadProcPidFile(kIgnoredPid, "oom_score_adj"))
548       .WillRepeatedly(Return("905"));
549   EXPECT_CALL(*data_source, OpenProcTaskDir(kIgnoredPid))
550       .WillRepeatedly(Invoke([&fake_ignored_proc_task](int32_t) {
551         return base::ScopedDir(opendir(fake_ignored_proc_task.path().c_str()));
552       }));
553   EXPECT_CALL(*data_source, ReadProcPidFile(kIgnoredPid, "stat"))
554       .WillRepeatedly(
555           Return("5 (pid_5) S 1 5 0 0 -1 4210944 2197 2451 0 1 99 99 4"));
556   EXPECT_CALL(*data_source, ReadProcPidFile(kIgnoredTid, "time_in_state"))
557       .Times(2)
558       .WillRepeatedly(
559           Return("cpu0\n300000 10\n748800 0\ncpu1\n300000 00\n652800 20\n"));
560 
561   data_source->Start();
562   task_runner_.RunUntilCheckpoint("all_done");
563   data_source->Flush(1 /* FlushRequestId */, []() {});
564 
565   // Collect all process packets order by their timestamp and pid.
566   using TimestampPid = std::pair</* timestamp */ uint64_t, /* pid */ int32_t>;
567   std::map<TimestampPid, protos::gen::ProcessStats::Process> processes_map;
568   for (const auto& packet : writer_raw_->GetAllTracePackets())
569     for (const auto& process : packet.process_stats().processes())
570       processes_map.insert({{packet.timestamp(), process.pid()}, process});
571   std::vector<protos::gen::ProcessStats::Process> processes;
572   for (auto it : processes_map)
573     processes.push_back(it.second);
574 
575   // 4 packets for pid=1, 2 packets for pid=5.
576   ASSERT_EQ(processes.size(), 6u);
577 
578   auto compare_tid = [](protos::gen::ProcessStats_Thread& l,
579                         protos::gen::ProcessStats_Thread& r) {
580     return l.tid() < r.tid();
581   };
582 
583   // First pull has all threads and all frequencies.
584   // Check pid = 1.
585   auto threads = processes[0].threads();
586   EXPECT_EQ(threads.size(), 2u);
587   std::sort(threads.begin(), threads.end(), compare_tid);
588   auto thread = threads[0];
589   EXPECT_EQ(thread.tid(), 1);
590   EXPECT_THAT(thread.cpu_freq_indices(), ElementsAre(1u, 3u, 10u));
591   EXPECT_THAT(thread.cpu_freq_ticks(), ElementsAre(1, 1, 5));
592   EXPECT_THAT(thread.cpu_freq_full(), true);
593   thread = threads[1];
594   EXPECT_EQ(thread.tid(), 2);
595   EXPECT_THAT(thread.cpu_freq_indices(), ElementsAre(1u, 10u, 11u));
596   EXPECT_THAT(thread.cpu_freq_ticks(), ElementsAre(10, 50, 60));
597   EXPECT_THAT(thread.cpu_freq_full(), true);
598   // Check pid = 5.
599   threads = processes[1].threads();
600   EXPECT_EQ(threads.size(), 1u);
601   thread = threads[0];
602   EXPECT_EQ(thread.tid(), 5);
603   EXPECT_THAT(thread.cpu_freq_indices(), ElementsAre(1u, 11u));
604   EXPECT_THAT(thread.cpu_freq_ticks(), ElementsAre(10, 20));
605   EXPECT_THAT(thread.cpu_freq_full(), true);
606 
607   // Second pull has only one thread that changed.
608   threads = processes[2].threads();
609   EXPECT_EQ(threads.size(), 1u);
610   thread = threads[0];
611   EXPECT_EQ(thread.tid(), 2);
612   EXPECT_THAT(thread.cpu_freq_indices(), ElementsAre(1u, 6u, 10u));
613   EXPECT_THAT(thread.cpu_freq_ticks(), ElementsAre(20, 30, 100));
614   // Value for cpu_freq index 11 did not change.
615   EXPECT_THAT(thread.has_cpu_freq_full(), false);
616 
617   // Third pull has all thread because cache was cleared.
618   // Check pid = 1.
619   threads = processes[3].threads();
620   EXPECT_EQ(threads.size(), 2u);
621   std::sort(threads.begin(), threads.end(), compare_tid);
622   thread = threads[0];
623   EXPECT_EQ(thread.tid(), 1);
624   EXPECT_THAT(thread.cpu_freq_indices(), ElementsAre(1u, 3u, 10u));
625   EXPECT_THAT(thread.cpu_freq_ticks(), ElementsAre(1, 1, 5));
626   EXPECT_THAT(thread.cpu_freq_full(), true);
627   thread = threads[1];
628   EXPECT_EQ(thread.tid(), 2);
629   EXPECT_THAT(thread.cpu_freq_indices(), ElementsAre(1u, 6u, 10u, 11u));
630   EXPECT_THAT(thread.cpu_freq_ticks(), ElementsAre(200, 30, 100, 60));
631   EXPECT_THAT(thread.cpu_freq_full(), true);
632   // Check pid = 5.
633   threads = processes[4].threads();
634   EXPECT_EQ(threads.size(), 1u);
635   thread = threads[0];
636   EXPECT_EQ(thread.tid(), 5);
637   EXPECT_THAT(thread.cpu_freq_indices(), ElementsAre(1u, 11u));
638   EXPECT_THAT(thread.cpu_freq_ticks(), ElementsAre(10, 20));
639   EXPECT_THAT(thread.cpu_freq_full(), true);
640 
641   // Forth full has only one thread that changed.
642   threads = processes[5].threads();
643   EXPECT_EQ(threads.size(), 1u);
644   thread = threads[0];
645   EXPECT_EQ(thread.tid(), 2);
646   EXPECT_THAT(thread.cpu_freq_indices(), ElementsAre(1u, 6u, 10u, 11u));
647   EXPECT_THAT(thread.cpu_freq_ticks(), ElementsAre(300, 40, 200, 70));
648   // All non-zero values for all cpu_freq indices changed. It is an exhaustive
649   // snapshot.
650   EXPECT_THAT(thread.cpu_freq_full(), true);
651 
652   for (const std::string& path : dirs_to_delete)
653     base::Rmdir(path);
654 }
655 
656 }  // namespace
657 }  // namespace perfetto
658