1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "perfetto/ext/base/watchdog.h"
18
19 #if PERFETTO_BUILDFLAG(PERFETTO_WATCHDOG)
20
21 #include <fcntl.h>
22 #include <inttypes.h>
23 #include <signal.h>
24 #include <stdint.h>
25
26 #include <fstream>
27 #include <thread>
28
29 #include "perfetto/base/build_config.h"
30 #include "perfetto/base/logging.h"
31 #include "perfetto/base/thread_utils.h"
32 #include "perfetto/ext/base/file_utils.h"
33 #include "perfetto/ext/base/scoped_file.h"
34 #include "perfetto/ext/base/utils.h"
35
36 namespace perfetto {
37 namespace base {
38
39 namespace {
40
41 constexpr uint32_t kDefaultPollingInterval = 30 * 1000;
42
IsMultipleOf(uint32_t number,uint32_t divisor)43 bool IsMultipleOf(uint32_t number, uint32_t divisor) {
44 return number >= divisor && number % divisor == 0;
45 }
46
MeanForArray(const uint64_t array[],size_t size)47 double MeanForArray(const uint64_t array[], size_t size) {
48 uint64_t total = 0;
49 for (size_t i = 0; i < size; i++) {
50 total += array[i];
51 }
52 return static_cast<double>(total / size);
53
54 }
55
56 } // namespace
57
ReadProcStat(int fd,ProcStat * out)58 bool ReadProcStat(int fd, ProcStat* out) {
59 char c[512];
60 size_t c_pos = 0;
61 while (c_pos < sizeof(c) - 1) {
62 ssize_t rd = PERFETTO_EINTR(read(fd, c + c_pos, sizeof(c) - c_pos));
63 if (rd < 0) {
64 PERFETTO_ELOG("Failed to read stat file to enforce resource limits.");
65 return false;
66 }
67 if (rd == 0)
68 break;
69 c_pos += static_cast<size_t>(rd);
70 }
71 PERFETTO_CHECK(c_pos < sizeof(c));
72 c[c_pos] = '\0';
73
74 if (sscanf(c,
75 "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*u %*u %*u %*u %lu "
76 "%lu %*d %*d %*d %*d %*d %*d %*u %*u %ld",
77 &out->utime, &out->stime, &out->rss_pages) != 3) {
78 PERFETTO_ELOG("Invalid stat format: %s", c);
79 return false;
80 }
81 return true;
82 }
83
Watchdog(uint32_t polling_interval_ms)84 Watchdog::Watchdog(uint32_t polling_interval_ms)
85 : polling_interval_ms_(polling_interval_ms) {}
86
~Watchdog()87 Watchdog::~Watchdog() {
88 if (!thread_.joinable()) {
89 PERFETTO_DCHECK(!enabled_);
90 return;
91 }
92 PERFETTO_DCHECK(enabled_);
93 enabled_ = false;
94 exit_signal_.notify_one();
95 thread_.join();
96 }
97
GetInstance()98 Watchdog* Watchdog::GetInstance() {
99 static Watchdog* watchdog = new Watchdog(kDefaultPollingInterval);
100 return watchdog;
101 }
102
CreateFatalTimer(uint32_t ms)103 Watchdog::Timer Watchdog::CreateFatalTimer(uint32_t ms) {
104 if (!enabled_.load(std::memory_order_relaxed))
105 return Watchdog::Timer(0);
106
107 return Watchdog::Timer(ms);
108 }
109
Start()110 void Watchdog::Start() {
111 std::lock_guard<std::mutex> guard(mutex_);
112 if (thread_.joinable()) {
113 PERFETTO_DCHECK(enabled_);
114 } else {
115 PERFETTO_DCHECK(!enabled_);
116
117 #if PERFETTO_BUILDFLAG(PERFETTO_OS_LINUX) || \
118 PERFETTO_BUILDFLAG(PERFETTO_OS_ANDROID)
119 // Kick the thread to start running but only on Android or Linux.
120 enabled_ = true;
121 thread_ = std::thread(&Watchdog::ThreadMain, this);
122 #endif
123 }
124 }
125
SetMemoryLimit(uint64_t bytes,uint32_t window_ms)126 void Watchdog::SetMemoryLimit(uint64_t bytes, uint32_t window_ms) {
127 // Update the fields under the lock.
128 std::lock_guard<std::mutex> guard(mutex_);
129
130 PERFETTO_CHECK(IsMultipleOf(window_ms, polling_interval_ms_) || bytes == 0);
131
132 size_t size = bytes == 0 ? 0 : window_ms / polling_interval_ms_ + 1;
133 memory_window_bytes_.Reset(size);
134 memory_limit_bytes_ = bytes;
135 }
136
SetCpuLimit(uint32_t percentage,uint32_t window_ms)137 void Watchdog::SetCpuLimit(uint32_t percentage, uint32_t window_ms) {
138 std::lock_guard<std::mutex> guard(mutex_);
139
140 PERFETTO_CHECK(percentage <= 100);
141 PERFETTO_CHECK(IsMultipleOf(window_ms, polling_interval_ms_) ||
142 percentage == 0);
143
144 size_t size = percentage == 0 ? 0 : window_ms / polling_interval_ms_ + 1;
145 cpu_window_time_ticks_.Reset(size);
146 cpu_limit_percentage_ = percentage;
147 }
148
ThreadMain()149 void Watchdog::ThreadMain() {
150 base::ScopedFile stat_fd(base::OpenFile("/proc/self/stat", O_RDONLY));
151 if (!stat_fd) {
152 PERFETTO_ELOG("Failed to open stat file to enforce resource limits.");
153 return;
154 }
155
156 std::unique_lock<std::mutex> guard(mutex_);
157 for (;;) {
158 exit_signal_.wait_for(guard,
159 std::chrono::milliseconds(polling_interval_ms_));
160 if (!enabled_)
161 return;
162
163 lseek(stat_fd.get(), 0, SEEK_SET);
164
165 ProcStat stat;
166 if (!ReadProcStat(stat_fd.get(), &stat)) {
167 return;
168 }
169
170 uint64_t cpu_time = stat.utime + stat.stime;
171 uint64_t rss_bytes =
172 static_cast<uint64_t>(stat.rss_pages) * base::GetSysPageSize();
173
174 CheckMemory(rss_bytes);
175 CheckCpu(cpu_time);
176 }
177 }
178
CheckMemory(uint64_t rss_bytes)179 void Watchdog::CheckMemory(uint64_t rss_bytes) {
180 if (memory_limit_bytes_ == 0)
181 return;
182
183 // Add the current stat value to the ring buffer and check that the mean
184 // remains under our threshold.
185 if (memory_window_bytes_.Push(rss_bytes)) {
186 if (memory_window_bytes_.Mean() > static_cast<double>(memory_limit_bytes_)) {
187 PERFETTO_ELOG(
188 "Memory watchdog trigger. Memory window of %f bytes is above the "
189 "%" PRIu64 " bytes limit.",
190 memory_window_bytes_.Mean(), memory_limit_bytes_);
191 kill(getpid(), SIGABRT);
192 }
193 }
194 }
195
CheckCpu(uint64_t cpu_time)196 void Watchdog::CheckCpu(uint64_t cpu_time) {
197 if (cpu_limit_percentage_ == 0)
198 return;
199
200 // Add the cpu time to the ring buffer.
201 if (cpu_window_time_ticks_.Push(cpu_time)) {
202 // Compute the percentage over the whole window and check that it remains
203 // under the threshold.
204 uint64_t difference_ticks = cpu_window_time_ticks_.NewestWhenFull() -
205 cpu_window_time_ticks_.OldestWhenFull();
206 double window_interval_ticks =
207 (static_cast<double>(WindowTimeForRingBuffer(cpu_window_time_ticks_)) /
208 1000.0) *
209 static_cast<double>(sysconf(_SC_CLK_TCK));
210 double percentage = static_cast<double>(difference_ticks) /
211 static_cast<double>(window_interval_ticks) * 100;
212 if (percentage > cpu_limit_percentage_) {
213 PERFETTO_ELOG("CPU watchdog trigger. %f%% CPU use is above the %" PRIu32
214 "%% CPU limit.",
215 percentage, cpu_limit_percentage_);
216 kill(getpid(), SIGABRT);
217 }
218 }
219 }
220
WindowTimeForRingBuffer(const WindowedInterval & window)221 uint32_t Watchdog::WindowTimeForRingBuffer(const WindowedInterval& window) {
222 return static_cast<uint32_t>(window.size() - 1) * polling_interval_ms_;
223 }
224
Push(uint64_t sample)225 bool Watchdog::WindowedInterval::Push(uint64_t sample) {
226 // Add the sample to the current position in the ring buffer.
227 buffer_[position_] = sample;
228
229 // Update the position with next one circularily.
230 position_ = (position_ + 1) % size_;
231
232 // Set the filled flag the first time we wrap.
233 filled_ = filled_ || position_ == 0;
234 return filled_;
235 }
236
Mean() const237 double Watchdog::WindowedInterval::Mean() const {
238 return MeanForArray(buffer_.get(), size_);
239 }
240
Clear()241 void Watchdog::WindowedInterval::Clear() {
242 position_ = 0;
243 buffer_.reset(new uint64_t[size_]());
244 }
245
Reset(size_t new_size)246 void Watchdog::WindowedInterval::Reset(size_t new_size) {
247 position_ = 0;
248 size_ = new_size;
249 buffer_.reset(new_size == 0 ? nullptr : new uint64_t[new_size]());
250 }
251
Timer(uint32_t ms)252 Watchdog::Timer::Timer(uint32_t ms) {
253 if (!ms)
254 return; // No-op timer created when the watchdog is disabled.
255
256 struct sigevent sev = {};
257 timer_t timerid;
258 sev.sigev_notify = SIGEV_THREAD_ID;
259 sev._sigev_un._tid = base::GetThreadId();
260 sev.sigev_signo = SIGABRT;
261 PERFETTO_CHECK(timer_create(CLOCK_MONOTONIC, &sev, &timerid) != -1);
262 timerid_ = base::make_optional(timerid);
263 struct itimerspec its = {};
264 its.it_value.tv_sec = ms / 1000;
265 its.it_value.tv_nsec = 1000000L * (ms % 1000);
266 PERFETTO_CHECK(timer_settime(timerid_.value(), 0, &its, nullptr) != -1);
267 }
268
~Timer()269 Watchdog::Timer::~Timer() {
270 if (timerid_) {
271 timer_delete(timerid_.value());
272 }
273 }
274
Timer(Timer && other)275 Watchdog::Timer::Timer(Timer&& other) noexcept {
276 timerid_ = std::move(other.timerid_);
277 other.timerid_ = base::nullopt;
278 }
279
280 } // namespace base
281 } // namespace perfetto
282
283 #endif // PERFETTO_BUILDFLAG(PERFETTO_WATCHDOG)
284