1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "RecordReadThread.h"
18 
19 #include <sys/resource.h>
20 #include <unistd.h>
21 
22 #include <algorithm>
23 #include <unordered_map>
24 
25 #include "environment.h"
26 #include "event_type.h"
27 #include "record.h"
28 #include "utils.h"
29 
30 namespace simpleperf {
31 
32 static constexpr size_t kDefaultLowBufferLevel = 10 * 1024 * 1024u;
33 static constexpr size_t kDefaultCriticalBufferLevel = 5 * 1024 * 1024u;
34 
RecordBuffer(size_t buffer_size)35 RecordBuffer::RecordBuffer(size_t buffer_size)
36     : read_head_(0), write_head_(0), buffer_size_(buffer_size), buffer_(new char[buffer_size]) {}
37 
GetFreeSize() const38 size_t RecordBuffer::GetFreeSize() const {
39   size_t write_head = write_head_.load(std::memory_order_relaxed);
40   size_t read_head = read_head_.load(std::memory_order_relaxed);
41   size_t write_tail = read_head > 0 ? read_head - 1 : buffer_size_ - 1;
42   if (write_head <= write_tail) {
43     return write_tail - write_head;
44   }
45   return buffer_size_ - write_head + write_tail;
46 }
47 
AllocWriteSpace(size_t record_size)48 char* RecordBuffer::AllocWriteSpace(size_t record_size) {
49   size_t write_head = write_head_.load(std::memory_order_relaxed);
50   size_t read_head = read_head_.load(std::memory_order_acquire);
51   size_t write_tail = read_head > 0 ? read_head - 1 : buffer_size_ - 1;
52   cur_write_record_size_ = record_size;
53   if (write_head < write_tail) {
54     if (write_head + record_size > write_tail) {
55       return nullptr;
56     }
57   } else if (write_head + record_size > buffer_size_) {
58     // Not enough space at the end of the buffer, need to wrap to the start of the buffer.
59     if (write_tail < record_size) {
60       return nullptr;
61     }
62     if (buffer_size_ - write_head >= sizeof(perf_event_header)) {
63       // Set the size field in perf_event_header to 0. So GetCurrentRecord() can wrap to the start
64       // of the buffer when size is 0.
65       memset(buffer_.get() + write_head, 0, sizeof(perf_event_header));
66     }
67     cur_write_record_size_ += buffer_size_ - write_head;
68     write_head = 0;
69   }
70   return buffer_.get() + write_head;
71 }
72 
FinishWrite()73 void RecordBuffer::FinishWrite() {
74   size_t write_head = write_head_.load(std::memory_order_relaxed);
75   write_head = (write_head + cur_write_record_size_) % buffer_size_;
76   write_head_.store(write_head, std::memory_order_release);
77 }
78 
GetCurrentRecord()79 char* RecordBuffer::GetCurrentRecord() {
80   size_t write_head = write_head_.load(std::memory_order_acquire);
81   size_t read_head = read_head_.load(std::memory_order_relaxed);
82   if (read_head == write_head) {
83     return nullptr;
84   }
85   perf_event_header header;
86   if (read_head > write_head) {
87     if (buffer_size_ - read_head < sizeof(header) ||
88         (memcpy(&header, buffer_.get() + read_head, sizeof(header)) && header.size == 0)) {
89       // Need to wrap to the start of the buffer.
90       cur_read_record_size_ += buffer_size_ - read_head;
91       read_head = 0;
92       memcpy(&header, buffer_.get(), sizeof(header));
93     }
94   } else {
95     memcpy(&header, buffer_.get() + read_head, sizeof(header));
96   }
97   cur_read_record_size_ += header.size;
98   return buffer_.get() + read_head;
99 }
100 
MoveToNextRecord()101 void RecordBuffer::MoveToNextRecord() {
102   size_t read_head = read_head_.load(std::memory_order_relaxed);
103   read_head = (read_head + cur_read_record_size_) % buffer_size_;
104   read_head_.store(read_head, std::memory_order_release);
105   cur_read_record_size_ = 0;
106 }
107 
RecordParser(const perf_event_attr & attr)108 RecordParser::RecordParser(const perf_event_attr& attr)
109     : sample_type_(attr.sample_type),
110       sample_regs_count_(__builtin_popcountll(attr.sample_regs_user)) {
111   size_t pos = sizeof(perf_event_header);
112   uint64_t mask = PERF_SAMPLE_IDENTIFIER | PERF_SAMPLE_IP;
113   pos += __builtin_popcountll(sample_type_ & mask) * sizeof(uint64_t);
114   if (sample_type_ & PERF_SAMPLE_TID) {
115     pid_pos_in_sample_records_ = pos;
116     pos += sizeof(uint64_t);
117   }
118   if (sample_type_ & PERF_SAMPLE_TIME) {
119     time_pos_in_sample_records_ = pos;
120     pos += sizeof(uint64_t);
121   }
122   mask = PERF_SAMPLE_ADDR | PERF_SAMPLE_ID | PERF_SAMPLE_STREAM_ID | PERF_SAMPLE_CPU |
123          PERF_SAMPLE_PERIOD;
124   pos += __builtin_popcountll(sample_type_ & mask) * sizeof(uint64_t);
125   callchain_pos_in_sample_records_ = pos;
126   if ((sample_type_ & PERF_SAMPLE_TIME) && attr.sample_id_all) {
127     mask = PERF_SAMPLE_IDENTIFIER | PERF_SAMPLE_CPU | PERF_SAMPLE_STREAM_ID | PERF_SAMPLE_ID;
128     time_rpos_in_non_sample_records_ =
129         (__builtin_popcountll(sample_type_ & mask) + 1) * sizeof(uint64_t);
130   }
131 }
132 
GetTimePos(const perf_event_header & header) const133 size_t RecordParser::GetTimePos(const perf_event_header& header) const {
134   if (header.type == PERF_RECORD_SAMPLE) {
135     return time_pos_in_sample_records_;
136   }
137   if (time_rpos_in_non_sample_records_ != 0u &&
138       time_rpos_in_non_sample_records_ < header.size - sizeof(perf_event_header)) {
139     return header.size - time_rpos_in_non_sample_records_;
140   }
141   return 0;
142 }
143 
GetStackSizePos(const std::function<void (size_t,size_t,void *)> & read_record_fn) const144 size_t RecordParser::GetStackSizePos(
145     const std::function<void(size_t, size_t, void*)>& read_record_fn) const {
146   size_t pos = callchain_pos_in_sample_records_;
147   if (sample_type_ & PERF_SAMPLE_CALLCHAIN) {
148     uint64_t ip_nr;
149     read_record_fn(pos, sizeof(ip_nr), &ip_nr);
150     pos += (ip_nr + 1) * sizeof(uint64_t);
151   }
152   if (sample_type_ & PERF_SAMPLE_RAW) {
153     uint32_t size;
154     read_record_fn(pos, sizeof(size), &size);
155     pos += size + sizeof(uint32_t);
156   }
157   if (sample_type_ & PERF_SAMPLE_BRANCH_STACK) {
158     uint64_t stack_nr;
159     read_record_fn(pos, sizeof(stack_nr), &stack_nr);
160     pos += sizeof(uint64_t) + stack_nr * sizeof(BranchStackItemType);
161   }
162   if (sample_type_ & PERF_SAMPLE_REGS_USER) {
163     uint64_t abi;
164     read_record_fn(pos, sizeof(abi), &abi);
165     pos += (1 + (abi == 0 ? 0 : sample_regs_count_)) * sizeof(uint64_t);
166   }
167   return (sample_type_ & PERF_SAMPLE_STACK_USER) ? pos : 0;
168 }
169 
KernelRecordReader(EventFd * event_fd)170 KernelRecordReader::KernelRecordReader(EventFd* event_fd) : event_fd_(event_fd) {
171   size_t buffer_size;
172   buffer_ = event_fd_->GetMappedBuffer(buffer_size);
173   buffer_mask_ = buffer_size - 1;
174 }
175 
GetDataFromKernelBuffer()176 bool KernelRecordReader::GetDataFromKernelBuffer() {
177   data_size_ = event_fd_->GetAvailableMmapDataSize(data_pos_);
178   if (data_size_ == 0) {
179     return false;
180   }
181   init_data_size_ = data_size_;
182   record_header_.size = 0;
183   return true;
184 }
185 
ReadRecord(size_t pos,size_t size,void * dest)186 void KernelRecordReader::ReadRecord(size_t pos, size_t size, void* dest) {
187   pos = (pos + data_pos_) & buffer_mask_;
188   size_t copy_size = std::min(size, buffer_mask_ + 1 - pos);
189   memcpy(dest, buffer_ + pos, copy_size);
190   if (copy_size < size) {
191     memcpy(static_cast<char*>(dest) + copy_size, buffer_, size - copy_size);
192   }
193 }
194 
MoveToNextRecord(const RecordParser & parser)195 bool KernelRecordReader::MoveToNextRecord(const RecordParser& parser) {
196   data_pos_ = (data_pos_ + record_header_.size) & buffer_mask_;
197   data_size_ -= record_header_.size;
198   if (data_size_ == 0) {
199     event_fd_->DiscardMmapData(init_data_size_);
200     init_data_size_ = 0;
201     return false;
202   }
203   ReadRecord(0, sizeof(record_header_), &record_header_);
204   size_t time_pos = parser.GetTimePos(record_header_);
205   if (time_pos != 0) {
206     ReadRecord(time_pos, sizeof(record_time_), &record_time_);
207   }
208   return true;
209 }
210 
RecordReadThread(size_t record_buffer_size,const perf_event_attr & attr,size_t min_mmap_pages,size_t max_mmap_pages,size_t aux_buffer_size,bool allow_cutting_samples,bool exclude_perf)211 RecordReadThread::RecordReadThread(size_t record_buffer_size, const perf_event_attr& attr,
212                                    size_t min_mmap_pages, size_t max_mmap_pages,
213                                    size_t aux_buffer_size, bool allow_cutting_samples,
214                                    bool exclude_perf)
215     : record_buffer_(record_buffer_size),
216       record_parser_(attr),
217       attr_(attr),
218       min_mmap_pages_(min_mmap_pages),
219       max_mmap_pages_(max_mmap_pages),
220       aux_buffer_size_(aux_buffer_size) {
221   if (attr.sample_type & PERF_SAMPLE_STACK_USER) {
222     stack_size_in_sample_record_ = attr.sample_stack_user;
223   }
224   record_buffer_low_level_ = std::min(record_buffer_size / 4, kDefaultLowBufferLevel);
225   record_buffer_critical_level_ = std::min(record_buffer_size / 6, kDefaultCriticalBufferLevel);
226   if (!allow_cutting_samples) {
227     record_buffer_low_level_ = record_buffer_critical_level_;
228   }
229   if (exclude_perf) {
230     exclude_pid_ = getpid();
231   }
232 }
233 
~RecordReadThread()234 RecordReadThread::~RecordReadThread() {
235   if (read_thread_) {
236     StopReadThread();
237   }
238 }
239 
RegisterDataCallback(IOEventLoop & loop,const std::function<bool ()> & data_callback)240 bool RecordReadThread::RegisterDataCallback(IOEventLoop& loop,
241                                             const std::function<bool()>& data_callback) {
242   int cmd_fd[2];
243   int data_fd[2];
244   if (pipe2(cmd_fd, O_CLOEXEC) != 0 || pipe2(data_fd, O_CLOEXEC) != 0) {
245     PLOG(ERROR) << "pipe2";
246     return false;
247   }
248   read_cmd_fd_.reset(cmd_fd[0]);
249   write_cmd_fd_.reset(cmd_fd[1]);
250   cmd_ = NO_CMD;
251   read_data_fd_.reset(data_fd[0]);
252   write_data_fd_.reset(data_fd[1]);
253   has_data_notification_ = false;
254   if (!loop.AddReadEvent(read_data_fd_, data_callback)) {
255     return false;
256   }
257   read_thread_.reset(new std::thread([&]() { RunReadThread(); }));
258   return true;
259 }
260 
AddEventFds(const std::vector<EventFd * > & event_fds)261 bool RecordReadThread::AddEventFds(const std::vector<EventFd*>& event_fds) {
262   return SendCmdToReadThread(CMD_ADD_EVENT_FDS, const_cast<std::vector<EventFd*>*>(&event_fds));
263 }
264 
RemoveEventFds(const std::vector<EventFd * > & event_fds)265 bool RecordReadThread::RemoveEventFds(const std::vector<EventFd*>& event_fds) {
266   return SendCmdToReadThread(CMD_REMOVE_EVENT_FDS, const_cast<std::vector<EventFd*>*>(&event_fds));
267 }
268 
SyncKernelBuffer()269 bool RecordReadThread::SyncKernelBuffer() {
270   return SendCmdToReadThread(CMD_SYNC_KERNEL_BUFFER, nullptr);
271 }
272 
StopReadThread()273 bool RecordReadThread::StopReadThread() {
274   bool result = true;
275   if (read_thread_ != nullptr) {
276     result = SendCmdToReadThread(CMD_STOP_THREAD, nullptr);
277     if (result) {
278       read_thread_->join();
279       read_thread_ = nullptr;
280     }
281   }
282   return result;
283 }
284 
SendCmdToReadThread(Cmd cmd,void * cmd_arg)285 bool RecordReadThread::SendCmdToReadThread(Cmd cmd, void* cmd_arg) {
286   {
287     std::lock_guard<std::mutex> lock(cmd_mutex_);
288     cmd_ = cmd;
289     cmd_arg_ = cmd_arg;
290   }
291   char unused = 0;
292   if (TEMP_FAILURE_RETRY(write(write_cmd_fd_, &unused, 1)) != 1) {
293     return false;
294   }
295   std::unique_lock<std::mutex> lock(cmd_mutex_);
296   while (cmd_ != NO_CMD) {
297     cmd_finish_cond_.wait(lock);
298   }
299   return cmd_result_;
300 }
301 
GetRecord()302 std::unique_ptr<Record> RecordReadThread::GetRecord() {
303   record_buffer_.MoveToNextRecord();
304   char* p = record_buffer_.GetCurrentRecord();
305   if (p != nullptr) {
306     std::unique_ptr<Record> r = ReadRecordFromBuffer(attr_, p);
307     if (r->type() == PERF_RECORD_AUXTRACE) {
308       auto auxtrace = static_cast<AuxTraceRecord*>(r.get());
309       record_buffer_.AddCurrentRecordSize(auxtrace->data->aux_size);
310       auxtrace->location.addr = r->Binary() + r->size();
311     }
312     return r;
313   }
314   if (has_data_notification_) {
315     char unused;
316     TEMP_FAILURE_RETRY(read(read_data_fd_, &unused, 1));
317     has_data_notification_ = false;
318   }
319   return nullptr;
320 }
321 
RunReadThread()322 void RecordReadThread::RunReadThread() {
323   IncreaseThreadPriority();
324   IOEventLoop loop;
325   CHECK(loop.AddReadEvent(read_cmd_fd_, [&]() { return HandleCmd(loop); }));
326   loop.RunLoop();
327 }
328 
IncreaseThreadPriority()329 void RecordReadThread::IncreaseThreadPriority() {
330   // TODO: use real time priority for root.
331   rlimit rlim;
332   int result = getrlimit(RLIMIT_NICE, &rlim);
333   if (result == 0 && rlim.rlim_cur == 40) {
334     result = setpriority(PRIO_PROCESS, gettid(), -20);
335     if (result == 0) {
336       LOG(VERBOSE) << "Priority of record read thread is increased";
337     }
338   }
339 }
340 
GetCmd()341 RecordReadThread::Cmd RecordReadThread::GetCmd() {
342   std::lock_guard<std::mutex> lock(cmd_mutex_);
343   return cmd_;
344 }
345 
HandleCmd(IOEventLoop & loop)346 bool RecordReadThread::HandleCmd(IOEventLoop& loop) {
347   char unused;
348   TEMP_FAILURE_RETRY(read(read_cmd_fd_, &unused, 1));
349   bool result = true;
350   switch (GetCmd()) {
351     case CMD_ADD_EVENT_FDS:
352       result = HandleAddEventFds(loop, *static_cast<std::vector<EventFd*>*>(cmd_arg_));
353       break;
354     case CMD_REMOVE_EVENT_FDS:
355       result = HandleRemoveEventFds(*static_cast<std::vector<EventFd*>*>(cmd_arg_));
356       break;
357     case CMD_SYNC_KERNEL_BUFFER:
358       result = ReadRecordsFromKernelBuffer();
359       break;
360     case CMD_STOP_THREAD:
361       result = loop.ExitLoop();
362       break;
363     default:
364       LOG(ERROR) << "Unknown cmd: " << GetCmd();
365       result = false;
366       break;
367   }
368   std::lock_guard<std::mutex> lock(cmd_mutex_);
369   cmd_ = NO_CMD;
370   cmd_result_ = result;
371   cmd_finish_cond_.notify_one();
372   return true;
373 }
374 
HandleAddEventFds(IOEventLoop & loop,const std::vector<EventFd * > & event_fds)375 bool RecordReadThread::HandleAddEventFds(IOEventLoop& loop,
376                                          const std::vector<EventFd*>& event_fds) {
377   std::unordered_map<int, EventFd*> cpu_map;
378   for (size_t pages = max_mmap_pages_; pages >= min_mmap_pages_; pages >>= 1) {
379     bool success = true;
380     bool report_error = pages == min_mmap_pages_;
381     for (EventFd* fd : event_fds) {
382       auto it = cpu_map.find(fd->Cpu());
383       if (it == cpu_map.end()) {
384         if (!fd->CreateMappedBuffer(pages, report_error)) {
385           success = false;
386           break;
387         }
388         if (IsEtmEventType(fd->attr().type)) {
389           if (!fd->CreateAuxBuffer(aux_buffer_size_, report_error)) {
390             fd->DestroyMappedBuffer();
391             success = false;
392             break;
393           }
394         }
395         cpu_map[fd->Cpu()] = fd;
396       } else {
397         if (!fd->ShareMappedBuffer(*(it->second), pages == min_mmap_pages_)) {
398           success = false;
399           break;
400         }
401       }
402     }
403     if (success) {
404       LOG(VERBOSE) << "Each kernel buffer is " << pages << " pages.";
405       break;
406     }
407     for (auto& pair : cpu_map) {
408       pair.second->DestroyMappedBuffer();
409       pair.second->DestroyAuxBuffer();
410     }
411     cpu_map.clear();
412   }
413   if (cpu_map.empty()) {
414     return false;
415   }
416   for (auto& pair : cpu_map) {
417     if (!pair.second->StartPolling(loop, [this]() { return ReadRecordsFromKernelBuffer(); })) {
418       return false;
419     }
420     kernel_record_readers_.emplace_back(pair.second);
421   }
422   return true;
423 }
424 
HandleRemoveEventFds(const std::vector<EventFd * > & event_fds)425 bool RecordReadThread::HandleRemoveEventFds(const std::vector<EventFd*>& event_fds) {
426   for (auto& event_fd : event_fds) {
427     if (event_fd->HasMappedBuffer()) {
428       auto it = std::find_if(
429           kernel_record_readers_.begin(), kernel_record_readers_.end(),
430           [&](const KernelRecordReader& reader) { return reader.GetEventFd() == event_fd; });
431       if (it != kernel_record_readers_.end()) {
432         kernel_record_readers_.erase(it);
433         event_fd->StopPolling();
434         event_fd->DestroyMappedBuffer();
435         event_fd->DestroyAuxBuffer();
436       }
437     }
438   }
439   return true;
440 }
441 
CompareRecordTime(KernelRecordReader * r1,KernelRecordReader * r2)442 static bool CompareRecordTime(KernelRecordReader* r1, KernelRecordReader* r2) {
443   return r1->RecordTime() > r2->RecordTime();
444 }
445 
446 // When reading from mmap buffers, we prefer reading from all buffers at once rather than reading
447 // one buffer at a time. Because by reading all buffers at once, we can merge records from
448 // different buffers easily in memory. Otherwise, we have to sort records with greater effort.
ReadRecordsFromKernelBuffer()449 bool RecordReadThread::ReadRecordsFromKernelBuffer() {
450   do {
451     std::vector<KernelRecordReader*> readers;
452     for (auto& reader : kernel_record_readers_) {
453       if (reader.GetDataFromKernelBuffer()) {
454         readers.push_back(&reader);
455       }
456     }
457     bool has_data = false;
458     if (!readers.empty()) {
459       has_data = true;
460       if (readers.size() == 1u) {
461         // Only one buffer has data, process it directly.
462         while (readers[0]->MoveToNextRecord(record_parser_)) {
463           PushRecordToRecordBuffer(readers[0]);
464         }
465       } else {
466         // Use a binary heap to merge records from different buffers. As records from the same
467         // buffer are already ordered by time, we only need to merge the first record from all
468         // buffers. And each time a record is popped from the heap, we put the next record from its
469         // buffer into the heap.
470         for (auto& reader : readers) {
471           reader->MoveToNextRecord(record_parser_);
472         }
473         std::make_heap(readers.begin(), readers.end(), CompareRecordTime);
474         size_t size = readers.size();
475         while (size > 0) {
476           std::pop_heap(readers.begin(), readers.begin() + size, CompareRecordTime);
477           PushRecordToRecordBuffer(readers[size - 1]);
478           if (readers[size - 1]->MoveToNextRecord(record_parser_)) {
479             std::push_heap(readers.begin(), readers.begin() + size, CompareRecordTime);
480           } else {
481             size--;
482           }
483         }
484       }
485     }
486     ReadAuxDataFromKernelBuffer(&has_data);
487     if (!has_data) {
488       break;
489     }
490     if (!SendDataNotificationToMainThread()) {
491       return false;
492     }
493     // If there are no commands, we can loop until there is no more data from the kernel.
494   } while (GetCmd() == NO_CMD);
495   return true;
496 }
497 
PushRecordToRecordBuffer(KernelRecordReader * kernel_record_reader)498 void RecordReadThread::PushRecordToRecordBuffer(KernelRecordReader* kernel_record_reader) {
499   const perf_event_header& header = kernel_record_reader->RecordHeader();
500   if (header.type == PERF_RECORD_SAMPLE && exclude_pid_ != -1) {
501     uint32_t pid;
502     kernel_record_reader->ReadRecord(record_parser_.GetPidPosInSampleRecord(), sizeof(pid), &pid);
503     if (pid == exclude_pid_) {
504       return;
505     }
506   }
507   if (header.type == PERF_RECORD_SAMPLE && stack_size_in_sample_record_ > 1024) {
508     size_t free_size = record_buffer_.GetFreeSize();
509     if (free_size < record_buffer_critical_level_) {
510       // When the free size in record buffer is below critical level, drop sample records to save
511       // space for more important records (like mmap or fork records).
512       stat_.lost_samples++;
513       return;
514     }
515     size_t stack_size_limit = stack_size_in_sample_record_;
516     if (free_size < record_buffer_low_level_) {
517       // When the free size in record buffer is below low level, cut the stack data in sample
518       // records to 1K. This makes the unwinder unwind only part of the callchains, but hopefully
519       // the call chain joiner can complete the callchains.
520       stack_size_limit = 1024;
521     }
522     size_t stack_size_pos =
523         record_parser_.GetStackSizePos([&](size_t pos, size_t size, void* dest) {
524           return kernel_record_reader->ReadRecord(pos, size, dest);
525         });
526     uint64_t stack_size;
527     kernel_record_reader->ReadRecord(stack_size_pos, sizeof(stack_size), &stack_size);
528     if (stack_size > 0) {
529       size_t dyn_stack_size_pos = stack_size_pos + sizeof(stack_size) + stack_size;
530       uint64_t dyn_stack_size;
531       kernel_record_reader->ReadRecord(dyn_stack_size_pos, sizeof(dyn_stack_size), &dyn_stack_size);
532       if (dyn_stack_size == 0) {
533         // If stack_user_data.dyn_size == 0, it may be because the kernel misses the patch to
534         // update dyn_size, like in N9 (See b/22612370). So assume all stack data is valid if
535         // dyn_size == 0.
536         // TODO: Add cts test.
537         dyn_stack_size = stack_size;
538       }
539       // When simpleperf requests the kernel to dump 64K stack per sample, it will allocate 64K
540       // space in each sample to store stack data. However, a thread may use less stack than 64K.
541       // So not all the 64K stack data in a sample is valid, and we only need to keep valid stack
542       // data, whose size is dyn_stack_size.
543       uint64_t new_stack_size = std::min<uint64_t>(dyn_stack_size, stack_size_limit);
544       if (stack_size > new_stack_size) {
545         // Remove part of the stack data.
546         perf_event_header new_header = header;
547         new_header.size -= stack_size - new_stack_size;
548         char* p = record_buffer_.AllocWriteSpace(new_header.size);
549         if (p != nullptr) {
550           memcpy(p, &new_header, sizeof(new_header));
551           size_t pos = sizeof(new_header);
552           kernel_record_reader->ReadRecord(pos, stack_size_pos - pos, p + pos);
553           memcpy(p + stack_size_pos, &new_stack_size, sizeof(uint64_t));
554           pos = stack_size_pos + sizeof(uint64_t);
555           kernel_record_reader->ReadRecord(pos, new_stack_size, p + pos);
556           memcpy(p + pos + new_stack_size, &new_stack_size, sizeof(uint64_t));
557           record_buffer_.FinishWrite();
558           if (new_stack_size < dyn_stack_size) {
559             stat_.cut_stack_samples++;
560           }
561         } else {
562           stat_.lost_samples++;
563         }
564         return;
565       }
566     }
567   }
568   char* p = record_buffer_.AllocWriteSpace(header.size);
569   if (p != nullptr) {
570     kernel_record_reader->ReadRecord(0, header.size, p);
571     record_buffer_.FinishWrite();
572   } else {
573     if (header.type == PERF_RECORD_SAMPLE) {
574       stat_.lost_samples++;
575     } else {
576       stat_.lost_non_samples++;
577     }
578   }
579 }
580 
ReadAuxDataFromKernelBuffer(bool * has_data)581 void RecordReadThread::ReadAuxDataFromKernelBuffer(bool* has_data) {
582   for (auto& reader : kernel_record_readers_) {
583     EventFd* event_fd = reader.GetEventFd();
584     if (event_fd->HasAuxBuffer()) {
585       char* buf[2];
586       size_t size[2];
587       uint64_t offset = event_fd->GetAvailableAuxData(&buf[0], &size[0], &buf[1], &size[1]);
588       size_t aux_size = size[0] + size[1];
589       if (aux_size == 0) {
590         continue;
591       }
592       *has_data = true;
593       AuxTraceRecord auxtrace(Align(aux_size, 8), offset, event_fd->Cpu(), 0, event_fd->Cpu());
594       size_t alloc_size = auxtrace.size() + auxtrace.data->aux_size;
595       if (record_buffer_.GetFreeSize() < alloc_size + record_buffer_critical_level_) {
596         stat_.lost_aux_data_size += aux_size;
597       } else {
598         char* p = record_buffer_.AllocWriteSpace(alloc_size);
599         CHECK(p != nullptr);
600         MoveToBinaryFormat(auxtrace.Binary(), auxtrace.size(), p);
601         MoveToBinaryFormat(buf[0], size[0], p);
602         if (size[1] != 0) {
603           MoveToBinaryFormat(buf[1], size[1], p);
604         }
605         size_t pad_size = auxtrace.data->aux_size - aux_size;
606         if (pad_size != 0) {
607           uint64_t pad = 0;
608           memcpy(p, &pad, pad_size);
609         }
610         record_buffer_.FinishWrite();
611         stat_.aux_data_size += aux_size;
612         LOG(DEBUG) << "record aux data " << aux_size << " bytes";
613       }
614       event_fd->DiscardAuxData(aux_size);
615     }
616   }
617 }
618 
SendDataNotificationToMainThread()619 bool RecordReadThread::SendDataNotificationToMainThread() {
620   if (!has_data_notification_.load(std::memory_order_relaxed)) {
621     has_data_notification_ = true;
622     char unused = 0;
623     if (TEMP_FAILURE_RETRY(write(write_data_fd_, &unused, 1)) != 1) {
624       PLOG(ERROR) << "write";
625       return false;
626     }
627   }
628   return true;
629 }
630 
631 }  // namespace simpleperf
632