1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "src/trace_processor/importers/proto/proto_trace_parser.h"
18 
19 #include <inttypes.h>
20 #include <string.h>
21 
22 #include <string>
23 
24 #include "perfetto/base/logging.h"
25 #include "perfetto/ext/base/metatrace_events.h"
26 #include "perfetto/ext/base/string_utils.h"
27 #include "perfetto/ext/base/string_view.h"
28 #include "perfetto/ext/base/string_writer.h"
29 #include "perfetto/ext/base/utils.h"
30 #include "perfetto/ext/base/uuid.h"
31 #include "perfetto/trace_processor/status.h"
32 
33 #include "src/trace_processor/importers/common/args_tracker.h"
34 #include "src/trace_processor/importers/common/clock_tracker.h"
35 #include "src/trace_processor/importers/common/event_tracker.h"
36 #include "src/trace_processor/importers/common/process_tracker.h"
37 #include "src/trace_processor/importers/common/slice_tracker.h"
38 #include "src/trace_processor/importers/common/track_tracker.h"
39 #include "src/trace_processor/importers/config.descriptor.h"
40 #include "src/trace_processor/importers/ftrace/ftrace_module.h"
41 #include "src/trace_processor/importers/proto/heap_profile_tracker.h"
42 #include "src/trace_processor/importers/proto/metadata_tracker.h"
43 #include "src/trace_processor/importers/proto/packet_sequence_state.h"
44 #include "src/trace_processor/importers/proto/profile_packet_utils.h"
45 #include "src/trace_processor/importers/proto/profiler_util.h"
46 #include "src/trace_processor/importers/proto/stack_profile_tracker.h"
47 #include "src/trace_processor/storage/metadata.h"
48 #include "src/trace_processor/storage/stats.h"
49 #include "src/trace_processor/tables/profiler_tables.h"
50 #include "src/trace_processor/timestamped_trace_piece.h"
51 #include "src/trace_processor/types/trace_processor_context.h"
52 #include "src/trace_processor/types/variadic.h"
53 #include "src/trace_processor/util/descriptors.h"
54 #include "src/trace_processor/util/protozero_to_text.h"
55 
56 #include "protos/perfetto/common/builtin_clock.pbzero.h"
57 #include "protos/perfetto/common/trace_stats.pbzero.h"
58 #include "protos/perfetto/config/trace_config.pbzero.h"
59 #include "protos/perfetto/trace/chrome/chrome_trace_event.pbzero.h"
60 #include "protos/perfetto/trace/interned_data/interned_data.pbzero.h"
61 #include "protos/perfetto/trace/perfetto/perfetto_metatrace.pbzero.h"
62 #include "protos/perfetto/trace/profiling/deobfuscation.pbzero.h"
63 #include "protos/perfetto/trace/profiling/profile_common.pbzero.h"
64 #include "protos/perfetto/trace/profiling/profile_packet.pbzero.h"
65 #include "protos/perfetto/trace/profiling/smaps.pbzero.h"
66 #include "protos/perfetto/trace/trace.pbzero.h"
67 #include "protos/perfetto/trace/trace_packet.pbzero.h"
68 
69 namespace perfetto {
70 namespace trace_processor {
71 
ProtoTraceParser(TraceProcessorContext * context)72 ProtoTraceParser::ProtoTraceParser(TraceProcessorContext* context)
73     : context_(context),
74       metatrace_id_(context->storage->InternString("metatrace")),
75       data_name_id_(context->storage->InternString("data")),
76       raw_chrome_metadata_event_id_(
77           context->storage->InternString("chrome_event.metadata")),
78       raw_chrome_legacy_system_trace_event_id_(
79           context->storage->InternString("chrome_event.legacy_system_trace")),
80       raw_chrome_legacy_user_trace_event_id_(
81           context->storage->InternString("chrome_event.legacy_user_trace")) {
82   // TODO(140860736): Once we support null values for
83   // stack_profile_frame.symbol_set_id remove this hack
84   context_->storage->mutable_symbol_table()->Insert(
85       {0, kNullStringId, kNullStringId, 0});
86 }
87 
88 ProtoTraceParser::~ProtoTraceParser() = default;
89 
ParseTracePacket(int64_t ts,TimestampedTracePiece ttp)90 void ProtoTraceParser::ParseTracePacket(int64_t ts, TimestampedTracePiece ttp) {
91   const TracePacketData* data = nullptr;
92   if (ttp.type == TimestampedTracePiece::Type::kTracePacket) {
93     data = &ttp.packet_data;
94   } else {
95     PERFETTO_DCHECK(ttp.type == TimestampedTracePiece::Type::kTrackEvent);
96     data = ttp.track_event_data.get();
97   }
98 
99   const TraceBlobView& blob = data->packet;
100   protos::pbzero::TracePacket::Decoder packet(blob.data(), blob.length());
101 
102   ParseTracePacketImpl(ts, ttp, data->sequence_state.get(), packet);
103 
104   // TODO(lalitm): maybe move this to the flush method in the trace processor
105   // once we have it. This may reduce performance in the ArgsTracker though so
106   // needs to be handled carefully.
107   context_->args_tracker->Flush();
108   PERFETTO_DCHECK(!packet.bytes_left());
109 }
110 
ParseTracePacketImpl(int64_t ts,const TimestampedTracePiece & ttp,PacketSequenceStateGeneration * sequence_state,const protos::pbzero::TracePacket::Decoder & packet)111 void ProtoTraceParser::ParseTracePacketImpl(
112     int64_t ts,
113     const TimestampedTracePiece& ttp,
114     PacketSequenceStateGeneration* sequence_state,
115     const protos::pbzero::TracePacket::Decoder& packet) {
116   // This needs to get handled both by the HeapGraphModule and
117   // ProtoTraceParser (for StackProfileTracker).
118   if (packet.has_deobfuscation_mapping()) {
119     ParseDeobfuscationMapping(ts, sequence_state,
120                               packet.trusted_packet_sequence_id(),
121                               packet.deobfuscation_mapping());
122   }
123 
124   // Chrome doesn't honor the one-of in TracePacket for this field and sets it
125   // together with chrome_metadata, which is handled by a module. Thus, we have
126   // to parse this field before the modules get to parse other fields.
127   // TODO(crbug/1194914): Move this back after the modules (or into a separate
128   // module) once the Chrome-side fix has propagated into all release channels.
129   if (packet.has_chrome_events()) {
130     ParseChromeEvents(ts, packet.chrome_events());
131   }
132 
133   // TODO(eseckler): Propagate statuses from modules.
134   auto& modules = context_->modules_by_field;
135   for (uint32_t field_id = 1; field_id < modules.size(); ++field_id) {
136     if (!modules[field_id].empty() && packet.Get(field_id).valid()) {
137       for (ProtoImporterModule* module : modules[field_id])
138         module->ParsePacket(packet, ttp, field_id);
139       return;
140     }
141   }
142 
143   if (packet.has_trace_stats())
144     ParseTraceStats(packet.trace_stats());
145 
146   if (packet.has_profile_packet()) {
147     ParseProfilePacket(ts, sequence_state, packet.trusted_packet_sequence_id(),
148                        packet.profile_packet());
149   }
150 
151   if (packet.has_perfetto_metatrace()) {
152     ParseMetatraceEvent(ts, packet.perfetto_metatrace());
153   }
154 
155   if (packet.has_trace_config()) {
156     ParseTraceConfig(packet.trace_config());
157   }
158 
159   if (packet.has_module_symbols()) {
160     ParseModuleSymbols(packet.module_symbols());
161   }
162 
163   if (packet.has_smaps_packet()) {
164     ParseSmapsPacket(ts, packet.smaps_packet());
165   }
166 }
167 
ParseFtracePacket(uint32_t cpu,int64_t,TimestampedTracePiece ttp)168 void ProtoTraceParser::ParseFtracePacket(uint32_t cpu,
169                                          int64_t /*ts*/,
170                                          TimestampedTracePiece ttp) {
171   PERFETTO_DCHECK(ttp.type == TimestampedTracePiece::Type::kFtraceEvent ||
172                   ttp.type == TimestampedTracePiece::Type::kInlineSchedSwitch ||
173                   ttp.type == TimestampedTracePiece::Type::kInlineSchedWaking);
174   PERFETTO_DCHECK(context_->ftrace_module);
175   context_->ftrace_module->ParseFtracePacket(cpu, ttp);
176 
177   // TODO(lalitm): maybe move this to the flush method in the trace processor
178   // once we have it. This may reduce performance in the ArgsTracker though so
179   // needs to be handled carefully.
180   context_->args_tracker->Flush();
181 }
182 
ParseTraceStats(ConstBytes blob)183 void ProtoTraceParser::ParseTraceStats(ConstBytes blob) {
184   protos::pbzero::TraceStats::Decoder evt(blob.data, blob.size);
185   auto* storage = context_->storage.get();
186   storage->SetStats(stats::traced_producers_connected,
187                     static_cast<int64_t>(evt.producers_connected()));
188   storage->SetStats(stats::traced_data_sources_registered,
189                     static_cast<int64_t>(evt.data_sources_registered()));
190   storage->SetStats(stats::traced_data_sources_seen,
191                     static_cast<int64_t>(evt.data_sources_seen()));
192   storage->SetStats(stats::traced_tracing_sessions,
193                     static_cast<int64_t>(evt.tracing_sessions()));
194   storage->SetStats(stats::traced_total_buffers,
195                     static_cast<int64_t>(evt.total_buffers()));
196   storage->SetStats(stats::traced_chunks_discarded,
197                     static_cast<int64_t>(evt.chunks_discarded()));
198   storage->SetStats(stats::traced_patches_discarded,
199                     static_cast<int64_t>(evt.patches_discarded()));
200 
201   int buf_num = 0;
202   for (auto it = evt.buffer_stats(); it; ++it, ++buf_num) {
203     protos::pbzero::TraceStats::BufferStats::Decoder buf(*it);
204     storage->SetIndexedStats(stats::traced_buf_buffer_size, buf_num,
205                              static_cast<int64_t>(buf.buffer_size()));
206     storage->SetIndexedStats(stats::traced_buf_bytes_written, buf_num,
207                              static_cast<int64_t>(buf.bytes_written()));
208     storage->SetIndexedStats(stats::traced_buf_bytes_overwritten, buf_num,
209                              static_cast<int64_t>(buf.bytes_overwritten()));
210     storage->SetIndexedStats(stats::traced_buf_bytes_read, buf_num,
211                              static_cast<int64_t>(buf.bytes_read()));
212     storage->SetIndexedStats(stats::traced_buf_padding_bytes_written, buf_num,
213                              static_cast<int64_t>(buf.padding_bytes_written()));
214     storage->SetIndexedStats(stats::traced_buf_padding_bytes_cleared, buf_num,
215                              static_cast<int64_t>(buf.padding_bytes_cleared()));
216     storage->SetIndexedStats(stats::traced_buf_chunks_written, buf_num,
217                              static_cast<int64_t>(buf.chunks_written()));
218     storage->SetIndexedStats(stats::traced_buf_chunks_rewritten, buf_num,
219                              static_cast<int64_t>(buf.chunks_rewritten()));
220     storage->SetIndexedStats(stats::traced_buf_chunks_overwritten, buf_num,
221                              static_cast<int64_t>(buf.chunks_overwritten()));
222     storage->SetIndexedStats(stats::traced_buf_chunks_discarded, buf_num,
223                              static_cast<int64_t>(buf.chunks_discarded()));
224     storage->SetIndexedStats(stats::traced_buf_chunks_read, buf_num,
225                              static_cast<int64_t>(buf.chunks_read()));
226     storage->SetIndexedStats(
227         stats::traced_buf_chunks_committed_out_of_order, buf_num,
228         static_cast<int64_t>(buf.chunks_committed_out_of_order()));
229     storage->SetIndexedStats(stats::traced_buf_write_wrap_count, buf_num,
230                              static_cast<int64_t>(buf.write_wrap_count()));
231     storage->SetIndexedStats(stats::traced_buf_patches_succeeded, buf_num,
232                              static_cast<int64_t>(buf.patches_succeeded()));
233     storage->SetIndexedStats(stats::traced_buf_patches_failed, buf_num,
234                              static_cast<int64_t>(buf.patches_failed()));
235     storage->SetIndexedStats(stats::traced_buf_readaheads_succeeded, buf_num,
236                              static_cast<int64_t>(buf.readaheads_succeeded()));
237     storage->SetIndexedStats(stats::traced_buf_readaheads_failed, buf_num,
238                              static_cast<int64_t>(buf.readaheads_failed()));
239     storage->SetIndexedStats(
240         stats::traced_buf_trace_writer_packet_loss, buf_num,
241         static_cast<int64_t>(buf.trace_writer_packet_loss()));
242   }
243 }
244 
ParseProfilePacket(int64_t ts,PacketSequenceStateGeneration * sequence_state,uint32_t seq_id,ConstBytes blob)245 void ProtoTraceParser::ParseProfilePacket(
246     int64_t ts,
247     PacketSequenceStateGeneration* sequence_state,
248     uint32_t seq_id,
249     ConstBytes blob) {
250   protos::pbzero::ProfilePacket::Decoder packet(blob.data, blob.size);
251   context_->heap_profile_tracker->SetProfilePacketIndex(seq_id, packet.index());
252 
253   for (auto it = packet.strings(); it; ++it) {
254     protos::pbzero::InternedString::Decoder entry(*it);
255 
256     const char* str = reinterpret_cast<const char*>(entry.str().data);
257     auto str_view = base::StringView(str, entry.str().size);
258     sequence_state->state()->sequence_stack_profile_tracker().AddString(
259         entry.iid(), str_view);
260   }
261 
262   for (auto it = packet.mappings(); it; ++it) {
263     protos::pbzero::Mapping::Decoder entry(*it);
264     SequenceStackProfileTracker::SourceMapping src_mapping =
265         ProfilePacketUtils::MakeSourceMapping(entry);
266     sequence_state->state()->sequence_stack_profile_tracker().AddMapping(
267         entry.iid(), src_mapping);
268   }
269 
270   for (auto it = packet.frames(); it; ++it) {
271     protos::pbzero::Frame::Decoder entry(*it);
272     SequenceStackProfileTracker::SourceFrame src_frame =
273         ProfilePacketUtils::MakeSourceFrame(entry);
274     sequence_state->state()->sequence_stack_profile_tracker().AddFrame(
275         entry.iid(), src_frame);
276   }
277 
278   for (auto it = packet.callstacks(); it; ++it) {
279     protos::pbzero::Callstack::Decoder entry(*it);
280     SequenceStackProfileTracker::SourceCallstack src_callstack =
281         ProfilePacketUtils::MakeSourceCallstack(entry);
282     sequence_state->state()->sequence_stack_profile_tracker().AddCallstack(
283         entry.iid(), src_callstack);
284   }
285 
286   for (auto it = packet.process_dumps(); it; ++it) {
287     protos::pbzero::ProfilePacket::ProcessHeapSamples::Decoder entry(*it);
288 
289     auto maybe_timestamp = context_->clock_tracker->ToTraceTime(
290         protos::pbzero::BUILTIN_CLOCK_MONOTONIC_COARSE,
291         static_cast<int64_t>(entry.timestamp()));
292 
293     // ToTraceTime() increments the clock_sync_failure error stat in this case.
294     if (!maybe_timestamp)
295       continue;
296 
297     int64_t timestamp = *maybe_timestamp;
298 
299     int pid = static_cast<int>(entry.pid());
300     context_->storage->SetIndexedStats(stats::heapprofd_last_profile_timestamp,
301                                        pid, ts);
302 
303     if (entry.disconnected())
304       context_->storage->IncrementIndexedStats(
305           stats::heapprofd_client_disconnected, pid);
306     if (entry.buffer_corrupted())
307       context_->storage->IncrementIndexedStats(
308           stats::heapprofd_buffer_corrupted, pid);
309     if (entry.buffer_overran() ||
310         entry.client_error() ==
311             protos::pbzero::ProfilePacket::ProcessHeapSamples::
312                 CLIENT_ERROR_HIT_TIMEOUT) {
313       context_->storage->IncrementIndexedStats(stats::heapprofd_buffer_overran,
314                                                pid);
315     }
316     if (entry.client_error()) {
317       context_->storage->SetIndexedStats(stats::heapprofd_client_error, pid,
318                                          entry.client_error());
319     }
320     if (entry.rejected_concurrent())
321       context_->storage->IncrementIndexedStats(
322           stats::heapprofd_rejected_concurrent, pid);
323     if (entry.hit_guardrail())
324       context_->storage->IncrementIndexedStats(stats::heapprofd_hit_guardrail,
325                                                pid);
326     if (entry.orig_sampling_interval_bytes()) {
327       context_->storage->SetIndexedStats(
328           stats::heapprofd_sampling_interval_adjusted, pid,
329           static_cast<int64_t>(entry.sampling_interval_bytes()) -
330               static_cast<int64_t>(entry.orig_sampling_interval_bytes()));
331     }
332 
333     protos::pbzero::ProfilePacket::ProcessStats::Decoder stats(entry.stats());
334     context_->storage->IncrementIndexedStats(
335         stats::heapprofd_unwind_time_us, static_cast<int>(entry.pid()),
336         static_cast<int64_t>(stats.total_unwinding_time_us()));
337     context_->storage->IncrementIndexedStats(
338         stats::heapprofd_unwind_samples, static_cast<int>(entry.pid()),
339         static_cast<int64_t>(stats.heap_samples()));
340     context_->storage->IncrementIndexedStats(
341         stats::heapprofd_client_spinlock_blocked, static_cast<int>(entry.pid()),
342         static_cast<int64_t>(stats.client_spinlock_blocked_us()));
343 
344     // orig_sampling_interval_bytes was introduced slightly after a bug with
345     // self_max_count was fixed in the producer. We use this as a proxy
346     // whether or not we are getting this data from a fixed producer or not.
347     bool trustworthy_max_count = entry.orig_sampling_interval_bytes() > 0;
348 
349     for (auto sample_it = entry.samples(); sample_it; ++sample_it) {
350       protos::pbzero::ProfilePacket::HeapSample::Decoder sample(*sample_it);
351 
352       HeapProfileTracker::SourceAllocation src_allocation;
353       src_allocation.pid = entry.pid();
354       if (entry.heap_name().size != 0) {
355         src_allocation.heap_name =
356             context_->storage->InternString(entry.heap_name());
357       } else {
358         src_allocation.heap_name = context_->storage->InternString("malloc");
359       }
360       src_allocation.timestamp = timestamp;
361       src_allocation.callstack_id = sample.callstack_id();
362       if (sample.has_self_max()) {
363         src_allocation.self_allocated = sample.self_max();
364         if (trustworthy_max_count)
365           src_allocation.alloc_count = sample.self_max_count();
366       } else {
367         src_allocation.self_allocated = sample.self_allocated();
368         src_allocation.self_freed = sample.self_freed();
369         src_allocation.alloc_count = sample.alloc_count();
370         src_allocation.free_count = sample.free_count();
371       }
372 
373       context_->heap_profile_tracker->StoreAllocation(seq_id, src_allocation);
374     }
375   }
376   if (!packet.continued()) {
377     PERFETTO_CHECK(sequence_state);
378     ProfilePacketInternLookup intern_lookup(sequence_state);
379     context_->heap_profile_tracker->FinalizeProfile(
380         seq_id, &sequence_state->state()->sequence_stack_profile_tracker(),
381         &intern_lookup);
382   }
383 }
384 
ParseDeobfuscationMapping(int64_t,PacketSequenceStateGeneration *,uint32_t,ConstBytes blob)385 void ProtoTraceParser::ParseDeobfuscationMapping(int64_t,
386                                                  PacketSequenceStateGeneration*,
387                                                  uint32_t /* seq_id */,
388                                                  ConstBytes blob) {
389   protos::pbzero::DeobfuscationMapping::Decoder deobfuscation_mapping(
390       blob.data, blob.size);
391   if (deobfuscation_mapping.package_name().size == 0)
392     return;
393 
394   auto opt_package_name_id = context_->storage->string_pool().GetId(
395       deobfuscation_mapping.package_name());
396   auto opt_memfd_id = context_->storage->string_pool().GetId("memfd");
397   if (!opt_package_name_id && !opt_memfd_id)
398     return;
399 
400   for (auto class_it = deobfuscation_mapping.obfuscated_classes(); class_it;
401        ++class_it) {
402     protos::pbzero::ObfuscatedClass::Decoder cls(*class_it);
403     for (auto member_it = cls.obfuscated_methods(); member_it; ++member_it) {
404       protos::pbzero::ObfuscatedMember::Decoder member(*member_it);
405       std::string merged_obfuscated = cls.obfuscated_name().ToStdString() +
406                                       "." +
407                                       member.obfuscated_name().ToStdString();
408       auto merged_obfuscated_id = context_->storage->string_pool().GetId(
409           base::StringView(merged_obfuscated));
410       if (!merged_obfuscated_id)
411         continue;
412       std::string merged_deobfuscated =
413           FullyQualifiedDeobfuscatedName(cls, member);
414 
415       std::vector<tables::StackProfileFrameTable::Id> frames;
416       if (opt_package_name_id) {
417         const std::vector<tables::StackProfileFrameTable::Id>* pkg_frames =
418             context_->global_stack_profile_tracker->JavaFramesForName(
419                 {*merged_obfuscated_id, *opt_package_name_id});
420         if (pkg_frames) {
421           frames.insert(frames.end(), pkg_frames->begin(), pkg_frames->end());
422         }
423       }
424       if (opt_memfd_id) {
425         const std::vector<tables::StackProfileFrameTable::Id>* memfd_frames =
426             context_->global_stack_profile_tracker->JavaFramesForName(
427                 {*merged_obfuscated_id, *opt_memfd_id});
428         if (memfd_frames) {
429           frames.insert(frames.end(), memfd_frames->begin(),
430                         memfd_frames->end());
431         }
432       }
433 
434       for (tables::StackProfileFrameTable::Id frame_id : frames) {
435         auto* frames_tbl =
436             context_->storage->mutable_stack_profile_frame_table();
437         frames_tbl->mutable_deobfuscated_name()->Set(
438             *frames_tbl->id().IndexOf(frame_id),
439             context_->storage->InternString(
440                 base::StringView(merged_deobfuscated)));
441       }
442     }
443   }
444 }
445 
ParseChromeEvents(int64_t ts,ConstBytes blob)446 void ProtoTraceParser::ParseChromeEvents(int64_t ts, ConstBytes blob) {
447   TraceStorage* storage = context_->storage.get();
448   protos::pbzero::ChromeEventBundle::Decoder bundle(blob.data, blob.size);
449   ArgsTracker args(context_);
450   if (bundle.has_metadata()) {
451     RawId id = storage->mutable_raw_table()
452                    ->Insert({ts, raw_chrome_metadata_event_id_, 0, 0})
453                    .id;
454     auto inserter = args.AddArgsTo(id);
455 
456     uint32_t bundle_index =
457         context_->metadata_tracker->IncrementChromeMetadataBundleCount();
458 
459     // The legacy untyped metadata is proxied via a special event in the raw
460     // table to JSON export.
461     for (auto it = bundle.metadata(); it; ++it) {
462       protos::pbzero::ChromeMetadata::Decoder metadata(*it);
463       Variadic value;
464       if (metadata.has_string_value()) {
465         value =
466             Variadic::String(storage->InternString(metadata.string_value()));
467       } else if (metadata.has_int_value()) {
468         value = Variadic::Integer(metadata.int_value());
469       } else if (metadata.has_bool_value()) {
470         value = Variadic::Integer(metadata.bool_value());
471       } else if (metadata.has_json_value()) {
472         value = Variadic::Json(storage->InternString(metadata.json_value()));
473       } else {
474         context_->storage->IncrementStats(stats::empty_chrome_metadata);
475         continue;
476       }
477 
478       StringId name_id = storage->InternString(metadata.name());
479       args.AddArgsTo(id).AddArg(name_id, value);
480 
481       char buffer[2048];
482       base::StringWriter writer(buffer, sizeof(buffer));
483       writer.AppendString("cr-");
484       // If we have data from multiple Chrome instances, append a suffix
485       // to differentiate them.
486       if (bundle_index > 1) {
487         writer.AppendUnsignedInt(bundle_index);
488         writer.AppendChar('-');
489       }
490       writer.AppendString(metadata.name());
491 
492       auto metadata_id = storage->InternString(writer.GetStringView());
493       context_->metadata_tracker->SetDynamicMetadata(metadata_id, value);
494     }
495   }
496 
497   if (bundle.has_legacy_ftrace_output()) {
498     RawId id =
499         storage->mutable_raw_table()
500             ->Insert({ts, raw_chrome_legacy_system_trace_event_id_, 0, 0})
501             .id;
502 
503     std::string data;
504     for (auto it = bundle.legacy_ftrace_output(); it; ++it) {
505       data += (*it).ToStdString();
506     }
507     Variadic value =
508         Variadic::String(storage->InternString(base::StringView(data)));
509     args.AddArgsTo(id).AddArg(data_name_id_, value);
510   }
511 
512   if (bundle.has_legacy_json_trace()) {
513     for (auto it = bundle.legacy_json_trace(); it; ++it) {
514       protos::pbzero::ChromeLegacyJsonTrace::Decoder legacy_trace(*it);
515       if (legacy_trace.type() !=
516           protos::pbzero::ChromeLegacyJsonTrace::USER_TRACE) {
517         continue;
518       }
519       RawId id =
520           storage->mutable_raw_table()
521               ->Insert({ts, raw_chrome_legacy_user_trace_event_id_, 0, 0})
522               .id;
523       Variadic value =
524           Variadic::String(storage->InternString(legacy_trace.data()));
525       args.AddArgsTo(id).AddArg(data_name_id_, value);
526     }
527   }
528 }
529 
ParseMetatraceEvent(int64_t ts,ConstBytes blob)530 void ProtoTraceParser::ParseMetatraceEvent(int64_t ts, ConstBytes blob) {
531   protos::pbzero::PerfettoMetatrace::Decoder event(blob.data, blob.size);
532   auto utid = context_->process_tracker->GetOrCreateThread(event.thread_id());
533 
534   StringId cat_id = metatrace_id_;
535   StringId name_id = kNullStringId;
536   char fallback[64];
537 
538   // This function inserts the args from the proto into the args table.
539   // Args inserted with the same key multiple times are treated as an array:
540   // this function correctly creates the key and flat key for each arg array.
541   auto args_fn = [this, &event](ArgsTracker::BoundInserter* inserter) {
542     using Arg = std::pair<StringId, StringId>;
543 
544     // First, get a list of all the args so we can group them by key.
545     std::vector<Arg> interned;
546     for (auto it = event.args(); it; ++it) {
547       protos::pbzero::PerfettoMetatrace::Arg::Decoder arg_proto(*it);
548       StringId key = context_->storage->InternString(arg_proto.key());
549       StringId value = context_->storage->InternString(arg_proto.value());
550       interned.emplace_back(key, value);
551     }
552 
553     // We stable sort insted of sorting here to avoid changing the order of the
554     // args in arrays.
555     std::stable_sort(interned.begin(), interned.end(),
556                      [](const Arg& a, const Arg& b) {
557                        return a.first.raw_id() < b.second.raw_id();
558                      });
559 
560     // Compute the correct key for each arg, possibly adding an index to
561     // the end of the key if needed.
562     char buffer[2048];
563     uint32_t current_idx = 0;
564     for (auto it = interned.begin(); it != interned.end(); ++it) {
565       auto next = it + 1;
566       StringId key = it->first;
567       StringId next_key = next == interned.end() ? kNullStringId : next->first;
568 
569       if (key != next_key && current_idx == 0) {
570         inserter->AddArg(key, Variadic::String(it->second));
571       } else {
572         constexpr size_t kMaxIndexSize = 20;
573         base::StringView key_str = context_->storage->GetString(key);
574         if (key_str.size() >= sizeof(buffer) - kMaxIndexSize) {
575           PERFETTO_DLOG("Ignoring arg with unreasonbly large size");
576           continue;
577         }
578 
579         base::StringWriter writer(buffer, sizeof(buffer));
580         writer.AppendString(key_str);
581         writer.AppendChar('[');
582         writer.AppendUnsignedInt(current_idx);
583         writer.AppendChar(']');
584 
585         StringId new_key =
586             context_->storage->InternString(writer.GetStringView());
587         inserter->AddArg(key, new_key, Variadic::String(it->second));
588 
589         current_idx = key == next_key ? current_idx + 1 : 0;
590       }
591     }
592   };
593 
594   if (event.has_event_id() || event.has_event_name()) {
595     if (event.has_event_id()) {
596       auto eid = event.event_id();
597       if (eid < metatrace::EVENTS_MAX) {
598         name_id = context_->storage->InternString(metatrace::kEventNames[eid]);
599       } else {
600         sprintf(fallback, "Event %d", eid);
601         name_id = context_->storage->InternString(fallback);
602       }
603     } else {
604       name_id = context_->storage->InternString(event.event_name());
605     }
606     TrackId track_id = context_->track_tracker->InternThreadTrack(utid);
607     context_->slice_tracker->Scoped(ts, track_id, cat_id, name_id,
608                                     event.event_duration_ns(), args_fn);
609   } else if (event.has_counter_id() || event.has_counter_name()) {
610     if (event.has_counter_id()) {
611       auto cid = event.counter_id();
612       if (cid < metatrace::COUNTERS_MAX) {
613         name_id =
614             context_->storage->InternString(metatrace::kCounterNames[cid]);
615       } else {
616         sprintf(fallback, "Counter %d", cid);
617         name_id = context_->storage->InternString(fallback);
618       }
619     } else {
620       name_id = context_->storage->InternString(event.counter_name());
621     }
622     TrackId track =
623         context_->track_tracker->InternThreadCounterTrack(name_id, utid);
624     auto opt_id =
625         context_->event_tracker->PushCounter(ts, event.counter_value(), track);
626     if (opt_id) {
627       auto inserter = context_->args_tracker->AddArgsTo(*opt_id);
628       args_fn(&inserter);
629     }
630   }
631 
632   if (event.has_overruns())
633     context_->storage->IncrementStats(stats::metatrace_overruns);
634 }
635 
ParseTraceConfig(ConstBytes blob)636 void ProtoTraceParser::ParseTraceConfig(ConstBytes blob) {
637   protos::pbzero::TraceConfig::Decoder trace_config(blob.data, blob.size);
638 
639   // TODO(eseckler): Propagate statuses from modules.
640   for (auto& module : context_->modules) {
641     module->ParseTraceConfig(trace_config);
642   }
643 
644   int64_t uuid_msb = trace_config.trace_uuid_msb();
645   int64_t uuid_lsb = trace_config.trace_uuid_lsb();
646   if (uuid_msb != 0 || uuid_lsb != 0) {
647     base::Uuid uuid(uuid_lsb, uuid_msb);
648     std::string str = uuid.ToPrettyString();
649     StringId id = context_->storage->InternString(base::StringView(str));
650     context_->metadata_tracker->SetMetadata(metadata::trace_uuid,
651                                             Variadic::String(id));
652   }
653 
654   if (trace_config.has_unique_session_name()) {
655     StringId id = context_->storage->InternString(
656         base::StringView(trace_config.unique_session_name()));
657     context_->metadata_tracker->SetMetadata(metadata::unique_session_name,
658                                             Variadic::String(id));
659   }
660 
661   DescriptorPool pool;
662   pool.AddFromFileDescriptorSet(kConfigDescriptor.data(),
663                                 kConfigDescriptor.size());
664 
665   std::string text = protozero_to_text::ProtozeroToText(
666       pool, ".perfetto.protos.TraceConfig", blob,
667       protozero_to_text::kIncludeNewLines);
668   StringId id = context_->storage->InternString(base::StringView(text));
669   context_->metadata_tracker->SetMetadata(metadata::trace_config_pbtxt,
670                                           Variadic::String(id));
671 }
672 
ParseModuleSymbols(ConstBytes blob)673 void ProtoTraceParser::ParseModuleSymbols(ConstBytes blob) {
674   protos::pbzero::ModuleSymbols::Decoder module_symbols(blob.data, blob.size);
675   StringId build_id;
676   // TODO(b/148109467): Remove workaround once all active Chrome versions
677   // write raw bytes instead of a string as build_id.
678   if (module_symbols.build_id().size == 33) {
679     build_id = context_->storage->InternString(module_symbols.build_id());
680   } else {
681     build_id = context_->storage->InternString(base::StringView(base::ToHex(
682         module_symbols.build_id().data, module_symbols.build_id().size)));
683   }
684 
685   auto mapping_ids = context_->global_stack_profile_tracker->FindMappingRow(
686       context_->storage->InternString(module_symbols.path()), build_id);
687   if (mapping_ids.empty()) {
688     context_->storage->IncrementStats(stats::stackprofile_invalid_mapping_id);
689     return;
690   }
691   for (auto addr_it = module_symbols.address_symbols(); addr_it; ++addr_it) {
692     protos::pbzero::AddressSymbols::Decoder address_symbols(*addr_it);
693 
694     uint32_t symbol_set_id = context_->storage->symbol_table().row_count();
695 
696     bool has_lines = false;
697     for (auto line_it = address_symbols.lines(); line_it; ++line_it) {
698       protos::pbzero::Line::Decoder line(*line_it);
699       context_->storage->mutable_symbol_table()->Insert(
700           {symbol_set_id, context_->storage->InternString(line.function_name()),
701            context_->storage->InternString(line.source_file_name()),
702            line.line_number()});
703       has_lines = true;
704     }
705     if (!has_lines) {
706       continue;
707     }
708     bool frame_found = false;
709     for (MappingId mapping_id : mapping_ids) {
710       std::vector<FrameId> frame_ids =
711           context_->global_stack_profile_tracker->FindFrameIds(
712               mapping_id, address_symbols.address());
713 
714       for (const FrameId frame_id : frame_ids) {
715         auto* frames = context_->storage->mutable_stack_profile_frame_table();
716         uint32_t frame_row = *frames->id().IndexOf(frame_id);
717         frames->mutable_symbol_set_id()->Set(frame_row, symbol_set_id);
718         frame_found = true;
719       }
720     }
721 
722     if (!frame_found) {
723       context_->storage->IncrementStats(stats::stackprofile_invalid_frame_id);
724       continue;
725     }
726   }
727 }
728 
ParseSmapsPacket(int64_t ts,ConstBytes blob)729 void ProtoTraceParser::ParseSmapsPacket(int64_t ts, ConstBytes blob) {
730   protos::pbzero::SmapsPacket::Decoder sp(blob.data, blob.size);
731   auto upid = context_->process_tracker->GetOrCreateProcess(sp.pid());
732 
733   for (auto it = sp.entries(); it; ++it) {
734     protos::pbzero::SmapsEntry::Decoder e(*it);
735     context_->storage->mutable_profiler_smaps_table()->Insert(
736         {upid, ts, context_->storage->InternString(e.path()),
737          static_cast<int64_t>(e.size_kb()),
738          static_cast<int64_t>(e.private_dirty_kb()),
739          static_cast<int64_t>(e.swap_kb()),
740          context_->storage->InternString(e.file_name()),
741          static_cast<int64_t>(e.start_address()),
742          static_cast<int64_t>(e.module_timestamp()),
743          context_->storage->InternString(e.module_debugid()),
744          context_->storage->InternString(e.module_debug_path()),
745          static_cast<int32_t>(e.protection_flags()),
746          static_cast<int64_t>(e.private_clean_resident_kb()),
747          static_cast<int64_t>(e.shared_dirty_resident_kb()),
748          static_cast<int64_t>(e.shared_clean_resident_kb()),
749          static_cast<int64_t>(e.locked_kb()),
750          static_cast<int64_t>(e.proportional_resident_kb())});
751   }
752 }
753 
754 }  // namespace trace_processor
755 }  // namespace perfetto
756