1 /*
2 * Copyright (C) 2021 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/util/proto_to_args_parser.h"
18
19 #include "perfetto/ext/base/string_view.h"
20 #include "perfetto/protozero/scattered_heap_buffer.h"
21 #include "protos/perfetto/common/descriptor.pbzero.h"
22 #include "protos/perfetto/trace/track_event/source_location.pbzero.h"
23 #include "src/protozero/test/example_proto/test_messages.pbzero.h"
24 #include "src/trace_processor/importers/common/trace_blob_view.h"
25 #include "src/trace_processor/importers/proto/packet_sequence_state.h"
26 #include "src/trace_processor/test_messages.descriptor.h"
27 #include "test/gtest_and_gmock.h"
28
29 #include <sstream>
30
31 namespace perfetto {
32 namespace trace_processor {
33 namespace util {
34 namespace {
35
36 constexpr size_t kChunkSize = 42;
37
38 using ::testing::_;
39 using ::testing::Eq;
40 using ::testing::Invoke;
41 using ::testing::NiceMock;
42
43 class ProtoToArgsParserTest : public ::testing::Test,
44 public ProtoToArgsParser::Delegate {
45 protected:
ProtoToArgsParserTest()46 ProtoToArgsParserTest() {}
47
args() const48 const std::vector<std::string>& args() const { return args_; }
49
AddInternedSourceLocation(uint64_t iid,TraceBlobView data)50 void AddInternedSourceLocation(uint64_t iid, TraceBlobView data) {
51 interned_source_locations_[iid] = std::unique_ptr<InternedMessageView>(
52 new InternedMessageView(std::move(data)));
53 }
54
55 private:
56 using Key = ProtoToArgsParser::Key;
57
AddInteger(const Key & key,int64_t value)58 void AddInteger(const Key& key, int64_t value) override {
59 std::stringstream ss;
60 ss << key.flat_key << " " << key.key << " " << value;
61 args_.push_back(ss.str());
62 }
63
AddUnsignedInteger(const Key & key,uint64_t value)64 void AddUnsignedInteger(const Key& key, uint64_t value) override {
65 std::stringstream ss;
66 ss << key.flat_key << " " << key.key << " " << value;
67 args_.push_back(ss.str());
68 }
69
AddString(const Key & key,const protozero::ConstChars & value)70 void AddString(const Key& key, const protozero::ConstChars& value) override {
71 std::stringstream ss;
72 ss << key.flat_key << " " << key.key << " " << value.ToStdString();
73 args_.push_back(ss.str());
74 }
75
AddDouble(const Key & key,double value)76 void AddDouble(const Key& key, double value) override {
77 std::stringstream ss;
78 ss << key.flat_key << " " << key.key << " " << value;
79 args_.push_back(ss.str());
80 }
81
AddPointer(const Key & key,const void * value)82 void AddPointer(const Key& key, const void* value) override {
83 std::stringstream ss;
84 ss << key.flat_key << " " << key.key << " " << std::hex
85 << reinterpret_cast<uintptr_t>(value) << std::dec;
86 args_.push_back(ss.str());
87 }
88
AddBoolean(const Key & key,bool value)89 void AddBoolean(const Key& key, bool value) override {
90 std::stringstream ss;
91 ss << key.flat_key << " " << key.key << " " << (value ? "true" : "false");
92 args_.push_back(ss.str());
93 }
94
AddJson(const Key & key,const protozero::ConstChars & value)95 void AddJson(const Key& key, const protozero::ConstChars& value) override {
96 std::stringstream ss;
97 ss << key.flat_key << " " << key.key << " " << std::hex
98 << value.ToStdString() << std::dec;
99 args_.push_back(ss.str());
100 }
101
GetInternedMessageView(uint32_t field_id,uint64_t iid)102 InternedMessageView* GetInternedMessageView(uint32_t field_id,
103 uint64_t iid) override {
104 if (field_id != protos::pbzero::InternedData::kSourceLocationsFieldNumber)
105 return nullptr;
106 return interned_source_locations_.at(iid).get();
107 }
108
109 std::vector<std::string> args_;
110 std::map<uint64_t, std::unique_ptr<InternedMessageView>>
111 interned_source_locations_;
112 };
113
TEST_F(ProtoToArgsParserTest,EnsureTestMessageProtoParses)114 TEST_F(ProtoToArgsParserTest, EnsureTestMessageProtoParses) {
115 DescriptorPool pool;
116 auto status = pool.AddFromFileDescriptorSet(kTestMessagesDescriptor.data(),
117 kTestMessagesDescriptor.size());
118 ProtoToArgsParser parser(pool);
119 EXPECT_TRUE(status.ok()) << "Failed to parse kTestMessagesDescriptor: "
120 << status.message();
121 }
122
TEST_F(ProtoToArgsParserTest,BasicSingleLayerProto)123 TEST_F(ProtoToArgsParserTest, BasicSingleLayerProto) {
124 using namespace protozero::test::protos::pbzero;
125 protozero::HeapBuffered<EveryField> msg{kChunkSize, kChunkSize};
126 msg->set_field_int32(-1);
127 msg->set_field_int64(-333123456789ll);
128 msg->set_field_uint32(600);
129 msg->set_field_uint64(333123456789ll);
130 msg->set_field_sint32(-5);
131 msg->set_field_sint64(-9000);
132 msg->set_field_fixed32(12345);
133 msg->set_field_fixed64(444123450000ll);
134 msg->set_field_sfixed32(-69999);
135 msg->set_field_sfixed64(-200);
136 msg->set_field_double(0.5555);
137 msg->set_field_bool(true);
138 msg->set_small_enum(SmallEnum::TO_BE);
139 msg->set_signed_enum(SignedEnum::NEGATIVE);
140 msg->set_big_enum(BigEnum::BEGIN);
141 msg->set_nested_enum(EveryField::PONG);
142 msg->set_field_float(3.14f);
143 msg->set_field_string("FizzBuzz");
144 msg->add_repeated_int32(1);
145 msg->add_repeated_int32(-1);
146 msg->add_repeated_int32(100);
147 msg->add_repeated_int32(2000000);
148
149 auto binary_proto = msg.SerializeAsArray();
150
151 DescriptorPool pool;
152 auto status = pool.AddFromFileDescriptorSet(kTestMessagesDescriptor.data(),
153 kTestMessagesDescriptor.size());
154 ProtoToArgsParser parser(pool);
155 ASSERT_TRUE(status.ok()) << "Failed to parse kTestMessagesDescriptor: "
156 << status.message();
157
158 status = parser.ParseMessage(
159 protozero::ConstBytes{binary_proto.data(), binary_proto.size()},
160 ".protozero.test.protos.EveryField", nullptr, *this);
161
162 EXPECT_TRUE(status.ok())
163 << "InternProtoFieldsIntoArgsTable failed with error: "
164 << status.message();
165
166 EXPECT_THAT(
167 args(),
168 testing::ElementsAre(
169 "field_int32 field_int32 -1", "field_int64 field_int64 -333123456789",
170 "field_uint32 field_uint32 600",
171 "field_uint64 field_uint64 333123456789",
172 "field_sint32 field_sint32 -5", "field_sint64 field_sint64 -9000",
173 "field_fixed32 field_fixed32 12345",
174 "field_fixed64 field_fixed64 444123450000",
175 "field_sfixed32 field_sfixed32 -69999",
176 "field_sfixed64 field_sfixed64 -200",
177 "field_double field_double 0.5555", "field_bool field_bool true",
178 "small_enum small_enum TO_BE", "signed_enum signed_enum NEGATIVE",
179 "big_enum big_enum BEGIN", "nested_enum nested_enum PONG",
180 "field_float field_float 3.14", "field_string field_string FizzBuzz",
181 "repeated_int32 repeated_int32[0] 1",
182 "repeated_int32 repeated_int32[1] -1",
183 "repeated_int32 repeated_int32[2] 100",
184 "repeated_int32 repeated_int32[3] 2000000"));
185 }
186
TEST_F(ProtoToArgsParserTest,NestedProto)187 TEST_F(ProtoToArgsParserTest, NestedProto) {
188 using namespace protozero::test::protos::pbzero;
189 protozero::HeapBuffered<NestedA> msg{kChunkSize, kChunkSize};
190 msg->set_super_nested()->set_value_c(3);
191
192 auto binary_proto = msg.SerializeAsArray();
193
194 DescriptorPool pool;
195 auto status = pool.AddFromFileDescriptorSet(kTestMessagesDescriptor.data(),
196 kTestMessagesDescriptor.size());
197 ProtoToArgsParser parser(pool);
198 ASSERT_TRUE(status.ok()) << "Failed to parse kTestMessagesDescriptor: "
199 << status.message();
200
201 status = parser.ParseMessage(
202 protozero::ConstBytes{binary_proto.data(), binary_proto.size()},
203 ".protozero.test.protos.NestedA", nullptr, *this);
204 EXPECT_TRUE(status.ok())
205 << "InternProtoFieldsIntoArgsTable failed with error: "
206 << status.message();
207 EXPECT_THAT(args(), testing::ElementsAre(
208 "super_nested.value_c super_nested.value_c 3"));
209 }
210
TEST_F(ProtoToArgsParserTest,CamelCaseFieldsProto)211 TEST_F(ProtoToArgsParserTest, CamelCaseFieldsProto) {
212 using namespace protozero::test::protos::pbzero;
213 protozero::HeapBuffered<CamelCaseFields> msg{kChunkSize, kChunkSize};
214 msg->set_barbaz(true);
215 msg->set_moomoo(true);
216 msg->set___bigbang(true);
217
218 auto binary_proto = msg.SerializeAsArray();
219
220 DescriptorPool pool;
221 auto status = pool.AddFromFileDescriptorSet(kTestMessagesDescriptor.data(),
222 kTestMessagesDescriptor.size());
223 ProtoToArgsParser parser(pool);
224 ASSERT_TRUE(status.ok()) << "Failed to parse kTestMessagesDescriptor: "
225 << status.message();
226
227 status = parser.ParseMessage(
228 protozero::ConstBytes{binary_proto.data(), binary_proto.size()},
229 ".protozero.test.protos.CamelCaseFields", nullptr, *this);
230 EXPECT_TRUE(status.ok())
231 << "InternProtoFieldsIntoArgsTable failed with error: "
232 << status.message();
233 EXPECT_THAT(args(),
234 testing::ElementsAre("barBaz barBaz true", "MooMoo MooMoo true",
235 "__bigBang __bigBang true"));
236 }
237
TEST_F(ProtoToArgsParserTest,NestedProtoParsingOverrideHandled)238 TEST_F(ProtoToArgsParserTest, NestedProtoParsingOverrideHandled) {
239 using namespace protozero::test::protos::pbzero;
240 protozero::HeapBuffered<NestedA> msg{kChunkSize, kChunkSize};
241 msg->set_super_nested()->set_value_c(3);
242
243 auto binary_proto = msg.SerializeAsArray();
244
245 DescriptorPool pool;
246 auto status = pool.AddFromFileDescriptorSet(kTestMessagesDescriptor.data(),
247 kTestMessagesDescriptor.size());
248 ProtoToArgsParser parser(pool);
249 ASSERT_TRUE(status.ok()) << "Failed to parse kTestMessagesDescriptor: "
250 << status.message();
251
252 parser.AddParsingOverride(
253 "super_nested.value_c",
254 [](const protozero::Field& field, ProtoToArgsParser::Delegate& writer) {
255 EXPECT_EQ(field.type(), protozero::proto_utils::ProtoWireType::kVarInt);
256 std::string key = "super_nested.value_b.replaced";
257 writer.AddInteger({key, key}, field.as_int32());
258 // We've handled this field by adding the desired args.
259 return base::OkStatus();
260 });
261
262 status = parser.ParseMessage(
263 protozero::ConstBytes{binary_proto.data(), binary_proto.size()},
264 ".protozero.test.protos.NestedA", nullptr, *this);
265 EXPECT_TRUE(status.ok())
266 << "InternProtoFieldsIntoArgsTable failed with error: "
267 << status.message();
268 EXPECT_THAT(
269 args(),
270 testing::ElementsAre(
271 "super_nested.value_b.replaced super_nested.value_b.replaced 3"));
272 }
273
TEST_F(ProtoToArgsParserTest,NestedProtoParsingOverrideSkipped)274 TEST_F(ProtoToArgsParserTest, NestedProtoParsingOverrideSkipped) {
275 using namespace protozero::test::protos::pbzero;
276 protozero::HeapBuffered<NestedA> msg{kChunkSize, kChunkSize};
277 msg->set_super_nested()->set_value_c(3);
278
279 auto binary_proto = msg.SerializeAsArray();
280
281 DescriptorPool pool;
282 auto status = pool.AddFromFileDescriptorSet(kTestMessagesDescriptor.data(),
283 kTestMessagesDescriptor.size());
284 ProtoToArgsParser parser(pool);
285 ASSERT_TRUE(status.ok()) << "Failed to parse kTestMessagesDescriptor: "
286 << status.message();
287
288 parser.AddParsingOverride(
289 "super_nested.value_c",
290 [](const protozero::Field& field, ProtoToArgsParser::Delegate&) {
291 static int val = 0;
292 ++val;
293 EXPECT_EQ(1, val);
294 EXPECT_EQ(field.type(), protozero::proto_utils::ProtoWireType::kVarInt);
295 return base::nullopt;
296 });
297
298 status = parser.ParseMessage(
299 protozero::ConstBytes{binary_proto.data(), binary_proto.size()},
300 ".protozero.test.protos.NestedA", nullptr, *this);
301 EXPECT_TRUE(status.ok())
302 << "InternProtoFieldsIntoArgsTable failed with error: "
303 << status.message();
304 EXPECT_THAT(args(), testing::ElementsAre(
305 "super_nested.value_c super_nested.value_c 3"));
306 }
307
TEST_F(ProtoToArgsParserTest,LookingUpInternedStateParsingOverride)308 TEST_F(ProtoToArgsParserTest, LookingUpInternedStateParsingOverride) {
309 using namespace protozero::test::protos::pbzero;
310 // The test proto, we will use |value_c| as the source_location iid.
311 protozero::HeapBuffered<NestedA> msg{kChunkSize, kChunkSize};
312 msg->set_super_nested()->set_value_c(3);
313 auto binary_proto = msg.SerializeAsArray();
314
315 // The interned source location.
316 protozero::HeapBuffered<protos::pbzero::SourceLocation> src_loc{kChunkSize,
317 kChunkSize};
318 const uint64_t kIid = 3;
319 src_loc->set_iid(kIid);
320 src_loc->set_file_name("test_file_name");
321 // We need to update sequence_state to point to it.
322 auto binary_data = src_loc.SerializeAsArray();
323 std::unique_ptr<uint8_t[]> buffer(new uint8_t[binary_data.size()]);
324 for (size_t i = 0; i < binary_data.size(); ++i) {
325 buffer.get()[i] = binary_data[i];
326 }
327 TraceBlobView blob(std::move(buffer), 0, binary_data.size());
328 AddInternedSourceLocation(kIid, std::move(blob));
329
330 DescriptorPool pool;
331 auto status = pool.AddFromFileDescriptorSet(kTestMessagesDescriptor.data(),
332 kTestMessagesDescriptor.size());
333 ASSERT_TRUE(status.ok()) << "Failed to parse kTestMessagesDescriptor: "
334 << status.message();
335
336 ProtoToArgsParser parser(pool);
337 // Now we override the behaviour of |value_c| so we can expand the iid into
338 // multiple args rows.
339 parser.AddParsingOverride(
340 "super_nested.value_c",
341 [](const protozero::Field& field, ProtoToArgsParser::Delegate& delegate)
342 -> base::Optional<base::Status> {
343 auto* decoder = delegate.GetInternedMessage(
344 protos::pbzero::InternedData::kSourceLocations, field.as_uint64());
345 if (!decoder) {
346 // Lookup failed fall back on default behaviour.
347 return base::nullopt;
348 }
349 delegate.AddString(ProtoToArgsParser::Key("file_name"),
350 protozero::ConstChars{"file", 4});
351 delegate.AddInteger(ProtoToArgsParser::Key("line_number"), 2);
352 return base::OkStatus();
353 });
354
355 status = parser.ParseMessage(
356 protozero::ConstBytes{binary_proto.data(), binary_proto.size()},
357 ".protozero.test.protos.NestedA", nullptr, *this);
358 EXPECT_TRUE(status.ok())
359 << "InternProtoFieldsIntoArgsTable failed with error: "
360 << status.message();
361 EXPECT_THAT(args(), testing::ElementsAre("file_name file_name file",
362 "line_number line_number 2"));
363 }
364
365 } // namespace
366 } // namespace util
367 } // namespace trace_processor
368 } // namespace perfetto
369