1 /*
2 * Copyright (C) 2021 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "perfetto/ext/base/file_utils.h"
18 #include "perfetto/ext/base/getopt.h"
19 #include "perfetto/ext/base/scoped_file.h"
20 #include "perfetto/ext/base/string_utils.h"
21 #include "perfetto/ext/base/version.h"
22 #include "src/protozero/filtering/filter_util.h"
23 #include "src/protozero/filtering/message_filter.h"
24
25 namespace perfetto {
26 namespace proto_filter {
27 namespace {
28
29 const char kUsage[] =
30 R"(Usage: proto_filter [-s schema_in] [-i message in] [-o message out] [-f filter in] [-F filter out] [-T filter_oct_out] [-d --dedupe] [-I proto include path] [-r root message]
31
32 -s --schema-in: Path to the root .proto file. Required for most operations
33 -I --proto_path: Extra include directory for proto includes. If omitted assumed CWD.
34 -r --root_message: Fully qualified name for the root proto message (e.g. perfetto.protos.Trace)
35 If omitted the first message defined in the schema will be used.
36 -i --msg_in: Path of a binary-encoded proto message which will be filtered.
37 -o --msg_out: Path of the binary-encoded filtered proto message written in output.
38 -f --filter_in: Path of a filter bytecode file previously generated by this tool.
39 -F --filter_out: Path of the filter bytecode file generated from the --schema-in definition.
40 -T --filter_oct_out: Like --filter_out, but emits a octal-escaped C string suitable for .pbtx.
41 -d --dedupe: Minimize filter size by deduping leaf messages with same field ids.
42
43 Example usage:
44
45 # Convert a .proto schema file into a diff-friendly list of messages/fields>
46
47 proto_filter -r perfetto.protos.Trace -s protos/perfetto/trace/trace.proto
48
49 # Generate the filter bytecode from a .proto schema
50
51 proto_filter -r perfetto.protos.Trace -s protos/perfetto/trace/trace.proto \
52 -F /tmp/bytecode [--dedupe]
53
54 # List the used/filtered fields from a trace file
55
56 proto_filter -r perfetto.protos.Trace -s protos/perfetto/trace/trace.proto \
57 -i test/data/example_android_trace_30s.pb -f /tmp/bytecode
58
59 # Filter a trace using a filter bytecode
60
61 proto_filter -i test/data/example_android_trace_30s.pb -f /tmp/bytecode \
62 -o /tmp/filtered_trace
63 )";
64
Main(int argc,char ** argv)65 int Main(int argc, char** argv) {
66 static const option long_options[] = {
67 {"help", no_argument, nullptr, 'h'},
68 {"version", no_argument, nullptr, 'v'},
69 {"dedupe", no_argument, nullptr, 'd'},
70 {"proto_path", no_argument, nullptr, 'I'},
71 {"schema_in", no_argument, nullptr, 's'},
72 {"root_message", no_argument, nullptr, 'r'},
73 {"msg_in", no_argument, nullptr, 'i'},
74 {"msg_out", no_argument, nullptr, 'o'},
75 {"filter_in", no_argument, nullptr, 'f'},
76 {"filter_out", no_argument, nullptr, 'F'},
77 {"filter_oct_out", no_argument, nullptr, 'T'},
78 {nullptr, 0, nullptr, 0}};
79
80 std::string msg_in;
81 std::string msg_out;
82 std::string filter_in;
83 std::string schema_in;
84 std::string filter_out;
85 std::string filter_oct_out;
86 std::string proto_path;
87 std::string root_message_arg;
88 bool dedupe = false;
89
90 for (;;) {
91 int option =
92 getopt_long(argc, argv, "hvdI:s:r:i:o:f:F:T:", long_options, nullptr);
93
94 if (option == -1)
95 break; // EOF.
96
97 if (option == 'v') {
98 printf("%s\n", base::GetVersionString());
99 exit(0);
100 }
101
102 if (option == 'd') {
103 dedupe = true;
104 continue;
105 }
106
107 if (option == 'I') {
108 proto_path = optarg;
109 continue;
110 }
111
112 if (option == 's') {
113 schema_in = optarg;
114 continue;
115 }
116
117 if (option == 'r') {
118 root_message_arg = optarg;
119 continue;
120 }
121
122 if (option == 'i') {
123 msg_in = optarg;
124 continue;
125 }
126
127 if (option == 'o') {
128 msg_out = optarg;
129 continue;
130 }
131
132 if (option == 'f') {
133 filter_in = optarg;
134 continue;
135 }
136
137 if (option == 'F') {
138 filter_out = optarg;
139 continue;
140 }
141
142 if (option == 'T') {
143 filter_oct_out = optarg;
144 continue;
145 }
146
147 if (option == 'h') {
148 fprintf(stdout, kUsage);
149 exit(0);
150 }
151
152 fprintf(stderr, kUsage);
153 exit(1);
154 }
155
156 if (msg_in.empty() && filter_in.empty() && schema_in.empty()) {
157 fprintf(stderr, kUsage);
158 return 1;
159 }
160
161 std::string msg_in_data;
162 if (!msg_in.empty()) {
163 PERFETTO_LOG("Loading proto-encoded message from %s", msg_in.c_str());
164 if (!base::ReadFile(msg_in, &msg_in_data)) {
165 PERFETTO_ELOG("Could not open message file %s", msg_in.c_str());
166 return 1;
167 }
168 }
169
170 protozero::FilterUtil filter;
171 if (!schema_in.empty()) {
172 PERFETTO_LOG("Loading proto schema from %s", schema_in.c_str());
173 if (!filter.LoadMessageDefinition(schema_in, root_message_arg,
174 proto_path)) {
175 PERFETTO_ELOG("Failed to parse proto schema from %s", schema_in.c_str());
176 return 1;
177 }
178 if (dedupe)
179 filter.Dedupe();
180 }
181
182 protozero::MessageFilter msg_filter;
183 std::string filter_data;
184 std::string filter_data_src;
185 if (!filter_in.empty()) {
186 PERFETTO_LOG("Loading filter bytecode from %s", filter_in.c_str());
187 if (!base::ReadFile(filter_in, &filter_data)) {
188 PERFETTO_ELOG("Could not open filter file %s", filter_in.c_str());
189 return 1;
190 }
191 filter_data_src = filter_in;
192 } else if (!schema_in.empty()) {
193 PERFETTO_LOG("Generating filter bytecode from %s", schema_in.c_str());
194 filter_data = filter.GenerateFilterBytecode();
195 filter_data_src = schema_in;
196 }
197
198 if (!filter_data.empty()) {
199 const uint8_t* data = reinterpret_cast<const uint8_t*>(filter_data.data());
200 if (!msg_filter.LoadFilterBytecode(data, filter_data.size())) {
201 PERFETTO_ELOG("Failed to parse filter bytecode from %s",
202 filter_data_src.c_str());
203 return 1;
204 }
205 }
206
207 // Write the filter bytecode in output.
208 if (!filter_out.empty()) {
209 auto fd = base::OpenFile(filter_out, O_WRONLY | O_TRUNC | O_CREAT, 0644);
210 if (!fd) {
211 PERFETTO_ELOG("Could not open filter out path %s", filter_out.c_str());
212 return 1;
213 }
214 PERFETTO_LOG("Writing filter bytecode (%zu bytes) into %s",
215 filter_data.size(), filter_out.c_str());
216 base::WriteAll(*fd, filter_data.data(), filter_data.size());
217 }
218
219 if (!filter_oct_out.empty()) {
220 auto fd =
221 base::OpenFile(filter_oct_out, O_WRONLY | O_TRUNC | O_CREAT, 0644);
222 if (!fd) {
223 PERFETTO_ELOG("Could not open filter out path %s",
224 filter_oct_out.c_str());
225 return 1;
226 }
227 std::string oct_str;
228 oct_str.reserve(filter_data.size() * 4 + 64);
229 oct_str.append("trace_filter{\n bytecode: \"");
230 for (char c : filter_data) {
231 uint8_t octect = static_cast<uint8_t>(c);
232 char buf[5]{'\\', '0', '0', '0', 0};
233 for (uint8_t i = 0; i < 3; ++i) {
234 buf[3 - i] = static_cast<char>('0' + static_cast<uint8_t>(octect) % 8);
235 octect /= 8;
236 }
237 oct_str.append(buf);
238 }
239 oct_str.append("\"\n}\n");
240 PERFETTO_LOG("Writing filter bytecode (%zu bytes) into %s", oct_str.size(),
241 filter_oct_out.c_str());
242 base::WriteAll(*fd, oct_str.data(), oct_str.size());
243 }
244
245 // Apply the filter to the input message (if any).
246 std::vector<uint8_t> msg_filtered_data;
247 if (!msg_in.empty()) {
248 PERFETTO_LOG("Applying filter %s to proto message %s",
249 filter_data_src.c_str(), msg_in.c_str());
250 msg_filter.enable_field_usage_tracking(true);
251 auto res = msg_filter.FilterMessage(msg_in_data.data(), msg_in_data.size());
252 if (res.error)
253 PERFETTO_FATAL("Filtering failed");
254 msg_filtered_data.insert(msg_filtered_data.end(), res.data.get(),
255 res.data.get() + res.size);
256 }
257
258 // Write out the filtered message.
259 if (!msg_out.empty()) {
260 PERFETTO_LOG("Writing filtered proto bytes (%zu bytes) into %s",
261 msg_filtered_data.size(), msg_out.c_str());
262 auto fd = base::OpenFile(msg_out, O_WRONLY | O_CREAT, 0644);
263 base::WriteAll(*fd, msg_filtered_data.data(), msg_filtered_data.size());
264 }
265
266 if (!msg_in.empty()) {
267 const auto& field_usage_map = msg_filter.field_usage();
268 for (const auto& it : field_usage_map) {
269 const std::string& field_path_varint = it.first;
270 int32_t num_occurrences = it.second;
271 std::string path_str = filter.LookupField(field_path_varint);
272 printf("%-100s %s %d\n", path_str.c_str(),
273 num_occurrences < 0 ? "DROP" : "PASS", std::abs(num_occurrences));
274 }
275 } else if (!schema_in.empty()) {
276 filter.PrintAsText();
277 }
278
279 if ((!filter_out.empty() || !filter_oct_out.empty()) && !dedupe) {
280 PERFETTO_ELOG(
281 "Warning: looks like you are generating a filter without --dedupe. For "
282 "production use cases, --dedupe can make the output bytecode "
283 "significantly smaller.");
284 }
285 return 0;
286 }
287
288 } // namespace
289 } // namespace proto_filter
290 } // namespace perfetto
291
main(int argc,char ** argv)292 int main(int argc, char** argv) {
293 return perfetto::proto_filter::Main(argc, argv);
294 }
295