1 /*
2  * Copyright (C) 2017, The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "Collation.h"
18 
19 #include <stdio.h>
20 
21 #include <map>
22 
23 #include "frameworks/base/cmds/statsd/src/atoms.pb.h"
24 
25 namespace android {
26 namespace stats_log_api_gen {
27 
28 using google::protobuf::EnumDescriptor;
29 using google::protobuf::FieldDescriptor;
30 using google::protobuf::FileDescriptor;
31 using google::protobuf::SourceLocation;
32 using std::make_shared;
33 using std::map;
34 
35 const bool dbg = false;
36 
37 //
38 // AtomDecl class
39 //
40 
AtomDecl()41 AtomDecl::AtomDecl() : code(0), name() {
42 }
43 
AtomDecl(const AtomDecl & that)44 AtomDecl::AtomDecl(const AtomDecl& that)
45     : code(that.code),
46       name(that.name),
47       message(that.message),
48       fields(that.fields),
49       fieldNumberToAnnotations(that.fieldNumberToAnnotations),
50       primaryFields(that.primaryFields),
51       exclusiveField(that.exclusiveField),
52       defaultState(that.defaultState),
53       triggerStateReset(that.triggerStateReset),
54       nested(that.nested),
55       uidField(that.uidField) {
56 }
57 
AtomDecl(int c,const string & n,const string & m)58 AtomDecl::AtomDecl(int c, const string& n, const string& m) : code(c), name(n), message(m) {
59 }
60 
~AtomDecl()61 AtomDecl::~AtomDecl() {
62 }
63 
64 /**
65  * Print an error message for a FieldDescriptor, including the file name and
66  * line number.
67  */
print_error(const FieldDescriptor * field,const char * format,...)68 static void print_error(const FieldDescriptor* field, const char* format, ...) {
69     const Descriptor* message = field->containing_type();
70     const FileDescriptor* file = message->file();
71 
72     SourceLocation loc;
73     if (field->GetSourceLocation(&loc)) {
74         // TODO: this will work if we can figure out how to pass
75         // --include_source_info to protoc
76         fprintf(stderr, "%s:%d: ", file->name().c_str(), loc.start_line);
77     } else {
78         fprintf(stderr, "%s: ", file->name().c_str());
79     }
80     va_list args;
81     va_start(args, format);
82     vfprintf(stderr, format, args);
83     va_end(args);
84 }
85 
86 /**
87  * Convert a protobuf type into a java type.
88  */
java_type(const FieldDescriptor * field)89 static java_type_t java_type(const FieldDescriptor* field) {
90     int protoType = field->type();
91     switch (protoType) {
92         case FieldDescriptor::TYPE_DOUBLE:
93             return JAVA_TYPE_DOUBLE;
94         case FieldDescriptor::TYPE_FLOAT:
95             return JAVA_TYPE_FLOAT;
96         case FieldDescriptor::TYPE_INT64:
97             return JAVA_TYPE_LONG;
98         case FieldDescriptor::TYPE_UINT64:
99             return JAVA_TYPE_LONG;
100         case FieldDescriptor::TYPE_INT32:
101             return JAVA_TYPE_INT;
102         case FieldDescriptor::TYPE_FIXED64:
103             return JAVA_TYPE_LONG;
104         case FieldDescriptor::TYPE_FIXED32:
105             return JAVA_TYPE_INT;
106         case FieldDescriptor::TYPE_BOOL:
107             return JAVA_TYPE_BOOLEAN;
108         case FieldDescriptor::TYPE_STRING:
109             return JAVA_TYPE_STRING;
110         case FieldDescriptor::TYPE_GROUP:
111             return JAVA_TYPE_UNKNOWN;
112         case FieldDescriptor::TYPE_MESSAGE:
113             // TODO: not the final package name
114             if (field->message_type()->full_name() == "android.os.statsd.AttributionNode") {
115                 return JAVA_TYPE_ATTRIBUTION_CHAIN;
116             } else if (field->message_type()->full_name() == "android.os.statsd.KeyValuePair") {
117                 return JAVA_TYPE_KEY_VALUE_PAIR;
118             } else if (field->options().GetExtension(os::statsd::log_mode) ==
119                        os::statsd::LogMode::MODE_BYTES) {
120                 return JAVA_TYPE_BYTE_ARRAY;
121             } else {
122                 return JAVA_TYPE_OBJECT;
123             }
124         case FieldDescriptor::TYPE_BYTES:
125             return JAVA_TYPE_BYTE_ARRAY;
126         case FieldDescriptor::TYPE_UINT32:
127             return JAVA_TYPE_INT;
128         case FieldDescriptor::TYPE_ENUM:
129             return JAVA_TYPE_ENUM;
130         case FieldDescriptor::TYPE_SFIXED32:
131             return JAVA_TYPE_INT;
132         case FieldDescriptor::TYPE_SFIXED64:
133             return JAVA_TYPE_LONG;
134         case FieldDescriptor::TYPE_SINT32:
135             return JAVA_TYPE_INT;
136         case FieldDescriptor::TYPE_SINT64:
137             return JAVA_TYPE_LONG;
138         default:
139             return JAVA_TYPE_UNKNOWN;
140     }
141 }
142 
143 /**
144  * Gather the enums info.
145  */
collate_enums(const EnumDescriptor & enumDescriptor,AtomField * atomField)146 void collate_enums(const EnumDescriptor& enumDescriptor, AtomField* atomField) {
147     for (int i = 0; i < enumDescriptor.value_count(); i++) {
148         atomField->enumValues[enumDescriptor.value(i)->number()] =
149                 enumDescriptor.value(i)->name().c_str();
150     }
151 }
152 
addAnnotationToAtomDecl(AtomDecl * atomDecl,const int fieldNumber,const AnnotationId annotationId,const AnnotationType annotationType,const AnnotationValue annotationValue)153 static void addAnnotationToAtomDecl(AtomDecl* atomDecl, const int fieldNumber,
154                                     const AnnotationId annotationId,
155                                     const AnnotationType annotationType,
156                                     const AnnotationValue annotationValue) {
157     if (dbg) {
158         printf("   Adding annotation to %s: [%d] = {id: %d, type: %d}\n", atomDecl->name.c_str(),
159                fieldNumber, annotationId, annotationType);
160     }
161     atomDecl->fieldNumberToAnnotations[fieldNumber].insert(
162             make_shared<Annotation>(annotationId, atomDecl->code, annotationType, annotationValue));
163 }
164 
collate_field_annotations(AtomDecl * atomDecl,const FieldDescriptor * field,const int fieldNumber,const java_type_t & javaType)165 static int collate_field_annotations(AtomDecl* atomDecl, const FieldDescriptor* field,
166                                      const int fieldNumber, const java_type_t& javaType) {
167     int errorCount = 0;
168 
169     if (field->options().HasExtension(os::statsd::state_field_option)) {
170         const os::statsd::StateAtomFieldOption& stateFieldOption =
171                 field->options().GetExtension(os::statsd::state_field_option);
172         const bool primaryField = stateFieldOption.primary_field();
173         const bool exclusiveState = stateFieldOption.exclusive_state();
174         const bool primaryFieldFirstUid = stateFieldOption.primary_field_first_uid();
175 
176         // Check the field is only one of primaryField, exclusiveState, or primaryFieldFirstUid.
177         if (primaryField + primaryFieldFirstUid + exclusiveState > 1) {
178             print_error(field,
179                         "Field can be max 1 of primary_field, exclusive_state, "
180                         "or primary_field_first_uid: '%s'\n",
181                         atomDecl->message.c_str());
182             errorCount++;
183         }
184 
185         if (primaryField) {
186             if (javaType == JAVA_TYPE_UNKNOWN || javaType == JAVA_TYPE_ATTRIBUTION_CHAIN ||
187                 javaType == JAVA_TYPE_OBJECT || javaType == JAVA_TYPE_BYTE_ARRAY) {
188                 print_error(field, "Invalid primary state field: '%s'\n",
189                             atomDecl->message.c_str());
190                 errorCount++;
191             } else {
192                 atomDecl->primaryFields.push_back(fieldNumber);
193                 addAnnotationToAtomDecl(atomDecl, fieldNumber, ANNOTATION_ID_PRIMARY_FIELD,
194                                         ANNOTATION_TYPE_BOOL, AnnotationValue(true));
195             }
196         }
197 
198         if (primaryFieldFirstUid) {
199             if (javaType != JAVA_TYPE_ATTRIBUTION_CHAIN) {
200                 print_error(field,
201                             "PRIMARY_FIELD_FIRST_UID annotation is only for AttributionChains: "
202                             "'%s'\n",
203                             atomDecl->message.c_str());
204                 errorCount++;
205             } else {
206                 atomDecl->primaryFields.push_back(FIRST_UID_IN_CHAIN_ID);
207                 addAnnotationToAtomDecl(atomDecl, fieldNumber,
208                                         ANNOTATION_ID_PRIMARY_FIELD_FIRST_UID, ANNOTATION_TYPE_BOOL,
209                                         AnnotationValue(true));
210             }
211         }
212 
213         if (exclusiveState) {
214             if (javaType == JAVA_TYPE_UNKNOWN || javaType == JAVA_TYPE_ATTRIBUTION_CHAIN ||
215                 javaType == JAVA_TYPE_OBJECT || javaType == JAVA_TYPE_BYTE_ARRAY) {
216                 print_error(field, "Invalid exclusive state field: '%s'\n",
217                             atomDecl->message.c_str());
218                 errorCount++;
219             }
220 
221             if (atomDecl->exclusiveField != 0) {
222                 print_error(field,
223                             "Cannot have more than one exclusive state field in an "
224                             "atom: '%s'\n",
225                             atomDecl->message.c_str());
226                 errorCount++;
227             } else {
228                 atomDecl->exclusiveField = fieldNumber;
229                 addAnnotationToAtomDecl(atomDecl, fieldNumber, ANNOTATION_ID_EXCLUSIVE_STATE,
230                                         ANNOTATION_TYPE_BOOL, AnnotationValue(true));
231             }
232 
233             if (stateFieldOption.has_default_state_value()) {
234                 const int defaultState = stateFieldOption.default_state_value();
235                 atomDecl->defaultState = defaultState;
236 
237                 addAnnotationToAtomDecl(atomDecl, fieldNumber, ANNOTATION_ID_DEFAULT_STATE,
238                                         ANNOTATION_TYPE_INT, AnnotationValue(defaultState));
239             }
240 
241             if (stateFieldOption.has_trigger_state_reset_value()) {
242                 const int triggerStateReset = stateFieldOption.trigger_state_reset_value();
243 
244                 atomDecl->triggerStateReset = triggerStateReset;
245                 addAnnotationToAtomDecl(atomDecl, fieldNumber, ANNOTATION_ID_TRIGGER_STATE_RESET,
246                                         ANNOTATION_TYPE_INT, AnnotationValue(triggerStateReset));
247             }
248 
249             if (stateFieldOption.has_nested()) {
250                 const bool nested = stateFieldOption.nested();
251                 atomDecl->nested = nested;
252 
253                 addAnnotationToAtomDecl(atomDecl, fieldNumber, ANNOTATION_ID_STATE_NESTED,
254                                         ANNOTATION_TYPE_BOOL, AnnotationValue(nested));
255             }
256         }
257     }
258 
259     if (field->options().GetExtension(os::statsd::is_uid) == true) {
260         if (javaType != JAVA_TYPE_INT) {
261             print_error(field, "is_uid annotation can only be applied to int32 fields: '%s'\n",
262                         atomDecl->message.c_str());
263             errorCount++;
264         }
265 
266         if (atomDecl->uidField == 0) {
267             atomDecl->uidField = fieldNumber;
268 
269             addAnnotationToAtomDecl(atomDecl, fieldNumber, ANNOTATION_ID_IS_UID,
270                                     ANNOTATION_TYPE_BOOL, AnnotationValue(true));
271         } else {
272             print_error(field,
273                         "Cannot have more than one field in an atom with is_uid "
274                         "annotation: '%s'\n",
275                         atomDecl->message.c_str());
276             errorCount++;
277         }
278     }
279 
280     return errorCount;
281 }
282 
283 /**
284  * Gather the info about an atom proto.
285  */
collate_atom(const Descriptor * atom,AtomDecl * atomDecl,vector<java_type_t> * signature)286 int collate_atom(const Descriptor* atom, AtomDecl* atomDecl, vector<java_type_t>* signature) {
287     int errorCount = 0;
288 
289     // Build a sorted list of the fields. Descriptor has them in source file
290     // order.
291     map<int, const FieldDescriptor*> fields;
292     for (int j = 0; j < atom->field_count(); j++) {
293         const FieldDescriptor* field = atom->field(j);
294         fields[field->number()] = field;
295     }
296 
297     // Check that the parameters start at 1 and go up sequentially.
298     int expectedNumber = 1;
299     for (map<int, const FieldDescriptor*>::const_iterator it = fields.begin(); it != fields.end();
300          it++) {
301         const int number = it->first;
302         const FieldDescriptor* field = it->second;
303         if (number != expectedNumber) {
304             print_error(field,
305                         "Fields must be numbered consecutively starting at 1:"
306                         " '%s' is %d but should be %d\n",
307                         field->name().c_str(), number, expectedNumber);
308             errorCount++;
309             expectedNumber = number;
310             continue;
311         }
312         expectedNumber++;
313     }
314 
315     // Check that only allowed types are present. Remove any invalid ones.
316     for (map<int, const FieldDescriptor*>::const_iterator it = fields.begin(); it != fields.end();
317          it++) {
318         const FieldDescriptor* field = it->second;
319         bool isBinaryField = field->options().GetExtension(os::statsd::log_mode) ==
320                              os::statsd::LogMode::MODE_BYTES;
321 
322         java_type_t javaType = java_type(field);
323 
324         if (javaType == JAVA_TYPE_UNKNOWN) {
325             print_error(field, "Unknown type for field: %s\n", field->name().c_str());
326             errorCount++;
327             continue;
328         } else if (javaType == JAVA_TYPE_OBJECT && atomDecl->code < PULL_ATOM_START_ID) {
329             // Allow attribution chain, but only at position 1.
330             print_error(field, "Message type not allowed for field in pushed atoms: %s\n",
331                         field->name().c_str());
332             errorCount++;
333             continue;
334         } else if (javaType == JAVA_TYPE_BYTE_ARRAY && !isBinaryField) {
335             print_error(field, "Raw bytes type not allowed for field: %s\n", field->name().c_str());
336             errorCount++;
337             continue;
338         }
339 
340         if (isBinaryField && javaType != JAVA_TYPE_BYTE_ARRAY) {
341             print_error(field, "Cannot mark field %s as bytes.\n", field->name().c_str());
342             errorCount++;
343             continue;
344         }
345 
346         // Doubles are not supported yet.
347         if (javaType == JAVA_TYPE_DOUBLE) {
348             print_error(field,
349                         "Doubles are not supported in atoms. Please change field %s "
350                         "to float\n",
351                         field->name().c_str());
352             errorCount++;
353             continue;
354         }
355 
356         if (field->is_repeated() &&
357             !(javaType == JAVA_TYPE_ATTRIBUTION_CHAIN || javaType == JAVA_TYPE_KEY_VALUE_PAIR)) {
358             print_error(field,
359                         "Repeated fields are not supported in atoms. Please make "
360                         "field %s not "
361                         "repeated.\n",
362                         field->name().c_str());
363             errorCount++;
364             continue;
365         }
366     }
367 
368     // Check that if there's an attribution chain, it's at position 1.
369     for (map<int, const FieldDescriptor*>::const_iterator it = fields.begin(); it != fields.end();
370          it++) {
371         int number = it->first;
372         if (number != 1) {
373             const FieldDescriptor* field = it->second;
374             java_type_t javaType = java_type(field);
375             if (javaType == JAVA_TYPE_ATTRIBUTION_CHAIN) {
376                 print_error(field,
377                             "AttributionChain fields must have field id 1, in message: '%s'\n",
378                             atom->name().c_str());
379                 errorCount++;
380             }
381         }
382     }
383 
384     // Build the type signature and the atom data.
385     for (map<int, const FieldDescriptor*>::const_iterator it = fields.begin(); it != fields.end();
386          it++) {
387         const FieldDescriptor* field = it->second;
388         java_type_t javaType = java_type(field);
389         bool isBinaryField = field->options().GetExtension(os::statsd::log_mode) ==
390                              os::statsd::LogMode::MODE_BYTES;
391 
392         AtomField atField(field->name(), javaType);
393 
394         if (javaType == JAVA_TYPE_ENUM) {
395             // All enums are treated as ints when it comes to function signatures.
396             collate_enums(*field->enum_type(), &atField);
397         }
398 
399         // Generate signature for pushed atoms
400         if (atomDecl->code < PULL_ATOM_START_ID) {
401             if (javaType == JAVA_TYPE_ENUM) {
402                 // All enums are treated as ints when it comes to function signatures.
403                 signature->push_back(JAVA_TYPE_INT);
404             } else if (javaType == JAVA_TYPE_OBJECT && isBinaryField) {
405                 signature->push_back(JAVA_TYPE_BYTE_ARRAY);
406             } else {
407                 signature->push_back(javaType);
408             }
409         }
410 
411         atomDecl->fields.push_back(atField);
412 
413         errorCount += collate_field_annotations(atomDecl, field, it->first, javaType);
414     }
415 
416     return errorCount;
417 }
418 
419 // This function flattens the fields of the AttributionNode proto in an Atom
420 // proto and generates the corresponding atom decl and signature.
get_non_chained_node(const Descriptor * atom,AtomDecl * atomDecl,vector<java_type_t> * signature)421 bool get_non_chained_node(const Descriptor* atom, AtomDecl* atomDecl,
422                           vector<java_type_t>* signature) {
423     // Build a sorted list of the fields. Descriptor has them in source file
424     // order.
425     map<int, const FieldDescriptor*> fields;
426     for (int j = 0; j < atom->field_count(); j++) {
427         const FieldDescriptor* field = atom->field(j);
428         fields[field->number()] = field;
429     }
430 
431     AtomDecl attributionDecl;
432     vector<java_type_t> attributionSignature;
433     collate_atom(android::os::statsd::AttributionNode::descriptor(), &attributionDecl,
434                  &attributionSignature);
435 
436     // Build the type signature and the atom data.
437     bool has_attribution_node = false;
438     for (map<int, const FieldDescriptor*>::const_iterator it = fields.begin(); it != fields.end();
439          it++) {
440         const FieldDescriptor* field = it->second;
441         java_type_t javaType = java_type(field);
442         if (javaType == JAVA_TYPE_ATTRIBUTION_CHAIN) {
443             atomDecl->fields.insert(atomDecl->fields.end(), attributionDecl.fields.begin(),
444                                     attributionDecl.fields.end());
445             signature->insert(signature->end(), attributionSignature.begin(),
446                               attributionSignature.end());
447             has_attribution_node = true;
448 
449         } else {
450             AtomField atField(field->name(), javaType);
451             if (javaType == JAVA_TYPE_ENUM) {
452                 // All enums are treated as ints when it comes to function signatures.
453                 signature->push_back(JAVA_TYPE_INT);
454                 collate_enums(*field->enum_type(), &atField);
455             } else {
456                 signature->push_back(javaType);
457             }
458             atomDecl->fields.push_back(atField);
459         }
460     }
461     return has_attribution_node;
462 }
463 
populateFieldNumberToAtomDeclSet(const shared_ptr<AtomDecl> & atomDecl,FieldNumberToAtomDeclSet * fieldNumberToAtomDeclSet)464 static void populateFieldNumberToAtomDeclSet(const shared_ptr<AtomDecl>& atomDecl,
465                                              FieldNumberToAtomDeclSet* fieldNumberToAtomDeclSet) {
466     for (FieldNumberToAnnotations::const_iterator it = atomDecl->fieldNumberToAnnotations.begin();
467          it != atomDecl->fieldNumberToAnnotations.end(); it++) {
468         const int fieldNumber = it->first;
469         (*fieldNumberToAtomDeclSet)[fieldNumber].insert(atomDecl);
470     }
471 }
472 
473 /**
474  * Gather the info about the atoms.
475  */
collate_atoms(const Descriptor * descriptor,const string & moduleName,Atoms * atoms)476 int collate_atoms(const Descriptor* descriptor, const string& moduleName, Atoms* atoms) {
477     int errorCount = 0;
478 
479     for (int i = 0; i < descriptor->field_count(); i++) {
480         const FieldDescriptor* atomField = descriptor->field(i);
481 
482         if (moduleName != DEFAULT_MODULE_NAME) {
483             const int moduleCount = atomField->options().ExtensionSize(os::statsd::module);
484             int j;
485             for (j = 0; j < moduleCount; ++j) {
486                 const string atomModuleName =
487                         atomField->options().GetExtension(os::statsd::module, j);
488                 if (atomModuleName == moduleName) {
489                     break;
490                 }
491             }
492 
493             // This atom is not in the module we're interested in; skip it.
494             if (moduleCount == j) {
495                 if (dbg) {
496                     printf("   Skipping %s (%d)\n", atomField->name().c_str(), atomField->number());
497                 }
498                 continue;
499             }
500         }
501 
502         if (dbg) {
503             printf("   %s (%d)\n", atomField->name().c_str(), atomField->number());
504         }
505 
506         // StatsEvent only has one oneof, which contains only messages. Don't allow
507         // other types.
508         if (atomField->type() != FieldDescriptor::TYPE_MESSAGE) {
509             print_error(atomField,
510                         "Bad type for atom. StatsEvent can only have message type "
511                         "fields: %s\n",
512                         atomField->name().c_str());
513             errorCount++;
514             continue;
515         }
516 
517         const Descriptor* atom = atomField->message_type();
518         shared_ptr<AtomDecl> atomDecl =
519                 make_shared<AtomDecl>(atomField->number(), atomField->name(), atom->name());
520 
521         if (atomDecl->code < PULL_ATOM_START_ID &&
522             atomField->options().GetExtension(os::statsd::truncate_timestamp)) {
523             addAnnotationToAtomDecl(atomDecl.get(), ATOM_ID_FIELD_NUMBER,
524                                     ANNOTATION_ID_TRUNCATE_TIMESTAMP, ANNOTATION_TYPE_BOOL,
525                                     AnnotationValue(true));
526             if (dbg) {
527                 printf("%s can have timestamp truncated\n", atomField->name().c_str());
528             }
529         }
530 
531         vector<java_type_t> signature;
532         errorCount += collate_atom(atom, atomDecl.get(), &signature);
533         if (atomDecl->primaryFields.size() != 0 && atomDecl->exclusiveField == 0) {
534             print_error(atomField, "Cannot have a primary field without an exclusive field: %s\n",
535                         atomField->name().c_str());
536             errorCount++;
537             continue;
538         }
539 
540         FieldNumberToAtomDeclSet& fieldNumberToAtomDeclSet = atoms->signatureInfoMap[signature];
541         populateFieldNumberToAtomDeclSet(atomDecl, &fieldNumberToAtomDeclSet);
542 
543         atoms->decls.insert(atomDecl);
544 
545         shared_ptr<AtomDecl> nonChainedAtomDecl =
546                 make_shared<AtomDecl>(atomField->number(), atomField->name(), atom->name());
547         vector<java_type_t> nonChainedSignature;
548         if (get_non_chained_node(atom, nonChainedAtomDecl.get(), &nonChainedSignature)) {
549             FieldNumberToAtomDeclSet& nonChainedFieldNumberToAtomDeclSet =
550                     atoms->nonChainedSignatureInfoMap[nonChainedSignature];
551             populateFieldNumberToAtomDeclSet(nonChainedAtomDecl,
552                                              &nonChainedFieldNumberToAtomDeclSet);
553 
554             atoms->non_chained_decls.insert(nonChainedAtomDecl);
555         }
556     }
557 
558     if (dbg) {
559         printf("signatures = [\n");
560         for (SignatureInfoMap::const_iterator it = atoms->signatureInfoMap.begin();
561              it != atoms->signatureInfoMap.end(); it++) {
562             printf("   ");
563             for (vector<java_type_t>::const_iterator jt = it->first.begin(); jt != it->first.end();
564                  jt++) {
565                 printf(" %d", (int)*jt);
566             }
567             printf("\n");
568         }
569         printf("]\n");
570     }
571 
572     return errorCount;
573 }
574 
575 }  // namespace stats_log_api_gen
576 }  // namespace android
577