1 // Copyright (C) 2021 The Android Open Source Project
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //      http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include <gtest/gtest.h>
16 
17 #include "src/StatsLogProcessor.h"
18 #include "tests/statsd_test_util.h"
19 
20 namespace android {
21 namespace os {
22 namespace statsd {
23 
24 #ifdef __ANDROID__
25 
26 using namespace std;
27 
28 namespace {
29 
CreateTestAtomReportedEvent(const uint64_t timestampNs,const long longField,const string & stringField)30 unique_ptr<LogEvent> CreateTestAtomReportedEvent(const uint64_t timestampNs, const long longField,
31                                                  const string& stringField) {
32     return CreateTestAtomReportedEvent(
33             timestampNs, /* attributionUids */ {1001},
34             /* attributionTags */ {"app1"}, /* intField */ 0, longField, /* floatField */ 0.0f,
35             stringField, /* boolField */ false, TestAtomReported::OFF, /* bytesField */ {},
36             /* repeatedIntField */ {}, /* repeatedLongField */ {}, /* repeatedFloatField */ {},
37             /* repeatedStringField */ {}, /* repeatedBoolField */ {},
38             /* repeatedBoolFieldLength */ 0, /* repeatedEnumField */ {});
39 }
40 
41 }  // anonymous namespace.
42 
43 class KllMetricE2eTest : public ::testing::Test {
44 protected:
SetUp()45     void SetUp() override {
46         key = ConfigKey(123, 987);
47         bucketStartTimeNs = getElapsedRealtimeNs();
48         bucketSizeNs = TimeUnitToBucketSizeInMillis(TEN_MINUTES) * 1000000LL;
49         whatMatcher = CreateScreenBrightnessChangedAtomMatcher();
50         metric = createKllMetric("ScreenBrightness", whatMatcher, /*valueField=*/1,
51                                  /*condition=*/nullopt);
52 
53         *config.add_atom_matcher() = whatMatcher;
54         *config.add_kll_metric() = metric;
55 
56         events.push_back(CreateScreenBrightnessChangedEvent(bucketStartTimeNs + 5 * NS_PER_SEC, 5));
57         events.push_back(
58                 CreateScreenBrightnessChangedEvent(bucketStartTimeNs + 15 * NS_PER_SEC, 15));
59         events.push_back(
60                 CreateScreenBrightnessChangedEvent(bucketStartTimeNs + 25 * NS_PER_SEC, 40));
61     }
62 
63     ConfigKey key;
64     uint64_t bucketStartTimeNs;
65     uint64_t bucketSizeNs;
66     AtomMatcher whatMatcher;
67     KllMetric metric;
68     StatsdConfig config;
69     vector<unique_ptr<LogEvent>> events;
70 };
71 
TEST_F(KllMetricE2eTest,TestSimpleMetric)72 TEST_F(KllMetricE2eTest, TestSimpleMetric) {
73     const sp<StatsLogProcessor> processor =
74             CreateStatsLogProcessor(bucketStartTimeNs, bucketStartTimeNs, config, key);
75 
76     for (auto& event : events) {
77         processor->OnLogEvent(event.get());
78     }
79 
80     uint64_t dumpTimeNs = bucketStartTimeNs + bucketSizeNs;
81     ConfigMetricsReportList reports;
82     vector<uint8_t> buffer;
83     processor->onDumpReport(key, dumpTimeNs, /*include_current_bucket*/ false, true, ADB_DUMP, FAST,
84                             &buffer);
85     EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
86     backfillDimensionPath(&reports);
87     backfillStringInReport(&reports);
88     backfillStartEndTimestamp(&reports);
89     ASSERT_EQ(reports.reports_size(), 1);
90 
91     ConfigMetricsReport report = reports.reports(0);
92     ASSERT_EQ(report.metrics_size(), 1);
93     StatsLogReport metricReport = report.metrics(0);
94     EXPECT_TRUE(metricReport.has_estimated_data_bytes());
95     EXPECT_EQ(metricReport.metric_id(), metric.id());
96     EXPECT_TRUE(metricReport.has_kll_metrics());
97     ASSERT_EQ(metricReport.kll_metrics().data_size(), 1);
98     KllMetricData data = metricReport.kll_metrics().data(0);
99     ASSERT_EQ(data.bucket_info_size(), 1);
100     KllBucketInfo bucket = data.bucket_info(0);
101     EXPECT_EQ(bucket.start_bucket_elapsed_nanos(), bucketStartTimeNs);
102     EXPECT_EQ(bucket.end_bucket_elapsed_nanos(), bucketStartTimeNs + bucketSizeNs);
103     EXPECT_EQ(bucket.sketches_size(), 1);
104     EXPECT_EQ(metricReport.kll_metrics().skipped_size(), 0);
105 }
106 
TEST_F(KllMetricE2eTest,TestMetricWithDimensions)107 TEST_F(KllMetricE2eTest, TestMetricWithDimensions) {
108     whatMatcher = CreateSimpleAtomMatcher("TestAtomReported", util::TEST_ATOM_REPORTED);
109     metric = createKllMetric("TestAtomMetric", whatMatcher, /* kllField */ 3,
110                              /* condition */ nullopt);
111 
112     *metric.mutable_dimensions_in_what() =
113             CreateDimensions(util::TEST_ATOM_REPORTED, {5 /* string_field */});
114 
115     config.clear_atom_matcher();
116     *config.add_atom_matcher() = whatMatcher;
117 
118     config.clear_kll_metric();
119     *config.add_kll_metric() = metric;
120 
121     events.clear();
122     events.push_back(CreateTestAtomReportedEvent(bucketStartTimeNs + 5 * NS_PER_SEC, 5l, "dim_1"));
123     events.push_back(CreateTestAtomReportedEvent(bucketStartTimeNs + 15 * NS_PER_SEC, 6l, "dim_2"));
124     events.push_back(CreateTestAtomReportedEvent(bucketStartTimeNs + 25 * NS_PER_SEC, 7l, "dim_1"));
125 
126     const sp<StatsLogProcessor> processor =
127             CreateStatsLogProcessor(bucketStartTimeNs, bucketStartTimeNs, config, key);
128 
129     for (auto& event : events) {
130         processor->OnLogEvent(event.get());
131     }
132 
133     uint64_t dumpTimeNs = bucketStartTimeNs + bucketSizeNs;
134     ConfigMetricsReportList reports;
135     vector<uint8_t> buffer;
136     processor->onDumpReport(key, dumpTimeNs, /*include_current_bucket*/ false, true, ADB_DUMP, FAST,
137                             &buffer);
138     EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
139     backfillDimensionPath(&reports);
140     backfillStringInReport(&reports);
141     backfillStartEndTimestamp(&reports);
142     ASSERT_EQ(reports.reports_size(), 1);
143 
144     ConfigMetricsReport report = reports.reports(0);
145     ASSERT_EQ(report.metrics_size(), 1);
146     StatsLogReport metricReport = report.metrics(0);
147     EXPECT_EQ(metricReport.metric_id(), metric.id());
148     EXPECT_TRUE(metricReport.has_kll_metrics());
149     ASSERT_EQ(metricReport.kll_metrics().data_size(), 2);
150 
151     KllMetricData data = metricReport.kll_metrics().data(0);
152     ASSERT_EQ(data.bucket_info_size(), 1);
153     KllBucketInfo bucket = data.bucket_info(0);
154     EXPECT_EQ(bucket.start_bucket_elapsed_nanos(), bucketStartTimeNs);
155     EXPECT_EQ(bucket.end_bucket_elapsed_nanos(), bucketStartTimeNs + bucketSizeNs);
156     EXPECT_EQ(bucket.sketches_size(), 1);
157     EXPECT_EQ(metricReport.kll_metrics().skipped_size(), 0);
158     EXPECT_EQ(data.dimensions_in_what().field(), util::TEST_ATOM_REPORTED);
159     ASSERT_EQ(data.dimensions_in_what().value_tuple().dimensions_value_size(), 1);
160     EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).field(), 5);
161     EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).value_str(), "dim_1");
162 
163     data = metricReport.kll_metrics().data(1);
164     ASSERT_EQ(data.bucket_info_size(), 1);
165     bucket = data.bucket_info(0);
166     EXPECT_EQ(bucket.start_bucket_elapsed_nanos(), bucketStartTimeNs);
167     EXPECT_EQ(bucket.end_bucket_elapsed_nanos(), bucketStartTimeNs + bucketSizeNs);
168     EXPECT_EQ(bucket.sketches_size(), 1);
169     EXPECT_EQ(metricReport.kll_metrics().skipped_size(), 0);
170     EXPECT_EQ(data.dimensions_in_what().field(), util::TEST_ATOM_REPORTED);
171     ASSERT_EQ(data.dimensions_in_what().value_tuple().dimensions_value_size(), 1);
172     EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).field(), 5);
173     EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).value_str(), "dim_2");
174 }
175 
TEST_F(KllMetricE2eTest,TestInitWithKllFieldPositionALL)176 TEST_F(KllMetricE2eTest, TestInitWithKllFieldPositionALL) {
177     // Create config.
178     StatsdConfig config;
179 
180     AtomMatcher testAtomReportedMatcher =
181             CreateSimpleAtomMatcher("TestAtomReportedMatcher", util::TEST_ATOM_REPORTED);
182     *config.add_atom_matcher() = testAtomReportedMatcher;
183 
184     // Create kll metric.
185     int64_t metricId = 123456;
186     KllMetric* kllMetric = config.add_kll_metric();
187     kllMetric->set_id(metricId);
188     kllMetric->set_bucket(TimeUnit::FIVE_MINUTES);
189     kllMetric->set_what(testAtomReportedMatcher.id());
190     *kllMetric->mutable_kll_field() = CreateRepeatedDimensions(
191             util::TEST_ATOM_REPORTED, {9 /*repeated_int_field*/}, {Position::ALL});
192 
193     // Initialize StatsLogProcessor.
194     const uint64_t bucketStartTimeNs = 10000000000;  // 0:10
195     int uid = 12345;
196     int64_t cfgId = 98765;
197     ConfigKey cfgKey(uid, cfgId);
198     sp<StatsLogProcessor> processor =
199             CreateStatsLogProcessor(bucketStartTimeNs, bucketStartTimeNs, config, cfgKey);
200 
201     // Config initialization fails.
202     ASSERT_EQ(0, processor->mMetricsManagers.size());
203 }
204 
TEST_F(KllMetricE2eTest,TestDimensionalSampling)205 TEST_F(KllMetricE2eTest, TestDimensionalSampling) {
206     ShardOffsetProvider::getInstance().setShardOffset(5);
207 
208     // Create config.
209     StatsdConfig config;
210 
211     AtomMatcher bleScanResultReceivedMatcher = CreateSimpleAtomMatcher(
212             "BleScanResultReceivedAtomMatcher", util::BLE_SCAN_RESULT_RECEIVED);
213     *config.add_atom_matcher() = bleScanResultReceivedMatcher;
214 
215     // Create kll metric.
216     KllMetric sampledKllMetric =
217             createKllMetric("KllSampledBleScanResultsPerUid", bleScanResultReceivedMatcher,
218                             /*num_results=*/2, nullopt);
219     *sampledKllMetric.mutable_dimensions_in_what() =
220             CreateAttributionUidDimensions(util::BLE_SCAN_RESULT_RECEIVED, {Position::FIRST});
221     *sampledKllMetric.mutable_dimensional_sampling_info()->mutable_sampled_what_field() =
222             CreateAttributionUidDimensions(util::BLE_SCAN_RESULT_RECEIVED, {Position::FIRST});
223     sampledKllMetric.mutable_dimensional_sampling_info()->set_shard_count(2);
224     *config.add_kll_metric() = sampledKllMetric;
225 
226     // Initialize StatsLogProcessor.
227     const uint64_t bucketStartTimeNs = 10000000000;  // 0:10
228     int uid = 12345;
229     int64_t cfgId = 98765;
230     ConfigKey cfgKey(uid, cfgId);
231 
232     sp<StatsLogProcessor> processor = CreateStatsLogProcessor(
233             bucketStartTimeNs, bucketStartTimeNs, config, cfgKey, nullptr, 0, new UidMap());
234 
235     int appUid1 = 1001;  // odd hash value
236     int appUid2 = 1002;  // even hash value
237     int appUid3 = 1003;  // odd hash value
238     std::vector<std::unique_ptr<LogEvent>> events;
239 
240     events.push_back(CreateBleScanResultReceivedEvent(bucketStartTimeNs + 20 * NS_PER_SEC,
241                                                       {appUid1}, {"tag1"}, 10));
242     events.push_back(CreateBleScanResultReceivedEvent(bucketStartTimeNs + 40 * NS_PER_SEC,
243                                                       {appUid2}, {"tag2"}, 10));
244     events.push_back(CreateBleScanResultReceivedEvent(bucketStartTimeNs + 60 * NS_PER_SEC,
245                                                       {appUid3}, {"tag3"}, 10));
246 
247     events.push_back(CreateBleScanResultReceivedEvent(bucketStartTimeNs + 120 * NS_PER_SEC,
248                                                       {appUid1}, {"tag1"}, 11));
249     events.push_back(CreateBleScanResultReceivedEvent(bucketStartTimeNs + 140 * NS_PER_SEC,
250                                                       {appUid2}, {"tag2"}, 12));
251     events.push_back(CreateBleScanResultReceivedEvent(bucketStartTimeNs + 160 * NS_PER_SEC,
252                                                       {appUid3}, {"tag3"}, 13));
253 
254     // Send log events to StatsLogProcessor.
255     for (auto& event : events) {
256         processor->OnLogEvent(event.get());
257     }
258 
259     // Check dump report.
260     vector<uint8_t> buffer;
261     ConfigMetricsReportList reports;
262     processor->onDumpReport(cfgKey, bucketStartTimeNs + bucketSizeNs + 1, false, true, ADB_DUMP,
263                             FAST, &buffer);
264     ASSERT_GT(buffer.size(), 0);
265     EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
266     backfillDimensionPath(&reports);
267     backfillStringInReport(&reports);
268     backfillStartEndTimestamp(&reports);
269     backfillAggregatedAtoms(&reports);
270 
271     ConfigMetricsReport report = reports.reports(0);
272     ASSERT_EQ(report.metrics_size(), 1);
273     StatsLogReport metricReport = report.metrics(0);
274     EXPECT_EQ(metricReport.metric_id(), sampledKllMetric.id());
275     EXPECT_TRUE(metricReport.has_kll_metrics());
276     StatsLogReport::KllMetricDataWrapper kllMetrics;
277     sortMetricDataByDimensionsValue(metricReport.kll_metrics(), &kllMetrics);
278     ASSERT_EQ(kllMetrics.data_size(), 2);
279     EXPECT_EQ(kllMetrics.skipped_size(), 0);
280 
281     // Only Uid 1 and 3 are logged. (odd hash value) + (offset of 5) % (shard count of 2) = 0
282     KllMetricData data = kllMetrics.data(0);
283     ValidateAttributionUidDimension(data.dimensions_in_what(), util::BLE_SCAN_RESULT_RECEIVED,
284                                     appUid1);
285     ValidateKllBucket(data.bucket_info(0), bucketStartTimeNs, bucketStartTimeNs + bucketSizeNs, {2},
286                       0);
287 
288     data = kllMetrics.data(1);
289     ValidateAttributionUidDimension(data.dimensions_in_what(), util::BLE_SCAN_RESULT_RECEIVED,
290                                     appUid3);
291     ValidateKllBucket(data.bucket_info(0), bucketStartTimeNs, bucketStartTimeNs + bucketSizeNs, {2},
292                       0);
293 }
294 
295 #else
296 GTEST_LOG_(INFO) << "This test does nothing.\n";
297 #endif
298 
299 }  // namespace statsd
300 }  // namespace os
301 }  // namespace android
302