1 /*
2  * Copyright (C) 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include <gtest/gtest_prod.h>
20 
21 #include <optional>
22 
23 #include "ValueMetricProducer.h"
24 
25 namespace android {
26 namespace os {
27 namespace statsd {
28 
29 // TODO(b/185796344): don't use Value from FieldValue.
30 using ValueBases = std::vector<std::optional<Value>>;
31 class NumericValueMetricProducer : public ValueMetricProducer<Value, ValueBases> {
32 public:
33     NumericValueMetricProducer(const ConfigKey& key, const ValueMetric& valueMetric,
34                                const uint64_t protoHash, const PullOptions& pullOptions,
35                                const BucketOptions& bucketOptions, const WhatOptions& whatOptions,
36                                const ConditionOptions& conditionOptions,
37                                const StateOptions& stateOptions,
38                                const ActivationOptions& activationOptions,
39                                const GuardrailOptions& guardrailOptions,
40                                const wp<ConfigMetadataProvider> configMetadataProvider);
41 
42     // Process data pulled on bucket boundary.
43     void onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& allData, PullResult pullResult,
44                       int64_t originalPullTimeNs) override;
45 
46     // Determine if metric needs to pull
isPullNeeded()47     bool isPullNeeded() const override {
48         std::lock_guard<std::mutex> lock(mMutex);
49         return mIsActive && (mCondition == ConditionState::kTrue);
50     }
51 
getMetricType()52     inline MetricType getMetricType() const override {
53         return METRIC_TYPE_VALUE;
54     }
55 
56 protected:
57 private:
58     void prepareFirstBucketLocked() override;
59 
getConditionIdForMetric(const StatsdConfig & config,const int configIndex)60     inline optional<int64_t> getConditionIdForMetric(const StatsdConfig& config,
61                                                      const int configIndex) const override {
62         const ValueMetric& metric = config.value_metric(configIndex);
63         return metric.has_condition() ? make_optional(metric.condition()) : nullopt;
64     }
65 
getWhatAtomMatcherIdForMetric(const StatsdConfig & config,const int configIndex)66     inline int64_t getWhatAtomMatcherIdForMetric(const StatsdConfig& config,
67                                                  const int configIndex) const override {
68         return config.value_metric(configIndex).what();
69     }
70 
getConditionLinksForMetric(const StatsdConfig & config,const int configIndex)71     inline ConditionLinks getConditionLinksForMetric(const StatsdConfig& config,
72                                                      const int configIndex) const override {
73         return config.value_metric(configIndex).links();
74     }
75 
76     void onActiveStateChangedInternalLocked(const int64_t eventTimeNs,
77                                             const bool isActive) override;
78 
79     // Only called when mIsActive and the event is NOT too late.
80     void onConditionChangedInternalLocked(const ConditionState oldCondition,
81                                           const ConditionState newCondition,
82                                           const int64_t eventTimeNs) override;
83 
aggregatedValueToString(const Value & value)84     inline std::string aggregatedValueToString(const Value& value) const override {
85         return value.toString();
86     }
87 
88     // Mark the data as invalid.
89     void invalidateCurrentBucket(const int64_t dropTimeNs, const BucketDropReason reason) override;
90 
91     // Reset diff base and mHasGlobalBase
92     void resetBase();
93 
94     // Calculate previous bucket end time based on current time.
95     int64_t calcPreviousBucketEndTime(const int64_t currentTimeNs);
96 
multipleBucketsSkipped(const int64_t numBucketsForward)97     inline bool multipleBucketsSkipped(const int64_t numBucketsForward) const override {
98         return numBucketsForward > 1 && (isPulled() || mUseDiff);
99     }
100 
101     // Process events retrieved from a pull.
102     void accumulateEvents(const std::vector<std::shared_ptr<LogEvent>>& allData,
103                           int64_t originalPullTimeNs, int64_t eventElapsedTimeNs);
104 
105     void closeCurrentBucket(const int64_t eventTimeNs,
106                             const int64_t nextBucketStartTimeNs) override;
107 
108     PastBucket<Value> buildPartialBucket(int64_t bucketEndTime,
109                                          std::vector<Interval>& intervals) override;
110 
111     bool valuePassesThreshold(const Interval& interval) const;
112 
113     Value getFinalValue(const Interval& interval) const;
114 
115     void initNextSlicedBucket(int64_t nextBucketStartTimeNs) override;
116 
117     void appendToFullBucket(const bool isFullBucketReached);
118 
119     bool hitFullBucketGuardRailLocked(const MetricDimensionKey& newKey);
120 
canSkipLogEventLocked(const MetricDimensionKey & eventKey,const bool condition,int64_t eventTimeNs,const map<int,HashableDimensionKey> & statePrimaryKeys)121     inline bool canSkipLogEventLocked(
122             const MetricDimensionKey& eventKey, const bool condition, int64_t eventTimeNs,
123             const map<int, HashableDimensionKey>& statePrimaryKeys) const override {
124         // For pushed metrics, can only skip if condition is false.
125         // For pulled metrics, can only skip if metric is not diffed and condition is false or
126         // unknown.
127         return (!isPulled() && !condition) ||
128                (isPulled() && !mUseDiff && mCondition != ConditionState::kTrue);
129     }
130 
131     bool aggregateFields(const int64_t eventTimeNs, const MetricDimensionKey& eventKey,
132                          const LogEvent& event, std::vector<Interval>& intervals,
133                          ValueBases& bases) override;
134 
135     void pullAndMatchEventsLocked(const int64_t timestampNs) override;
136 
137     DumpProtoFields getDumpProtoFields() const override;
138 
139     void writePastBucketAggregateToProto(const int aggIndex, const Value& value,
140                                          const int sampleSize,
141                                          ProtoOutputStream* const protoOutput) const override;
142 
143     // Internal function to calculate the current used bytes.
144     size_t byteSizeLocked() const override;
145 
146     void combineValueFields(pair<LogEvent, std::vector<int>>& eventValues, const LogEvent& newEvent,
147                             const std::vector<int>& newValueIndices) const;
148 
getAggregationTypeLocked(int index)149     ValueMetric::AggregationType getAggregationTypeLocked(int index) const {
150         return mAggregationTypes.size() == 1 ? mAggregationTypes[0] : mAggregationTypes[index];
151     }
152 
153     size_t getAggregatedValueSize(const Value& value) const override;
154 
hasAvgAggregationType(const vector<ValueMetric::AggregationType> aggregationTypes)155     bool hasAvgAggregationType(const vector<ValueMetric::AggregationType> aggregationTypes) const {
156         for (const int aggType : aggregationTypes) {
157             if (aggType == ValueMetric_AggregationType_AVG) {
158                 return true;
159             }
160         }
161         return false;
162     }
163 
164     const bool mUseAbsoluteValueOnReset;
165 
166     const std::vector<ValueMetric::AggregationType> mAggregationTypes;
167 
168     const bool mIncludeSampleSize;
169 
170     const bool mUseDiff;
171 
172     const ValueMetric::ValueDirection mValueDirection;
173 
174     const bool mSkipZeroDiffOutput;
175 
176     // If true, use a zero value as base to compute the diff.
177     // This is used for new keys which are present in the new data but was not
178     // present in the base data.
179     // The default base will only be used if we have a global base.
180     const bool mUseZeroDefaultBase;
181 
182     // For pulled metrics, this is always set to true whenever a pull succeeds.
183     // It is set to false when a pull fails, or upon condition change to false.
184     // This is used to decide if we have the right base data to compute the
185     // diff against.
186     bool mHasGlobalBase;
187 
188     const int64_t mMaxPullDelayNs;
189 
190     // Deduped value fields for matching.
191     const std::vector<Matcher> mDedupedFieldMatchers;
192 
193     // For anomaly detection.
194     std::unordered_map<MetricDimensionKey, int64_t> mCurrentFullBucket;
195 
196     FRIEND_TEST(NumericValueMetricProducerTest, TestAnomalyDetection);
197     FRIEND_TEST(NumericValueMetricProducerTest, TestBaseSetOnConditionChange);
198     FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundariesOnConditionChange);
199     FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundaryNoCondition);
200     FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundaryWithCondition);
201     FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundaryWithCondition2);
202     FRIEND_TEST(NumericValueMetricProducerTest, TestBucketInvalidIfGlobalBaseIsNotSet);
203     FRIEND_TEST(NumericValueMetricProducerTest, TestCalcPreviousBucketEndTime);
204     FRIEND_TEST(NumericValueMetricProducerTest, TestDataIsNotUpdatedWhenNoConditionChanged);
205     FRIEND_TEST(NumericValueMetricProducerTest, TestEmptyDataResetsBase_onBucketBoundary);
206     FRIEND_TEST(NumericValueMetricProducerTest, TestEmptyDataResetsBase_onConditionChanged);
207     FRIEND_TEST(NumericValueMetricProducerTest, TestEmptyDataResetsBase_onDataPulled);
208     FRIEND_TEST(NumericValueMetricProducerTest, TestEventsWithNonSlicedCondition);
209     FRIEND_TEST(NumericValueMetricProducerTest, TestFirstBucket);
210     FRIEND_TEST(NumericValueMetricProducerTest, TestLateOnDataPulledWithDiff);
211     FRIEND_TEST(NumericValueMetricProducerTest, TestLateOnDataPulledWithoutDiff);
212     FRIEND_TEST(NumericValueMetricProducerTest, TestPartialResetOnBucketBoundaries);
213     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_bucketBoundaryFalse);
214     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_bucketBoundaryTrue);
215     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_withFailure);
216     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_withMultipleConditionChanges);
217     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_withoutCondition);
218     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsNoCondition);
219     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsTakeAbsoluteValueOnReset);
220     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsTakeZeroOnReset);
221     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsWithFiltering);
222     FRIEND_TEST(NumericValueMetricProducerTest, TestPulledWithAppUpgradeDisabled);
223     FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateAvg);
224     FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateMax);
225     FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateMin);
226     FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateSum);
227     FRIEND_TEST(NumericValueMetricProducerTest, TestPushedEventsWithCondition);
228     FRIEND_TEST(NumericValueMetricProducerTest, TestPushedEventsWithoutCondition);
229     FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullDelayExceeded);
230     FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullFailAfterConditionChange);
231     FRIEND_TEST(NumericValueMetricProducerTest,
232                 TestResetBaseOnPullFailAfterConditionChange_EndOfBucket);
233     FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullFailBeforeConditionChange);
234     FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullTooLate);
235     FRIEND_TEST(NumericValueMetricProducerTest, TestSampleSize);
236     FRIEND_TEST(NumericValueMetricProducerTest, TestSkipZeroDiffOutput);
237     FRIEND_TEST(NumericValueMetricProducerTest, TestSkipZeroDiffOutputMultiValue);
238     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedState);
239     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMap);
240     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithPrimaryField_WithDimensions);
241     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithCondition);
242     FRIEND_TEST(NumericValueMetricProducerTest, TestTrimUnusedDimensionKey);
243     FRIEND_TEST(NumericValueMetricProducerTest, TestUseZeroDefaultBase);
244     FRIEND_TEST(NumericValueMetricProducerTest, TestUseZeroDefaultBaseWithPullFailures);
245     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMultipleDimensions);
246     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMissingDataInStateChange);
247     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithDataMissingInConditionChange);
248     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMissingDataThenFlushBucket);
249     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithNoPullOnBucketBoundary);
250     FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithConditionFalseMultipleBuckets);
251     FRIEND_TEST(NumericValueMetricProducerTest,
252                 TestSlicedStateWithMultipleDimensionsMissingDataInPull);
253     FRIEND_TEST(NumericValueMetricProducerTest, TestUploadThreshold);
254     FRIEND_TEST(NumericValueMetricProducerTest, TestMultipleAggTypesPulled);
255     FRIEND_TEST(NumericValueMetricProducerTest, TestMultipleAggTypesPushed);
256 
257     FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenOneConditionFailed);
258     FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenInitialPullFailed);
259     FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenLastPullFailed);
260     FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenGuardRailHit);
261     FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop,
262                 TestInvalidBucketWhenDumpReportRequested);
263     FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop,
264                 TestInvalidBucketWhenAccumulateEventWrongBucket);
265     FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop,
266                 TestInvalidBucketWhenMultipleBucketsSkipped);
267 
268     FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestBucketBoundariesOnPartialBucket);
269     FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket,
270                 TestFullBucketResetWhenLastBucketInvalid);
271     FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPartialBucketCreated);
272     FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPushedEvents);
273     FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPulledValue);
274     FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPulledValueWhileConditionFalse);
275 
276     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
277                 TestAlarmLatePullWhileConditionTrue);
278     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
279                 TestAlarmLatePullWithConditionChanged);
280     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
281                 TestAlarmLatePullWhileConditionFalse);
282     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
283                 TestLatePullOnConditionChangeFalse);
284     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
285                 TestLatePullOnConditionChangeTrue);
286     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestAlarmLatePullNoCondition);
287     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
288                 TestAlarmLatePullNoConditionWithSkipped);
289     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
290                 TestThresholdNotDefinedNoUpload);
291     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestThresholdDefinedZero);
292     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
293                 TestThresholdUploadPassWhenEqual);
294     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection,
295                 TestThresholdUploadPassWhenGreater);
296     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestThresholdUploadSkip);
297     FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestLateStateChangeSlicedAtoms);
298 
299     FRIEND_TEST(NumericValueMetricProducerTest, TestSubsetDimensions);
300 
301     FRIEND_TEST(ConfigUpdateTest, TestUpdateValueMetrics);
302 
303     FRIEND_TEST(MetricsManagerUtilDimLimitTest, TestDimLimit);
304 
305     FRIEND_TEST(ConfigUpdateDimLimitTest, TestDimLimit);
306 
307     FRIEND_TEST(ValueMetricE2eTest, TestInitWithMultipleAggTypes);
308     FRIEND_TEST(ValueMetricE2eTest, TestInitWithDefaultAggType);
309 
310     friend class NumericValueMetricProducerTestHelper;
311 };
312 
313 }  // namespace statsd
314 }  // namespace os
315 }  // namespace android
316