1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <unordered_map> 20 21 #include <android/util/ProtoOutputStream.h> 22 #include <gtest/gtest_prod.h> 23 #include "../condition/ConditionTracker.h" 24 #include "../external/PullDataReceiver.h" 25 #include "../external/StatsPullerManager.h" 26 #include "../matchers/matcher_util.h" 27 #include "../matchers/EventMatcherWizard.h" 28 #include "MetricProducer.h" 29 #include "src/statsd_config.pb.h" 30 #include "../stats_util.h" 31 32 namespace android { 33 namespace os { 34 namespace statsd { 35 36 struct GaugeAtom { GaugeAtomGaugeAtom37 GaugeAtom(const std::shared_ptr<vector<FieldValue>>& fields, int64_t elapsedTimeNs) 38 : mFields(fields), mElapsedTimestampNs(elapsedTimeNs) { 39 } 40 std::shared_ptr<vector<FieldValue>> mFields; 41 int64_t mElapsedTimestampNs; 42 }; 43 44 struct GaugeBucket { 45 int64_t mBucketStartNs; 46 int64_t mBucketEndNs; 47 std::vector<GaugeAtom> mGaugeAtoms; 48 49 // Maps the field/value pairs of an atom to a list of timestamps used to deduplicate atoms. 50 std::unordered_map<AtomDimensionKey, std::vector<int64_t>> mAggregatedAtoms; 51 }; 52 53 typedef std::unordered_map<MetricDimensionKey, std::vector<GaugeAtom>> 54 DimToGaugeAtomsMap; 55 56 // This gauge metric producer first register the puller to automatically pull the gauge at the 57 // beginning of each bucket. If the condition is met, insert it to the bucket info. Otherwise 58 // proactively pull the gauge when the condition is changed to be true. Therefore, the gauge metric 59 // producer always reports the gauge at the earliest time of the bucket when the condition is met. 60 class GaugeMetricProducer : public MetricProducer, public virtual PullDataReceiver { 61 public: 62 GaugeMetricProducer( 63 const ConfigKey& key, const GaugeMetric& gaugeMetric, int conditionIndex, 64 const vector<ConditionState>& initialConditionCache, 65 const sp<ConditionWizard>& conditionWizard, const uint64_t protoHash, 66 const int whatMatcherIndex, const sp<EventMatcherWizard>& matcherWizard, 67 const int pullTagId, int triggerAtomId, int atomId, const int64_t timeBaseNs, 68 int64_t startTimeNs, const sp<StatsPullerManager>& pullerManager, 69 const wp<ConfigMetadataProvider> configMetadataProvider, 70 const std::unordered_map<int, std::shared_ptr<Activation>>& eventActivationMap = {}, 71 const std::unordered_map<int, std::vector<std::shared_ptr<Activation>>>& 72 eventDeactivationMap = {}, 73 const size_t dimensionSoftLimit = StatsdStats::kDimensionKeySizeSoftLimit, 74 const size_t dimensionHardLimit = StatsdStats::kDimensionKeySizeHardLimit); 75 76 virtual ~GaugeMetricProducer(); 77 78 // Handles when the pulled data arrives. 79 void onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& data, PullResult pullResult, 80 int64_t originalPullTimeNs) override; 81 82 // Determine if metric needs to pull isPullNeeded()83 bool isPullNeeded() const override { 84 std::lock_guard<std::mutex> lock(mMutex); 85 return mIsActive && (mCondition == ConditionState::kTrue) && 86 shouldKeepRandomSample(mPullProbability); 87 }; 88 89 // GaugeMetric needs to immediately trigger another pull when we create the partial bucket. notifyAppUpgradeInternalLocked(int64_t eventTimeNs)90 void notifyAppUpgradeInternalLocked(int64_t eventTimeNs) override { 91 flushLocked(eventTimeNs); 92 if (mIsPulled && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE && mIsActive) { 93 pullAndMatchEventsLocked(eventTimeNs); 94 } 95 }; 96 97 // GaugeMetric needs to immediately trigger another pull when we create the partial bucket. onStatsdInitCompleted(int64_t eventTimeNs)98 void onStatsdInitCompleted(int64_t eventTimeNs) override { 99 ATRACE_CALL(); 100 std::lock_guard<std::mutex> lock(mMutex); 101 flushLocked(eventTimeNs); 102 if (mIsPulled && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE && mIsActive) { 103 pullAndMatchEventsLocked(eventTimeNs); 104 } 105 }; 106 getMetricType()107 MetricType getMetricType() const override { 108 return METRIC_TYPE_GAUGE; 109 } 110 111 protected: 112 void onMatchedLogEventInternalLocked( 113 const size_t matcherIndex, const MetricDimensionKey& eventKey, 114 const ConditionKey& conditionKey, bool condition, const LogEvent& event, 115 const std::map<int, HashableDimensionKey>& statePrimaryKeys) override; 116 117 private: 118 void onDumpReportLocked(const int64_t dumpTimeNs, 119 const bool include_current_partial_bucket, 120 const bool erase_data, 121 const DumpLatency dumpLatency, 122 std::set<string> *str_set, 123 android::util::ProtoOutputStream* protoOutput) override; 124 void clearPastBucketsLocked(const int64_t dumpTimeNs) override; 125 126 // Internal interface to handle condition change. 127 void onConditionChangedLocked(const bool conditionMet, int64_t eventTime) override; 128 129 // Internal interface to handle active state change. 130 void onActiveStateChangedLocked(const int64_t eventTimeNs, const bool isActive) override; 131 132 // Internal interface to handle sliced condition change. 133 void onSlicedConditionMayChangeLocked(bool overallCondition, int64_t eventTime) override; 134 135 // Internal function to calculate the current used bytes. 136 size_t byteSizeLocked() const override; 137 138 void dumpStatesLocked(int out, bool verbose) const override; 139 140 void dropDataLocked(const int64_t dropTimeNs) override; 141 142 // Util function to flush the old packet. 143 void flushIfNeededLocked(int64_t eventTime) override; 144 145 void flushCurrentBucketLocked(int64_t eventTimeNs, int64_t nextBucketStartTimeNs) override; 146 147 void prepareFirstBucketLocked() override; 148 149 // Only call if mCondition == ConditionState::kTrue && metric is active. 150 void pullAndMatchEventsLocked(const int64_t timestampNs); 151 152 size_t computeGaugeBucketSizeLocked( 153 const bool isFullBucket, const MetricDimensionKey& dimKey, const bool isFirstBucket, 154 const std::unordered_map<AtomDimensionKey, std::vector<int64_t>>& aggregatedAtoms) 155 const; 156 157 optional<InvalidConfigReason> onConfigUpdatedLocked( 158 const StatsdConfig& config, int configIndex, int metricIndex, 159 const std::vector<sp<AtomMatchingTracker>>& allAtomMatchingTrackers, 160 const std::unordered_map<int64_t, int>& oldAtomMatchingTrackerMap, 161 const std::unordered_map<int64_t, int>& newAtomMatchingTrackerMap, 162 const sp<EventMatcherWizard>& matcherWizard, 163 const std::vector<sp<ConditionTracker>>& allConditionTrackers, 164 const std::unordered_map<int64_t, int>& conditionTrackerMap, 165 const sp<ConditionWizard>& wizard, 166 const std::unordered_map<int64_t, int>& metricToActivationMap, 167 std::unordered_map<int, std::vector<int>>& trackerToMetricMap, 168 std::unordered_map<int, std::vector<int>>& conditionToMetricMap, 169 std::unordered_map<int, std::vector<int>>& activationAtomTrackerToMetricMap, 170 std::unordered_map<int, std::vector<int>>& deactivationAtomTrackerToMetricMap, 171 std::vector<int>& metricsWithActivation) override; 172 isRandomNSamples()173 inline bool isRandomNSamples() const { 174 return (mTriggerAtomId == -1 && mSamplingType == GaugeMetric::FIRST_N_SAMPLES) || 175 mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE; 176 } 177 178 int mWhatMatcherIndex; 179 180 sp<EventMatcherWizard> mEventMatcherWizard; 181 182 sp<StatsPullerManager> mPullerManager; 183 // tagId for pulled data. -1 if this is not pulled 184 const int mPullTagId; 185 186 // tagId for atoms that trigger the pulling, if any 187 const int mTriggerAtomId; 188 189 // tagId for output atom 190 const int mAtomId; 191 192 // if this is pulled metric 193 const bool mIsPulled; 194 195 // Save the past buckets and we can clear when the StatsLogReport is dumped. 196 std::unordered_map<MetricDimensionKey, std::vector<GaugeBucket>> mPastBuckets; 197 198 // The current partial bucket. 199 std::shared_ptr<DimToGaugeAtomsMap> mCurrentSlicedBucket; 200 201 // The current full bucket for anomaly detection. This is updated to the latest value seen for 202 // this slice (ie, for partial buckets, we use the last partial bucket in this full bucket). 203 std::shared_ptr<DimToValMap> mCurrentSlicedBucketForAnomaly; 204 205 const int64_t mMinBucketSizeNs; 206 207 // Translate Atom based bucket to single numeric value bucket for anomaly and updates the map 208 // for each slice with the latest value. 209 void updateCurrentSlicedBucketForAnomaly(); 210 211 // Allowlist of fields to report. Empty means all are reported. 212 std::vector<Matcher> mFieldMatchers; 213 214 GaugeMetric::SamplingType mSamplingType; 215 216 const int64_t mMaxPullDelayNs; 217 218 // apply an allowlist on the original input 219 std::shared_ptr<vector<FieldValue>> getGaugeFields(const LogEvent& event); 220 221 // Util function to check whether the specified dimension hits the guardrail. 222 bool hitGuardRailLocked(const MetricDimensionKey& newKey); 223 224 static const size_t kBucketSize = sizeof(GaugeBucket{}); 225 226 const size_t mDimensionSoftLimit; 227 228 const size_t mDimensionHardLimit; 229 230 const size_t mGaugeAtomsPerDimensionLimit; 231 232 // Tracks if the dimension guardrail has been hit in the current report. 233 bool mDimensionGuardrailHit; 234 235 const int mSamplingPercentage; 236 237 const int mPullProbability; 238 239 FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsWithCondition); 240 FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsWithSlicedCondition); 241 FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsNoCondition); 242 FRIEND_TEST(GaugeMetricProducerTest, TestPulledWithAppUpgradeDisabled); 243 FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsAnomalyDetection); 244 FRIEND_TEST(GaugeMetricProducerTest, TestFirstBucket); 245 FRIEND_TEST(GaugeMetricProducerTest, TestPullOnTrigger); 246 FRIEND_TEST(GaugeMetricProducerTest, TestPullNWithoutTrigger); 247 FRIEND_TEST(GaugeMetricProducerTest, TestRemoveDimensionInOutput); 248 FRIEND_TEST(GaugeMetricProducerTest, TestPullDimensionalSampling); 249 250 FRIEND_TEST(GaugeMetricProducerTest_PartialBucket, TestPushedEvents); 251 FRIEND_TEST(GaugeMetricProducerTest_PartialBucket, TestPulled); 252 253 FRIEND_TEST(ConfigUpdateTest, TestUpdateGaugeMetrics); 254 255 FRIEND_TEST(MetricsManagerUtilDimLimitTest, TestDimLimit); 256 257 FRIEND_TEST(ConfigUpdateDimLimitTest, TestDimLimit); 258 }; 259 260 } // namespace statsd 261 } // namespace os 262 } // namespace android 263