1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include <gtest/gtest_prod.h>
20 #include "config/ConfigListener.h"
21 #include "logd/LogEvent.h"
22 #include "metrics/MetricsManager.h"
23 #include "packages/UidMap.h"
24 #include "external/StatsPullerManager.h"
25 
26 #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h"
27 #include "frameworks/base/cmds/statsd/src/statsd_metadata.pb.h"
28 
29 #include <stdio.h>
30 #include <unordered_map>
31 
32 namespace android {
33 namespace os {
34 namespace statsd {
35 
36 
37 class StatsLogProcessor : public ConfigListener, public virtual PackageInfoListener {
38 public:
39     StatsLogProcessor(const sp<UidMap>& uidMap, const sp<StatsPullerManager>& pullerManager,
40                       const sp<AlarmMonitor>& anomalyAlarmMonitor,
41                       const sp<AlarmMonitor>& subscriberTriggerAlarmMonitor,
42                       const int64_t timeBaseNs,
43                       const std::function<bool(const ConfigKey&)>& sendBroadcast,
44                       const std::function<bool(const int&,
45                                                const vector<int64_t>&)>& sendActivationBroadcast);
46     virtual ~StatsLogProcessor();
47 
48     void OnLogEvent(LogEvent* event);
49 
50     void OnConfigUpdated(const int64_t timestampNs, const ConfigKey& key,
51                          const StatsdConfig& config);
52     void OnConfigRemoved(const ConfigKey& key);
53 
54     size_t GetMetricsSize(const ConfigKey& key) const;
55 
56     void GetActiveConfigs(const int uid, vector<int64_t>& outActiveConfigs);
57 
58     void onDumpReport(const ConfigKey& key, const int64_t dumpTimeNs,
59                       const bool include_current_partial_bucket, const bool erase_data,
60                       const DumpReportReason dumpReportReason,
61                       const DumpLatency dumpLatency,
62                       vector<uint8_t>* outData);
63     void onDumpReport(const ConfigKey& key, const int64_t dumpTimeNs,
64                       const bool include_current_partial_bucket, const bool erase_data,
65                       const DumpReportReason dumpReportReason,
66                       const DumpLatency dumpLatency,
67                       ProtoOutputStream* proto);
68 
69     /* Tells MetricsManager that the alarms in alarmSet have fired. Modifies anomaly alarmSet. */
70     void onAnomalyAlarmFired(
71             const int64_t& timestampNs,
72             unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet);
73 
74     /* Tells MetricsManager that the alarms in alarmSet have fired. Modifies periodic alarmSet. */
75     void onPeriodicAlarmFired(
76             const int64_t& timestampNs,
77             unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet);
78 
79     /* Flushes data to disk. Data on memory will be gone after written to disk. */
80     void WriteDataToDisk(const DumpReportReason dumpReportReason,
81                          const DumpLatency dumpLatency);
82 
83     /* Persist configs containing metrics with active activations to disk. */
84     void SaveActiveConfigsToDisk(int64_t currentTimeNs);
85 
86     /* Writes the current active status/ttl for all configs and metrics to ProtoOutputStream. */
87     void WriteActiveConfigsToProtoOutputStream(
88             int64_t currentTimeNs, const DumpReportReason reason, ProtoOutputStream* proto);
89 
90     /* Load configs containing metrics with active activations from disk. */
91     void LoadActiveConfigsFromDisk();
92 
93     /* Persist metadata for configs and metrics to disk. */
94     void SaveMetadataToDisk(int64_t currentWallClockTimeNs, int64_t systemElapsedTimeNs);
95 
96     /* Writes the statsd metadata for all configs and metrics to StatsMetadataList. */
97     void WriteMetadataToProto(int64_t currentWallClockTimeNs,
98                               int64_t systemElapsedTimeNs,
99                               metadata::StatsMetadataList* metadataList);
100 
101     /* Load stats metadata for configs and metrics from disk. */
102     void LoadMetadataFromDisk(int64_t currentWallClockTimeNs,
103                               int64_t systemElapsedTimeNs);
104 
105     /* Sets the metadata for all configs and metrics */
106     void SetMetadataState(const metadata::StatsMetadataList& statsMetadataList,
107                           int64_t currentWallClockTimeNs,
108                           int64_t systemElapsedTimeNs);
109 
110     /* Sets the active status/ttl for all configs and metrics to the status in ActiveConfigList. */
111     void SetConfigsActiveState(const ActiveConfigList& activeConfigList, int64_t currentTimeNs);
112 
113     /* Notify all MetricsManagers of app upgrades */
114     void notifyAppUpgrade(const int64_t& eventTimeNs, const string& apk, const int uid,
115                           const int64_t version) override;
116 
117     /* Notify all MetricsManagers of app removals */
118     void notifyAppRemoved(const int64_t& eventTimeNs, const string& apk, const int uid) override;
119 
120     /* Notify all MetricsManagers of uid map snapshots received */
121     void onUidMapReceived(const int64_t& eventTimeNs) override;
122 
123     /* Notify all metrics managers of boot completed
124      * This will force a bucket split when the boot is finished.
125      */
126     void onStatsdInitCompleted(const int64_t& elapsedTimeNs);
127 
128     // Reset all configs.
129     void resetConfigs();
130 
getUidMap()131     inline sp<UidMap> getUidMap() {
132         return mUidMap;
133     }
134 
135     void dumpStates(int outFd, bool verbose);
136 
137     void informPullAlarmFired(const int64_t timestampNs);
138 
139     int64_t getLastReportTimeNs(const ConfigKey& key);
140 
setPrintLogs(bool enabled)141     inline void setPrintLogs(bool enabled) {
142 #ifdef VERY_VERBOSE_PRINTING
143         std::lock_guard<std::mutex> lock(mMetricsMutex);
144         mPrintAllLogs = enabled;
145 #endif
146     }
147 
148     // Add a specific config key to the possible configs to dump ASAP.
149     void noteOnDiskData(const ConfigKey& key);
150 
151 private:
152     // For testing only.
getAnomalyAlarmMonitor()153     inline sp<AlarmMonitor> getAnomalyAlarmMonitor() const {
154         return mAnomalyAlarmMonitor;
155     }
156 
getPeriodicAlarmMonitor()157     inline sp<AlarmMonitor> getPeriodicAlarmMonitor() const {
158         return mPeriodicAlarmMonitor;
159     }
160 
161     mutable mutex mMetricsMutex;
162 
163     std::unordered_map<ConfigKey, sp<MetricsManager>> mMetricsManagers;
164 
165     std::unordered_map<ConfigKey, int64_t> mLastBroadcastTimes;
166 
167     // Last time we sent a broadcast to this uid that the active configs had changed.
168     std::unordered_map<int, int64_t> mLastActivationBroadcastTimes;
169 
170     // Tracks when we last checked the bytes consumed for each config key.
171     std::unordered_map<ConfigKey, int64_t> mLastByteSizeTimes;
172 
173     // Tracks which config keys has metric reports on disk
174     std::set<ConfigKey> mOnDiskDataConfigs;
175 
176     sp<UidMap> mUidMap;  // Reference to the UidMap to lookup app name and version for each uid.
177 
178     sp<StatsPullerManager> mPullerManager;  // Reference to StatsPullerManager
179 
180     sp<AlarmMonitor> mAnomalyAlarmMonitor;
181 
182     sp<AlarmMonitor> mPeriodicAlarmMonitor;
183 
184     void OnLogEvent(LogEvent* event, int64_t elapsedRealtimeNs);
185 
186     void resetIfConfigTtlExpiredLocked(const int64_t timestampNs);
187 
188     void OnConfigUpdatedLocked(
189         const int64_t currentTimestampNs, const ConfigKey& key, const StatsdConfig& config);
190 
191     void GetActiveConfigsLocked(const int uid, vector<int64_t>& outActiveConfigs);
192 
193     void WriteActiveConfigsToProtoOutputStreamLocked(
194             int64_t currentTimeNs, const DumpReportReason reason, ProtoOutputStream* proto);
195 
196     void SetConfigsActiveStateLocked(const ActiveConfigList& activeConfigList,
197                                      int64_t currentTimeNs);
198 
199     void SetMetadataStateLocked(const metadata::StatsMetadataList& statsMetadataList,
200                                 int64_t currentWallClockTimeNs,
201                                 int64_t systemElapsedTimeNs);
202 
203     void WriteMetadataToProtoLocked(int64_t currentWallClockTimeNs,
204                                     int64_t systemElapsedTimeNs,
205                                     metadata::StatsMetadataList* metadataList);
206 
207     void WriteDataToDiskLocked(const DumpReportReason dumpReportReason,
208                                const DumpLatency dumpLatency);
209 
210     void WriteDataToDiskLocked(const ConfigKey& key, const int64_t timestampNs,
211                                const DumpReportReason dumpReportReason,
212                                const DumpLatency dumpLatency);
213 
214     void onConfigMetricsReportLocked(
215             const ConfigKey& key, const int64_t dumpTimeStampNs,
216             const bool include_current_partial_bucket, const bool erase_data,
217             const DumpReportReason dumpReportReason, const DumpLatency dumpLatency,
218             /*if dataSavedToDisk is true, it indicates the caller will write the data to disk
219              (e.g., before reboot). So no need to further persist local history.*/
220             const bool dataSavedToDisk, vector<uint8_t>* proto);
221 
222     /* Check if we should send a broadcast if approaching memory limits and if we're over, we
223      * actually delete the data. */
224     void flushIfNecessaryLocked(const ConfigKey& key, MetricsManager& metricsManager);
225 
226     // Maps the isolated uid in the log event to host uid if the log event contains uid fields.
227     void mapIsolatedUidToHostUidIfNecessaryLocked(LogEvent* event) const;
228 
229     // Handler over the isolated uid change event.
230     void onIsolatedUidChangedEventLocked(const LogEvent& event);
231 
232     // Handler over the binary push state changed event.
233     void onBinaryPushStateChangedEventLocked(LogEvent* event);
234 
235     // Handler over the watchdog rollback occurred event.
236     void onWatchdogRollbackOccurredLocked(LogEvent* event);
237 
238     // Updates train info on disk based on binary push state changed info and
239     // write disk info into parameters.
240     void getAndUpdateTrainInfoOnDisk(bool is_rollback, InstallTrainInfo* trainInfoIn);
241 
242     // Gets experiment ids on disk for associated train and updates them
243     // depending on rollback type. Then writes them back to disk and returns
244     // them.
245     std::vector<int64_t> processWatchdogRollbackOccurred(const int32_t rollbackTypeIn,
246                                                           const string& packageName);
247 
248     // Reset all configs.
249     void resetConfigsLocked(const int64_t timestampNs);
250     // Reset the specified configs.
251     void resetConfigsLocked(const int64_t timestampNs, const std::vector<ConfigKey>& configs);
252 
253     // Function used to send a broadcast so that receiver for the config key can call getData
254     // to retrieve the stored data.
255     std::function<bool(const ConfigKey& key)> mSendBroadcast;
256 
257     // Function used to send a broadcast so that receiver can be notified of which configs
258     // are currently active.
259     std::function<bool(const int& uid, const vector<int64_t>& configIds)> mSendActivationBroadcast;
260 
261     const int64_t mTimeBaseNs;
262 
263     // Largest timestamp of the events that we have processed.
264     int64_t mLargestTimestampSeen = 0;
265 
266     int64_t mLastTimestampSeen = 0;
267 
268     int64_t mLastPullerCacheClearTimeSec = 0;
269 
270     // Last time we wrote data to disk.
271     int64_t mLastWriteTimeNs = 0;
272 
273     // Last time we wrote active metrics to disk.
274     int64_t mLastActiveMetricsWriteNs = 0;
275 
276     //Last time we wrote metadata to disk.
277     int64_t mLastMetadataWriteNs = 0;
278 
279 #ifdef VERY_VERBOSE_PRINTING
280     bool mPrintAllLogs = false;
281 #endif
282 
283     FRIEND_TEST(StatsLogProcessorTest, TestOutOfOrderLogs);
284     FRIEND_TEST(StatsLogProcessorTest, TestRateLimitByteSize);
285     FRIEND_TEST(StatsLogProcessorTest, TestRateLimitBroadcast);
286     FRIEND_TEST(StatsLogProcessorTest, TestDropWhenByteSizeTooLarge);
287     FRIEND_TEST(StatsLogProcessorTest, InvalidConfigRemoved);
288     FRIEND_TEST(StatsLogProcessorTest, TestActiveConfigMetricDiskWriteRead);
289     FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBoot);
290     FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBootMultipleActivations);
291     FRIEND_TEST(StatsLogProcessorTest,
292             TestActivationOnBootMultipleActivationsDifferentActivationTypes);
293     FRIEND_TEST(StatsLogProcessorTest, TestActivationsPersistAcrossSystemServerRestart);
294 
295     FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration1);
296     FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration2);
297     FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration3);
298     FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration1);
299     FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration2);
300     FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration3);
301     FRIEND_TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks1);
302     FRIEND_TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks2);
303     FRIEND_TEST(AttributionE2eTest, TestAttributionMatchAndSliceByFirstUid);
304     FRIEND_TEST(AttributionE2eTest, TestAttributionMatchAndSliceByChain);
305     FRIEND_TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent);
306     FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents);
307     FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvent_LateAlarm);
308     FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEventsWithActivation);
309     FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEventsNoCondition);
310     FRIEND_TEST(GaugeMetricE2eTest, TestConditionChangeToTrueSamplePulledEvents);
311 
312     FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_single_bucket);
313     FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_multiple_buckets);
314     FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk_no_data_written);
315     FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk);
316     FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_load_refractory_from_disk);
317     FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_single_bucket);
318     FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_multiple_buckets);
319     FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_long_refractory_period);
320 
321     FRIEND_TEST(AlarmE2eTest, TestMultipleAlarms);
322     FRIEND_TEST(ConfigTtlE2eTest, TestCountMetric);
323     FRIEND_TEST(MetricActivationE2eTest, TestCountMetric);
324     FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithOneDeactivation);
325     FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoDeactivations);
326     FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithSameDeactivation);
327     FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoMetricsTwoDeactivations);
328 
329     FRIEND_TEST(CountMetricE2eTest, TestInitialConditionChanges);
330     FRIEND_TEST(CountMetricE2eTest, TestSlicedState);
331     FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithMap);
332     FRIEND_TEST(CountMetricE2eTest, TestMultipleSlicedStates);
333     FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithPrimaryFields);
334 
335     FRIEND_TEST(DurationMetricE2eTest, TestOneBucket);
336     FRIEND_TEST(DurationMetricE2eTest, TestTwoBuckets);
337     FRIEND_TEST(DurationMetricE2eTest, TestWithActivation);
338     FRIEND_TEST(DurationMetricE2eTest, TestWithCondition);
339     FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedCondition);
340     FRIEND_TEST(DurationMetricE2eTest, TestWithActivationAndSlicedCondition);
341     FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedState);
342     FRIEND_TEST(DurationMetricE2eTest, TestWithConditionAndSlicedState);
343     FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedStateMapped);
344     FRIEND_TEST(DurationMetricE2eTest, TestSlicedStatePrimaryFieldsNotSubsetDimInWhat);
345     FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedStatePrimaryFieldsSubset);
346 
347     FRIEND_TEST(ValueMetricE2eTest, TestInitialConditionChanges);
348     FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents);
349     FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents_LateAlarm);
350     FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents_WithActivation);
351     FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState);
352     FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithDimensions);
353     FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithIncorrectDimensions);
354 };
355 
356 }  // namespace statsd
357 }  // namespace os
358 }  // namespace android
359