1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.net; 18 19 import static android.net.NetworkStats.TAG_NONE; 20 import static android.net.TrafficStats.KB_IN_BYTES; 21 import static android.net.TrafficStats.MB_IN_BYTES; 22 import static android.text.format.DateUtils.YEAR_IN_MILLIS; 23 import static com.android.internal.util.Preconditions.checkNotNull; 24 25 import android.annotation.Nullable; 26 import android.net.NetworkStats; 27 import android.net.NetworkStats.NonMonotonicObserver; 28 import android.net.NetworkStatsHistory; 29 import android.net.NetworkTemplate; 30 import android.net.TrafficStats; 31 import android.os.DropBoxManager; 32 import android.util.Log; 33 import android.util.MathUtils; 34 import android.util.Slog; 35 36 import com.android.internal.net.VpnInfo; 37 import com.android.internal.util.FileRotator; 38 import com.android.internal.util.IndentingPrintWriter; 39 import com.google.android.collect.Sets; 40 41 import java.io.ByteArrayOutputStream; 42 import java.io.DataOutputStream; 43 import java.io.File; 44 import java.io.IOException; 45 import java.io.InputStream; 46 import java.io.OutputStream; 47 import java.io.PrintWriter; 48 import java.lang.ref.WeakReference; 49 import java.util.Arrays; 50 import java.util.HashSet; 51 import java.util.Map; 52 53 import libcore.io.IoUtils; 54 55 /** 56 * Logic to record deltas between periodic {@link NetworkStats} snapshots into 57 * {@link NetworkStatsHistory} that belong to {@link NetworkStatsCollection}. 58 * Keeps pending changes in memory until they pass a specific threshold, in 59 * bytes. Uses {@link FileRotator} for persistence logic if present. 60 * <p> 61 * Not inherently thread safe. 62 */ 63 public class NetworkStatsRecorder { 64 private static final String TAG = "NetworkStatsRecorder"; 65 private static final boolean LOGD = false; 66 private static final boolean LOGV = false; 67 68 private static final String TAG_NETSTATS_DUMP = "netstats_dump"; 69 70 /** Dump before deleting in {@link #recoverFromWtf()}. */ 71 private static final boolean DUMP_BEFORE_DELETE = true; 72 73 private final FileRotator mRotator; 74 private final NonMonotonicObserver<String> mObserver; 75 private final DropBoxManager mDropBox; 76 private final String mCookie; 77 78 private final long mBucketDuration; 79 private final boolean mOnlyTags; 80 81 private long mPersistThresholdBytes = 2 * MB_IN_BYTES; 82 private NetworkStats mLastSnapshot; 83 84 private final NetworkStatsCollection mPending; 85 private final NetworkStatsCollection mSinceBoot; 86 87 private final CombiningRewriter mPendingRewriter; 88 89 private WeakReference<NetworkStatsCollection> mComplete; 90 91 /** 92 * Non-persisted recorder, with only one bucket. Used by {@link NetworkStatsObservers}. 93 */ NetworkStatsRecorder()94 public NetworkStatsRecorder() { 95 mRotator = null; 96 mObserver = null; 97 mDropBox = null; 98 mCookie = null; 99 100 // set the bucket big enough to have all data in one bucket, but allow some 101 // slack to avoid overflow 102 mBucketDuration = YEAR_IN_MILLIS; 103 mOnlyTags = false; 104 105 mPending = null; 106 mSinceBoot = new NetworkStatsCollection(mBucketDuration); 107 108 mPendingRewriter = null; 109 } 110 111 /** 112 * Persisted recorder. 113 */ NetworkStatsRecorder(FileRotator rotator, NonMonotonicObserver<String> observer, DropBoxManager dropBox, String cookie, long bucketDuration, boolean onlyTags)114 public NetworkStatsRecorder(FileRotator rotator, NonMonotonicObserver<String> observer, 115 DropBoxManager dropBox, String cookie, long bucketDuration, boolean onlyTags) { 116 mRotator = checkNotNull(rotator, "missing FileRotator"); 117 mObserver = checkNotNull(observer, "missing NonMonotonicObserver"); 118 mDropBox = checkNotNull(dropBox, "missing DropBoxManager"); 119 mCookie = cookie; 120 121 mBucketDuration = bucketDuration; 122 mOnlyTags = onlyTags; 123 124 mPending = new NetworkStatsCollection(bucketDuration); 125 mSinceBoot = new NetworkStatsCollection(bucketDuration); 126 127 mPendingRewriter = new CombiningRewriter(mPending); 128 } 129 setPersistThreshold(long thresholdBytes)130 public void setPersistThreshold(long thresholdBytes) { 131 if (LOGV) Slog.v(TAG, "setPersistThreshold() with " + thresholdBytes); 132 mPersistThresholdBytes = MathUtils.constrain( 133 thresholdBytes, 1 * KB_IN_BYTES, 100 * MB_IN_BYTES); 134 } 135 resetLocked()136 public void resetLocked() { 137 mLastSnapshot = null; 138 if (mPending != null) { 139 mPending.reset(); 140 } 141 if (mSinceBoot != null) { 142 mSinceBoot.reset(); 143 } 144 if (mComplete != null) { 145 mComplete.clear(); 146 } 147 } 148 getTotalSinceBootLocked(NetworkTemplate template)149 public NetworkStats.Entry getTotalSinceBootLocked(NetworkTemplate template) { 150 return mSinceBoot.getSummary(template, Long.MIN_VALUE, Long.MAX_VALUE, 151 NetworkStatsAccess.Level.DEVICE).getTotal(null); 152 } 153 getSinceBoot()154 public NetworkStatsCollection getSinceBoot() { 155 return mSinceBoot; 156 } 157 158 /** 159 * Load complete history represented by {@link FileRotator}. Caches 160 * internally as a {@link WeakReference}, and updated with future 161 * {@link #recordSnapshotLocked(NetworkStats, Map, long)} snapshots as long 162 * as reference is valid. 163 */ getOrLoadCompleteLocked()164 public NetworkStatsCollection getOrLoadCompleteLocked() { 165 checkNotNull(mRotator, "missing FileRotator"); 166 NetworkStatsCollection res = mComplete != null ? mComplete.get() : null; 167 if (res == null) { 168 res = loadLocked(Long.MIN_VALUE, Long.MAX_VALUE); 169 mComplete = new WeakReference<NetworkStatsCollection>(res); 170 } 171 return res; 172 } 173 getOrLoadPartialLocked(long start, long end)174 public NetworkStatsCollection getOrLoadPartialLocked(long start, long end) { 175 checkNotNull(mRotator, "missing FileRotator"); 176 NetworkStatsCollection res = mComplete != null ? mComplete.get() : null; 177 if (res == null) { 178 res = loadLocked(start, end); 179 } 180 return res; 181 } 182 loadLocked(long start, long end)183 private NetworkStatsCollection loadLocked(long start, long end) { 184 if (LOGD) Slog.d(TAG, "loadLocked() reading from disk for " + mCookie); 185 final NetworkStatsCollection res = new NetworkStatsCollection(mBucketDuration); 186 try { 187 mRotator.readMatching(res, start, end); 188 res.recordCollection(mPending); 189 } catch (IOException e) { 190 Log.wtf(TAG, "problem completely reading network stats", e); 191 recoverFromWtf(); 192 } catch (OutOfMemoryError e) { 193 Log.wtf(TAG, "problem completely reading network stats", e); 194 recoverFromWtf(); 195 } 196 return res; 197 } 198 199 /** 200 * Record any delta that occurred since last {@link NetworkStats} snapshot, 201 * using the given {@link Map} to identify network interfaces. First 202 * snapshot is considered bootstrap, and is not counted as delta. 203 * 204 * @param vpnArray Optional info about the currently active VPN, if any. This is used to 205 * redistribute traffic from the VPN app to the underlying responsible apps. 206 * This should always be set to null if the provided snapshot is aggregated 207 * across all UIDs (e.g. contains UID_ALL buckets), regardless of VPN state. 208 */ recordSnapshotLocked(NetworkStats snapshot, Map<String, NetworkIdentitySet> ifaceIdent, @Nullable VpnInfo[] vpnArray, long currentTimeMillis)209 public void recordSnapshotLocked(NetworkStats snapshot, 210 Map<String, NetworkIdentitySet> ifaceIdent, @Nullable VpnInfo[] vpnArray, 211 long currentTimeMillis) { 212 final HashSet<String> unknownIfaces = Sets.newHashSet(); 213 214 // skip recording when snapshot missing 215 if (snapshot == null) return; 216 217 // assume first snapshot is bootstrap and don't record 218 if (mLastSnapshot == null) { 219 mLastSnapshot = snapshot; 220 return; 221 } 222 223 final NetworkStatsCollection complete = mComplete != null ? mComplete.get() : null; 224 225 final NetworkStats delta = NetworkStats.subtract( 226 snapshot, mLastSnapshot, mObserver, mCookie); 227 final long end = currentTimeMillis; 228 final long start = end - delta.getElapsedRealtime(); 229 230 if (vpnArray != null) { 231 for (VpnInfo info : vpnArray) { 232 delta.migrateTun(info.ownerUid, info.vpnIface, info.primaryUnderlyingIface); 233 } 234 } 235 236 NetworkStats.Entry entry = null; 237 for (int i = 0; i < delta.size(); i++) { 238 entry = delta.getValues(i, entry); 239 final NetworkIdentitySet ident = ifaceIdent.get(entry.iface); 240 if (ident == null) { 241 unknownIfaces.add(entry.iface); 242 continue; 243 } 244 245 // skip when no delta occurred 246 if (entry.isEmpty()) continue; 247 248 // only record tag data when requested 249 if ((entry.tag == TAG_NONE) != mOnlyTags) { 250 if (mPending != null) { 251 mPending.recordData(ident, entry.uid, entry.set, entry.tag, start, end, entry); 252 } 253 254 // also record against boot stats when present 255 if (mSinceBoot != null) { 256 mSinceBoot.recordData(ident, entry.uid, entry.set, entry.tag, start, end, entry); 257 } 258 259 // also record against complete dataset when present 260 if (complete != null) { 261 complete.recordData(ident, entry.uid, entry.set, entry.tag, start, end, entry); 262 } 263 } 264 } 265 266 mLastSnapshot = snapshot; 267 268 if (LOGV && unknownIfaces.size() > 0) { 269 Slog.w(TAG, "unknown interfaces " + unknownIfaces + ", ignoring those stats"); 270 } 271 } 272 273 /** 274 * Consider persisting any pending deltas, if they are beyond 275 * {@link #mPersistThresholdBytes}. 276 */ maybePersistLocked(long currentTimeMillis)277 public void maybePersistLocked(long currentTimeMillis) { 278 checkNotNull(mRotator, "missing FileRotator"); 279 final long pendingBytes = mPending.getTotalBytes(); 280 if (pendingBytes >= mPersistThresholdBytes) { 281 forcePersistLocked(currentTimeMillis); 282 } else { 283 mRotator.maybeRotate(currentTimeMillis); 284 } 285 } 286 287 /** 288 * Force persisting any pending deltas. 289 */ forcePersistLocked(long currentTimeMillis)290 public void forcePersistLocked(long currentTimeMillis) { 291 checkNotNull(mRotator, "missing FileRotator"); 292 if (mPending.isDirty()) { 293 if (LOGD) Slog.d(TAG, "forcePersistLocked() writing for " + mCookie); 294 try { 295 mRotator.rewriteActive(mPendingRewriter, currentTimeMillis); 296 mRotator.maybeRotate(currentTimeMillis); 297 mPending.reset(); 298 } catch (IOException e) { 299 Log.wtf(TAG, "problem persisting pending stats", e); 300 recoverFromWtf(); 301 } catch (OutOfMemoryError e) { 302 Log.wtf(TAG, "problem persisting pending stats", e); 303 recoverFromWtf(); 304 } 305 } 306 } 307 308 /** 309 * Remove the given UID from all {@link FileRotator} history, migrating it 310 * to {@link TrafficStats#UID_REMOVED}. 311 */ removeUidsLocked(int[] uids)312 public void removeUidsLocked(int[] uids) { 313 if (mRotator != null) { 314 try { 315 // Rewrite all persisted data to migrate UID stats 316 mRotator.rewriteAll(new RemoveUidRewriter(mBucketDuration, uids)); 317 } catch (IOException e) { 318 Log.wtf(TAG, "problem removing UIDs " + Arrays.toString(uids), e); 319 recoverFromWtf(); 320 } catch (OutOfMemoryError e) { 321 Log.wtf(TAG, "problem removing UIDs " + Arrays.toString(uids), e); 322 recoverFromWtf(); 323 } 324 } 325 326 // Remove any pending stats 327 if (mPending != null) { 328 mPending.removeUids(uids); 329 } 330 if (mSinceBoot != null) { 331 mSinceBoot.removeUids(uids); 332 } 333 334 // Clear UID from current stats snapshot 335 if (mLastSnapshot != null) { 336 mLastSnapshot = mLastSnapshot.withoutUids(uids); 337 } 338 339 final NetworkStatsCollection complete = mComplete != null ? mComplete.get() : null; 340 if (complete != null) { 341 complete.removeUids(uids); 342 } 343 } 344 345 /** 346 * Rewriter that will combine current {@link NetworkStatsCollection} values 347 * with anything read from disk, and write combined set to disk. Clears the 348 * original {@link NetworkStatsCollection} when finished writing. 349 */ 350 private static class CombiningRewriter implements FileRotator.Rewriter { 351 private final NetworkStatsCollection mCollection; 352 CombiningRewriter(NetworkStatsCollection collection)353 public CombiningRewriter(NetworkStatsCollection collection) { 354 mCollection = checkNotNull(collection, "missing NetworkStatsCollection"); 355 } 356 357 @Override reset()358 public void reset() { 359 // ignored 360 } 361 362 @Override read(InputStream in)363 public void read(InputStream in) throws IOException { 364 mCollection.read(in); 365 } 366 367 @Override shouldWrite()368 public boolean shouldWrite() { 369 return true; 370 } 371 372 @Override write(OutputStream out)373 public void write(OutputStream out) throws IOException { 374 mCollection.write(new DataOutputStream(out)); 375 mCollection.reset(); 376 } 377 } 378 379 /** 380 * Rewriter that will remove any {@link NetworkStatsHistory} attributed to 381 * the requested UID, only writing data back when modified. 382 */ 383 public static class RemoveUidRewriter implements FileRotator.Rewriter { 384 private final NetworkStatsCollection mTemp; 385 private final int[] mUids; 386 RemoveUidRewriter(long bucketDuration, int[] uids)387 public RemoveUidRewriter(long bucketDuration, int[] uids) { 388 mTemp = new NetworkStatsCollection(bucketDuration); 389 mUids = uids; 390 } 391 392 @Override reset()393 public void reset() { 394 mTemp.reset(); 395 } 396 397 @Override read(InputStream in)398 public void read(InputStream in) throws IOException { 399 mTemp.read(in); 400 mTemp.clearDirty(); 401 mTemp.removeUids(mUids); 402 } 403 404 @Override shouldWrite()405 public boolean shouldWrite() { 406 return mTemp.isDirty(); 407 } 408 409 @Override write(OutputStream out)410 public void write(OutputStream out) throws IOException { 411 mTemp.write(new DataOutputStream(out)); 412 } 413 } 414 importLegacyNetworkLocked(File file)415 public void importLegacyNetworkLocked(File file) throws IOException { 416 checkNotNull(mRotator, "missing FileRotator"); 417 418 // legacy file still exists; start empty to avoid double importing 419 mRotator.deleteAll(); 420 421 final NetworkStatsCollection collection = new NetworkStatsCollection(mBucketDuration); 422 collection.readLegacyNetwork(file); 423 424 final long startMillis = collection.getStartMillis(); 425 final long endMillis = collection.getEndMillis(); 426 427 if (!collection.isEmpty()) { 428 // process legacy data, creating active file at starting time, then 429 // using end time to possibly trigger rotation. 430 mRotator.rewriteActive(new CombiningRewriter(collection), startMillis); 431 mRotator.maybeRotate(endMillis); 432 } 433 } 434 importLegacyUidLocked(File file)435 public void importLegacyUidLocked(File file) throws IOException { 436 checkNotNull(mRotator, "missing FileRotator"); 437 438 // legacy file still exists; start empty to avoid double importing 439 mRotator.deleteAll(); 440 441 final NetworkStatsCollection collection = new NetworkStatsCollection(mBucketDuration); 442 collection.readLegacyUid(file, mOnlyTags); 443 444 final long startMillis = collection.getStartMillis(); 445 final long endMillis = collection.getEndMillis(); 446 447 if (!collection.isEmpty()) { 448 // process legacy data, creating active file at starting time, then 449 // using end time to possibly trigger rotation. 450 mRotator.rewriteActive(new CombiningRewriter(collection), startMillis); 451 mRotator.maybeRotate(endMillis); 452 } 453 } 454 dumpLocked(IndentingPrintWriter pw, boolean fullHistory)455 public void dumpLocked(IndentingPrintWriter pw, boolean fullHistory) { 456 if (mPending != null) { 457 pw.print("Pending bytes: "); pw.println(mPending.getTotalBytes()); 458 } 459 if (fullHistory) { 460 pw.println("Complete history:"); 461 getOrLoadCompleteLocked().dump(pw); 462 } else { 463 pw.println("History since boot:"); 464 mSinceBoot.dump(pw); 465 } 466 } 467 dumpCheckin(PrintWriter pw, long start, long end)468 public void dumpCheckin(PrintWriter pw, long start, long end) { 469 // Only load and dump stats from the requested window 470 getOrLoadPartialLocked(start, end).dumpCheckin(pw, start, end); 471 } 472 473 /** 474 * Recover from {@link FileRotator} failure by dumping state to 475 * {@link DropBoxManager} and deleting contents. 476 */ recoverFromWtf()477 private void recoverFromWtf() { 478 if (DUMP_BEFORE_DELETE) { 479 final ByteArrayOutputStream os = new ByteArrayOutputStream(); 480 try { 481 mRotator.dumpAll(os); 482 } catch (IOException e) { 483 // ignore partial contents 484 os.reset(); 485 } finally { 486 IoUtils.closeQuietly(os); 487 } 488 mDropBox.addData(TAG_NETSTATS_DUMP, os.toByteArray(), 0); 489 } 490 491 mRotator.deleteAll(); 492 } 493 } 494