1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.net; 18 19 import static android.annotation.SystemApi.Client.MODULE_LIBRARIES; 20 import static android.net.NetworkStats.DEFAULT_NETWORK_NO; 21 import static android.net.NetworkStats.DEFAULT_NETWORK_YES; 22 import static android.net.NetworkStats.IFACE_ALL; 23 import static android.net.NetworkStats.METERED_NO; 24 import static android.net.NetworkStats.METERED_YES; 25 import static android.net.NetworkStats.ROAMING_NO; 26 import static android.net.NetworkStats.ROAMING_YES; 27 import static android.net.NetworkStats.SET_ALL; 28 import static android.net.NetworkStats.SET_DEFAULT; 29 import static android.net.NetworkStats.SET_FOREGROUND; 30 import static android.net.NetworkStats.TAG_NONE; 31 import static android.net.NetworkStats.UID_ALL; 32 import static android.net.NetworkTemplate.MATCH_BLUETOOTH; 33 import static android.net.NetworkTemplate.MATCH_ETHERNET; 34 import static android.net.NetworkTemplate.MATCH_MOBILE; 35 import static android.net.NetworkTemplate.MATCH_PROXY; 36 import static android.net.NetworkTemplate.MATCH_WIFI; 37 import static android.net.TrafficStats.UID_REMOVED; 38 import static android.text.format.DateUtils.WEEK_IN_MILLIS; 39 40 import static com.android.net.module.util.NetworkStatsUtils.multiplySafeByRational; 41 42 import android.annotation.NonNull; 43 import android.annotation.Nullable; 44 import android.annotation.SystemApi; 45 import android.net.NetworkStats.State; 46 import android.net.NetworkStatsHistory.Entry; 47 import android.os.Binder; 48 import android.service.NetworkStatsCollectionKeyProto; 49 import android.service.NetworkStatsCollectionProto; 50 import android.service.NetworkStatsCollectionStatsProto; 51 import android.telephony.SubscriptionPlan; 52 import android.text.format.DateUtils; 53 import android.util.ArrayMap; 54 import android.util.ArraySet; 55 import android.util.AtomicFile; 56 import android.util.IndentingPrintWriter; 57 import android.util.Log; 58 import android.util.Range; 59 import android.util.proto.ProtoOutputStream; 60 61 import com.android.internal.annotations.VisibleForTesting; 62 import com.android.internal.util.FileRotator; 63 import com.android.modules.utils.FastDataInput; 64 import com.android.net.module.util.CollectionUtils; 65 import com.android.net.module.util.NetworkStatsUtils; 66 67 import libcore.io.IoUtils; 68 69 import java.io.BufferedInputStream; 70 import java.io.DataInput; 71 import java.io.DataInputStream; 72 import java.io.DataOutput; 73 import java.io.DataOutputStream; 74 import java.io.File; 75 import java.io.FileNotFoundException; 76 import java.io.IOException; 77 import java.io.InputStream; 78 import java.io.OutputStream; 79 import java.io.PrintWriter; 80 import java.net.ProtocolException; 81 import java.time.ZonedDateTime; 82 import java.util.ArrayList; 83 import java.util.Collections; 84 import java.util.HashMap; 85 import java.util.Iterator; 86 import java.util.List; 87 import java.util.Map; 88 import java.util.Objects; 89 import java.util.Set; 90 91 /** 92 * Collection of {@link NetworkStatsHistory}, stored based on combined key of 93 * {@link NetworkIdentitySet}, UID, set, and tag. Knows how to persist itself. 94 * 95 * @hide 96 */ 97 @SystemApi(client = MODULE_LIBRARIES) 98 public class NetworkStatsCollection implements FileRotator.Reader, FileRotator.Writer { 99 private static final String TAG = NetworkStatsCollection.class.getSimpleName(); 100 /** File header magic number: "ANET" */ 101 private static final int FILE_MAGIC = 0x414E4554; 102 103 private static final int VERSION_NETWORK_INIT = 1; 104 105 private static final int VERSION_UID_INIT = 1; 106 private static final int VERSION_UID_WITH_IDENT = 2; 107 private static final int VERSION_UID_WITH_TAG = 3; 108 private static final int VERSION_UID_WITH_SET = 4; 109 110 private static final int VERSION_UNIFIED_INIT = 16; 111 112 private ArrayMap<Key, NetworkStatsHistory> mStats = new ArrayMap<>(); 113 114 private final long mBucketDurationMillis; 115 116 private long mStartMillis; 117 private long mEndMillis; 118 private long mTotalBytes; 119 private boolean mDirty; 120 private final boolean mUseFastDataInput; 121 122 /** 123 * Construct a {@link NetworkStatsCollection} object. 124 * 125 * @param bucketDurationMillis duration of the buckets in this object, in milliseconds. 126 * @hide 127 */ NetworkStatsCollection(long bucketDurationMillis)128 public NetworkStatsCollection(long bucketDurationMillis) { 129 this(bucketDurationMillis, false /* useFastDataInput */); 130 } 131 132 /** 133 * Construct a {@link NetworkStatsCollection} object. 134 * 135 * @param bucketDurationMillis duration of the buckets in this object, in milliseconds. 136 * @param useFastDataInput true if using {@link FastDataInput} is preferred. Otherwise, false. 137 * @hide 138 */ NetworkStatsCollection(long bucketDurationMillis, boolean useFastDataInput)139 public NetworkStatsCollection(long bucketDurationMillis, boolean useFastDataInput) { 140 mBucketDurationMillis = bucketDurationMillis; 141 mUseFastDataInput = useFastDataInput; 142 reset(); 143 } 144 145 /** @hide */ clear()146 public void clear() { 147 reset(); 148 } 149 150 /** @hide */ reset()151 public void reset() { 152 mStats.clear(); 153 mStartMillis = Long.MAX_VALUE; 154 mEndMillis = Long.MIN_VALUE; 155 mTotalBytes = 0; 156 mDirty = false; 157 } 158 159 /** @hide */ getStartMillis()160 public long getStartMillis() { 161 return mStartMillis; 162 } 163 164 /** 165 * Return first atomic bucket in this collection, which is more conservative 166 * than {@link #mStartMillis}. 167 * @hide 168 */ getFirstAtomicBucketMillis()169 public long getFirstAtomicBucketMillis() { 170 if (mStartMillis == Long.MAX_VALUE) { 171 return Long.MAX_VALUE; 172 } else { 173 return mStartMillis + mBucketDurationMillis; 174 } 175 } 176 177 /** @hide */ getEndMillis()178 public long getEndMillis() { 179 return mEndMillis; 180 } 181 182 /** @hide */ getTotalBytes()183 public long getTotalBytes() { 184 return mTotalBytes; 185 } 186 187 /** @hide */ isDirty()188 public boolean isDirty() { 189 return mDirty; 190 } 191 192 /** @hide */ clearDirty()193 public void clearDirty() { 194 mDirty = false; 195 } 196 197 /** @hide */ isEmpty()198 public boolean isEmpty() { 199 return mStartMillis == Long.MAX_VALUE && mEndMillis == Long.MIN_VALUE; 200 } 201 202 /** @hide */ 203 @VisibleForTesting roundUp(long time)204 public long roundUp(long time) { 205 if (time == Long.MIN_VALUE || time == Long.MAX_VALUE 206 || time == SubscriptionPlan.TIME_UNKNOWN) { 207 return time; 208 } else { 209 final long mod = time % mBucketDurationMillis; 210 if (mod > 0) { 211 time -= mod; 212 time += mBucketDurationMillis; 213 } 214 return time; 215 } 216 } 217 218 /** @hide */ 219 @VisibleForTesting roundDown(long time)220 public long roundDown(long time) { 221 if (time == Long.MIN_VALUE || time == Long.MAX_VALUE 222 || time == SubscriptionPlan.TIME_UNKNOWN) { 223 return time; 224 } else { 225 final long mod = time % mBucketDurationMillis; 226 if (mod > 0) { 227 time -= mod; 228 } 229 return time; 230 } 231 } 232 233 /** @hide */ getRelevantUids(@etworkStatsAccess.Level int accessLevel)234 public int[] getRelevantUids(@NetworkStatsAccess.Level int accessLevel) { 235 return getRelevantUids(accessLevel, Binder.getCallingUid()); 236 } 237 238 /** @hide */ getRelevantUids(@etworkStatsAccess.Level int accessLevel, final int callerUid)239 public int[] getRelevantUids(@NetworkStatsAccess.Level int accessLevel, 240 final int callerUid) { 241 final ArrayList<Integer> uids = new ArrayList<>(); 242 for (int i = 0; i < mStats.size(); i++) { 243 final Key key = mStats.keyAt(i); 244 if (NetworkStatsAccess.isAccessibleToUser(key.uid, callerUid, accessLevel)) { 245 int j = Collections.binarySearch(uids, new Integer(key.uid)); 246 247 if (j < 0) { 248 j = ~j; 249 uids.add(j, key.uid); 250 } 251 } 252 } 253 return CollectionUtils.toIntArray(uids); 254 } 255 256 /** 257 * Combine all {@link NetworkStatsHistory} in this collection which match 258 * the requested parameters. 259 * @hide 260 */ getHistory(NetworkTemplate template, SubscriptionPlan augmentPlan, int uid, int set, int tag, int fields, long start, long end, @NetworkStatsAccess.Level int accessLevel, int callerUid)261 public NetworkStatsHistory getHistory(NetworkTemplate template, SubscriptionPlan augmentPlan, 262 int uid, int set, int tag, int fields, long start, long end, 263 @NetworkStatsAccess.Level int accessLevel, int callerUid) { 264 if (!NetworkStatsAccess.isAccessibleToUser(uid, callerUid, accessLevel)) { 265 throw new SecurityException("Network stats history of uid " + uid 266 + " is forbidden for caller " + callerUid); 267 } 268 269 // 180 days of history should be enough for anyone; if we end up needing 270 // more, we'll dynamically grow the history object. 271 final int bucketEstimate = (int) NetworkStatsUtils.constrain( 272 ((end - start) / mBucketDurationMillis), 0, 273 (180 * DateUtils.DAY_IN_MILLIS) / mBucketDurationMillis); 274 final NetworkStatsHistory combined = new NetworkStatsHistory( 275 mBucketDurationMillis, bucketEstimate, fields); 276 277 // shortcut when we know stats will be empty 278 if (start == end) return combined; 279 280 // Figure out the window of time that we should be augmenting (if any) 281 long augmentStart = SubscriptionPlan.TIME_UNKNOWN; 282 long augmentEnd = (augmentPlan != null) ? augmentPlan.getDataUsageTime() 283 : SubscriptionPlan.TIME_UNKNOWN; 284 // And if augmenting, we might need to collect more data to adjust with 285 long collectStart = start; 286 long collectEnd = end; 287 288 if (augmentEnd != SubscriptionPlan.TIME_UNKNOWN) { 289 final Iterator<Range<ZonedDateTime>> it = augmentPlan.cycleIterator(); 290 while (it.hasNext()) { 291 final Range<ZonedDateTime> cycle = it.next(); 292 final long cycleStart = cycle.getLower().toInstant().toEpochMilli(); 293 final long cycleEnd = cycle.getUpper().toInstant().toEpochMilli(); 294 if (cycleStart <= augmentEnd && augmentEnd < cycleEnd) { 295 augmentStart = cycleStart; 296 collectStart = Long.min(collectStart, augmentStart); 297 collectEnd = Long.max(collectEnd, augmentEnd); 298 break; 299 } 300 } 301 } 302 303 if (augmentStart != SubscriptionPlan.TIME_UNKNOWN) { 304 // Shrink augmentation window so we don't risk undercounting. 305 augmentStart = roundUp(augmentStart); 306 augmentEnd = roundDown(augmentEnd); 307 // Grow collection window so we get all the stats needed. 308 collectStart = roundDown(collectStart); 309 collectEnd = roundUp(collectEnd); 310 } 311 312 for (int i = 0; i < mStats.size(); i++) { 313 final Key key = mStats.keyAt(i); 314 if (key.uid == uid && NetworkStats.setMatches(set, key.set) && key.tag == tag 315 && templateMatches(template, key.ident)) { 316 final NetworkStatsHistory value = mStats.valueAt(i); 317 combined.recordHistory(value, collectStart, collectEnd); 318 } 319 } 320 321 if (augmentStart != SubscriptionPlan.TIME_UNKNOWN) { 322 final NetworkStatsHistory.Entry entry = combined.getValues( 323 augmentStart, augmentEnd, null); 324 325 // If we don't have any recorded data for this time period, give 326 // ourselves something to scale with. 327 if (entry.rxBytes == 0 || entry.txBytes == 0) { 328 combined.recordData(augmentStart, augmentEnd, 329 new NetworkStats.Entry(IFACE_ALL, UID_ALL, SET_DEFAULT, TAG_NONE, 330 METERED_NO, ROAMING_NO, DEFAULT_NETWORK_NO, 1L, 0L, 1L, 0L, 0L)); 331 combined.getValues(augmentStart, augmentEnd, entry); 332 } 333 334 final long rawBytes = (entry.rxBytes + entry.txBytes) == 0 ? 1 : 335 (entry.rxBytes + entry.txBytes); 336 final long rawRxBytes = entry.rxBytes == 0 ? 1 : entry.rxBytes; 337 final long rawTxBytes = entry.txBytes == 0 ? 1 : entry.txBytes; 338 final long targetBytes = augmentPlan.getDataUsageBytes(); 339 340 final long targetRxBytes = multiplySafeByRational(targetBytes, rawRxBytes, rawBytes); 341 final long targetTxBytes = multiplySafeByRational(targetBytes, rawTxBytes, rawBytes); 342 343 344 // Scale all matching buckets to reach anchor target 345 final long beforeTotal = combined.getTotalBytes(); 346 for (int i = 0; i < combined.size(); i++) { 347 combined.getValues(i, entry); 348 if (entry.bucketStart >= augmentStart 349 && entry.bucketStart + entry.bucketDuration <= augmentEnd) { 350 entry.rxBytes = multiplySafeByRational( 351 targetRxBytes, entry.rxBytes, rawRxBytes); 352 entry.txBytes = multiplySafeByRational( 353 targetTxBytes, entry.txBytes, rawTxBytes); 354 // We purposefully clear out packet counters to indicate 355 // that this data has been augmented. 356 entry.rxPackets = 0; 357 entry.txPackets = 0; 358 combined.setValues(i, entry); 359 } 360 } 361 362 final long deltaTotal = combined.getTotalBytes() - beforeTotal; 363 if (deltaTotal != 0) { 364 Log.d(TAG, "Augmented network usage by " + deltaTotal + " bytes"); 365 } 366 367 // Finally we can slice data as originally requested 368 final NetworkStatsHistory sliced = new NetworkStatsHistory( 369 mBucketDurationMillis, bucketEstimate, fields); 370 sliced.recordHistory(combined, start, end); 371 return sliced; 372 } else { 373 return combined; 374 } 375 } 376 377 /** 378 * Summarize all {@link NetworkStatsHistory} in this collection which match 379 * the requested parameters across the requested range. 380 * 381 * @param template - a predicate for filtering netstats. 382 * @param start - start of the range, timestamp in milliseconds since the epoch. 383 * @param end - end of the range, timestamp in milliseconds since the epoch. 384 * @param accessLevel - caller access level. 385 * @param callerUid - caller UID. 386 * @hide 387 */ getSummary(NetworkTemplate template, long start, long end, @NetworkStatsAccess.Level int accessLevel, int callerUid)388 public NetworkStats getSummary(NetworkTemplate template, long start, long end, 389 @NetworkStatsAccess.Level int accessLevel, int callerUid) { 390 final long now = System.currentTimeMillis(); 391 392 final NetworkStats stats = new NetworkStats(end - start, 24); 393 394 // shortcut when we know stats will be empty 395 if (start == end) return stats; 396 397 final NetworkStats.Entry entry = new NetworkStats.Entry(); 398 NetworkStatsHistory.Entry historyEntry = null; 399 400 for (int i = 0; i < mStats.size(); i++) { 401 final Key key = mStats.keyAt(i); 402 if (templateMatches(template, key.ident) 403 && NetworkStatsAccess.isAccessibleToUser(key.uid, callerUid, accessLevel) 404 && key.set < NetworkStats.SET_DEBUG_START) { 405 final NetworkStatsHistory value = mStats.valueAt(i); 406 historyEntry = value.getValues(start, end, now, historyEntry); 407 408 entry.iface = IFACE_ALL; 409 entry.uid = key.uid; 410 entry.set = key.set; 411 entry.tag = key.tag; 412 entry.defaultNetwork = key.ident.areAllMembersOnDefaultNetwork() 413 ? DEFAULT_NETWORK_YES : DEFAULT_NETWORK_NO; 414 entry.metered = key.ident.isAnyMemberMetered() ? METERED_YES : METERED_NO; 415 entry.roaming = key.ident.isAnyMemberRoaming() ? ROAMING_YES : ROAMING_NO; 416 entry.rxBytes = historyEntry.rxBytes; 417 entry.rxPackets = historyEntry.rxPackets; 418 entry.txBytes = historyEntry.txBytes; 419 entry.txPackets = historyEntry.txPackets; 420 entry.operations = historyEntry.operations; 421 422 if (!entry.isEmpty()) { 423 stats.combineValues(entry); 424 } 425 } 426 } 427 428 return stats; 429 } 430 431 /** 432 * Record given {@link android.net.NetworkStats.Entry} into this collection. 433 * @hide 434 */ recordData(NetworkIdentitySet ident, int uid, int set, int tag, long start, long end, NetworkStats.Entry entry)435 public void recordData(NetworkIdentitySet ident, int uid, int set, int tag, long start, 436 long end, NetworkStats.Entry entry) { 437 final NetworkStatsHistory history = findOrCreateHistory(ident, uid, set, tag); 438 history.recordData(start, end, entry); 439 noteRecordedHistory(history.getStart(), history.getEnd(), entry.rxBytes + entry.txBytes); 440 } 441 442 /** 443 * Record given {@link NetworkStatsHistory} into this collection. 444 * 445 * @hide 446 */ recordHistory(@onNull Key key, @NonNull NetworkStatsHistory history)447 public void recordHistory(@NonNull Key key, @NonNull NetworkStatsHistory history) { 448 Objects.requireNonNull(key); 449 Objects.requireNonNull(history); 450 if (history.size() == 0) return; 451 noteRecordedHistory(history.getStart(), history.getEnd(), history.getTotalBytes()); 452 453 NetworkStatsHistory target = mStats.get(key); 454 if (target == null) { 455 target = new NetworkStatsHistory(history.getBucketDuration()); 456 mStats.put(key, target); 457 } 458 target.recordEntireHistory(history); 459 } 460 461 /** 462 * Record all {@link NetworkStatsHistory} contained in the given collection 463 * into this collection. 464 * 465 * @hide 466 */ recordCollection(@onNull NetworkStatsCollection another)467 public void recordCollection(@NonNull NetworkStatsCollection another) { 468 Objects.requireNonNull(another); 469 for (int i = 0; i < another.mStats.size(); i++) { 470 final Key key = another.mStats.keyAt(i); 471 final NetworkStatsHistory value = another.mStats.valueAt(i); 472 recordHistory(key, value); 473 } 474 } 475 findOrCreateHistory( NetworkIdentitySet ident, int uid, int set, int tag)476 private NetworkStatsHistory findOrCreateHistory( 477 NetworkIdentitySet ident, int uid, int set, int tag) { 478 final Key key = new Key(ident, uid, set, tag); 479 final NetworkStatsHistory existing = mStats.get(key); 480 481 // update when no existing, or when bucket duration changed 482 NetworkStatsHistory updated = null; 483 if (existing == null) { 484 updated = new NetworkStatsHistory(mBucketDurationMillis, 10); 485 } else if (existing.getBucketDuration() != mBucketDurationMillis) { 486 updated = new NetworkStatsHistory(existing, mBucketDurationMillis); 487 } 488 489 if (updated != null) { 490 mStats.put(key, updated); 491 return updated; 492 } else { 493 return existing; 494 } 495 } 496 497 /** @hide */ 498 @Override read(InputStream in)499 public void read(InputStream in) throws IOException { 500 if (mUseFastDataInput) { 501 read(FastDataInput.obtain(in)); 502 } else { 503 read((DataInput) new DataInputStream(in)); 504 } 505 } 506 read(DataInput in)507 private void read(DataInput in) throws IOException { 508 // verify file magic header intact 509 final int magic = in.readInt(); 510 if (magic != FILE_MAGIC) { 511 throw new ProtocolException("unexpected magic: " + magic); 512 } 513 514 final int version = in.readInt(); 515 switch (version) { 516 case VERSION_UNIFIED_INIT: { 517 // uid := size *(NetworkIdentitySet size *(uid set tag NetworkStatsHistory)) 518 final int identSize = in.readInt(); 519 for (int i = 0; i < identSize; i++) { 520 final NetworkIdentitySet ident = new NetworkIdentitySet(in); 521 522 final int size = in.readInt(); 523 for (int j = 0; j < size; j++) { 524 final int uid = in.readInt(); 525 final int set = in.readInt(); 526 final int tag = in.readInt(); 527 528 final Key key = new Key(ident, uid, set, tag); 529 final NetworkStatsHistory history = new NetworkStatsHistory(in); 530 recordHistory(key, history); 531 } 532 } 533 break; 534 } 535 default: { 536 throw new ProtocolException("unexpected version: " + version); 537 } 538 } 539 } 540 541 /** @hide */ 542 @Override write(OutputStream out)543 public void write(OutputStream out) throws IOException { 544 write((DataOutput) new DataOutputStream(out)); 545 out.flush(); 546 } 547 write(DataOutput out)548 private void write(DataOutput out) throws IOException { 549 // cluster key lists grouped by ident 550 final HashMap<NetworkIdentitySet, ArrayList<Key>> keysByIdent = new HashMap<>(); 551 for (Key key : mStats.keySet()) { 552 ArrayList<Key> keys = keysByIdent.get(key.ident); 553 if (keys == null) { 554 keys = new ArrayList<>(); 555 keysByIdent.put(key.ident, keys); 556 } 557 keys.add(key); 558 } 559 560 out.writeInt(FILE_MAGIC); 561 out.writeInt(VERSION_UNIFIED_INIT); 562 563 out.writeInt(keysByIdent.size()); 564 for (NetworkIdentitySet ident : keysByIdent.keySet()) { 565 final ArrayList<Key> keys = keysByIdent.get(ident); 566 ident.writeToStream(out); 567 568 out.writeInt(keys.size()); 569 for (Key key : keys) { 570 final NetworkStatsHistory history = mStats.get(key); 571 out.writeInt(key.uid); 572 out.writeInt(key.set); 573 out.writeInt(key.tag); 574 history.writeToStream(out); 575 } 576 } 577 } 578 579 /** 580 * Read legacy network summary statistics file format into the collection, 581 * See {@code NetworkStatsService#maybeUpgradeLegacyStatsLocked}. 582 * 583 * @deprecated 584 * @hide 585 */ 586 @Deprecated readLegacyNetwork(File file)587 public void readLegacyNetwork(File file) throws IOException { 588 final AtomicFile inputFile = new AtomicFile(file); 589 590 DataInputStream in = null; 591 try { 592 in = new DataInputStream(new BufferedInputStream(inputFile.openRead())); 593 594 // verify file magic header intact 595 final int magic = in.readInt(); 596 if (magic != FILE_MAGIC) { 597 throw new ProtocolException("unexpected magic: " + magic); 598 } 599 600 final int version = in.readInt(); 601 switch (version) { 602 case VERSION_NETWORK_INIT: { 603 // network := size *(NetworkIdentitySet NetworkStatsHistory) 604 final int size = in.readInt(); 605 for (int i = 0; i < size; i++) { 606 final NetworkIdentitySet ident = new NetworkIdentitySet(in); 607 final NetworkStatsHistory history = new NetworkStatsHistory(in); 608 609 final Key key = new Key(ident, UID_ALL, SET_ALL, TAG_NONE); 610 recordHistory(key, history); 611 } 612 break; 613 } 614 default: { 615 throw new ProtocolException("unexpected version: " + version); 616 } 617 } 618 } catch (FileNotFoundException e) { 619 // missing stats is okay, probably first boot 620 } finally { 621 IoUtils.closeQuietly(in); 622 } 623 } 624 625 /** 626 * Read legacy Uid statistics file format into the collection, 627 * See {@code NetworkStatsService#maybeUpgradeLegacyStatsLocked}. 628 * 629 * @deprecated 630 * @hide 631 */ 632 @Deprecated readLegacyUid(File file, boolean onlyTags)633 public void readLegacyUid(File file, boolean onlyTags) throws IOException { 634 final AtomicFile inputFile = new AtomicFile(file); 635 636 DataInputStream in = null; 637 try { 638 in = new DataInputStream(new BufferedInputStream(inputFile.openRead())); 639 640 // verify file magic header intact 641 final int magic = in.readInt(); 642 if (magic != FILE_MAGIC) { 643 throw new ProtocolException("unexpected magic: " + magic); 644 } 645 646 final int version = in.readInt(); 647 switch (version) { 648 case VERSION_UID_INIT: { 649 // uid := size *(UID NetworkStatsHistory) 650 651 // drop this data version, since we don't have a good 652 // mapping into NetworkIdentitySet. 653 break; 654 } 655 case VERSION_UID_WITH_IDENT: { 656 // uid := size *(NetworkIdentitySet size *(UID NetworkStatsHistory)) 657 658 // drop this data version, since this version only existed 659 // for a short time. 660 break; 661 } 662 case VERSION_UID_WITH_TAG: 663 case VERSION_UID_WITH_SET: { 664 // uid := size *(NetworkIdentitySet size *(uid set tag NetworkStatsHistory)) 665 final int identSize = in.readInt(); 666 for (int i = 0; i < identSize; i++) { 667 final NetworkIdentitySet ident = new NetworkIdentitySet(in); 668 669 final int size = in.readInt(); 670 for (int j = 0; j < size; j++) { 671 final int uid = in.readInt(); 672 final int set = (version >= VERSION_UID_WITH_SET) ? in.readInt() 673 : SET_DEFAULT; 674 final int tag = in.readInt(); 675 676 final Key key = new Key(ident, uid, set, tag); 677 final NetworkStatsHistory history = new NetworkStatsHistory(in); 678 679 if ((tag == TAG_NONE) != onlyTags) { 680 recordHistory(key, history); 681 } 682 } 683 } 684 break; 685 } 686 default: { 687 throw new ProtocolException("unexpected version: " + version); 688 } 689 } 690 } catch (FileNotFoundException e) { 691 // missing stats is okay, probably first boot 692 } finally { 693 IoUtils.closeQuietly(in); 694 } 695 } 696 697 /** 698 * Remove any {@link NetworkStatsHistory} attributed to the requested UID, 699 * moving any {@link NetworkStats#TAG_NONE} series to 700 * {@link TrafficStats#UID_REMOVED}. 701 * @hide 702 */ removeUids(int[] uids)703 public void removeUids(int[] uids) { 704 final ArrayList<Key> knownKeys = new ArrayList<>(); 705 knownKeys.addAll(mStats.keySet()); 706 707 // migrate all UID stats into special "removed" bucket 708 for (Key key : knownKeys) { 709 if (CollectionUtils.contains(uids, key.uid)) { 710 // only migrate combined TAG_NONE history 711 if (key.tag == TAG_NONE) { 712 final NetworkStatsHistory uidHistory = mStats.get(key); 713 final NetworkStatsHistory removedHistory = findOrCreateHistory( 714 key.ident, UID_REMOVED, SET_DEFAULT, TAG_NONE); 715 removedHistory.recordEntireHistory(uidHistory); 716 } 717 mStats.remove(key); 718 mDirty = true; 719 } 720 } 721 } 722 723 /** 724 * Remove histories which contains or is before the cutoff timestamp. 725 * @hide 726 */ removeHistoryBefore(long cutoffMillis)727 public void removeHistoryBefore(long cutoffMillis) { 728 final ArrayList<Key> knownKeys = new ArrayList<>(); 729 knownKeys.addAll(mStats.keySet()); 730 731 for (Key key : knownKeys) { 732 final NetworkStatsHistory history = mStats.get(key); 733 if (history.getStart() > cutoffMillis) continue; 734 735 history.removeBucketsStartingBefore(cutoffMillis); 736 if (history.size() == 0) { 737 mStats.remove(key); 738 } 739 mDirty = true; 740 } 741 } 742 noteRecordedHistory(long startMillis, long endMillis, long totalBytes)743 private void noteRecordedHistory(long startMillis, long endMillis, long totalBytes) { 744 if (startMillis < mStartMillis) mStartMillis = startMillis; 745 if (endMillis > mEndMillis) mEndMillis = endMillis; 746 mTotalBytes += totalBytes; 747 mDirty = true; 748 } 749 estimateBuckets()750 private int estimateBuckets() { 751 return (int) (Math.min(mEndMillis - mStartMillis, WEEK_IN_MILLIS * 5) 752 / mBucketDurationMillis); 753 } 754 getSortedKeys()755 private ArrayList<Key> getSortedKeys() { 756 final ArrayList<Key> keys = new ArrayList<>(); 757 keys.addAll(mStats.keySet()); 758 Collections.sort(keys, (left, right) -> Key.compare(left, right)); 759 return keys; 760 } 761 762 /** @hide */ dump(IndentingPrintWriter pw)763 public void dump(IndentingPrintWriter pw) { 764 for (Key key : getSortedKeys()) { 765 pw.print("ident="); pw.print(key.ident.toString()); 766 pw.print(" uid="); pw.print(key.uid); 767 pw.print(" set="); pw.print(NetworkStats.setToString(key.set)); 768 pw.print(" tag="); pw.println(NetworkStats.tagToString(key.tag)); 769 770 final NetworkStatsHistory history = mStats.get(key); 771 pw.increaseIndent(); 772 history.dump(pw, true); 773 pw.decreaseIndent(); 774 } 775 } 776 777 /** @hide */ dumpDebug(ProtoOutputStream proto, long tag)778 public void dumpDebug(ProtoOutputStream proto, long tag) { 779 final long start = proto.start(tag); 780 781 for (Key key : getSortedKeys()) { 782 final long startStats = proto.start(NetworkStatsCollectionProto.STATS); 783 784 // Key 785 final long startKey = proto.start(NetworkStatsCollectionStatsProto.KEY); 786 key.ident.dumpDebug(proto, NetworkStatsCollectionKeyProto.IDENTITY); 787 proto.write(NetworkStatsCollectionKeyProto.UID, key.uid); 788 proto.write(NetworkStatsCollectionKeyProto.SET, key.set); 789 proto.write(NetworkStatsCollectionKeyProto.TAG, key.tag); 790 proto.end(startKey); 791 792 // Value 793 final NetworkStatsHistory history = mStats.get(key); 794 history.dumpDebug(proto, NetworkStatsCollectionStatsProto.HISTORY); 795 proto.end(startStats); 796 } 797 798 proto.end(start); 799 } 800 801 /** @hide */ dumpCheckin(PrintWriter pw, long start, long end)802 public void dumpCheckin(PrintWriter pw, long start, long end) { 803 dumpCheckin(pw, start, end, new NetworkTemplate.Builder(MATCH_MOBILE) 804 .setMeteredness(METERED_YES).build(), "cell"); 805 dumpCheckin(pw, start, end, new NetworkTemplate.Builder(MATCH_WIFI).build(), "wifi"); 806 dumpCheckin(pw, start, end, new NetworkTemplate.Builder(MATCH_ETHERNET).build(), "eth"); 807 dumpCheckin(pw, start, end, new NetworkTemplate.Builder(MATCH_BLUETOOTH).build(), "bt"); 808 dumpCheckin(pw, start, end, new NetworkTemplate.Builder(MATCH_PROXY).build(), "proxy"); 809 } 810 811 /** 812 * Dump all contained stats that match requested parameters, but group 813 * together all matching {@link NetworkTemplate} under a single prefix. 814 */ dumpCheckin(PrintWriter pw, long start, long end, NetworkTemplate groupTemplate, String groupPrefix)815 private void dumpCheckin(PrintWriter pw, long start, long end, NetworkTemplate groupTemplate, 816 String groupPrefix) { 817 final ArrayMap<Key, NetworkStatsHistory> grouped = new ArrayMap<>(); 818 819 // Walk through all history, grouping by matching network templates 820 for (int i = 0; i < mStats.size(); i++) { 821 final Key key = mStats.keyAt(i); 822 final NetworkStatsHistory value = mStats.valueAt(i); 823 824 if (!templateMatches(groupTemplate, key.ident)) continue; 825 if (key.set >= NetworkStats.SET_DEBUG_START) continue; 826 827 final Key groupKey = new Key(new NetworkIdentitySet(), key.uid, key.set, key.tag); 828 NetworkStatsHistory groupHistory = grouped.get(groupKey); 829 if (groupHistory == null) { 830 groupHistory = new NetworkStatsHistory(value.getBucketDuration()); 831 grouped.put(groupKey, groupHistory); 832 } 833 groupHistory.recordHistory(value, start, end); 834 } 835 836 for (int i = 0; i < grouped.size(); i++) { 837 final Key key = grouped.keyAt(i); 838 final NetworkStatsHistory value = grouped.valueAt(i); 839 840 if (value.size() == 0) continue; 841 842 pw.print("c,"); 843 pw.print(groupPrefix); pw.print(','); 844 pw.print(key.uid); pw.print(','); 845 pw.print(NetworkStats.setToCheckinString(key.set)); pw.print(','); 846 pw.print(key.tag); 847 pw.println(); 848 849 value.dumpCheckin(pw); 850 } 851 } 852 853 /** 854 * Test if given {@link NetworkTemplate} matches any {@link NetworkIdentity} 855 * in the given {@link NetworkIdentitySet}. 856 */ templateMatches(NetworkTemplate template, NetworkIdentitySet identSet)857 private static boolean templateMatches(NetworkTemplate template, NetworkIdentitySet identSet) { 858 for (NetworkIdentity ident : identSet) { 859 if (template.matches(ident)) { 860 return true; 861 } 862 } 863 return false; 864 } 865 866 /** 867 * Get the all historical stats of the collection {@link NetworkStatsCollection}. 868 * 869 * @return All {@link NetworkStatsHistory} in this collection. 870 */ 871 @NonNull getEntries()872 public Map<Key, NetworkStatsHistory> getEntries() { 873 return new ArrayMap(mStats); 874 } 875 876 /** 877 * Builder class for {@link NetworkStatsCollection}. 878 */ 879 public static final class Builder { 880 private final long mBucketDurationMillis; 881 private final ArrayMap<Key, NetworkStatsHistory> mEntries = new ArrayMap<>(); 882 883 /** 884 * Creates a new Builder with given bucket duration. 885 * 886 * @param bucketDuration Duration of the buckets of the object, in milliseconds. 887 */ Builder(long bucketDurationMillis)888 public Builder(long bucketDurationMillis) { 889 mBucketDurationMillis = bucketDurationMillis; 890 } 891 892 /** 893 * Add association of the history with the specified key in this map. 894 * 895 * @param key The object used to identify a network, see {@link Key}. 896 * If history already exists for this key, then the passed-in history is appended 897 * to the previously-passed in history. The caller must ensure that the history 898 * passed-in timestamps are greater than all previously-passed-in timestamps. 899 * @param history {@link NetworkStatsHistory} instance associated to the given {@link Key}. 900 * @return The builder object. 901 */ 902 @NonNull addEntry(@onNull Key key, @NonNull NetworkStatsHistory history)903 public NetworkStatsCollection.Builder addEntry(@NonNull Key key, 904 @NonNull NetworkStatsHistory history) { 905 Objects.requireNonNull(key); 906 Objects.requireNonNull(history); 907 final List<Entry> historyEntries = history.getEntries(); 908 final NetworkStatsHistory existing = mEntries.get(key); 909 910 final int size = historyEntries.size() + ((existing != null) ? existing.size() : 0); 911 final NetworkStatsHistory.Builder historyBuilder = 912 new NetworkStatsHistory.Builder(mBucketDurationMillis, size); 913 914 // TODO: this simply appends the entries to any entries that were already present in 915 // the builder, which requires the caller to pass in entries in order. We might be 916 // able to do better with something like recordHistory. 917 if (existing != null) { 918 for (Entry entry : existing.getEntries()) { 919 historyBuilder.addEntry(entry); 920 } 921 } 922 923 for (Entry entry : historyEntries) { 924 historyBuilder.addEntry(entry); 925 } 926 927 mEntries.put(key, historyBuilder.build()); 928 return this; 929 } 930 931 /** 932 * Builds the instance of the {@link NetworkStatsCollection}. 933 * 934 * @return the built instance of {@link NetworkStatsCollection}. 935 */ 936 @NonNull build()937 public NetworkStatsCollection build() { 938 final NetworkStatsCollection collection = 939 new NetworkStatsCollection(mBucketDurationMillis); 940 for (int i = 0; i < mEntries.size(); i++) { 941 collection.recordHistory(mEntries.keyAt(i), mEntries.valueAt(i)); 942 } 943 return collection; 944 } 945 } 946 947 str(NetworkStatsCollection.Key key)948 private static String str(NetworkStatsCollection.Key key) { 949 StringBuilder sb = new StringBuilder() 950 .append(key.ident.toString()) 951 .append(" uid=").append(key.uid); 952 if (key.set != SET_FOREGROUND) { 953 sb.append(" set=").append(key.set); 954 } 955 if (key.tag != 0) { 956 sb.append(" tag=").append(key.tag); 957 } 958 return sb.toString(); 959 } 960 961 // The importer will modify some keys when importing them. 962 // In order to keep the comparison code simple, add such special cases here and simply 963 // ignore them. This should not impact fidelity much because the start/end checks and the total 964 // bytes check still need to pass. couldKeyChangeOnImport(NetworkStatsCollection.Key key)965 private static boolean couldKeyChangeOnImport(NetworkStatsCollection.Key key) { 966 if (key.ident.isEmpty()) return false; 967 final NetworkIdentity firstIdent = key.ident.iterator().next(); 968 969 // Non-mobile network with non-empty RAT type. 970 // This combination is invalid and the NetworkIdentity.Builder will throw if it is passed 971 // in, but it looks like it was previously possible to persist it to disk. The importer sets 972 // the RAT type to NETWORK_TYPE_ALL. 973 if (firstIdent.getType() != ConnectivityManager.TYPE_MOBILE 974 && firstIdent.getRatType() != NetworkTemplate.NETWORK_TYPE_ALL) { 975 return true; 976 } 977 978 return false; 979 } 980 981 /** 982 * Compare two {@link NetworkStatsCollection} instances and returning a human-readable 983 * string description of difference for debugging purpose. 984 * 985 * @hide 986 */ 987 @Nullable compareStats(NetworkStatsCollection migrated, NetworkStatsCollection legacy, boolean allowKeyChange)988 public static String compareStats(NetworkStatsCollection migrated, 989 NetworkStatsCollection legacy, boolean allowKeyChange) { 990 final Map<NetworkStatsCollection.Key, NetworkStatsHistory> migEntries = 991 migrated.getEntries(); 992 final Map<NetworkStatsCollection.Key, NetworkStatsHistory> legEntries = legacy.getEntries(); 993 994 final ArraySet<NetworkStatsCollection.Key> unmatchedLegKeys = 995 new ArraySet<>(legEntries.keySet()); 996 997 for (NetworkStatsCollection.Key legKey : legEntries.keySet()) { 998 final NetworkStatsHistory legHistory = legEntries.get(legKey); 999 final NetworkStatsHistory migHistory = migEntries.get(legKey); 1000 1001 if (migHistory == null && allowKeyChange && couldKeyChangeOnImport(legKey)) { 1002 unmatchedLegKeys.remove(legKey); 1003 continue; 1004 } 1005 1006 if (migHistory == null) { 1007 return "Missing migrated history for legacy key " + str(legKey) 1008 + ", legacy history was " + legHistory; 1009 } 1010 if (!migHistory.isSameAs(legHistory)) { 1011 return "Difference in history for key " + legKey + "; legacy history " + legHistory 1012 + ", migrated history " + migHistory; 1013 } 1014 unmatchedLegKeys.remove(legKey); 1015 } 1016 1017 if (!unmatchedLegKeys.isEmpty()) { 1018 final NetworkStatsHistory first = legEntries.get(unmatchedLegKeys.valueAt(0)); 1019 return "Found unmatched legacy keys: count=" + unmatchedLegKeys.size() 1020 + ", first unmatched collection " + first; 1021 } 1022 1023 if (migrated.getStartMillis() != legacy.getStartMillis() 1024 || migrated.getEndMillis() != legacy.getEndMillis()) { 1025 return "Start / end of the collections " 1026 + migrated.getStartMillis() + "/" + legacy.getStartMillis() + " and " 1027 + migrated.getEndMillis() + "/" + legacy.getEndMillis() 1028 + " don't match"; 1029 } 1030 1031 if (migrated.getTotalBytes() != legacy.getTotalBytes()) { 1032 return "Total bytes " + migrated.getTotalBytes() + " and " + legacy.getTotalBytes() 1033 + " don't match for collections with start/end " 1034 + migrated.getStartMillis() 1035 + "/" + legacy.getStartMillis(); 1036 } 1037 1038 return null; 1039 } 1040 1041 /** 1042 * the identifier that associate with the {@link NetworkStatsHistory} object to identify 1043 * a certain record in the {@link NetworkStatsCollection} object. 1044 */ 1045 public static final class Key { 1046 /** @hide */ 1047 public final NetworkIdentitySet ident; 1048 /** @hide */ 1049 public final int uid; 1050 /** @hide */ 1051 public final int set; 1052 /** @hide */ 1053 public final int tag; 1054 1055 private final int mHashCode; 1056 1057 /** 1058 * Construct a {@link Key} object. 1059 * 1060 * @param ident a Set of {@link NetworkIdentity} that associated with the record. 1061 * @param uid Uid of the record. 1062 * @param set Set of the record, see {@code NetworkStats#SET_*}. 1063 * @param tag Tag of the record, see {@link TrafficStats#setThreadStatsTag(int)}. 1064 */ Key(@onNull Set<NetworkIdentity> ident, int uid, @State int set, int tag)1065 public Key(@NonNull Set<NetworkIdentity> ident, int uid, @State int set, int tag) { 1066 this(new NetworkIdentitySet(Objects.requireNonNull(ident)), uid, set, tag); 1067 } 1068 1069 /** @hide */ Key(@onNull NetworkIdentitySet ident, int uid, int set, int tag)1070 public Key(@NonNull NetworkIdentitySet ident, int uid, int set, int tag) { 1071 this.ident = Objects.requireNonNull(ident); 1072 this.uid = uid; 1073 this.set = set; 1074 this.tag = tag; 1075 mHashCode = Objects.hash(ident, uid, set, tag); 1076 } 1077 1078 @Override hashCode()1079 public int hashCode() { 1080 return mHashCode; 1081 } 1082 1083 @Override equals(@ullable Object obj)1084 public boolean equals(@Nullable Object obj) { 1085 if (obj instanceof Key) { 1086 final Key key = (Key) obj; 1087 return uid == key.uid && set == key.set && tag == key.tag 1088 && Objects.equals(ident, key.ident); 1089 } 1090 return false; 1091 } 1092 1093 /** @hide */ compare(@onNull Key left, @NonNull Key right)1094 public static int compare(@NonNull Key left, @NonNull Key right) { 1095 Objects.requireNonNull(left); 1096 Objects.requireNonNull(right); 1097 int res = 0; 1098 if (left.ident != null && right.ident != null) { 1099 res = NetworkIdentitySet.compare(left.ident, right.ident); 1100 } 1101 if (res == 0) { 1102 res = Integer.compare(left.uid, right.uid); 1103 } 1104 if (res == 0) { 1105 res = Integer.compare(left.set, right.set); 1106 } 1107 if (res == 0) { 1108 res = Integer.compare(left.tag, right.tag); 1109 } 1110 return res; 1111 } 1112 } 1113 } 1114