1 /**
2  * Copyright (C) 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License"); you may not
5  * use this file except in compliance with the License. You may obtain a copy
6  * of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13  * License for the specific language governing permissions and limitations
14  * under the License.
15  */
16 
17 package com.android.server.usage;
18 
19 import android.app.usage.TimeSparseArray;
20 import android.app.usage.UsageEvents;
21 import android.app.usage.UsageStats;
22 import android.app.usage.UsageStatsManager;
23 import android.os.Build;
24 import android.os.SystemProperties;
25 import android.util.ArrayMap;
26 import android.util.AtomicFile;
27 import android.util.Slog;
28 import android.util.SparseArray;
29 import android.util.TimeUtils;
30 
31 import com.android.internal.annotations.VisibleForTesting;
32 import com.android.internal.util.ArrayUtils;
33 import com.android.internal.util.IndentingPrintWriter;
34 
35 import libcore.io.IoUtils;
36 
37 import java.io.BufferedReader;
38 import java.io.BufferedWriter;
39 import java.io.ByteArrayInputStream;
40 import java.io.ByteArrayOutputStream;
41 import java.io.DataInputStream;
42 import java.io.DataOutputStream;
43 import java.io.File;
44 import java.io.FileInputStream;
45 import java.io.FileNotFoundException;
46 import java.io.FileOutputStream;
47 import java.io.FileReader;
48 import java.io.FileWriter;
49 import java.io.FilenameFilter;
50 import java.io.IOException;
51 import java.io.InputStream;
52 import java.io.OutputStream;
53 import java.nio.file.Files;
54 import java.nio.file.StandardCopyOption;
55 import java.util.ArrayList;
56 import java.util.HashMap;
57 import java.util.List;
58 
59 /**
60  * Provides an interface to query for UsageStat data from a Protocol Buffer database.
61  *
62  * Prior to version 4, UsageStatsDatabase used XML to store Usage Stats data to disk.
63  * When the UsageStatsDatabase version is upgraded, the files on disk are migrated to the new
64  * version on init. The steps of migration are as follows:
65  * 1) Check if version upgrade breadcrumb exists on disk, if so skip to step 4.
66  * 2) Move current files to a timestamped backup directory.
67  * 3) Write a temporary breadcrumb file with some info about the backup directory.
68  * 4) Deserialize the backup files in the timestamped backup folder referenced by the breadcrumb.
69  * 5) Reserialize the data read from the file with the new version format and replace the old files
70  * 6) Repeat Step 3 and 4 for each file in the backup folder.
71  * 7) Update the version file with the new version and build fingerprint.
72  * 8) Delete the time stamped backup folder (unless flagged to be kept).
73  * 9) Delete the breadcrumb file.
74  *
75  * Performing the upgrade steps in this order, protects against unexpected shutdowns mid upgrade
76  *
77  * The backup directory will contain directories with timestamp names. If the upgrade breadcrumb
78  * exists on disk, it will contain a timestamp which will match one of the backup directories. The
79  * breadcrumb will also contain a version number which will denote how the files in the backup
80  * directory should be deserialized.
81  */
82 public class UsageStatsDatabase {
83     private static final int DEFAULT_CURRENT_VERSION = 5;
84     /**
85      * Current version of the backup schema
86      *
87      * @hide
88      */
89     @VisibleForTesting
90     public static final int BACKUP_VERSION = 4;
91 
92     @VisibleForTesting
93     static final int[] MAX_FILES_PER_INTERVAL_TYPE = new int[]{100, 50, 12, 10};
94 
95     // Key under which the payload blob is stored
96     // same as UsageStatsBackupHelper.KEY_USAGE_STATS
97     static final String KEY_USAGE_STATS = "usage_stats";
98 
99     // Persist versioned backup files.
100     // Should be false, except when testing new versions
101     static final boolean KEEP_BACKUP_DIR = false;
102 
103     private static final String TAG = "UsageStatsDatabase";
104     private static final boolean DEBUG = UsageStatsService.DEBUG;
105     private static final String BAK_SUFFIX = ".bak";
106     private static final String CHECKED_IN_SUFFIX = UsageStatsXml.CHECKED_IN_SUFFIX;
107     private static final String RETENTION_LEN_KEY = "ro.usagestats.chooser.retention";
108     private static final int SELECTION_LOG_RETENTION_LEN =
109             SystemProperties.getInt(RETENTION_LEN_KEY, 14);
110 
111     private final Object mLock = new Object();
112     private final File[] mIntervalDirs;
113     @VisibleForTesting
114     final TimeSparseArray<AtomicFile>[] mSortedStatFiles;
115     private final UnixCalendar mCal;
116     private final File mVersionFile;
117     private final File mBackupsDir;
118     // If this file exists on disk, UsageStatsDatabase is in the middle of migrating files to a new
119     // version. If this file exists on boot, the upgrade was interrupted and needs to be picked up
120     // where it left off.
121     private final File mUpdateBreadcrumb;
122     // Current version of the database files schema
123     private int mCurrentVersion;
124     private boolean mFirstUpdate;
125     private boolean mNewUpdate;
126     private boolean mUpgradePerformed;
127 
128     // The obfuscated packages to tokens mappings file
129     private final File mPackageMappingsFile;
130     // Holds all of the data related to the obfuscated packages and their token mappings.
131     final PackagesTokenData mPackagesTokenData = new PackagesTokenData();
132 
133     /**
134      * UsageStatsDatabase constructor that allows setting the version number.
135      * This should only be used for testing.
136      *
137      * @hide
138      */
139     @VisibleForTesting
UsageStatsDatabase(File dir, int version)140     public UsageStatsDatabase(File dir, int version) {
141         mIntervalDirs = new File[]{
142             new File(dir, "daily"),
143             new File(dir, "weekly"),
144             new File(dir, "monthly"),
145             new File(dir, "yearly"),
146         };
147         mCurrentVersion = version;
148         mVersionFile = new File(dir, "version");
149         mBackupsDir = new File(dir, "backups");
150         mUpdateBreadcrumb = new File(dir, "breadcrumb");
151         mSortedStatFiles = new TimeSparseArray[mIntervalDirs.length];
152         mPackageMappingsFile = new File(dir, "mappings");
153         mCal = new UnixCalendar(0);
154     }
155 
UsageStatsDatabase(File dir)156     public UsageStatsDatabase(File dir) {
157         this(dir, DEFAULT_CURRENT_VERSION);
158     }
159 
160     /**
161      * Initialize any directories required and index what stats are available.
162      */
init(long currentTimeMillis)163     public void init(long currentTimeMillis) {
164         synchronized (mLock) {
165             for (File f : mIntervalDirs) {
166                 f.mkdirs();
167                 if (!f.exists()) {
168                     throw new IllegalStateException("Failed to create directory "
169                             + f.getAbsolutePath());
170                 }
171             }
172 
173             checkVersionAndBuildLocked();
174             indexFilesLocked();
175 
176             // Delete files that are in the future.
177             for (TimeSparseArray<AtomicFile> files : mSortedStatFiles) {
178                 final int startIndex = files.closestIndexOnOrAfter(currentTimeMillis);
179                 if (startIndex < 0) {
180                     continue;
181                 }
182 
183                 final int fileCount = files.size();
184                 for (int i = startIndex; i < fileCount; i++) {
185                     files.valueAt(i).delete();
186                 }
187 
188                 // Remove in a separate loop because any accesses (valueAt)
189                 // will cause a gc in the SparseArray and mess up the order.
190                 for (int i = startIndex; i < fileCount; i++) {
191                     files.removeAt(i);
192                 }
193             }
194         }
195     }
196 
197     public interface CheckinAction {
checkin(IntervalStats stats)198         boolean checkin(IntervalStats stats);
199     }
200 
201     /**
202      * Calls {@link CheckinAction#checkin(IntervalStats)} on the given {@link CheckinAction}
203      * for all {@link IntervalStats} that haven't been checked-in.
204      * If any of the calls to {@link CheckinAction#checkin(IntervalStats)} returns false or throws
205      * an exception, the check-in will be aborted.
206      *
207      * @param checkinAction The callback to run when checking-in {@link IntervalStats}.
208      * @return true if the check-in succeeded.
209      */
checkinDailyFiles(CheckinAction checkinAction)210     public boolean checkinDailyFiles(CheckinAction checkinAction) {
211         synchronized (mLock) {
212             final TimeSparseArray<AtomicFile> files =
213                     mSortedStatFiles[UsageStatsManager.INTERVAL_DAILY];
214             final int fileCount = files.size();
215 
216             // We may have holes in the checkin (if there was an error)
217             // so find the last checked-in file and go from there.
218             int lastCheckin = -1;
219             for (int i = 0; i < fileCount - 1; i++) {
220                 if (files.valueAt(i).getBaseFile().getPath().endsWith(CHECKED_IN_SUFFIX)) {
221                     lastCheckin = i;
222                 }
223             }
224 
225             final int start = lastCheckin + 1;
226             if (start == fileCount - 1) {
227                 return true;
228             }
229 
230             try {
231                 for (int i = start; i < fileCount - 1; i++) {
232                     final IntervalStats stats = new IntervalStats();
233                     readLocked(files.valueAt(i), stats);
234                     if (!checkinAction.checkin(stats)) {
235                         return false;
236                     }
237                 }
238             } catch (Exception e) {
239                 Slog.e(TAG, "Failed to check-in", e);
240                 return false;
241             }
242 
243             // We have successfully checked-in the stats, so rename the files so that they
244             // are marked as checked-in.
245             for (int i = start; i < fileCount - 1; i++) {
246                 final AtomicFile file = files.valueAt(i);
247                 final File checkedInFile = new File(
248                         file.getBaseFile().getPath() + CHECKED_IN_SUFFIX);
249                 if (!file.getBaseFile().renameTo(checkedInFile)) {
250                     // We must return success, as we've already marked some files as checked-in.
251                     // It's better to repeat ourselves than to lose data.
252                     Slog.e(TAG, "Failed to mark file " + file.getBaseFile().getPath()
253                             + " as checked-in");
254                     return true;
255                 }
256 
257                 // AtomicFile needs to set a new backup path with the same -c extension, so
258                 // we replace the old AtomicFile with the updated one.
259                 files.setValueAt(i, new AtomicFile(checkedInFile));
260             }
261         }
262         return true;
263     }
264 
265     /** @hide */
266     @VisibleForTesting
forceIndexFiles()267     void forceIndexFiles() {
268         synchronized (mLock) {
269             indexFilesLocked();
270         }
271     }
272 
indexFilesLocked()273     private void indexFilesLocked() {
274         final FilenameFilter backupFileFilter = new FilenameFilter() {
275             @Override
276             public boolean accept(File dir, String name) {
277                 return !name.endsWith(BAK_SUFFIX);
278             }
279         };
280         // Index the available usage stat files on disk.
281         for (int i = 0; i < mSortedStatFiles.length; i++) {
282             if (mSortedStatFiles[i] == null) {
283                 mSortedStatFiles[i] = new TimeSparseArray<>();
284             } else {
285                 mSortedStatFiles[i].clear();
286             }
287             File[] files = mIntervalDirs[i].listFiles(backupFileFilter);
288             if (files != null) {
289                 if (DEBUG) {
290                     Slog.d(TAG, "Found " + files.length + " stat files for interval " + i);
291                 }
292                 final int len = files.length;
293                 for (int j = 0; j < len; j++) {
294                     final File f = files[j];
295                     final AtomicFile af = new AtomicFile(f);
296                     try {
297                         mSortedStatFiles[i].put(parseBeginTime(af), af);
298                     } catch (IOException e) {
299                         Slog.e(TAG, "failed to index file: " + f, e);
300                     }
301                 }
302 
303                 // only keep the max allowed number of files for each interval type.
304                 final int toDelete = mSortedStatFiles[i].size() - MAX_FILES_PER_INTERVAL_TYPE[i];
305                 if (toDelete > 0) {
306                     for (int j = 0; j < toDelete; j++) {
307                         mSortedStatFiles[i].valueAt(0).delete();
308                         mSortedStatFiles[i].removeAt(0);
309                     }
310                     Slog.d(TAG, "Deleted " + toDelete + " stat files for interval " + i);
311                 }
312             }
313         }
314     }
315 
316     /**
317      * Is this the first update to the system from L to M?
318      */
isFirstUpdate()319     boolean isFirstUpdate() {
320         return mFirstUpdate;
321     }
322 
323     /**
324      * Is this a system update since we started tracking build fingerprint in the version file?
325      */
isNewUpdate()326     boolean isNewUpdate() {
327         return mNewUpdate;
328     }
329 
330     /**
331      * Was an upgrade performed when this database was initialized?
332      */
wasUpgradePerformed()333     boolean wasUpgradePerformed() {
334         return mUpgradePerformed;
335     }
336 
checkVersionAndBuildLocked()337     private void checkVersionAndBuildLocked() {
338         int version;
339         String buildFingerprint;
340         String currentFingerprint = getBuildFingerprint();
341         mFirstUpdate = true;
342         mNewUpdate = true;
343         try (BufferedReader reader = new BufferedReader(new FileReader(mVersionFile))) {
344             version = Integer.parseInt(reader.readLine());
345             buildFingerprint = reader.readLine();
346             if (buildFingerprint != null) {
347                 mFirstUpdate = false;
348             }
349             if (currentFingerprint.equals(buildFingerprint)) {
350                 mNewUpdate = false;
351             }
352         } catch (NumberFormatException | IOException e) {
353             version = 0;
354         }
355 
356         if (version != mCurrentVersion) {
357             Slog.i(TAG, "Upgrading from version " + version + " to " + mCurrentVersion);
358             if (!mUpdateBreadcrumb.exists()) {
359                 try {
360                     doUpgradeLocked(version);
361                 } catch (Exception e) {
362                     Slog.e(TAG,
363                             "Failed to upgrade from version " + version + " to " + mCurrentVersion,
364                             e);
365                     // Fallback to previous version.
366                     mCurrentVersion = version;
367                     return;
368                 }
369             } else {
370                 Slog.i(TAG, "Version upgrade breadcrumb found on disk! Continuing version upgrade");
371             }
372         }
373 
374         if (mUpdateBreadcrumb.exists()) {
375             int previousVersion;
376             long token;
377             try (BufferedReader reader = new BufferedReader(
378                     new FileReader(mUpdateBreadcrumb))) {
379                 token = Long.parseLong(reader.readLine());
380                 previousVersion = Integer.parseInt(reader.readLine());
381             } catch (NumberFormatException | IOException e) {
382                 Slog.e(TAG, "Failed read version upgrade breadcrumb");
383                 throw new RuntimeException(e);
384             }
385             if (mCurrentVersion >= 4) {
386                 continueUpgradeLocked(previousVersion, token);
387             } else {
388                 Slog.wtf(TAG, "Attempting to upgrade to an unsupported version: "
389                         + mCurrentVersion);
390             }
391         }
392 
393         if (version != mCurrentVersion || mNewUpdate) {
394             try (BufferedWriter writer = new BufferedWriter(new FileWriter(mVersionFile))) {
395                 writer.write(Integer.toString(mCurrentVersion));
396                 writer.write("\n");
397                 writer.write(currentFingerprint);
398                 writer.write("\n");
399                 writer.flush();
400             } catch (IOException e) {
401                 Slog.e(TAG, "Failed to write new version");
402                 throw new RuntimeException(e);
403             }
404         }
405 
406         if (mUpdateBreadcrumb.exists()) {
407             // Files should be up to date with current version. Clear the version update breadcrumb
408             mUpdateBreadcrumb.delete();
409             // update mUpgradePerformed after breadcrumb is deleted to indicate a successful upgrade
410             mUpgradePerformed = true;
411         }
412 
413         if (mBackupsDir.exists() && !KEEP_BACKUP_DIR) {
414             mUpgradePerformed = true; // updated here to ensure that data is cleaned up
415             deleteDirectory(mBackupsDir);
416         }
417     }
418 
getBuildFingerprint()419     private String getBuildFingerprint() {
420         return Build.VERSION.RELEASE + ";"
421                 + Build.VERSION.CODENAME + ";"
422                 + Build.VERSION.INCREMENTAL;
423     }
424 
doUpgradeLocked(int thisVersion)425     private void doUpgradeLocked(int thisVersion) {
426         if (thisVersion < 2) {
427             // Delete all files if we are version 0. This is a pre-release version,
428             // so this is fine.
429             Slog.i(TAG, "Deleting all usage stats files");
430             for (int i = 0; i < mIntervalDirs.length; i++) {
431                 File[] files = mIntervalDirs[i].listFiles();
432                 if (files != null) {
433                     for (File f : files) {
434                         f.delete();
435                     }
436                 }
437             }
438         } else {
439             // Create a dir in backups based on current timestamp
440             final long token = System.currentTimeMillis();
441             final File backupDir = new File(mBackupsDir, Long.toString(token));
442             backupDir.mkdirs();
443             if (!backupDir.exists()) {
444                 throw new IllegalStateException(
445                         "Failed to create backup directory " + backupDir.getAbsolutePath());
446             }
447             try {
448                 Files.copy(mVersionFile.toPath(),
449                         new File(backupDir, mVersionFile.getName()).toPath(),
450                         StandardCopyOption.REPLACE_EXISTING);
451             } catch (IOException e) {
452                 Slog.e(TAG, "Failed to back up version file : " + mVersionFile.toString());
453                 throw new RuntimeException(e);
454             }
455 
456             for (int i = 0; i < mIntervalDirs.length; i++) {
457                 final File backupIntervalDir = new File(backupDir, mIntervalDirs[i].getName());
458                 backupIntervalDir.mkdir();
459 
460                 if (!backupIntervalDir.exists()) {
461                     throw new IllegalStateException(
462                             "Failed to create interval backup directory "
463                                     + backupIntervalDir.getAbsolutePath());
464                 }
465                 File[] files = mIntervalDirs[i].listFiles();
466                 if (files != null) {
467                     for (int j = 0; j < files.length; j++) {
468                         final File backupFile = new File(backupIntervalDir, files[j].getName());
469                         if (DEBUG) {
470                             Slog.d(TAG, "Creating versioned (" + Integer.toString(thisVersion)
471                                     + ") backup of " + files[j].toString()
472                                     + " stat files for interval "
473                                     + i + " to " + backupFile.toString());
474                         }
475 
476                         try {
477                             // Backup file should not already exist, but make sure it doesn't
478                             Files.move(files[j].toPath(), backupFile.toPath(),
479                                     StandardCopyOption.REPLACE_EXISTING);
480                         } catch (IOException e) {
481                             Slog.e(TAG, "Failed to back up file : " + files[j].toString());
482                             throw new RuntimeException(e);
483                         }
484                     }
485                 }
486             }
487 
488             // Leave a breadcrumb behind noting that all the usage stats have been moved to a backup
489             BufferedWriter writer = null;
490             try {
491                 writer = new BufferedWriter(new FileWriter(mUpdateBreadcrumb));
492                 writer.write(Long.toString(token));
493                 writer.write("\n");
494                 writer.write(Integer.toString(thisVersion));
495                 writer.write("\n");
496                 writer.flush();
497             } catch (IOException e) {
498                 Slog.e(TAG, "Failed to write new version upgrade breadcrumb");
499                 throw new RuntimeException(e);
500             } finally {
501                 IoUtils.closeQuietly(writer);
502             }
503         }
504     }
505 
continueUpgradeLocked(int version, long token)506     private void continueUpgradeLocked(int version, long token) {
507         if (version <= 3) {
508             Slog.w(TAG, "Reading UsageStats as XML; current database version: " + mCurrentVersion);
509         }
510         final File backupDir = new File(mBackupsDir, Long.toString(token));
511 
512         // Upgrade step logic for the entire usage stats directory, not individual interval dirs.
513         if (version >= 5) {
514             readMappingsLocked();
515         }
516 
517         // Read each file in the backup according to the version and write to the interval
518         // directories in the current versions format
519         for (int i = 0; i < mIntervalDirs.length; i++) {
520             final File backedUpInterval = new File(backupDir, mIntervalDirs[i].getName());
521             File[] files = backedUpInterval.listFiles();
522             if (files != null) {
523                 for (int j = 0; j < files.length; j++) {
524                     if (DEBUG) {
525                         Slog.d(TAG,
526                                 "Upgrading " + files[j].toString() + " to version ("
527                                         + Integer.toString(
528                                         mCurrentVersion) + ") for interval " + i);
529                     }
530                     try {
531                         IntervalStats stats = new IntervalStats();
532                         readLocked(new AtomicFile(files[j]), stats, version, mPackagesTokenData);
533                         // Upgrade to version 5+.
534                         // Future version upgrades should add additional logic here to upgrade.
535                         if (mCurrentVersion >= 5) {
536                             // Create the initial obfuscated packages map.
537                             stats.obfuscateData(mPackagesTokenData);
538                         }
539                         writeLocked(new AtomicFile(new File(mIntervalDirs[i],
540                                 Long.toString(stats.beginTime))), stats, mCurrentVersion,
541                                 mPackagesTokenData);
542                     } catch (Exception e) {
543                         // This method is called on boot, log the exception and move on
544                         Slog.e(TAG, "Failed to upgrade backup file : " + files[j].toString());
545                     }
546                 }
547             }
548         }
549 
550         // Upgrade step logic for the entire usage stats directory, not individual interval dirs.
551         if (mCurrentVersion >= 5) {
552             try {
553                 writeMappingsLocked();
554             } catch (IOException e) {
555                 Slog.e(TAG, "Failed to write the tokens mappings file.");
556             }
557         }
558     }
559 
560     /**
561      * Returns the token mapped to the package removed or {@code PackagesTokenData.UNASSIGNED_TOKEN}
562      * if not mapped.
563      */
onPackageRemoved(String packageName, long timeRemoved)564     int onPackageRemoved(String packageName, long timeRemoved) {
565         synchronized (mLock) {
566             final int tokenRemoved = mPackagesTokenData.removePackage(packageName, timeRemoved);
567             try {
568                 writeMappingsLocked();
569             } catch (Exception e) {
570                 Slog.w(TAG, "Unable to update package mappings on disk after removing token "
571                         + tokenRemoved);
572             }
573             return tokenRemoved;
574         }
575     }
576 
577     /**
578      * Reads all the usage stats data on disk and rewrites it with any data related to uninstalled
579      * packages omitted. Returns {@code true} on success, {@code false} otherwise.
580      */
pruneUninstalledPackagesData()581     boolean pruneUninstalledPackagesData() {
582         synchronized (mLock) {
583             for (int i = 0; i < mIntervalDirs.length; i++) {
584                 final File[] files = mIntervalDirs[i].listFiles();
585                 if (files == null) {
586                     continue;
587                 }
588                 for (int j = 0; j < files.length; j++) {
589                     try {
590                         final IntervalStats stats = new IntervalStats();
591                         final AtomicFile atomicFile = new AtomicFile(files[j]);
592                         if (!readLocked(atomicFile, stats, mCurrentVersion, mPackagesTokenData)) {
593                             continue; // no data was omitted when read so no need to rewrite
594                         }
595                         // Any data related to packages that have been removed would have failed
596                         // the deobfuscation step on read so the IntervalStats object here only
597                         // contains data for packages that are currently installed - all we need
598                         // to do here is write the data back to disk.
599                         writeLocked(atomicFile, stats, mCurrentVersion, mPackagesTokenData);
600                     } catch (Exception e) {
601                         Slog.e(TAG, "Failed to prune data from: " + files[j].toString());
602                         return false;
603                     }
604                 }
605             }
606 
607             try {
608                 writeMappingsLocked();
609             } catch (IOException e) {
610                 Slog.e(TAG, "Failed to write package mappings after pruning data.");
611                 return false;
612             }
613             return true;
614         }
615     }
616 
617     /**
618      * Iterates through all the files on disk and prunes any data that belongs to packages that have
619      * been uninstalled (packages that are not in the given list).
620      * Note: this should only be called once, when there has been a database upgrade.
621      *
622      * @param installedPackages map of installed packages (package_name:package_install_time)
623      */
prunePackagesDataOnUpgrade(HashMap<String, Long> installedPackages)624     void prunePackagesDataOnUpgrade(HashMap<String, Long> installedPackages) {
625         if (ArrayUtils.isEmpty(installedPackages)) {
626             return;
627         }
628         synchronized (mLock) {
629             for (int i = 0; i < mIntervalDirs.length; i++) {
630                 final File[] files = mIntervalDirs[i].listFiles();
631                 if (files == null) {
632                     continue;
633                 }
634                 for (int j = 0; j < files.length; j++) {
635                     try {
636                         final IntervalStats stats = new IntervalStats();
637                         final AtomicFile atomicFile = new AtomicFile(files[j]);
638                         readLocked(atomicFile, stats, mCurrentVersion, mPackagesTokenData);
639                         if (!pruneStats(installedPackages, stats)) {
640                             continue; // no stats were pruned so no need to rewrite
641                         }
642                         writeLocked(atomicFile, stats, mCurrentVersion, mPackagesTokenData);
643                     } catch (Exception e) {
644                         Slog.e(TAG, "Failed to prune data from: " + files[j].toString());
645                     }
646                 }
647             }
648         }
649     }
650 
pruneStats(HashMap<String, Long> installedPackages, IntervalStats stats)651     private boolean pruneStats(HashMap<String, Long> installedPackages, IntervalStats stats) {
652         boolean dataPruned = false;
653 
654         // prune old package usage stats
655         for (int i = stats.packageStats.size() - 1; i >= 0; i--) {
656             final UsageStats usageStats = stats.packageStats.valueAt(i);
657             final Long timeInstalled = installedPackages.get(usageStats.mPackageName);
658             if (timeInstalled == null || timeInstalled > usageStats.mEndTimeStamp) {
659                 stats.packageStats.removeAt(i);
660                 dataPruned = true;
661             }
662         }
663         if (dataPruned) {
664             // ensure old stats don't linger around during the obfuscation step on write
665             stats.packageStatsObfuscated.clear();
666         }
667 
668         // prune old events
669         for (int i = stats.events.size() - 1; i >= 0; i--) {
670             final UsageEvents.Event event = stats.events.get(i);
671             final Long timeInstalled = installedPackages.get(event.mPackage);
672             if (timeInstalled == null || timeInstalled > event.mTimeStamp) {
673                 stats.events.remove(i);
674                 dataPruned = true;
675             }
676         }
677 
678         return dataPruned;
679     }
680 
onTimeChanged(long timeDiffMillis)681     public void onTimeChanged(long timeDiffMillis) {
682         synchronized (mLock) {
683             StringBuilder logBuilder = new StringBuilder();
684             logBuilder.append("Time changed by ");
685             TimeUtils.formatDuration(timeDiffMillis, logBuilder);
686             logBuilder.append(".");
687 
688             int filesDeleted = 0;
689             int filesMoved = 0;
690 
691             for (TimeSparseArray<AtomicFile> files : mSortedStatFiles) {
692                 final int fileCount = files.size();
693                 for (int i = 0; i < fileCount; i++) {
694                     final AtomicFile file = files.valueAt(i);
695                     final long newTime = files.keyAt(i) + timeDiffMillis;
696                     if (newTime < 0) {
697                         filesDeleted++;
698                         file.delete();
699                     } else {
700                         try {
701                             file.openRead().close();
702                         } catch (IOException e) {
703                             // Ignore, this is just to make sure there are no backups.
704                         }
705 
706                         String newName = Long.toString(newTime);
707                         if (file.getBaseFile().getName().endsWith(CHECKED_IN_SUFFIX)) {
708                             newName = newName + CHECKED_IN_SUFFIX;
709                         }
710 
711                         final File newFile = new File(file.getBaseFile().getParentFile(), newName);
712                         filesMoved++;
713                         file.getBaseFile().renameTo(newFile);
714                     }
715                 }
716                 files.clear();
717             }
718 
719             logBuilder.append(" files deleted: ").append(filesDeleted);
720             logBuilder.append(" files moved: ").append(filesMoved);
721             Slog.i(TAG, logBuilder.toString());
722 
723             // Now re-index the new files.
724             indexFilesLocked();
725         }
726     }
727 
728     /**
729      * Get the latest stats that exist for this interval type.
730      */
getLatestUsageStats(int intervalType)731     public IntervalStats getLatestUsageStats(int intervalType) {
732         synchronized (mLock) {
733             if (intervalType < 0 || intervalType >= mIntervalDirs.length) {
734                 throw new IllegalArgumentException("Bad interval type " + intervalType);
735             }
736 
737             final int fileCount = mSortedStatFiles[intervalType].size();
738             if (fileCount == 0) {
739                 return null;
740             }
741 
742             try {
743                 final AtomicFile f = mSortedStatFiles[intervalType].valueAt(fileCount - 1);
744                 IntervalStats stats = new IntervalStats();
745                 readLocked(f, stats);
746                 return stats;
747             } catch (Exception e) {
748                 Slog.e(TAG, "Failed to read usage stats file", e);
749             }
750         }
751         return null;
752     }
753 
754     /**
755      * Filter out those stats from the given stats that belong to removed packages. Filtering out
756      * all of the stats at once has an amortized cost for future calls.
757      */
filterStats(IntervalStats stats)758     void filterStats(IntervalStats stats) {
759         if (mPackagesTokenData.removedPackagesMap.isEmpty()) {
760             return;
761         }
762         final ArrayMap<String, Long> removedPackagesMap = mPackagesTokenData.removedPackagesMap;
763 
764         // filter out package usage stats
765         final int removedPackagesSize = removedPackagesMap.size();
766         for (int i = 0; i < removedPackagesSize; i++) {
767             final String removedPackage = removedPackagesMap.keyAt(i);
768             final UsageStats usageStats = stats.packageStats.get(removedPackage);
769             if (usageStats != null && usageStats.mEndTimeStamp < removedPackagesMap.valueAt(i)) {
770                 stats.packageStats.remove(removedPackage);
771             }
772         }
773 
774         // filter out events
775         for (int i = stats.events.size() - 1; i >= 0; i--) {
776             final UsageEvents.Event event = stats.events.get(i);
777             final Long timeRemoved = removedPackagesMap.get(event.mPackage);
778             if (timeRemoved != null && timeRemoved > event.mTimeStamp) {
779                 stats.events.remove(i);
780             }
781         }
782     }
783 
784     /**
785      * Figures out what to extract from the given IntervalStats object.
786      */
787     public interface StatCombiner<T> {
788 
789         /**
790          * Implementations should extract interesting from <code>stats</code> and add it
791          * to the <code>accumulatedResult</code> list.
792          *
793          * If the <code>stats</code> object is mutable, <code>mutable</code> will be true,
794          * which means you should make a copy of the data before adding it to the
795          * <code>accumulatedResult</code> list.
796          *
797          * @param stats             The {@link IntervalStats} object selected.
798          * @param mutable           Whether or not the data inside the stats object is mutable.
799          * @param accumulatedResult The list to which to add extracted data.
800          */
combine(IntervalStats stats, boolean mutable, List<T> accumulatedResult)801         void combine(IntervalStats stats, boolean mutable, List<T> accumulatedResult);
802     }
803 
804     /**
805      * Find all {@link IntervalStats} for the given range and interval type.
806      */
queryUsageStats(int intervalType, long beginTime, long endTime, StatCombiner<T> combiner)807     public <T> List<T> queryUsageStats(int intervalType, long beginTime, long endTime,
808             StatCombiner<T> combiner) {
809         synchronized (mLock) {
810             if (intervalType < 0 || intervalType >= mIntervalDirs.length) {
811                 throw new IllegalArgumentException("Bad interval type " + intervalType);
812             }
813 
814             final TimeSparseArray<AtomicFile> intervalStats = mSortedStatFiles[intervalType];
815 
816             if (endTime <= beginTime) {
817                 if (DEBUG) {
818                     Slog.d(TAG, "endTime(" + endTime + ") <= beginTime(" + beginTime + ")");
819                 }
820                 return null;
821             }
822 
823             int startIndex = intervalStats.closestIndexOnOrBefore(beginTime);
824             if (startIndex < 0) {
825                 // All the stats available have timestamps after beginTime, which means they all
826                 // match.
827                 startIndex = 0;
828             }
829 
830             int endIndex = intervalStats.closestIndexOnOrBefore(endTime);
831             if (endIndex < 0) {
832                 // All the stats start after this range ends, so nothing matches.
833                 if (DEBUG) {
834                     Slog.d(TAG, "No results for this range. All stats start after.");
835                 }
836                 return null;
837             }
838 
839             if (intervalStats.keyAt(endIndex) == endTime) {
840                 // The endTime is exclusive, so if we matched exactly take the one before.
841                 endIndex--;
842                 if (endIndex < 0) {
843                     // All the stats start after this range ends, so nothing matches.
844                     if (DEBUG) {
845                         Slog.d(TAG, "No results for this range. All stats start after.");
846                     }
847                     return null;
848                 }
849             }
850 
851             final ArrayList<T> results = new ArrayList<>();
852             for (int i = startIndex; i <= endIndex; i++) {
853                 final AtomicFile f = intervalStats.valueAt(i);
854                 final IntervalStats stats = new IntervalStats();
855 
856                 if (DEBUG) {
857                     Slog.d(TAG, "Reading stat file " + f.getBaseFile().getAbsolutePath());
858                 }
859 
860                 try {
861                     readLocked(f, stats);
862                     if (beginTime < stats.endTime) {
863                         combiner.combine(stats, false, results);
864                     }
865                 } catch (Exception e) {
866                     Slog.e(TAG, "Failed to read usage stats file", e);
867                     // We continue so that we return results that are not
868                     // corrupt.
869                 }
870             }
871             return results;
872         }
873     }
874 
875     /**
876      * Find the interval that best matches this range.
877      *
878      * TODO(adamlesinski): Use endTimeStamp in best fit calculation.
879      */
findBestFitBucket(long beginTimeStamp, long endTimeStamp)880     public int findBestFitBucket(long beginTimeStamp, long endTimeStamp) {
881         synchronized (mLock) {
882             int bestBucket = -1;
883             long smallestDiff = Long.MAX_VALUE;
884             for (int i = mSortedStatFiles.length - 1; i >= 0; i--) {
885                 final int index = mSortedStatFiles[i].closestIndexOnOrBefore(beginTimeStamp);
886                 int size = mSortedStatFiles[i].size();
887                 if (index >= 0 && index < size) {
888                     // We have some results here, check if they are better than our current match.
889                     long diff = Math.abs(mSortedStatFiles[i].keyAt(index) - beginTimeStamp);
890                     if (diff < smallestDiff) {
891                         smallestDiff = diff;
892                         bestBucket = i;
893                     }
894                 }
895             }
896             return bestBucket;
897         }
898     }
899 
900     /**
901      * Remove any usage stat files that are too old.
902      */
prune(final long currentTimeMillis)903     public void prune(final long currentTimeMillis) {
904         synchronized (mLock) {
905             mCal.setTimeInMillis(currentTimeMillis);
906             mCal.addYears(-3);
907             pruneFilesOlderThan(mIntervalDirs[UsageStatsManager.INTERVAL_YEARLY],
908                     mCal.getTimeInMillis());
909 
910             mCal.setTimeInMillis(currentTimeMillis);
911             mCal.addMonths(-6);
912             pruneFilesOlderThan(mIntervalDirs[UsageStatsManager.INTERVAL_MONTHLY],
913                     mCal.getTimeInMillis());
914 
915             mCal.setTimeInMillis(currentTimeMillis);
916             mCal.addWeeks(-4);
917             pruneFilesOlderThan(mIntervalDirs[UsageStatsManager.INTERVAL_WEEKLY],
918                     mCal.getTimeInMillis());
919 
920             mCal.setTimeInMillis(currentTimeMillis);
921             mCal.addDays(-10);
922             pruneFilesOlderThan(mIntervalDirs[UsageStatsManager.INTERVAL_DAILY],
923                     mCal.getTimeInMillis());
924 
925             mCal.setTimeInMillis(currentTimeMillis);
926             mCal.addDays(-SELECTION_LOG_RETENTION_LEN);
927             for (int i = 0; i < mIntervalDirs.length; ++i) {
928                 pruneChooserCountsOlderThan(mIntervalDirs[i], mCal.getTimeInMillis());
929             }
930 
931             // We must re-index our file list or we will be trying to read
932             // deleted files.
933             indexFilesLocked();
934         }
935     }
936 
pruneFilesOlderThan(File dir, long expiryTime)937     private static void pruneFilesOlderThan(File dir, long expiryTime) {
938         File[] files = dir.listFiles();
939         if (files != null) {
940             for (File f : files) {
941                 long beginTime;
942                 try {
943                     beginTime = parseBeginTime(f);
944                 } catch (IOException e) {
945                     beginTime = 0;
946                 }
947 
948                 if (beginTime < expiryTime) {
949                     new AtomicFile(f).delete();
950                 }
951             }
952         }
953     }
954 
pruneChooserCountsOlderThan(File dir, long expiryTime)955     private void pruneChooserCountsOlderThan(File dir, long expiryTime) {
956         File[] files = dir.listFiles();
957         if (files != null) {
958             for (File f : files) {
959                 long beginTime;
960                 try {
961                     beginTime = parseBeginTime(f);
962                 } catch (IOException e) {
963                     beginTime = 0;
964                 }
965 
966                 if (beginTime < expiryTime) {
967                     try {
968                         final AtomicFile af = new AtomicFile(f);
969                         final IntervalStats stats = new IntervalStats();
970                         readLocked(af, stats);
971                         final int pkgCount = stats.packageStats.size();
972                         for (int i = 0; i < pkgCount; i++) {
973                             UsageStats pkgStats = stats.packageStats.valueAt(i);
974                             if (pkgStats.mChooserCounts != null) {
975                                 pkgStats.mChooserCounts.clear();
976                             }
977                         }
978                         writeLocked(af, stats);
979                     } catch (Exception e) {
980                         Slog.e(TAG, "Failed to delete chooser counts from usage stats file", e);
981                     }
982                 }
983             }
984         }
985     }
986 
987 
parseBeginTime(AtomicFile file)988     private static long parseBeginTime(AtomicFile file) throws IOException {
989         return parseBeginTime(file.getBaseFile());
990     }
991 
parseBeginTime(File file)992     private static long parseBeginTime(File file) throws IOException {
993         String name = file.getName();
994 
995         // Parse out the digits from the the front of the file name
996         for (int i = 0; i < name.length(); i++) {
997             final char c = name.charAt(i);
998             if (c < '0' || c > '9') {
999                 // found first char that is not a digit.
1000                 name = name.substring(0, i);
1001                 break;
1002             }
1003         }
1004 
1005         try {
1006             return Long.parseLong(name);
1007         } catch (NumberFormatException e) {
1008             throw new IOException(e);
1009         }
1010     }
1011 
writeLocked(AtomicFile file, IntervalStats stats)1012     private void writeLocked(AtomicFile file, IntervalStats stats)
1013             throws IOException, RuntimeException {
1014         if (mCurrentVersion <= 3) {
1015             Slog.wtf(TAG, "Attempting to write UsageStats as XML with version " + mCurrentVersion);
1016             return;
1017         }
1018         writeLocked(file, stats, mCurrentVersion, mPackagesTokenData);
1019     }
1020 
writeLocked(AtomicFile file, IntervalStats stats, int version, PackagesTokenData packagesTokenData)1021     private static void writeLocked(AtomicFile file, IntervalStats stats, int version,
1022             PackagesTokenData packagesTokenData) throws IOException, RuntimeException {
1023         FileOutputStream fos = file.startWrite();
1024         try {
1025             writeLocked(fos, stats, version, packagesTokenData);
1026             file.finishWrite(fos);
1027             fos = null;
1028         } finally {
1029             // When fos is null (successful write), this will no-op
1030             file.failWrite(fos);
1031         }
1032     }
1033 
writeLocked(OutputStream out, IntervalStats stats, int version, PackagesTokenData packagesTokenData)1034     private static void writeLocked(OutputStream out, IntervalStats stats, int version,
1035             PackagesTokenData packagesTokenData) throws RuntimeException {
1036         switch (version) {
1037             case 1:
1038             case 2:
1039             case 3:
1040                 Slog.wtf(TAG, "Attempting to write UsageStats as XML with version " + version);
1041                 break;
1042             case 4:
1043                 try {
1044                     UsageStatsProto.write(out, stats);
1045                 } catch (Exception e) {
1046                     Slog.e(TAG, "Unable to write interval stats to proto.", e);
1047                 }
1048                 break;
1049             case 5:
1050                 stats.obfuscateData(packagesTokenData);
1051                 try {
1052                     UsageStatsProtoV2.write(out, stats);
1053                 } catch (Exception e) {
1054                     Slog.e(TAG, "Unable to write interval stats to proto.", e);
1055                 }
1056                 break;
1057             default:
1058                 throw new RuntimeException(
1059                         "Unhandled UsageStatsDatabase version: " + Integer.toString(version)
1060                                 + " on write.");
1061         }
1062     }
1063 
1064     /**
1065      * Note: the data read from the given file will add to the IntervalStats object passed into this
1066      * method. It is up to the caller to ensure that this is the desired behavior - if not, the
1067      * caller should ensure that the data in the reused object is being cleared.
1068      */
readLocked(AtomicFile file, IntervalStats statsOut)1069     private void readLocked(AtomicFile file, IntervalStats statsOut)
1070             throws IOException, RuntimeException {
1071         if (mCurrentVersion <= 3) {
1072             Slog.wtf(TAG, "Reading UsageStats as XML; current database version: "
1073                     + mCurrentVersion);
1074         }
1075         readLocked(file, statsOut, mCurrentVersion, mPackagesTokenData);
1076     }
1077 
1078     /**
1079      * Returns {@code true} if any stats were omitted while reading, {@code false} otherwise.
1080      * <p/>
1081      * Note: the data read from the given file will add to the IntervalStats object passed into this
1082      * method. It is up to the caller to ensure that this is the desired behavior - if not, the
1083      * caller should ensure that the data in the reused object is being cleared.
1084      */
readLocked(AtomicFile file, IntervalStats statsOut, int version, PackagesTokenData packagesTokenData)1085     private static boolean readLocked(AtomicFile file, IntervalStats statsOut, int version,
1086             PackagesTokenData packagesTokenData) throws IOException, RuntimeException {
1087         boolean dataOmitted = false;
1088         try {
1089             FileInputStream in = file.openRead();
1090             try {
1091                 statsOut.beginTime = parseBeginTime(file);
1092                 dataOmitted = readLocked(in, statsOut, version, packagesTokenData);
1093                 statsOut.lastTimeSaved = file.getLastModifiedTime();
1094             } finally {
1095                 try {
1096                     in.close();
1097                 } catch (IOException e) {
1098                     // Empty
1099                 }
1100             }
1101         } catch (FileNotFoundException e) {
1102             Slog.e(TAG, "UsageStatsDatabase", e);
1103             throw e;
1104         }
1105         return dataOmitted;
1106     }
1107 
1108     /**
1109      * Returns {@code true} if any stats were omitted while reading, {@code false} otherwise.
1110      * <p/>
1111      * Note: the data read from the given file will add to the IntervalStats object passed into this
1112      * method. It is up to the caller to ensure that this is the desired behavior - if not, the
1113      * caller should ensure that the data in the reused object is being cleared.
1114      */
readLocked(InputStream in, IntervalStats statsOut, int version, PackagesTokenData packagesTokenData)1115     private static boolean readLocked(InputStream in, IntervalStats statsOut, int version,
1116             PackagesTokenData packagesTokenData) throws RuntimeException {
1117         boolean dataOmitted = false;
1118         switch (version) {
1119             case 1:
1120             case 2:
1121             case 3:
1122                 Slog.w(TAG, "Reading UsageStats as XML; database version: " + version);
1123                 try {
1124                     UsageStatsXml.read(in, statsOut);
1125                 } catch (Exception e) {
1126                     Slog.e(TAG, "Unable to read interval stats from XML", e);
1127                 }
1128                 break;
1129             case 4:
1130                 try {
1131                     UsageStatsProto.read(in, statsOut);
1132                 } catch (Exception e) {
1133                     Slog.e(TAG, "Unable to read interval stats from proto.", e);
1134                 }
1135                 break;
1136             case 5:
1137                 try {
1138                     UsageStatsProtoV2.read(in, statsOut);
1139                 } catch (Exception e) {
1140                     Slog.e(TAG, "Unable to read interval stats from proto.", e);
1141                 }
1142                 dataOmitted = statsOut.deobfuscateData(packagesTokenData);
1143                 break;
1144             default:
1145                 throw new RuntimeException(
1146                         "Unhandled UsageStatsDatabase version: " + Integer.toString(version)
1147                                 + " on read.");
1148         }
1149         return dataOmitted;
1150     }
1151 
1152     /**
1153      * Reads the obfuscated data file from disk containing the tokens to packages mappings and
1154      * rebuilds the packages to tokens mappings based on that data.
1155      */
readMappingsLocked()1156     public void readMappingsLocked() {
1157         if (!mPackageMappingsFile.exists()) {
1158             return; // package mappings file is missing - recreate mappings on next write.
1159         }
1160 
1161         try (FileInputStream in = new AtomicFile(mPackageMappingsFile).openRead()) {
1162             UsageStatsProtoV2.readObfuscatedData(in, mPackagesTokenData);
1163         } catch (Exception e) {
1164             Slog.e(TAG, "Failed to read the obfuscated packages mapping file.", e);
1165             return;
1166         }
1167 
1168         final SparseArray<ArrayList<String>> tokensToPackagesMap =
1169                 mPackagesTokenData.tokensToPackagesMap;
1170         final int tokensToPackagesMapSize = tokensToPackagesMap.size();
1171         for (int i = 0; i < tokensToPackagesMapSize; i++) {
1172             final int packageToken = tokensToPackagesMap.keyAt(i);
1173             final ArrayList<String> tokensMap = tokensToPackagesMap.valueAt(i);
1174             final ArrayMap<String, Integer> packageStringsMap = new ArrayMap<>();
1175             final int tokensMapSize = tokensMap.size();
1176             // package name will always be at index 0 but its token should not be 0
1177             packageStringsMap.put(tokensMap.get(0), packageToken);
1178             for (int j = 1; j < tokensMapSize; j++) {
1179                 packageStringsMap.put(tokensMap.get(j), j);
1180             }
1181             mPackagesTokenData.packagesToTokensMap.put(tokensMap.get(0), packageStringsMap);
1182         }
1183     }
1184 
writeMappingsLocked()1185     void writeMappingsLocked() throws IOException {
1186         final AtomicFile file = new AtomicFile(mPackageMappingsFile);
1187         FileOutputStream fos = file.startWrite();
1188         try {
1189             UsageStatsProtoV2.writeObfuscatedData(fos, mPackagesTokenData);
1190             file.finishWrite(fos);
1191             fos = null;
1192         } catch (Exception e) {
1193             Slog.e(TAG, "Unable to write obfuscated data to proto.", e);
1194         } finally {
1195             file.failWrite(fos);
1196         }
1197     }
1198 
obfuscateCurrentStats(IntervalStats[] currentStats)1199     void obfuscateCurrentStats(IntervalStats[] currentStats) {
1200         if (mCurrentVersion < 5) {
1201             return;
1202         }
1203         for (int i = 0; i < currentStats.length; i++) {
1204             final IntervalStats stats = currentStats[i];
1205             stats.obfuscateData(mPackagesTokenData);
1206         }
1207     }
1208 
1209     /**
1210      * Update the stats in the database. They may not be written to disk immediately.
1211      */
putUsageStats(int intervalType, IntervalStats stats)1212     public void putUsageStats(int intervalType, IntervalStats stats) throws IOException {
1213         if (stats == null) return;
1214         synchronized (mLock) {
1215             if (intervalType < 0 || intervalType >= mIntervalDirs.length) {
1216                 throw new IllegalArgumentException("Bad interval type " + intervalType);
1217             }
1218 
1219             AtomicFile f = mSortedStatFiles[intervalType].get(stats.beginTime);
1220             if (f == null) {
1221                 f = new AtomicFile(new File(mIntervalDirs[intervalType],
1222                         Long.toString(stats.beginTime)));
1223                 mSortedStatFiles[intervalType].put(stats.beginTime, f);
1224             }
1225 
1226             writeLocked(f, stats);
1227             stats.lastTimeSaved = f.getLastModifiedTime();
1228         }
1229     }
1230 
1231 
1232     /* Backup/Restore Code */
getBackupPayload(String key)1233     byte[] getBackupPayload(String key) {
1234         return getBackupPayload(key, BACKUP_VERSION);
1235     }
1236 
1237     /**
1238      * @hide
1239      */
1240     @VisibleForTesting
getBackupPayload(String key, int version)1241     public byte[] getBackupPayload(String key, int version) {
1242         if (version >= 1 && version <= 3) {
1243             Slog.wtf(TAG, "Attempting to backup UsageStats as XML with version " + version);
1244             return null;
1245         }
1246         synchronized (mLock) {
1247             ByteArrayOutputStream baos = new ByteArrayOutputStream();
1248             if (KEY_USAGE_STATS.equals(key)) {
1249                 prune(System.currentTimeMillis());
1250                 DataOutputStream out = new DataOutputStream(baos);
1251                 try {
1252                     out.writeInt(version);
1253 
1254                     out.writeInt(mSortedStatFiles[UsageStatsManager.INTERVAL_DAILY].size());
1255 
1256                     for (int i = 0; i < mSortedStatFiles[UsageStatsManager.INTERVAL_DAILY].size();
1257                             i++) {
1258                         writeIntervalStatsToStream(out,
1259                                 mSortedStatFiles[UsageStatsManager.INTERVAL_DAILY].valueAt(i),
1260                                 version);
1261                     }
1262 
1263                     out.writeInt(mSortedStatFiles[UsageStatsManager.INTERVAL_WEEKLY].size());
1264                     for (int i = 0; i < mSortedStatFiles[UsageStatsManager.INTERVAL_WEEKLY].size();
1265                             i++) {
1266                         writeIntervalStatsToStream(out,
1267                                 mSortedStatFiles[UsageStatsManager.INTERVAL_WEEKLY].valueAt(i),
1268                                 version);
1269                     }
1270 
1271                     out.writeInt(mSortedStatFiles[UsageStatsManager.INTERVAL_MONTHLY].size());
1272                     for (int i = 0; i < mSortedStatFiles[UsageStatsManager.INTERVAL_MONTHLY].size();
1273                             i++) {
1274                         writeIntervalStatsToStream(out,
1275                                 mSortedStatFiles[UsageStatsManager.INTERVAL_MONTHLY].valueAt(i),
1276                                 version);
1277                     }
1278 
1279                     out.writeInt(mSortedStatFiles[UsageStatsManager.INTERVAL_YEARLY].size());
1280                     for (int i = 0; i < mSortedStatFiles[UsageStatsManager.INTERVAL_YEARLY].size();
1281                             i++) {
1282                         writeIntervalStatsToStream(out,
1283                                 mSortedStatFiles[UsageStatsManager.INTERVAL_YEARLY].valueAt(i),
1284                                 version);
1285                     }
1286                     if (DEBUG) Slog.i(TAG, "Written " + baos.size() + " bytes of data");
1287                 } catch (IOException ioe) {
1288                     Slog.d(TAG, "Failed to write data to output stream", ioe);
1289                     baos.reset();
1290                 }
1291             }
1292             return baos.toByteArray();
1293         }
1294 
1295     }
1296 
1297     /**
1298      * @hide
1299      */
1300     @VisibleForTesting
applyRestoredPayload(String key, byte[] payload)1301     public void applyRestoredPayload(String key, byte[] payload) {
1302         synchronized (mLock) {
1303             if (KEY_USAGE_STATS.equals(key)) {
1304                 // Read stats files for the current device configs
1305                 IntervalStats dailyConfigSource =
1306                         getLatestUsageStats(UsageStatsManager.INTERVAL_DAILY);
1307                 IntervalStats weeklyConfigSource =
1308                         getLatestUsageStats(UsageStatsManager.INTERVAL_WEEKLY);
1309                 IntervalStats monthlyConfigSource =
1310                         getLatestUsageStats(UsageStatsManager.INTERVAL_MONTHLY);
1311                 IntervalStats yearlyConfigSource =
1312                         getLatestUsageStats(UsageStatsManager.INTERVAL_YEARLY);
1313 
1314                 try {
1315                     DataInputStream in = new DataInputStream(new ByteArrayInputStream(payload));
1316                     int backupDataVersion = in.readInt();
1317 
1318                     // Can't handle this backup set
1319                     if (backupDataVersion < 1 || backupDataVersion > BACKUP_VERSION) return;
1320 
1321                     // Delete all stats files
1322                     // Do this after reading version and before actually restoring
1323                     for (int i = 0; i < mIntervalDirs.length; i++) {
1324                         deleteDirectoryContents(mIntervalDirs[i]);
1325                     }
1326 
1327                     int fileCount = in.readInt();
1328                     for (int i = 0; i < fileCount; i++) {
1329                         IntervalStats stats = deserializeIntervalStats(getIntervalStatsBytes(in),
1330                                 backupDataVersion);
1331                         stats = mergeStats(stats, dailyConfigSource);
1332                         putUsageStats(UsageStatsManager.INTERVAL_DAILY, stats);
1333                     }
1334 
1335                     fileCount = in.readInt();
1336                     for (int i = 0; i < fileCount; i++) {
1337                         IntervalStats stats = deserializeIntervalStats(getIntervalStatsBytes(in),
1338                                 backupDataVersion);
1339                         stats = mergeStats(stats, weeklyConfigSource);
1340                         putUsageStats(UsageStatsManager.INTERVAL_WEEKLY, stats);
1341                     }
1342 
1343                     fileCount = in.readInt();
1344                     for (int i = 0; i < fileCount; i++) {
1345                         IntervalStats stats = deserializeIntervalStats(getIntervalStatsBytes(in),
1346                                 backupDataVersion);
1347                         stats = mergeStats(stats, monthlyConfigSource);
1348                         putUsageStats(UsageStatsManager.INTERVAL_MONTHLY, stats);
1349                     }
1350 
1351                     fileCount = in.readInt();
1352                     for (int i = 0; i < fileCount; i++) {
1353                         IntervalStats stats = deserializeIntervalStats(getIntervalStatsBytes(in),
1354                                 backupDataVersion);
1355                         stats = mergeStats(stats, yearlyConfigSource);
1356                         putUsageStats(UsageStatsManager.INTERVAL_YEARLY, stats);
1357                     }
1358                     if (DEBUG) Slog.i(TAG, "Completed Restoring UsageStats");
1359                 } catch (IOException ioe) {
1360                     Slog.d(TAG, "Failed to read data from input stream", ioe);
1361                 } finally {
1362                     indexFilesLocked();
1363                 }
1364             }
1365         }
1366     }
1367 
1368     /**
1369      * Get the Configuration Statistics from the current device statistics and merge them
1370      * with the backed up usage statistics.
1371      */
mergeStats(IntervalStats beingRestored, IntervalStats onDevice)1372     private IntervalStats mergeStats(IntervalStats beingRestored, IntervalStats onDevice) {
1373         if (onDevice == null) return beingRestored;
1374         if (beingRestored == null) return null;
1375         beingRestored.activeConfiguration = onDevice.activeConfiguration;
1376         beingRestored.configurations.putAll(onDevice.configurations);
1377         beingRestored.events.clear();
1378         beingRestored.events.merge(onDevice.events);
1379         return beingRestored;
1380     }
1381 
writeIntervalStatsToStream(DataOutputStream out, AtomicFile statsFile, int version)1382     private void writeIntervalStatsToStream(DataOutputStream out, AtomicFile statsFile, int version)
1383             throws IOException {
1384         IntervalStats stats = new IntervalStats();
1385         try {
1386             readLocked(statsFile, stats);
1387         } catch (IOException e) {
1388             Slog.e(TAG, "Failed to read usage stats file", e);
1389             out.writeInt(0);
1390             return;
1391         }
1392         sanitizeIntervalStatsForBackup(stats);
1393         byte[] data = serializeIntervalStats(stats, version);
1394         out.writeInt(data.length);
1395         out.write(data);
1396     }
1397 
getIntervalStatsBytes(DataInputStream in)1398     private static byte[] getIntervalStatsBytes(DataInputStream in) throws IOException {
1399         int length = in.readInt();
1400         byte[] buffer = new byte[length];
1401         in.read(buffer, 0, length);
1402         return buffer;
1403     }
1404 
sanitizeIntervalStatsForBackup(IntervalStats stats)1405     private static void sanitizeIntervalStatsForBackup(IntervalStats stats) {
1406         if (stats == null) return;
1407         stats.activeConfiguration = null;
1408         stats.configurations.clear();
1409         stats.events.clear();
1410     }
1411 
serializeIntervalStats(IntervalStats stats, int version)1412     private byte[] serializeIntervalStats(IntervalStats stats, int version) {
1413         ByteArrayOutputStream baos = new ByteArrayOutputStream();
1414         DataOutputStream out = new DataOutputStream(baos);
1415         try {
1416             out.writeLong(stats.beginTime);
1417             writeLocked(out, stats, version, mPackagesTokenData);
1418         } catch (Exception ioe) {
1419             Slog.d(TAG, "Serializing IntervalStats Failed", ioe);
1420             baos.reset();
1421         }
1422         return baos.toByteArray();
1423     }
1424 
deserializeIntervalStats(byte[] data, int version)1425     private IntervalStats deserializeIntervalStats(byte[] data, int version) {
1426         ByteArrayInputStream bais = new ByteArrayInputStream(data);
1427         DataInputStream in = new DataInputStream(bais);
1428         IntervalStats stats = new IntervalStats();
1429         try {
1430             stats.beginTime = in.readLong();
1431             readLocked(in, stats, version, mPackagesTokenData);
1432         } catch (Exception e) {
1433             Slog.d(TAG, "DeSerializing IntervalStats Failed", e);
1434             stats = null;
1435         }
1436         return stats;
1437     }
1438 
deleteDirectoryContents(File directory)1439     private static void deleteDirectoryContents(File directory) {
1440         File[] files = directory.listFiles();
1441         for (File file : files) {
1442             deleteDirectory(file);
1443         }
1444     }
1445 
deleteDirectory(File directory)1446     private static void deleteDirectory(File directory) {
1447         File[] files = directory.listFiles();
1448         if (files != null) {
1449             for (File file : files) {
1450                 if (!file.isDirectory()) {
1451                     file.delete();
1452                 } else {
1453                     deleteDirectory(file);
1454                 }
1455             }
1456         }
1457         directory.delete();
1458     }
1459 
1460     /**
1461      * Prints the obfuscated package mappings and a summary of the database files.
1462      * @param pw the print writer to print to
1463      */
dump(IndentingPrintWriter pw, boolean compact)1464     public void dump(IndentingPrintWriter pw, boolean compact) {
1465         synchronized (mLock) {
1466             pw.println();
1467             pw.println("UsageStatsDatabase:");
1468             pw.increaseIndent();
1469             dumpMappings(pw);
1470             pw.decreaseIndent();
1471             pw.println("Database Summary:");
1472             pw.increaseIndent();
1473             for (int i = 0; i < mSortedStatFiles.length; i++) {
1474                 final TimeSparseArray<AtomicFile> files = mSortedStatFiles[i];
1475                 final int size = files.size();
1476                 pw.print(UserUsageStatsService.intervalToString(i));
1477                 pw.print(" stats files: ");
1478                 pw.print(size);
1479                 pw.println(", sorted list of files:");
1480                 pw.increaseIndent();
1481                 for (int f = 0; f < size; f++) {
1482                     final long fileName = files.keyAt(f);
1483                     if (compact) {
1484                         pw.print(UserUsageStatsService.formatDateTime(fileName, false));
1485                     } else {
1486                         pw.printPair(Long.toString(fileName),
1487                                 UserUsageStatsService.formatDateTime(fileName, true));
1488                     }
1489                     pw.println();
1490                 }
1491                 pw.decreaseIndent();
1492             }
1493             pw.decreaseIndent();
1494         }
1495     }
1496 
dumpMappings(IndentingPrintWriter pw)1497     void dumpMappings(IndentingPrintWriter pw) {
1498         synchronized (mLock) {
1499             pw.println("Obfuscated Packages Mappings:");
1500             pw.increaseIndent();
1501             pw.println("Counter: " + mPackagesTokenData.counter);
1502             pw.println("Tokens Map Size: " + mPackagesTokenData.tokensToPackagesMap.size());
1503             for (int i = 0; i < mPackagesTokenData.tokensToPackagesMap.size(); i++) {
1504                 final int packageToken = mPackagesTokenData.tokensToPackagesMap.keyAt(i);
1505                 final String packageStrings = String.join(", ",
1506                         mPackagesTokenData.tokensToPackagesMap.valueAt(i));
1507                 pw.println("Token " + packageToken + ": [" + packageStrings + "]");
1508             }
1509             pw.println();
1510             pw.decreaseIndent();
1511         }
1512     }
1513 
readIntervalStatsForFile(int interval, long fileName)1514     IntervalStats readIntervalStatsForFile(int interval, long fileName) {
1515         synchronized (mLock) {
1516             final IntervalStats stats = new IntervalStats();
1517             try {
1518                 readLocked(mSortedStatFiles[interval].get(fileName, null), stats);
1519                 return stats;
1520             } catch (Exception e) {
1521                 return null;
1522             }
1523         }
1524     }
1525 }
1526