1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.providers.media; 18 19 import static com.android.providers.media.DatabaseHelper.DATA_MEDIA_XATTR_DIRECTORY_PATH; 20 import static com.android.providers.media.DatabaseHelper.EXTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX; 21 import static com.android.providers.media.DatabaseHelper.EXTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX; 22 import static com.android.providers.media.DatabaseHelper.INTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX; 23 import static com.android.providers.media.DatabaseHelper.INTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX; 24 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__EXTERNAL_PRIMARY; 25 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__INTERNAL; 26 import static com.android.providers.media.MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__PUBLIC; 27 import static com.android.providers.media.util.Logging.TAG; 28 29 import android.content.ContentValues; 30 import android.database.Cursor; 31 import android.database.sqlite.SQLiteDatabase; 32 import android.os.Build; 33 import android.os.CancellationSignal; 34 import android.os.Environment; 35 import android.os.ParcelFileDescriptor; 36 import android.os.SystemClock; 37 import android.os.SystemProperties; 38 import android.os.UserHandle; 39 import android.provider.MediaStore; 40 import android.system.ErrnoException; 41 import android.system.Os; 42 import android.system.OsConstants; 43 import android.util.Log; 44 import android.util.Pair; 45 46 import androidx.annotation.NonNull; 47 48 import com.android.providers.media.dao.FileRow; 49 import com.android.providers.media.fuse.FuseDaemon; 50 import com.android.providers.media.stableuris.dao.BackupIdRow; 51 import com.android.providers.media.util.StringUtils; 52 53 import com.google.common.base.Strings; 54 55 import java.io.File; 56 import java.io.FileNotFoundException; 57 import java.io.IOException; 58 import java.util.ArrayList; 59 import java.util.Arrays; 60 import java.util.HashSet; 61 import java.util.List; 62 import java.util.Locale; 63 import java.util.Map; 64 import java.util.Optional; 65 import java.util.Set; 66 import java.util.concurrent.ConcurrentHashMap; 67 import java.util.concurrent.TimeoutException; 68 import java.util.concurrent.atomic.AtomicInteger; 69 import java.util.stream.Collectors; 70 71 /** 72 * To ensure that the ids of MediaStore database uris are stable and reliable. 73 */ 74 public class DatabaseBackupAndRecovery { 75 76 private static final String LOWER_FS_RECOVERY_DIRECTORY_PATH = 77 "/data/media/" + UserHandle.myUserId() + "/.transforms/recovery"; 78 79 /** 80 * Path for storing owner id to owner package identifier relation and vice versa. 81 * Lower file system path is used as upper file system does not support xattrs. 82 */ 83 private static final String OWNER_RELATION_LOWER_FS_BACKUP_PATH = 84 "/data/media/" + UserHandle.myUserId() + "/.transforms/recovery/leveldb-ownership"; 85 86 private static final String INTERNAL_VOLUME_LOWER_FS_BACKUP_PATH = 87 LOWER_FS_RECOVERY_DIRECTORY_PATH + "/leveldb-internal"; 88 89 private static final String EXTERNAL_PRIMARY_VOLUME_LOWER_FS_BACKUP_PATH = 90 LOWER_FS_RECOVERY_DIRECTORY_PATH + "/leveldb-external_primary"; 91 92 /** 93 * Every LevelDB table name starts with this prefix. 94 */ 95 private static final String LEVEL_DB_PREFIX = "leveldb-"; 96 97 /** 98 * Frequency at which next value of owner id is backed up in the external storage. 99 */ 100 private static final int NEXT_OWNER_ID_BACKUP_FREQUENCY = 50; 101 102 /** 103 * Start value used for next owner id. 104 */ 105 private static final int NEXT_OWNER_ID_DEFAULT_VALUE = 0; 106 107 /** 108 * Key name of xattr used to set next owner id on ownership DB. 109 */ 110 private static final String NEXT_OWNER_ID_XATTR_KEY = "user.nextownerid"; 111 112 /** 113 * Key name of xattr used to store last modified generation number. 114 */ 115 private static final String LAST_BACKEDUP_GENERATION_XATTR_KEY = "user.lastbackedgeneration"; 116 117 /** 118 * External primary storage root path for given user. 119 */ 120 private static final String EXTERNAL_PRIMARY_ROOT_PATH = 121 "/storage/emulated/" + UserHandle.myUserId(); 122 123 /** 124 * Array of columns backed up in external storage. 125 */ 126 private static final String[] QUERY_COLUMNS = new String[]{ 127 MediaStore.Files.FileColumns._ID, 128 MediaStore.Files.FileColumns.DATA, 129 MediaStore.Files.FileColumns.IS_FAVORITE, 130 MediaStore.Files.FileColumns.IS_PENDING, 131 MediaStore.Files.FileColumns.IS_TRASHED, 132 MediaStore.Files.FileColumns.MEDIA_TYPE, 133 MediaStore.Files.FileColumns._USER_ID, 134 MediaStore.Files.FileColumns.DATE_EXPIRES, 135 MediaStore.Files.FileColumns.OWNER_PACKAGE_NAME, 136 MediaStore.Files.FileColumns.GENERATION_MODIFIED, 137 MediaStore.Files.FileColumns.VOLUME_NAME 138 }; 139 140 /** 141 * Wait time of 15 seconds in millis. 142 */ 143 private static final long WAIT_TIME_15_SECONDS_IN_MILLIS = 15000; 144 145 /** 146 * Number of records to read from leveldb in a JNI call. 147 */ 148 protected static final int LEVEL_DB_READ_LIMIT = 100; 149 150 /** 151 * Stores cached value of next owner id. This helps in improving performance by backing up next 152 * row id less frequently in the external storage. 153 */ 154 private AtomicInteger mNextOwnerId; 155 156 /** 157 * Stores value of next backup of owner id. 158 */ 159 private AtomicInteger mNextOwnerIdBackup; 160 private final ConfigStore mConfigStore; 161 private final VolumeCache mVolumeCache; 162 private Set<String> mSetupCompleteVolumes = ConcurrentHashMap.newKeySet(); 163 164 // Flag only used to enable/disable feature for testing 165 private boolean mIsStableUriEnabledForInternal = false; 166 167 // Flag only used to enable/disable feature for testing 168 private boolean mIsStableUriEnabledForExternal = false; 169 170 // Flag only used to enable/disable feature for testing 171 private boolean mIsStableUrisEnabledForPublic = false; 172 173 private static Map<String, String> sOwnerIdRelationMap; 174 175 public static final String STABLE_URI_INTERNAL_PROPERTY = 176 "persist.sys.fuse.backup.internal_db_backup"; 177 178 private static boolean STABLE_URI_INTERNAL_PROPERTY_VALUE = true; 179 180 public static final String STABLE_URI_EXTERNAL_PROPERTY = 181 "persist.sys.fuse.backup.external_volume_backup"; 182 183 private static boolean STABLE_URI_EXTERNAL_PROPERTY_VALUE = false; 184 185 public static final String STABLE_URI_PUBLIC_PROPERTY = 186 "persist.sys.fuse.backup.public_db_backup"; 187 188 private static boolean STABLE_URI_PUBLIC_PROPERTY_VALUE = false; 189 DatabaseBackupAndRecovery(ConfigStore configStore, VolumeCache volumeCache)190 protected DatabaseBackupAndRecovery(ConfigStore configStore, VolumeCache volumeCache) { 191 mConfigStore = configStore; 192 mVolumeCache = volumeCache; 193 } 194 195 /** 196 * Returns true if migration and recovery code flow for stable uris is enabled for given volume. 197 */ isStableUrisEnabled(String volumeName)198 protected boolean isStableUrisEnabled(String volumeName) { 199 // Check if flags are enabled for test for internal volume 200 if (MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName) 201 && mIsStableUriEnabledForInternal) { 202 return true; 203 } 204 // Check if flags are enabled for test for external primary volume 205 if (MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName) 206 && mIsStableUriEnabledForExternal) { 207 return true; 208 } 209 210 // Check if flags are enabled for test for external primary volume 211 if (!MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName) 212 && !MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName) 213 && mIsStableUrisEnabledForPublic) { 214 return true; 215 } 216 217 // Feature is disabled for below S due to vold mount issues. 218 if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) { 219 return false; 220 } 221 222 switch (volumeName) { 223 case MediaStore.VOLUME_INTERNAL: 224 return mIsStableUriEnabledForInternal 225 || mConfigStore.isStableUrisForInternalVolumeEnabled() 226 || SystemProperties.getBoolean(STABLE_URI_INTERNAL_PROPERTY, 227 /* defaultValue */ STABLE_URI_INTERNAL_PROPERTY_VALUE); 228 case MediaStore.VOLUME_EXTERNAL_PRIMARY: 229 return mIsStableUriEnabledForExternal 230 || mConfigStore.isStableUrisForExternalVolumeEnabled() 231 || SystemProperties.getBoolean(STABLE_URI_EXTERNAL_PROPERTY, 232 /* defaultValue */ STABLE_URI_EXTERNAL_PROPERTY_VALUE); 233 default: 234 // public volume 235 return mIsStableUrisEnabledForPublic 236 || isStableUrisEnabled(MediaStore.VOLUME_EXTERNAL_PRIMARY) 237 && mConfigStore.isStableUrisForPublicVolumeEnabled() 238 || SystemProperties.getBoolean(STABLE_URI_PUBLIC_PROPERTY, 239 /* defaultValue */ STABLE_URI_PUBLIC_PROPERTY_VALUE); 240 } 241 } 242 243 /** 244 * On device boot, leveldb setup is done as part of attachVolume call for primary external. 245 * Also, on device config flag change, we check if flag is enabled, if yes, we proceed to 246 * setup(no-op if connection already exists). So, we setup backup and recovery for internal 247 * volume on Media mount signal of EXTERNAL_PRIMARY. 248 */ setupVolumeDbBackupAndRecovery(String volumeName)249 protected synchronized void setupVolumeDbBackupAndRecovery(String volumeName) { 250 // Since internal volume does not have any fuse daemon thread, leveldb instance 251 // for internal volume is created by fuse daemon thread of EXTERNAL_PRIMARY. 252 if (MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName)) { 253 // Set backup only for external primary for now. 254 return; 255 } 256 // Do not create leveldb instance if stable uris is not enabled for internal volume. 257 if (!isStableUrisEnabled(MediaStore.VOLUME_INTERNAL)) { 258 // Return if we are not supporting backup for internal volume 259 return; 260 } 261 262 if (mSetupCompleteVolumes.contains(volumeName)) { 263 // Return if setup is already done 264 return; 265 } 266 267 final long startTime = SystemClock.elapsedRealtime(); 268 int vol = MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName) 269 ? MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__EXTERNAL_PRIMARY 270 : MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__PUBLIC; 271 try { 272 if (!new File(LOWER_FS_RECOVERY_DIRECTORY_PATH).exists()) { 273 new File(LOWER_FS_RECOVERY_DIRECTORY_PATH).mkdirs(); 274 Log.v(TAG, "Created recovery directory:" + LOWER_FS_RECOVERY_DIRECTORY_PATH); 275 } 276 FuseDaemon fuseDaemonExternalPrimary = getFuseDaemonForFileWithWait(new File( 277 DatabaseBackupAndRecovery.EXTERNAL_PRIMARY_ROOT_PATH)); 278 Log.d(TAG, "Received db backup Fuse Daemon for: " + volumeName); 279 if (MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName) && ( 280 isStableUrisEnabled(MediaStore.VOLUME_INTERNAL) || isStableUrisEnabled( 281 MediaStore.VOLUME_EXTERNAL_PRIMARY))) { 282 // Setup internal and external volumes 283 MediaProviderStatsLog.write( 284 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED, 285 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED__STATUS__ATTEMPTED, vol); 286 fuseDaemonExternalPrimary.setupVolumeDbBackup(); 287 mSetupCompleteVolumes.add(volumeName); 288 MediaProviderStatsLog.write( 289 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED, 290 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED__STATUS__SUCCESS, vol); 291 } else if (isStableUrisEnabled(volumeName)) { 292 // Setup public volume 293 FuseDaemon fuseDaemonPublicVolume = getFuseDaemonForPath( 294 getFuseFilePathFromVolumeName(volumeName)); 295 MediaProviderStatsLog.write( 296 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED, 297 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED__STATUS__ATTEMPTED, vol); 298 fuseDaemonPublicVolume.setupPublicVolumeDbBackup(volumeName); 299 mSetupCompleteVolumes.add(volumeName); 300 MediaProviderStatsLog.write( 301 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED, 302 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED__STATUS__SUCCESS, vol); 303 } else { 304 return; 305 } 306 } catch (IOException e) { 307 MediaProviderStatsLog.write( 308 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED, 309 MediaProviderStatsLog.BACKUP_SETUP_STATUS_REPORTED__STATUS__FAILURE, vol); 310 Log.e(TAG, "Failure in setting up backup and recovery for volume: " + volumeName, e); 311 return; 312 } finally { 313 Log.i(TAG, "Backup and recovery setup time taken in milliseconds:" + ( 314 SystemClock.elapsedRealtime() - startTime)); 315 } 316 Log.i(TAG, "Successfully set up backup and recovery for volume: " + volumeName); 317 } 318 319 /** 320 * Backs up databases to external storage to ensure stable URIs. 321 */ backupDatabases(DatabaseHelper internalDatabaseHelper, DatabaseHelper externalDatabaseHelper, CancellationSignal signal)322 public void backupDatabases(DatabaseHelper internalDatabaseHelper, 323 DatabaseHelper externalDatabaseHelper, CancellationSignal signal) { 324 setupVolumeDbBackupAndRecovery(MediaStore.VOLUME_EXTERNAL_PRIMARY); 325 Log.i(TAG, "Triggering database backup"); 326 backupInternalDatabase(internalDatabaseHelper, signal); 327 backupExternalDatabase(externalDatabaseHelper, MediaStore.VOLUME_EXTERNAL_PRIMARY, signal); 328 329 for (MediaVolume mediaVolume : mVolumeCache.getExternalVolumes()) { 330 if (mediaVolume.isPublicVolume()) { 331 setupVolumeDbBackupAndRecovery(mediaVolume.getName()); 332 backupExternalDatabase(externalDatabaseHelper, mediaVolume.getName(), signal); 333 } 334 } 335 } 336 readDataFromBackup(String volumeName, String filePath)337 protected Optional<BackupIdRow> readDataFromBackup(String volumeName, String filePath) { 338 if (!isStableUrisEnabled(volumeName)) { 339 return Optional.empty(); 340 } 341 342 try { 343 final String data = getFuseDaemonForPath(getFuseFilePathFromVolumeName(volumeName)) 344 .readBackedUpData(filePath); 345 if (data == null || data.isEmpty()) { 346 Log.w(TAG, "No backup found for path: " + filePath); 347 return Optional.empty(); 348 } 349 350 return Optional.of(BackupIdRow.deserialize(data)); 351 } catch (Exception e) { 352 Log.e(TAG, "Failure in getting backed up data for filePath: " + filePath, e); 353 return Optional.empty(); 354 } 355 } 356 backupInternalDatabase(DatabaseHelper internalDbHelper, CancellationSignal signal)357 protected synchronized void backupInternalDatabase(DatabaseHelper internalDbHelper, 358 CancellationSignal signal) { 359 if (!isStableUrisEnabled(MediaStore.VOLUME_INTERNAL) 360 || internalDbHelper.isDatabaseRecovering()) { 361 return; 362 } 363 364 if (!mSetupCompleteVolumes.contains(MediaStore.VOLUME_EXTERNAL_PRIMARY)) { 365 Log.w(TAG, 366 "Setup is not present for backup of internal and external primary volume."); 367 return; 368 } 369 370 FuseDaemon fuseDaemon; 371 try { 372 fuseDaemon = getFuseDaemonForPath(EXTERNAL_PRIMARY_ROOT_PATH); 373 } catch (FileNotFoundException e) { 374 Log.e(TAG, 375 "Fuse Daemon not found for primary external storage, skipping backing up of " 376 + "internal database.", 377 e); 378 return; 379 } 380 381 internalDbHelper.runWithTransaction((db) -> { 382 try (Cursor c = db.query(true, "files", QUERY_COLUMNS, null, null, null, null, null, 383 null, signal)) { 384 while (c.moveToNext()) { 385 backupDataValues(fuseDaemon, c); 386 } 387 Log.d(TAG, String.format(Locale.ROOT, 388 "Backed up %d rows of internal database to external storage on idle " 389 + "maintenance.", 390 c.getCount())); 391 } catch (Exception e) { 392 Log.e(TAG, "Failure in backing up internal database to external storage.", e); 393 } 394 return null; 395 }); 396 } 397 backupExternalDatabase(DatabaseHelper externalDbHelper, String volumeName, CancellationSignal signal)398 protected synchronized void backupExternalDatabase(DatabaseHelper externalDbHelper, 399 String volumeName, CancellationSignal signal) { 400 if (!isStableUrisEnabled(volumeName) 401 || externalDbHelper.isDatabaseRecovering()) { 402 return; 403 } 404 405 if (!mSetupCompleteVolumes.contains(volumeName)) { 406 return; 407 } 408 409 FuseDaemon fuseDaemonExternalPrimary; 410 try { 411 fuseDaemonExternalPrimary = getFuseDaemonForFileWithWait( 412 new File(EXTERNAL_PRIMARY_ROOT_PATH)); 413 } catch (FileNotFoundException e) { 414 Log.e(TAG, 415 "Fuse Daemon not found for primary external storage, skipping backing up of " 416 + volumeName, 417 e); 418 return; 419 } 420 FuseDaemon fuseDaemonPublicVolume; 421 if (!isInternalOrExternalPrimary(volumeName)) { 422 try { 423 fuseDaemonPublicVolume = getFuseDaemonForFileWithWait(new File( 424 getFuseFilePathFromVolumeName(volumeName))); 425 } catch (FileNotFoundException e) { 426 Log.e(TAG, 427 "Fuse Daemon not found for " 428 + getFuseFilePathFromVolumeName(volumeName) 429 + ", skipping backing up of " + volumeName, 430 e); 431 return; 432 } 433 } else { 434 fuseDaemonPublicVolume = null; 435 } 436 437 final String backupPath = 438 LOWER_FS_RECOVERY_DIRECTORY_PATH + "/" + LEVEL_DB_PREFIX + volumeName; 439 long lastBackedGenerationNumber = getLastBackedGenerationNumber(backupPath); 440 441 final String generationClause = MediaStore.Files.FileColumns.GENERATION_MODIFIED + " >= " 442 + lastBackedGenerationNumber; 443 final String volumeClause = MediaStore.Files.FileColumns.VOLUME_NAME + " = '" 444 + volumeName + "'"; 445 final String selectionClause = generationClause + " AND " + volumeClause; 446 447 externalDbHelper.runWithTransaction((db) -> { 448 long maxGeneration = lastBackedGenerationNumber; 449 Log.d(TAG, "Started to back up " + volumeName 450 + ", maxGeneration:" + maxGeneration); 451 try (Cursor c = db.query(true, "files", QUERY_COLUMNS, selectionClause, null, null, 452 null, MediaStore.MediaColumns.GENERATION_MODIFIED + " ASC", null, signal)) { 453 while (c.moveToNext()) { 454 if (signal != null && signal.isCanceled()) { 455 Log.i(TAG, "Received a cancellation signal during the DB " 456 + "backup process"); 457 break; 458 } 459 if (isInternalOrExternalPrimary(volumeName)) { 460 backupDataValues(fuseDaemonExternalPrimary, c); 461 } else { 462 // public volume 463 backupDataValues(fuseDaemonExternalPrimary, fuseDaemonPublicVolume, c); 464 } 465 maxGeneration = Math.max(maxGeneration, c.getLong(9)); 466 } 467 setXattr(backupPath, LAST_BACKEDUP_GENERATION_XATTR_KEY, 468 String.valueOf(maxGeneration - 1)); 469 Log.d(TAG, String.format(Locale.ROOT, 470 "Backed up %d rows of " + volumeName + " to external storage on idle " 471 + "maintenance.", 472 c.getCount())); 473 } catch (Exception e) { 474 Log.e(TAG, "Failure in backing up " + volumeName + " to external storage.", e); 475 return null; 476 } 477 return null; 478 }); 479 } 480 backupDataValues(FuseDaemon fuseDaemon, Cursor c)481 private void backupDataValues(FuseDaemon fuseDaemon, Cursor c) throws IOException { 482 backupDataValues(fuseDaemon, null, c); 483 } 484 backupDataValues(FuseDaemon externalPrimaryFuseDaemon, FuseDaemon publicVolumeFuseDaemon, Cursor c)485 private void backupDataValues(FuseDaemon externalPrimaryFuseDaemon, 486 FuseDaemon publicVolumeFuseDaemon, Cursor c) throws IOException { 487 final long id = c.getLong(0); 488 final String data = c.getString(1); 489 final boolean isFavorite = c.getInt(2) != 0; 490 final boolean isPending = c.getInt(3) != 0; 491 final boolean isTrashed = c.getInt(4) != 0; 492 final int mediaType = c.getInt(5); 493 final int userId = c.getInt(6); 494 final String dateExpires = c.getString(7); 495 final String ownerPackageName = c.getString(8); 496 final String volumeName = c.getString(10); 497 BackupIdRow backupIdRow = createBackupIdRow(externalPrimaryFuseDaemon, id, mediaType, 498 isFavorite, isPending, isTrashed, userId, dateExpires, ownerPackageName); 499 if (isInternalOrExternalPrimary(volumeName)) { 500 externalPrimaryFuseDaemon.backupVolumeDbData(volumeName, data, 501 BackupIdRow.serialize(backupIdRow)); 502 } else { 503 // public volume 504 publicVolumeFuseDaemon.backupVolumeDbData(volumeName, data, 505 BackupIdRow.serialize(backupIdRow)); 506 } 507 } 508 deleteBackupForVolume(String volumeName)509 protected void deleteBackupForVolume(String volumeName) { 510 File dbFilePath = new File( 511 String.format(Locale.ROOT, "%s/%s.db", LOWER_FS_RECOVERY_DIRECTORY_PATH, 512 volumeName)); 513 if (dbFilePath.exists()) { 514 dbFilePath.delete(); 515 } 516 } 517 readBackedUpFilePaths(String volumeName, String lastReadValue, int limit)518 protected String[] readBackedUpFilePaths(String volumeName, String lastReadValue, int limit) { 519 if (!isStableUrisEnabled(volumeName)) { 520 return new String[0]; 521 } 522 523 try { 524 return getFuseDaemonForPath(getFuseFilePathFromVolumeName(volumeName)) 525 .readBackedUpFilePaths(volumeName, lastReadValue, limit); 526 } catch (IOException e) { 527 Log.e(TAG, "Failure in reading backed up file paths for volume: " + volumeName, e); 528 return new String[0]; 529 } 530 } 531 updateNextRowIdXattr(DatabaseHelper helper, long id)532 protected void updateNextRowIdXattr(DatabaseHelper helper, long id) { 533 if (helper.isInternal()) { 534 updateNextRowIdForInternal(helper, id); 535 return; 536 } 537 538 if (!helper.isNextRowIdBackupEnabled()) { 539 return; 540 } 541 542 Optional<Long> nextRowIdBackupOptional = helper.getNextRowId(); 543 if (!nextRowIdBackupOptional.isPresent()) { 544 throw new RuntimeException( 545 String.format(Locale.ROOT, "Cannot find next row id xattr for %s.", 546 helper.getDatabaseName())); 547 } 548 549 if (id >= nextRowIdBackupOptional.get()) { 550 helper.backupNextRowId(id); 551 } 552 } 553 getLastBackedGenerationNumber(String backupPath)554 private long getLastBackedGenerationNumber(String backupPath) { 555 // Read last backed up generation number 556 Optional<Long> lastBackedUpGenNum = getXattrOfLongValue( 557 backupPath, LAST_BACKEDUP_GENERATION_XATTR_KEY); 558 long lastBackedGenerationNumber = lastBackedUpGenNum.isPresent() 559 ? lastBackedUpGenNum.get() : 0; 560 if (lastBackedGenerationNumber > 0) { 561 Log.i(TAG, "Last backed up generation number for " + backupPath + " is " 562 + lastBackedGenerationNumber); 563 } 564 return lastBackedGenerationNumber; 565 } 566 567 @NonNull getFuseDaemonForPath(@onNull String path)568 private FuseDaemon getFuseDaemonForPath(@NonNull String path) 569 throws FileNotFoundException { 570 return MediaProvider.getFuseDaemonForFile(new File(path), mVolumeCache); 571 } 572 updateNextRowIdAndSetDirty(@onNull DatabaseHelper helper, @NonNull FileRow oldRow, @NonNull FileRow newRow)573 protected void updateNextRowIdAndSetDirty(@NonNull DatabaseHelper helper, 574 @NonNull FileRow oldRow, @NonNull FileRow newRow) { 575 updateNextRowIdXattr(helper, newRow.getId()); 576 markBackupAsDirty(helper, oldRow); 577 } 578 579 /** 580 * Backs up DB data in external storage to recover in case of DB rollback. 581 */ backupVolumeDbData(DatabaseHelper databaseHelper, FileRow insertedRow)582 protected void backupVolumeDbData(DatabaseHelper databaseHelper, FileRow insertedRow) { 583 if (!isBackupUpdateAllowed(databaseHelper, insertedRow.getVolumeName())) { 584 return; 585 } 586 587 try { 588 FuseDaemon fuseDaemonExternalPrimary = getFuseDaemonForPath(EXTERNAL_PRIMARY_ROOT_PATH); 589 final BackupIdRow value = createBackupIdRow(fuseDaemonExternalPrimary, insertedRow); 590 if (isInternalOrExternalPrimary(insertedRow.getVolumeName())) { 591 fuseDaemonExternalPrimary.backupVolumeDbData(insertedRow.getVolumeName(), 592 insertedRow.getPath(), BackupIdRow.serialize(value)); 593 } else { 594 // public volume 595 final FuseDaemon fuseDaemonPublicVolume = getFuseDaemonForPath( 596 getFuseFilePathFromVolumeName(insertedRow.getVolumeName())); 597 fuseDaemonPublicVolume.backupVolumeDbData(insertedRow.getVolumeName(), 598 insertedRow.getPath(), BackupIdRow.serialize(value)); 599 } 600 } catch (Exception e) { 601 Log.e(TAG, "Failure in backing up data to external storage", e); 602 } 603 } 604 createBackupIdRow(FuseDaemon fuseDaemon, FileRow insertedRow)605 private BackupIdRow createBackupIdRow(FuseDaemon fuseDaemon, FileRow insertedRow) 606 throws IOException { 607 return createBackupIdRow(fuseDaemon, insertedRow.getId(), insertedRow.getMediaType(), 608 insertedRow.isFavorite(), insertedRow.isPending(), insertedRow.isTrashed(), 609 insertedRow.getUserId(), insertedRow.getDateExpires(), 610 insertedRow.getOwnerPackageName()); 611 } 612 createBackupIdRow(FuseDaemon fuseDaemon, long id, int mediaType, boolean isFavorite, boolean isPending, boolean isTrashed, int userId, String dateExpires, String ownerPackageName)613 private BackupIdRow createBackupIdRow(FuseDaemon fuseDaemon, long id, int mediaType, 614 boolean isFavorite, 615 boolean isPending, boolean isTrashed, int userId, String dateExpires, 616 String ownerPackageName) throws IOException { 617 BackupIdRow.Builder builder = BackupIdRow.newBuilder(id); 618 builder.setMediaType(mediaType); 619 builder.setIsFavorite(isFavorite ? 1 : 0); 620 builder.setIsPending(isPending ? 1 : 0); 621 builder.setIsTrashed(isTrashed ? 1 : 0); 622 builder.setUserId(userId); 623 builder.setDateExpires(dateExpires); 624 // We set owner package id instead of owner package name in the backup. When an 625 // application is uninstalled, all media rows corresponding to it will be orphaned and 626 // would have owner package name as null. This should not change if application is 627 // installed again. Therefore, we are storing owner id instead of owner package name. On 628 // package uninstallation, we delete the owner id relation from the backup. All rows 629 // recovered for orphaned owner ids will have package name as null. Since we also need to 630 // support cloned apps, we are storing a combination of owner package name and user id to 631 // uniquely identify a package. 632 builder.setOwnerPackagedId(getOwnerPackageId(fuseDaemon, ownerPackageName, userId)); 633 return builder.setIsDirty(false).build(); 634 } 635 636 getOwnerPackageId(FuseDaemon fuseDaemon, String ownerPackageName, int userId)637 private int getOwnerPackageId(FuseDaemon fuseDaemon, String ownerPackageName, int userId) 638 throws IOException { 639 if (Strings.isNullOrEmpty(ownerPackageName) || ownerPackageName.equalsIgnoreCase("null")) { 640 // We store -1 in the backup if owner package name is null. 641 return -1; 642 } 643 644 // Create identifier of format "owner_pkg_name::user_id". Tightly coupling owner package 645 // name and user id helps in handling app cloning scenarios. 646 String ownerPackageIdentifier = createOwnerPackageIdentifier(ownerPackageName, userId); 647 // Read any existing entry for given owner package name and user id 648 String ownerId = fuseDaemon.readFromOwnershipBackup(ownerPackageIdentifier); 649 if (!ownerId.trim().isEmpty()) { 650 // Use existing owner id if found and is positive 651 int val = Integer.parseInt(ownerId); 652 if (val >= 0) { 653 return val; 654 } 655 } 656 657 int nextOwnerId = getAndIncrementNextOwnerId(); 658 fuseDaemon.createOwnerIdRelation(String.valueOf(nextOwnerId), ownerPackageIdentifier); 659 Log.v(TAG, "Created relation b/w " + nextOwnerId + " and " + ownerPackageIdentifier); 660 return nextOwnerId; 661 } 662 createOwnerPackageIdentifier(String ownerPackageName, int userId)663 private String createOwnerPackageIdentifier(String ownerPackageName, int userId) { 664 return ownerPackageName.trim().concat("::").concat(String.valueOf(userId)); 665 } 666 getPackageNameAndUserId(String ownerPackageIdentifier)667 private Pair<String, Integer> getPackageNameAndUserId(String ownerPackageIdentifier) { 668 if (ownerPackageIdentifier.trim().isEmpty()) { 669 return Pair.create(null, null); 670 } 671 672 String[] arr = ownerPackageIdentifier.trim().split("::"); 673 return Pair.create(arr[0], Integer.valueOf(arr[1])); 674 } 675 getAndIncrementNextOwnerId()676 private synchronized int getAndIncrementNextOwnerId() { 677 // In synchronized block to avoid use of same owner id for multiple owner package relations 678 if (mNextOwnerId == null) { 679 Optional<Integer> nextOwnerIdOptional = getXattrOfIntegerValue( 680 OWNER_RELATION_LOWER_FS_BACKUP_PATH, 681 NEXT_OWNER_ID_XATTR_KEY); 682 mNextOwnerId = nextOwnerIdOptional.map(AtomicInteger::new).orElseGet( 683 () -> new AtomicInteger(NEXT_OWNER_ID_DEFAULT_VALUE)); 684 mNextOwnerIdBackup = new AtomicInteger(mNextOwnerId.get()); 685 } 686 if (mNextOwnerId.get() >= mNextOwnerIdBackup.get()) { 687 int nextBackup = mNextOwnerId.get() + NEXT_OWNER_ID_BACKUP_FREQUENCY; 688 updateNextOwnerId(nextBackup); 689 mNextOwnerIdBackup = new AtomicInteger(nextBackup); 690 } 691 int returnValue = mNextOwnerId.get(); 692 mNextOwnerId.set(returnValue + 1); 693 return returnValue; 694 } 695 updateNextOwnerId(int val)696 private void updateNextOwnerId(int val) { 697 setXattr(OWNER_RELATION_LOWER_FS_BACKUP_PATH, NEXT_OWNER_ID_XATTR_KEY, String.valueOf(val)); 698 Log.d(TAG, "Updated next owner id to: " + val); 699 } 700 removeOwnerIdToPackageRelation(String packageName, int userId)701 protected void removeOwnerIdToPackageRelation(String packageName, int userId) { 702 if (Strings.isNullOrEmpty(packageName) || packageName.equalsIgnoreCase("null") 703 || !isStableUrisEnabled(MediaStore.VOLUME_EXTERNAL_PRIMARY) 704 || !new File(OWNER_RELATION_LOWER_FS_BACKUP_PATH).exists() 705 || !mSetupCompleteVolumes.contains(MediaStore.VOLUME_EXTERNAL_PRIMARY)) { 706 return; 707 } 708 709 try { 710 FuseDaemon fuseDaemon = getFuseDaemonForPath(EXTERNAL_PRIMARY_ROOT_PATH); 711 String ownerPackageIdentifier = createOwnerPackageIdentifier(packageName, userId); 712 String ownerId = fuseDaemon.readFromOwnershipBackup(ownerPackageIdentifier); 713 714 fuseDaemon.removeOwnerIdRelation(ownerId, ownerPackageIdentifier); 715 } catch (Exception e) { 716 Log.e(TAG, "Failure in removing owner id to package relation", e); 717 } 718 } 719 720 /** 721 * Deletes backed up data(needed for recovery) from external storage. 722 */ deleteFromDbBackup(DatabaseHelper databaseHelper, FileRow deletedRow)723 protected void deleteFromDbBackup(DatabaseHelper databaseHelper, FileRow deletedRow) { 724 if (!isBackupUpdateAllowed(databaseHelper, deletedRow.getVolumeName())) { 725 return; 726 } 727 728 String deletedFilePath = deletedRow.getPath(); 729 if (deletedFilePath == null) { 730 return; 731 } 732 733 try { 734 getFuseDaemonForPath(getFuseFilePathFromVolumeName(deletedRow.getVolumeName())) 735 .deleteDbBackup(deletedFilePath); 736 } catch (IOException e) { 737 Log.w(TAG, "Failure in deleting backup data for key: " + deletedFilePath, e); 738 } 739 } 740 isBackupUpdateAllowed(DatabaseHelper databaseHelper, String volumeName)741 protected boolean isBackupUpdateAllowed(DatabaseHelper databaseHelper, String volumeName) { 742 // Backup only if stable uris is enabled, db is not recovering and backup setup is complete. 743 return isStableUrisEnabled(volumeName) && !databaseHelper.isDatabaseRecovering() 744 && mSetupCompleteVolumes.contains(volumeName); 745 } 746 isInternalOrExternalPrimary(String volumeName)747 private boolean isInternalOrExternalPrimary(String volumeName) { 748 if (Strings.isNullOrEmpty(volumeName)) { 749 // This should never happen 750 Log.e(TAG, "Volume name is " + volumeName + ", treating it as non public volume"); 751 return true; 752 } 753 return MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName) 754 || MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName); 755 } 756 updateNextRowIdForInternal(DatabaseHelper helper, long id)757 private void updateNextRowIdForInternal(DatabaseHelper helper, long id) { 758 if (!isStableUrisEnabled(MediaStore.VOLUME_INTERNAL)) { 759 return; 760 } 761 762 Optional<Long> nextRowIdBackupOptional = helper.getNextRowId(); 763 764 if (!nextRowIdBackupOptional.isPresent()) { 765 return; 766 } 767 768 if (id >= nextRowIdBackupOptional.get()) { 769 helper.backupNextRowId(id); 770 } 771 } 772 markBackupAsDirty(DatabaseHelper databaseHelper, FileRow updatedRow)773 private void markBackupAsDirty(DatabaseHelper databaseHelper, FileRow updatedRow) { 774 if (!isBackupUpdateAllowed(databaseHelper, updatedRow.getVolumeName())) { 775 return; 776 } 777 778 final String updatedFilePath = updatedRow.getPath(); 779 try { 780 getFuseDaemonForPath(getFuseFilePathFromVolumeName(updatedRow.getVolumeName())) 781 .backupVolumeDbData( 782 updatedRow.getVolumeName(), 783 updatedFilePath, 784 BackupIdRow.serialize(BackupIdRow.newBuilder(updatedRow.getId()) 785 .setIsDirty(true).build())); 786 } catch (IOException e) { 787 Log.e(TAG, "Failure in marking data as dirty to external storage for path:" 788 + updatedFilePath, e); 789 } 790 } 791 792 /** 793 * Reads value corresponding to given key from xattr on given path. 794 */ getXattr(String path, String key)795 static Optional<String> getXattr(String path, String key) { 796 try { 797 return Optional.of(Arrays.toString(Os.getxattr(path, key))); 798 } catch (Exception e) { 799 Log.w(TAG, String.format(Locale.ROOT, 800 "Exception encountered while reading xattr:%s from path:%s.", key, path)); 801 return Optional.empty(); 802 } 803 } 804 805 /** 806 * Reads long value corresponding to given key from xattr on given path. 807 */ getXattrOfLongValue(String path, String key)808 static Optional<Long> getXattrOfLongValue(String path, String key) { 809 try { 810 return Optional.of(Long.parseLong(new String(Os.getxattr(path, key)))); 811 } catch (Exception e) { 812 Log.w(TAG, String.format(Locale.ROOT, 813 "Exception encountered while reading xattr:%s from path:%s.", key, path)); 814 return Optional.empty(); 815 } 816 } 817 818 /** 819 * Reads integer value corresponding to given key from xattr on given path. 820 */ getXattrOfIntegerValue(String path, String key)821 static Optional<Integer> getXattrOfIntegerValue(String path, String key) { 822 try { 823 return Optional.of(Integer.parseInt(new String(Os.getxattr(path, key)))); 824 } catch (Exception e) { 825 Log.w(TAG, String.format(Locale.ROOT, 826 "Exception encountered while reading xattr:%s from path:%s.", key, path)); 827 return Optional.empty(); 828 } 829 } 830 831 /** 832 * Sets key and value as xattr on given path. 833 */ setXattr(String path, String key, String value)834 static boolean setXattr(String path, String key, String value) { 835 try (ParcelFileDescriptor pfd = ParcelFileDescriptor.open(new File(path), 836 ParcelFileDescriptor.MODE_READ_ONLY)) { 837 // Map id value to xattr key 838 Os.setxattr(path, key, value.getBytes(), 0); 839 Os.fsync(pfd.getFileDescriptor()); 840 Log.d(TAG, String.format("xattr set to %s for key:%s on path: %s.", value, key, path)); 841 return true; 842 } catch (Exception e) { 843 Log.e(TAG, String.format(Locale.ROOT, "Failed to set xattr:%s to %s for path: %s.", key, 844 value, path), e); 845 return false; 846 } 847 } 848 849 /** 850 * Deletes xattr with given key on given path. Becomes a no-op when xattr is not present. 851 */ removeXattr(String path, String key)852 static boolean removeXattr(String path, String key) { 853 try (ParcelFileDescriptor pfd = ParcelFileDescriptor.open(new File(path), 854 ParcelFileDescriptor.MODE_READ_ONLY)) { 855 Os.removexattr(path, key); 856 Os.fsync(pfd.getFileDescriptor()); 857 Log.d(TAG, String.format("xattr key:%s removed on path: %s.", key, path)); 858 return true; 859 } catch (Exception e) { 860 if (e instanceof ErrnoException) { 861 ErrnoException exception = (ErrnoException) e; 862 if (exception.errno == OsConstants.ENODATA) { 863 Log.w(TAG, String.format(Locale.ROOT, 864 "xattr:%s is not removed as it is not found on path: %s.", key, path)); 865 return true; 866 } 867 } 868 869 Log.e(TAG, String.format(Locale.ROOT, "Failed to remove xattr:%s for path: %s.", key, 870 path), e); 871 return false; 872 } 873 } 874 875 /** 876 * Lists xattrs of given path. 877 */ listXattr(String path)878 static List<String> listXattr(String path) { 879 try { 880 return Arrays.asList(Os.listxattr(path)); 881 } catch (Exception e) { 882 Log.e(TAG, "Exception in reading xattrs on path: " + path, e); 883 return new ArrayList<>(); 884 } 885 } 886 insertDataInDatabase(SQLiteDatabase db, BackupIdRow row, String filePath, String volumeName)887 protected boolean insertDataInDatabase(SQLiteDatabase db, BackupIdRow row, String filePath, 888 String volumeName) { 889 final ContentValues values = createValuesFromFileRow(row, filePath, volumeName); 890 return db.insert("files", null, values) != -1; 891 } 892 createValuesFromFileRow(BackupIdRow row, String filePath, String volumeName)893 private ContentValues createValuesFromFileRow(BackupIdRow row, String filePath, 894 String volumeName) { 895 ContentValues values = new ContentValues(); 896 values.put(MediaStore.Files.FileColumns._ID, row.getId()); 897 values.put(MediaStore.Files.FileColumns.IS_FAVORITE, row.getIsFavorite()); 898 values.put(MediaStore.Files.FileColumns.IS_PENDING, row.getIsPending()); 899 values.put(MediaStore.Files.FileColumns.IS_TRASHED, row.getIsTrashed()); 900 values.put(MediaStore.Files.FileColumns.DATA, filePath); 901 values.put(MediaStore.Files.FileColumns.VOLUME_NAME, volumeName); 902 values.put(MediaStore.Files.FileColumns._USER_ID, row.getUserId()); 903 values.put(MediaStore.Files.FileColumns.MEDIA_TYPE, row.getMediaType()); 904 if (!StringUtils.isNullOrEmpty(row.getDateExpires())) { 905 values.put(MediaStore.Files.FileColumns.DATE_EXPIRES, 906 Long.valueOf(row.getDateExpires())); 907 } 908 if (row.getOwnerPackageId() >= 0) { 909 Pair<String, Integer> ownerPackageNameAndUidPair = getOwnerPackageNameAndUidPair( 910 row.getOwnerPackageId()); 911 if (ownerPackageNameAndUidPair.first != null) { 912 values.put(MediaStore.Files.FileColumns.OWNER_PACKAGE_NAME, 913 ownerPackageNameAndUidPair.first); 914 } 915 if (ownerPackageNameAndUidPair.second != null) { 916 values.put(MediaStore.Files.FileColumns._USER_ID, 917 ownerPackageNameAndUidPair.second); 918 } 919 } 920 921 return values; 922 } 923 getOwnerPackageNameAndUidPair(int ownerPackageId)924 protected Pair<String, Integer> getOwnerPackageNameAndUidPair(int ownerPackageId) { 925 if (sOwnerIdRelationMap == null) { 926 try { 927 sOwnerIdRelationMap = readOwnerIdRelationsFromLevelDb(); 928 Log.v(TAG, "Cached owner id map"); 929 } catch (IOException e) { 930 Log.e(TAG, "Failure in reading owner details for owner id:" + ownerPackageId, e); 931 return Pair.create(null, null); 932 } 933 } 934 935 if (sOwnerIdRelationMap.containsKey(String.valueOf(ownerPackageId))) { 936 return getPackageNameAndUserId(sOwnerIdRelationMap.get(String.valueOf(ownerPackageId))); 937 } 938 939 return Pair.create(null, null); 940 } 941 readOwnerIdRelationsFromLevelDb()942 protected Map<String, String> readOwnerIdRelationsFromLevelDb() throws IOException { 943 return getFuseDaemonForPath(EXTERNAL_PRIMARY_ROOT_PATH).readOwnerIdRelations(); 944 } 945 readOwnerPackageName(String ownerId)946 protected String readOwnerPackageName(String ownerId) throws IOException { 947 Map<String, String> ownerIdRelationMap = readOwnerIdRelationsFromLevelDb(); 948 if (ownerIdRelationMap.containsKey(String.valueOf(ownerId))) { 949 return getPackageNameAndUserId(ownerIdRelationMap.get(ownerId)).first; 950 } 951 952 return null; 953 } 954 recoverData(SQLiteDatabase db, String volumeName)955 protected void recoverData(SQLiteDatabase db, String volumeName) throws Exception{ 956 if (!MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName) 957 && !MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName)) { 958 // todo: implement for public volume 959 return; 960 } 961 final long startTime = SystemClock.elapsedRealtime(); 962 final String fuseFilePath = getFuseFilePathFromVolumeName(volumeName); 963 // Wait for external primary to be attached as we use same thread for internal volume. 964 // Maximum wait for 10s 965 getFuseDaemonForFileWithWait(new File(fuseFilePath)); 966 if (!isBackupPresent(volumeName)) { 967 throw new FileNotFoundException("Backup file not found for " + volumeName); 968 } 969 970 Log.d(TAG, "Backup is present for " + volumeName); 971 try { 972 waitForVolumeToBeAttached(mSetupCompleteVolumes); 973 } catch (Exception e) { 974 throw new IllegalStateException( 975 "Volume not attached in given time. Cannot recover data.", e); 976 } 977 978 long rowsRecovered = 0; 979 long dirtyRowsCount = 0; 980 String[] backedUpFilePaths; 981 String lastReadValue = ""; 982 983 while (true) { 984 backedUpFilePaths = readBackedUpFilePaths(volumeName, lastReadValue, 985 LEVEL_DB_READ_LIMIT); 986 if (backedUpFilePaths.length == 0) { 987 break; 988 } 989 990 // Reset cached owner id relation map 991 sOwnerIdRelationMap = null; 992 for (String filePath : backedUpFilePaths) { 993 Optional<BackupIdRow> fileRow = readDataFromBackup(volumeName, filePath); 994 if (fileRow.isPresent()) { 995 if (fileRow.get().getIsDirty()) { 996 dirtyRowsCount++; 997 continue; 998 } 999 1000 if(insertDataInDatabase(db, fileRow.get(), filePath, volumeName)) { 1001 rowsRecovered++; 1002 } 1003 } 1004 } 1005 1006 // Read less rows than expected 1007 if (backedUpFilePaths.length < LEVEL_DB_READ_LIMIT) { 1008 break; 1009 } 1010 lastReadValue = backedUpFilePaths[backedUpFilePaths.length - 1]; 1011 } 1012 long recoveryTime = SystemClock.elapsedRealtime() - startTime; 1013 MediaProviderStatsLog.write(MediaProviderStatsLog.MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED, 1014 getVolumeNameForStatsLog(volumeName), recoveryTime, rowsRecovered, dirtyRowsCount); 1015 Log.i(TAG, String.format(Locale.ROOT, "%d rows recovered for volume:%s.", rowsRecovered, 1016 volumeName)); 1017 Log.i(TAG, String.format(Locale.ROOT, "Recovery time: %d ms", recoveryTime)); 1018 } 1019 resetLastBackedUpGenerationNumber(String volumeName)1020 void resetLastBackedUpGenerationNumber(String volumeName) { 1021 // Resetting generation number 1022 setXattr(LOWER_FS_RECOVERY_DIRECTORY_PATH + "/" + LEVEL_DB_PREFIX + volumeName, 1023 LAST_BACKEDUP_GENERATION_XATTR_KEY, String.valueOf(0)); 1024 Log.v(TAG, "Leveldb Last backed generation number reset done to 0 for " + volumeName); 1025 } 1026 isBackupPresent(String volumeName)1027 protected boolean isBackupPresent(String volumeName) { 1028 if (MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName)) { 1029 return new File(INTERNAL_VOLUME_LOWER_FS_BACKUP_PATH).exists(); 1030 } else if (MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName)) { 1031 return new File(EXTERNAL_PRIMARY_VOLUME_LOWER_FS_BACKUP_PATH).exists(); 1032 } 1033 1034 return false; 1035 } 1036 waitForVolumeToBeAttached(Set<String> setupCompleteVolumes)1037 protected void waitForVolumeToBeAttached(Set<String> setupCompleteVolumes) 1038 throws TimeoutException { 1039 long time = 0; 1040 // Wait of 10 seconds 1041 long waitTimeInMilliseconds = 10000; 1042 // Poll every 100 milliseconds 1043 long pollTime = 100; 1044 while (time <= waitTimeInMilliseconds) { 1045 if (setupCompleteVolumes.contains(MediaStore.VOLUME_EXTERNAL_PRIMARY)) { 1046 Log.i(TAG, "Found external primary volume attached."); 1047 return; 1048 } 1049 1050 SystemClock.sleep(pollTime); 1051 time += pollTime; 1052 } 1053 throw new TimeoutException("Timed out waiting for external primary setup"); 1054 } 1055 getFuseDaemonForFileWithWait(File fuseFilePath)1056 protected FuseDaemon getFuseDaemonForFileWithWait(File fuseFilePath) 1057 throws FileNotFoundException { 1058 pollForExternalStorageMountedState(); 1059 return MediaProvider.getFuseDaemonForFileWithWait(fuseFilePath, mVolumeCache, 1060 WAIT_TIME_15_SECONDS_IN_MILLIS); 1061 } 1062 setStableUrisGlobalFlag(String volumeName, boolean isEnabled)1063 protected void setStableUrisGlobalFlag(String volumeName, boolean isEnabled) { 1064 if (MediaStore.VOLUME_INTERNAL.equalsIgnoreCase(volumeName)) { 1065 mIsStableUriEnabledForInternal = isEnabled; 1066 } else if (MediaStore.VOLUME_EXTERNAL_PRIMARY.equalsIgnoreCase(volumeName)) { 1067 mIsStableUriEnabledForExternal = isEnabled; 1068 } else { 1069 mIsStableUrisEnabledForPublic = isEnabled; 1070 } 1071 } 1072 getVolumeNameForStatsLog(String volumeName)1073 private int getVolumeNameForStatsLog(String volumeName) { 1074 if (volumeName.equalsIgnoreCase(MediaStore.VOLUME_INTERNAL)) { 1075 return MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__INTERNAL; 1076 } else if (volumeName.equalsIgnoreCase(MediaStore.VOLUME_EXTERNAL_PRIMARY)) { 1077 return MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__EXTERNAL_PRIMARY; 1078 } 1079 1080 return MEDIA_PROVIDER_VOLUME_RECOVERY_REPORTED__VOLUME__PUBLIC; 1081 } 1082 getFuseFilePathFromVolumeName(String volumeName)1083 private static String getFuseFilePathFromVolumeName(String volumeName) { 1084 if (Strings.isNullOrEmpty(volumeName)) { 1085 // Returning EXTERNAL_PRIMARY_ROOT_PATH to avoid any regressions 1086 Log.e(TAG, "Trying to get a Fuse Daemon for volume name = " + volumeName); 1087 return EXTERNAL_PRIMARY_ROOT_PATH; 1088 } 1089 switch (volumeName) { 1090 case MediaStore.VOLUME_INTERNAL: 1091 case MediaStore.VOLUME_EXTERNAL_PRIMARY: 1092 return EXTERNAL_PRIMARY_ROOT_PATH; 1093 default: 1094 return "/storage/" + volumeName.toUpperCase(Locale.ROOT); 1095 } 1096 } 1097 1098 /** 1099 * Returns list of backed up files from external storage. 1100 */ getBackupFiles()1101 protected List<File> getBackupFiles() { 1102 return Arrays.asList(new File(LOWER_FS_RECOVERY_DIRECTORY_PATH).listFiles()); 1103 } 1104 1105 /** 1106 * Updates backup in external storage to the latest values. Deletes backup of old file path if 1107 * file path has changed. 1108 */ updateBackup(DatabaseHelper helper, FileRow oldRow, FileRow newRow)1109 public void updateBackup(DatabaseHelper helper, FileRow oldRow, FileRow newRow) { 1110 if (!isBackupUpdateAllowed(helper, newRow.getVolumeName())) { 1111 return; 1112 } 1113 1114 FuseDaemon fuseDaemon; 1115 try { 1116 fuseDaemon = getFuseDaemonForPath(getFuseFilePathFromVolumeName( 1117 newRow.getVolumeName())); 1118 } catch (FileNotFoundException e) { 1119 Log.e(TAG, 1120 "Fuse Daemon not found for primary external storage, skipping update of " 1121 + "backup.", 1122 e); 1123 return; 1124 } 1125 1126 helper.runWithTransaction((db) -> { 1127 try (Cursor c = db.query(true, "files", QUERY_COLUMNS, "_id=?", 1128 new String[]{String.valueOf(newRow.getId())}, null, null, null, 1129 null, null)) { 1130 if (c.moveToFirst()) { 1131 backupDataValues(fuseDaemon, c); 1132 String newPath = c.getString(1); 1133 if (oldRow.getPath() != null && !oldRow.getPath().equalsIgnoreCase(newPath)) { 1134 // If file path has changed, update leveldb backup to delete old path. 1135 deleteFromDbBackup(helper, oldRow); 1136 Log.v(TAG, "Deleted backup of old file path: " + oldRow.getPath()); 1137 } 1138 } 1139 } catch (Exception e) { 1140 Log.e(TAG, "Failure in updating row in external storage backup.", e); 1141 } 1142 return null; 1143 }); 1144 } 1145 1146 /** 1147 * Removes database recovery data for given user id. This is done when a user is removed. 1148 */ removeRecoveryDataForUserId(int removedUserId)1149 protected void removeRecoveryDataForUserId(int removedUserId) { 1150 String removeduserIdString = String.valueOf(removedUserId); 1151 removeXattr(DATA_MEDIA_XATTR_DIRECTORY_PATH, 1152 INTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX.concat( 1153 removeduserIdString)); 1154 removeXattr(DATA_MEDIA_XATTR_DIRECTORY_PATH, 1155 EXTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX.concat( 1156 removeduserIdString)); 1157 removeXattr(DATA_MEDIA_XATTR_DIRECTORY_PATH, 1158 INTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX.concat(removeduserIdString)); 1159 removeXattr(DATA_MEDIA_XATTR_DIRECTORY_PATH, 1160 EXTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX.concat(removeduserIdString)); 1161 Log.v(TAG, "Removed recovery data for user id: " + removedUserId); 1162 } 1163 1164 /** 1165 * Removes database recovery data for obsolete user id. It accepts list of valid/active users 1166 * and removes the recovery data for ones not present in this list. 1167 * This is done during an idle maintenance. 1168 */ removeRecoveryDataExceptValidUsers(List<String> validUsers)1169 protected void removeRecoveryDataExceptValidUsers(List<String> validUsers) { 1170 List<String> xattrList = listXattr(DATA_MEDIA_XATTR_DIRECTORY_PATH); 1171 Log.i(TAG, "Xattr list is " + xattrList); 1172 if (xattrList.isEmpty()) { 1173 return; 1174 } 1175 1176 Log.i(TAG, "Valid users list is " + validUsers); 1177 List<String> invalidUsers = getInvalidUsersList(xattrList, validUsers); 1178 Log.i(TAG, "Invalid users list is " + invalidUsers); 1179 for (String userIdToBeRemoved : invalidUsers) { 1180 if (userIdToBeRemoved != null && !userIdToBeRemoved.trim().isEmpty()) { 1181 removeRecoveryDataForUserId(Integer.parseInt(userIdToBeRemoved)); 1182 } 1183 } 1184 } 1185 getInvalidUsersList(List<String> recoveryData, List<String> validUsers)1186 protected static List<String> getInvalidUsersList(List<String> recoveryData, 1187 List<String> validUsers) { 1188 Set<String> presentUserIdsAsXattr = new HashSet<>(); 1189 for (String xattr : recoveryData) { 1190 if (xattr.startsWith(INTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX)) { 1191 presentUserIdsAsXattr.add( 1192 xattr.substring(INTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX.length())); 1193 } else if (xattr.startsWith(EXTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX)) { 1194 presentUserIdsAsXattr.add( 1195 xattr.substring(EXTERNAL_DB_NEXT_ROW_ID_XATTR_KEY_PREFIX.length())); 1196 } else if (xattr.startsWith(INTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX)) { 1197 presentUserIdsAsXattr.add( 1198 xattr.substring(INTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX.length())); 1199 } else if (xattr.startsWith(EXTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX)) { 1200 presentUserIdsAsXattr.add( 1201 xattr.substring(EXTERNAL_DB_SESSION_ID_XATTR_KEY_PREFIX.length())); 1202 } 1203 } 1204 // Remove valid users 1205 validUsers.forEach(presentUserIdsAsXattr::remove); 1206 return presentUserIdsAsXattr.stream().collect(Collectors.toList()); 1207 } 1208 pollForExternalStorageMountedState()1209 private static void pollForExternalStorageMountedState() { 1210 final File target = Environment.getExternalStorageDirectory(); 1211 for (int i = 0; i < WAIT_TIME_15_SECONDS_IN_MILLIS / 100; i++) { 1212 if (Environment.MEDIA_MOUNTED.equals(Environment.getExternalStorageState(target))) { 1213 return; 1214 } 1215 Log.v(TAG, "Waiting for external storage..."); 1216 SystemClock.sleep(100); 1217 } 1218 throw new RuntimeException("Timed out while waiting for ExternalStorageState " 1219 + "to be MEDIA_MOUNTED"); 1220 } 1221 1222 /** 1223 * Performs actions to be taken on volume unmount. 1224 * @param volumeName name of volume which is detached 1225 */ onDetachVolume(String volumeName)1226 public void onDetachVolume(String volumeName) { 1227 if (mSetupCompleteVolumes.contains(volumeName)) { 1228 mSetupCompleteVolumes.remove(volumeName); 1229 Log.v(TAG, 1230 "Removed leveldb connections from in memory setup cache for volume:" 1231 + volumeName); 1232 } 1233 } 1234 } 1235