1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.providers.media.scan; 18 19 import static android.media.MediaMetadataRetriever.METADATA_KEY_ALBUM; 20 import static android.media.MediaMetadataRetriever.METADATA_KEY_ALBUMARTIST; 21 import static android.media.MediaMetadataRetriever.METADATA_KEY_ARTIST; 22 import static android.media.MediaMetadataRetriever.METADATA_KEY_AUTHOR; 23 import static android.media.MediaMetadataRetriever.METADATA_KEY_BITRATE; 24 import static android.media.MediaMetadataRetriever.METADATA_KEY_CAPTURE_FRAMERATE; 25 import static android.media.MediaMetadataRetriever.METADATA_KEY_CD_TRACK_NUMBER; 26 import static android.media.MediaMetadataRetriever.METADATA_KEY_COLOR_RANGE; 27 import static android.media.MediaMetadataRetriever.METADATA_KEY_COLOR_STANDARD; 28 import static android.media.MediaMetadataRetriever.METADATA_KEY_COLOR_TRANSFER; 29 import static android.media.MediaMetadataRetriever.METADATA_KEY_COMPILATION; 30 import static android.media.MediaMetadataRetriever.METADATA_KEY_COMPOSER; 31 import static android.media.MediaMetadataRetriever.METADATA_KEY_DATE; 32 import static android.media.MediaMetadataRetriever.METADATA_KEY_DISC_NUMBER; 33 import static android.media.MediaMetadataRetriever.METADATA_KEY_DURATION; 34 import static android.media.MediaMetadataRetriever.METADATA_KEY_GENRE; 35 import static android.media.MediaMetadataRetriever.METADATA_KEY_IMAGE_HEIGHT; 36 import static android.media.MediaMetadataRetriever.METADATA_KEY_IMAGE_WIDTH; 37 import static android.media.MediaMetadataRetriever.METADATA_KEY_MIMETYPE; 38 import static android.media.MediaMetadataRetriever.METADATA_KEY_NUM_TRACKS; 39 import static android.media.MediaMetadataRetriever.METADATA_KEY_TITLE; 40 import static android.media.MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT; 41 import static android.media.MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION; 42 import static android.media.MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH; 43 import static android.media.MediaMetadataRetriever.METADATA_KEY_WRITER; 44 import static android.media.MediaMetadataRetriever.METADATA_KEY_YEAR; 45 import static android.provider.MediaStore.AUTHORITY; 46 import static android.provider.MediaStore.UNKNOWN_STRING; 47 import static android.text.format.DateUtils.HOUR_IN_MILLIS; 48 import static android.text.format.DateUtils.MINUTE_IN_MILLIS; 49 50 import android.content.ContentProviderClient; 51 import android.content.ContentProviderOperation; 52 import android.content.ContentProviderResult; 53 import android.content.ContentResolver; 54 import android.content.ContentUris; 55 import android.content.Context; 56 import android.content.OperationApplicationException; 57 import android.database.Cursor; 58 import android.database.sqlite.SQLiteDatabase; 59 import android.drm.DrmManagerClient; 60 import android.drm.DrmSupportInfo; 61 import android.media.ExifInterface; 62 import android.media.MediaMetadataRetriever; 63 import android.mtp.MtpConstants; 64 import android.net.Uri; 65 import android.os.Build; 66 import android.os.Bundle; 67 import android.os.CancellationSignal; 68 import android.os.Environment; 69 import android.os.OperationCanceledException; 70 import android.os.RemoteException; 71 import android.os.SystemClock; 72 import android.os.Trace; 73 import android.provider.MediaStore; 74 import android.provider.MediaStore.Audio.AudioColumns; 75 import android.provider.MediaStore.Audio.PlaylistsColumns; 76 import android.provider.MediaStore.Files.FileColumns; 77 import android.provider.MediaStore.Images.ImageColumns; 78 import android.provider.MediaStore.MediaColumns; 79 import android.provider.MediaStore.Video.VideoColumns; 80 import android.text.TextUtils; 81 import android.util.ArrayMap; 82 import android.util.ArraySet; 83 import android.util.Log; 84 import android.util.Pair; 85 86 import androidx.annotation.GuardedBy; 87 import androidx.annotation.NonNull; 88 import androidx.annotation.Nullable; 89 import androidx.annotation.VisibleForTesting; 90 91 import com.android.providers.media.util.DatabaseUtils; 92 import com.android.providers.media.util.ExifUtils; 93 import com.android.providers.media.util.FileUtils; 94 import com.android.providers.media.util.IsoInterface; 95 import com.android.providers.media.util.Logging; 96 import com.android.providers.media.util.LongArray; 97 import com.android.providers.media.util.Metrics; 98 import com.android.providers.media.util.MimeUtils; 99 import com.android.providers.media.util.XmpInterface; 100 101 import java.io.File; 102 import java.io.FileInputStream; 103 import java.io.IOException; 104 import java.nio.file.FileVisitResult; 105 import java.nio.file.FileVisitor; 106 import java.nio.file.Files; 107 import java.nio.file.Path; 108 import java.nio.file.attribute.BasicFileAttributes; 109 import java.text.ParseException; 110 import java.text.SimpleDateFormat; 111 import java.util.ArrayList; 112 import java.util.Arrays; 113 import java.util.Iterator; 114 import java.util.Locale; 115 import java.util.Map; 116 import java.util.Objects; 117 import java.util.Optional; 118 import java.util.Set; 119 import java.util.TimeZone; 120 import java.util.concurrent.locks.Lock; 121 import java.util.concurrent.locks.ReentrantLock; 122 import java.util.regex.Matcher; 123 import java.util.regex.Pattern; 124 125 /** 126 * Modern implementation of media scanner. 127 * <p> 128 * This is a bug-compatible reimplementation of the legacy media scanner, but 129 * written purely in managed code for better testability and long-term 130 * maintainability. 131 * <p> 132 * Initial tests shows it performing roughly on-par with the legacy scanner. 133 * <p> 134 * In general, we start by populating metadata based on file attributes, and 135 * then overwrite with any valid metadata found using 136 * {@link MediaMetadataRetriever}, {@link ExifInterface}, and 137 * {@link XmpInterface}, each with increasing levels of trust. 138 */ 139 public class ModernMediaScanner implements MediaScanner { 140 private static final String TAG = "ModernMediaScanner"; 141 private static final boolean LOGW = Log.isLoggable(TAG, Log.WARN); 142 private static final boolean LOGD = Log.isLoggable(TAG, Log.DEBUG); 143 private static final boolean LOGV = Log.isLoggable(TAG, Log.VERBOSE); 144 145 // TODO: refactor to use UPSERT once we have SQLite 3.24.0 146 147 // TODO: deprecate playlist editing 148 // TODO: deprecate PARENT column, since callers can't see directories 149 150 @GuardedBy("sDateFormat") 151 private static final SimpleDateFormat sDateFormat; 152 153 static { 154 sDateFormat = new SimpleDateFormat("yyyyMMdd'T'HHmmss"); 155 sDateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); 156 } 157 158 private static final int BATCH_SIZE = 32; 159 160 private static final Pattern PATTERN_VISIBLE = Pattern.compile( 161 "(?i)^/storage/[^/]+(?:/[0-9]+)?(?:/Android/sandbox/([^/]+))?$"); 162 private static final Pattern PATTERN_INVISIBLE = Pattern.compile( 163 "(?i)^/storage/[^/]+(?:/[0-9]+)?(?:/Android/sandbox/([^/]+))?/" + 164 "(?:(?:Android/(?:data|obb)$)|(?:(?:Movies|Music|Pictures)/.thumbnails$))"); 165 166 private static final Pattern PATTERN_YEAR = Pattern.compile("([1-9][0-9][0-9][0-9])"); 167 168 private static final Pattern PATTERN_ALBUM_ART = Pattern.compile( 169 "(?i)(?:(?:^folder|(?:^AlbumArt(?:(?:_\\{.*\\}_)?(?:small|large))?))(?:\\.jpg$)|(?:\\._.*))"); 170 171 private final Context mContext; 172 private final DrmManagerClient mDrmClient; 173 174 /** 175 * Map from volume name to signals that can be used to cancel any active 176 * scan operations on those volumes. 177 */ 178 @GuardedBy("mSignals") 179 private final ArrayMap<String, CancellationSignal> mSignals = new ArrayMap<>(); 180 181 /** 182 * Holder that contains a reference count of the number of threads 183 * interested in a specific directory, along with a lock to ensure that 184 * parallel scans don't overlap and confuse each other. 185 */ 186 private static class DirectoryLock { 187 public int count; 188 public final Lock lock = new ReentrantLock(); 189 } 190 191 /** 192 * Map from directory to locks designed to ensure that parallel scans don't 193 * overlap and confuse each other. 194 */ 195 @GuardedBy("mDirectoryLocks") 196 private final Map<Path, DirectoryLock> mDirectoryLocks = new ArrayMap<>(); 197 198 /** 199 * Set of MIME types that should be considered to be DRM, meaning we need to 200 * consult {@link DrmManagerClient} to obtain the actual MIME type. 201 */ 202 private final Set<String> mDrmMimeTypes = new ArraySet<>(); 203 ModernMediaScanner(Context context)204 public ModernMediaScanner(Context context) { 205 mContext = context; 206 mDrmClient = new DrmManagerClient(context); 207 208 // Dynamically collect the set of MIME types that should be considered 209 // to be DRM, as this can vary between devices 210 for (DrmSupportInfo info : mDrmClient.getAvailableDrmSupportInfo()) { 211 Iterator<String> mimeTypes = info.getMimeTypeIterator(); 212 while (mimeTypes.hasNext()) { 213 mDrmMimeTypes.add(mimeTypes.next()); 214 } 215 } 216 } 217 218 @Override getContext()219 public Context getContext() { 220 return mContext; 221 } 222 223 @Override scanDirectory(File file, int reason)224 public void scanDirectory(File file, int reason) { 225 try (Scan scan = new Scan(file, reason, /*ownerPackage*/ null)) { 226 scan.run(); 227 } catch (OperationCanceledException ignored) { 228 } 229 } 230 231 @Override scanFile(File file, int reason)232 public Uri scanFile(File file, int reason) { 233 return scanFile(file, reason, /*ownerPackage*/ null); 234 } 235 236 @Override scanFile(File file, int reason, @Nullable String ownerPackage)237 public Uri scanFile(File file, int reason, @Nullable String ownerPackage) { 238 try (Scan scan = new Scan(file, reason, ownerPackage)) { 239 scan.run(); 240 return scan.getFirstResult(); 241 } catch (OperationCanceledException ignored) { 242 return null; 243 } 244 } 245 246 @Override onDetachVolume(String volumeName)247 public void onDetachVolume(String volumeName) { 248 synchronized (mSignals) { 249 final CancellationSignal signal = mSignals.remove(volumeName); 250 if (signal != null) { 251 signal.cancel(); 252 } 253 } 254 } 255 getOrCreateSignal(String volumeName)256 private CancellationSignal getOrCreateSignal(String volumeName) { 257 synchronized (mSignals) { 258 CancellationSignal signal = mSignals.get(volumeName); 259 if (signal == null) { 260 signal = new CancellationSignal(); 261 mSignals.put(volumeName, signal); 262 } 263 return signal; 264 } 265 } 266 267 /** 268 * Individual scan request for a specific file or directory. When run it 269 * will traverse all included media files under the requested location, 270 * reconciling them against {@link MediaStore}. 271 */ 272 private class Scan implements Runnable, FileVisitor<Path>, AutoCloseable { 273 private final ContentProviderClient mClient; 274 private final ContentResolver mResolver; 275 276 private final File mRoot; 277 private final int mReason; 278 private final String mVolumeName; 279 private final Uri mFilesUri; 280 private final CancellationSignal mSignal; 281 private final String mOwnerPackage; 282 283 private final long mStartGeneration; 284 private final boolean mSingleFile; 285 private final Set<Path> mAcquiredDirectoryLocks = new ArraySet<>(); 286 private final ArrayList<ContentProviderOperation> mPending = new ArrayList<>(); 287 private LongArray mScannedIds = new LongArray(); 288 private LongArray mUnknownIds = new LongArray(); 289 290 private long mFirstId = -1; 291 292 private int mFileCount; 293 private int mInsertCount; 294 private int mUpdateCount; 295 private int mDeleteCount; 296 297 /** 298 * Tracks hidden directory and hidden subdirectories in a directory tree. A positive count 299 * indicates that one or more of the current file's parents is a hidden directory. 300 */ 301 private int mHiddenDirCount; 302 Scan(File root, int reason, @Nullable String ownerPackage)303 public Scan(File root, int reason, @Nullable String ownerPackage) { 304 Trace.beginSection("ctor"); 305 306 mClient = mContext.getContentResolver() 307 .acquireContentProviderClient(MediaStore.AUTHORITY); 308 mResolver = ContentResolver.wrap(mClient.getLocalContentProvider()); 309 310 mRoot = root; 311 mReason = reason; 312 mVolumeName = FileUtils.getVolumeName(mContext, root); 313 mFilesUri = MediaStore.Files.getContentUri(mVolumeName); 314 mSignal = getOrCreateSignal(mVolumeName); 315 316 mStartGeneration = MediaStore.getGeneration(mResolver, mVolumeName); 317 mSingleFile = mRoot.isFile(); 318 mOwnerPackage = ownerPackage; 319 320 Trace.endSection(); 321 } 322 323 @Override run()324 public void run() { 325 final long startTime = SystemClock.elapsedRealtime(); 326 327 // First, scan everything that should be visible under requested 328 // location, tracking scanned IDs along the way 329 walkFileTree(); 330 331 // Second, reconcile all items known in the database against all the 332 // items we scanned above 333 if (mSingleFile && mScannedIds.size() == 1) { 334 // We can safely skip this step if the scan targeted a single 335 // file which we scanned above 336 } else { 337 reconcileAndClean(); 338 } 339 340 // Third, resolve any playlists that we scanned 341 resolvePlaylists(); 342 343 if (!mSingleFile) { 344 final long durationMillis = SystemClock.elapsedRealtime() - startTime; 345 Metrics.logScan(mVolumeName, mReason, mFileCount, durationMillis, 346 mInsertCount, mUpdateCount, mDeleteCount); 347 } 348 } 349 walkFileTree()350 private void walkFileTree() { 351 mSignal.throwIfCanceled(); 352 final Pair<Boolean, Boolean> isDirScannableAndHidden = 353 shouldScanPathAndIsPathHidden(mSingleFile ? mRoot.getParentFile() : mRoot); 354 if (isDirScannableAndHidden.first) { 355 // This directory is scannable. 356 Trace.beginSection("walkFileTree"); 357 358 if (isDirScannableAndHidden.second) { 359 // This directory is hidden 360 mHiddenDirCount++; 361 } 362 if (mSingleFile) { 363 acquireDirectoryLock(mRoot.getParentFile().toPath()); 364 } 365 try { 366 Files.walkFileTree(mRoot.toPath(), this); 367 applyPending(); 368 } catch (IOException e) { 369 // This should never happen, so yell loudly 370 throw new IllegalStateException(e); 371 } finally { 372 if (mSingleFile) { 373 releaseDirectoryLock(mRoot.getParentFile().toPath()); 374 } 375 Trace.endSection(); 376 } 377 } 378 } 379 reconcileAndClean()380 private void reconcileAndClean() { 381 final long[] scannedIds = mScannedIds.toArray(); 382 Arrays.sort(scannedIds); 383 384 // The query phase is split from the delete phase so that our query 385 // remains stable if we need to paginate across multiple windows. 386 mSignal.throwIfCanceled(); 387 Trace.beginSection("reconcile"); 388 389 // Ignore abstract playlists which don't have files on disk 390 final String formatClause = "ifnull(" + FileColumns.FORMAT + "," 391 + MtpConstants.FORMAT_UNDEFINED + ") != " 392 + MtpConstants.FORMAT_ABSTRACT_AV_PLAYLIST; 393 final String dataClause = "(" + FileColumns.DATA + " LIKE ? ESCAPE '\\' OR " 394 + FileColumns.DATA + " LIKE ? ESCAPE '\\')"; 395 final String generationClause = FileColumns.GENERATION_ADDED + " <= " 396 + mStartGeneration; 397 final Bundle queryArgs = new Bundle(); 398 queryArgs.putString(ContentResolver.QUERY_ARG_SQL_SELECTION, 399 formatClause + " AND " + dataClause + " AND " + generationClause); 400 final String pathEscapedForLike = DatabaseUtils.escapeForLike(mRoot.getAbsolutePath()); 401 queryArgs.putStringArray(ContentResolver.QUERY_ARG_SQL_SELECTION_ARGS, 402 new String[] {pathEscapedForLike + "/%", pathEscapedForLike}); 403 queryArgs.putString(ContentResolver.QUERY_ARG_SQL_SORT_ORDER, 404 FileColumns._ID + " DESC"); 405 queryArgs.putInt(MediaStore.QUERY_ARG_MATCH_PENDING, MediaStore.MATCH_INCLUDE); 406 queryArgs.putInt(MediaStore.QUERY_ARG_MATCH_TRASHED, MediaStore.MATCH_INCLUDE); 407 queryArgs.putInt(MediaStore.QUERY_ARG_MATCH_FAVORITE, MediaStore.MATCH_INCLUDE); 408 409 try (Cursor c = mResolver.query(mFilesUri, new String[] { FileColumns._ID }, 410 queryArgs, mSignal)) { 411 while (c.moveToNext()) { 412 final long id = c.getLong(0); 413 if (Arrays.binarySearch(scannedIds, id) < 0) { 414 mUnknownIds.add(id); 415 } 416 } 417 } finally { 418 Trace.endSection(); 419 } 420 421 // Third, clean all the unknown database entries found above 422 mSignal.throwIfCanceled(); 423 Trace.beginSection("clean"); 424 try { 425 for (int i = 0; i < mUnknownIds.size(); i++) { 426 final long id = mUnknownIds.get(i); 427 if (LOGV) Log.v(TAG, "Cleaning " + id); 428 final Uri uri = MediaStore.Files.getContentUri(mVolumeName, id).buildUpon() 429 .appendQueryParameter(MediaStore.PARAM_DELETE_DATA, "false") 430 .build(); 431 addPending(ContentProviderOperation.newDelete(uri).build()); 432 maybeApplyPending(); 433 } 434 applyPending(); 435 } finally { 436 Trace.endSection(); 437 } 438 } 439 resolvePlaylists()440 private void resolvePlaylists() { 441 mSignal.throwIfCanceled(); 442 443 // Playlists aren't supported on internal storage, so bail early 444 if (MediaStore.VOLUME_INTERNAL.equals(mVolumeName)) return; 445 446 final Uri playlistsUri = MediaStore.Audio.Playlists.getContentUri(mVolumeName); 447 final Bundle queryArgs = new Bundle(); 448 queryArgs.putString(ContentResolver.QUERY_ARG_SQL_SELECTION, 449 FileColumns.GENERATION_MODIFIED + " > " + mStartGeneration); 450 try (Cursor c = mResolver.query(playlistsUri, new String[] { FileColumns._ID }, 451 queryArgs, mSignal)) { 452 while (c.moveToNext()) { 453 final long id = c.getLong(0); 454 MediaStore.resolvePlaylistMembers(mResolver, 455 ContentUris.withAppendedId(playlistsUri, id)); 456 } 457 } finally { 458 Trace.endSection(); 459 } 460 } 461 462 /** 463 * Create and acquire a lock on the given directory, giving the calling 464 * thread exclusive access to ensure that parallel scans don't overlap 465 * and confuse each other. 466 */ acquireDirectoryLock(@onNull Path dir)467 private void acquireDirectoryLock(@NonNull Path dir) { 468 Trace.beginSection("acquireDirectoryLock"); 469 DirectoryLock lock; 470 synchronized (mDirectoryLocks) { 471 lock = mDirectoryLocks.get(dir); 472 if (lock == null) { 473 lock = new DirectoryLock(); 474 mDirectoryLocks.put(dir, lock); 475 } 476 lock.count++; 477 } 478 lock.lock.lock(); 479 mAcquiredDirectoryLocks.add(dir); 480 Trace.endSection(); 481 } 482 483 /** 484 * Release a currently held lock on the given directory, releasing any 485 * other waiting parallel scans to proceed, and cleaning up data 486 * structures if no other threads are waiting. 487 */ releaseDirectoryLock(@onNull Path dir)488 private void releaseDirectoryLock(@NonNull Path dir) { 489 Trace.beginSection("releaseDirectoryLock"); 490 DirectoryLock lock; 491 synchronized (mDirectoryLocks) { 492 lock = mDirectoryLocks.get(dir); 493 if (lock == null) { 494 throw new IllegalStateException(); 495 } 496 if (--lock.count == 0) { 497 mDirectoryLocks.remove(dir); 498 } 499 } 500 lock.lock.unlock(); 501 mAcquiredDirectoryLocks.remove(dir); 502 Trace.endSection(); 503 } 504 505 @Override close()506 public void close() { 507 // Sanity check that we drained any pending operations 508 if (!mPending.isEmpty()) { 509 throw new IllegalStateException(); 510 } 511 512 // Release any locks we're still holding, typically when we 513 // encountered an exception; we snapshot the original list so we're 514 // not confused as it's mutated by release operations 515 for (Path dir : new ArraySet<>(mAcquiredDirectoryLocks)) { 516 releaseDirectoryLock(dir); 517 } 518 519 mClient.close(); 520 } 521 522 @Override preVisitDirectory(Path dir, BasicFileAttributes attrs)523 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) 524 throws IOException { 525 // Possibly bail before digging into each directory 526 mSignal.throwIfCanceled(); 527 528 if (!shouldScanDirectory(dir.toFile())) { 529 return FileVisitResult.SKIP_SUBTREE; 530 } 531 532 // Acquire lock on this directory to ensure parallel scans don't 533 // overlap and confuse each other 534 acquireDirectoryLock(dir); 535 536 if (FileUtils.isDirectoryHidden(dir.toFile())) { 537 mHiddenDirCount++; 538 } 539 540 // Scan this directory as a normal file so that "parent" database 541 // entries are created 542 return visitFile(dir, attrs); 543 } 544 545 @Override visitFile(Path file, BasicFileAttributes attrs)546 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) 547 throws IOException { 548 if (LOGV) Log.v(TAG, "Visiting " + file); 549 mFileCount++; 550 551 // Skip files that have already been scanned, and which haven't 552 // changed since they were last scanned 553 final File realFile = file.toFile(); 554 long existingId = -1; 555 556 String actualMimeType; 557 if (attrs.isDirectory()) { 558 actualMimeType = null; 559 } else { 560 actualMimeType = MimeUtils.resolveMimeType(realFile); 561 } 562 563 // Resolve the MIME type of DRM files before scanning them; if we 564 // have trouble then we'll continue scanning as a generic file 565 final boolean isDrm = mDrmMimeTypes.contains(actualMimeType); 566 if (isDrm) { 567 actualMimeType = mDrmClient.getOriginalMimeType(realFile.getPath()); 568 } 569 570 int actualMediaType = FileColumns.MEDIA_TYPE_NONE; 571 if (actualMimeType != null) { 572 actualMediaType = resolveMediaTypeFromFilePath(realFile, actualMimeType, 573 /*isHidden*/ mHiddenDirCount > 0); 574 } 575 576 Trace.beginSection("checkChanged"); 577 578 final Bundle queryArgs = new Bundle(); 579 queryArgs.putString(ContentResolver.QUERY_ARG_SQL_SELECTION, 580 FileColumns.DATA + "=?"); 581 queryArgs.putStringArray(ContentResolver.QUERY_ARG_SQL_SELECTION_ARGS, 582 new String[] { realFile.getAbsolutePath() }); 583 queryArgs.putInt(MediaStore.QUERY_ARG_MATCH_PENDING, MediaStore.MATCH_INCLUDE); 584 queryArgs.putInt(MediaStore.QUERY_ARG_MATCH_TRASHED, MediaStore.MATCH_INCLUDE); 585 queryArgs.putInt(MediaStore.QUERY_ARG_MATCH_FAVORITE, MediaStore.MATCH_INCLUDE); 586 final String[] projection = new String[] {FileColumns._ID, FileColumns.DATE_MODIFIED, 587 FileColumns.SIZE, FileColumns.MIME_TYPE, FileColumns.MEDIA_TYPE, 588 FileColumns.IS_PENDING}; 589 590 final Matcher matcher = FileUtils.PATTERN_EXPIRES_FILE.matcher(realFile.getName()); 591 // If IS_PENDING is set by FUSE, we should scan the file and update IS_PENDING to zero. 592 // Pending files from FUSE will not be rewritten to contain expiry timestamp. 593 boolean isPendingFromFuse = !matcher.matches(); 594 595 try (Cursor c = mResolver.query(mFilesUri, projection, queryArgs, mSignal)) { 596 if (c.moveToFirst()) { 597 existingId = c.getLong(0); 598 final long dateModified = c.getLong(1); 599 final long size = c.getLong(2); 600 final String mimeType = c.getString(3); 601 final int mediaType = c.getInt(4); 602 isPendingFromFuse &= c.getInt(5) != 0; 603 604 // Remember visiting this existing item, even if we skipped 605 // due to it being unchanged; this is needed so we don't 606 // delete the item during a later cleaning phase 607 mScannedIds.add(existingId); 608 609 // We also technically found our first result 610 if (mFirstId == -1) { 611 mFirstId = existingId; 612 } 613 614 final boolean sameTime = (lastModifiedTime(realFile, attrs) == dateModified); 615 final boolean sameSize = (attrs.size() == size); 616 final boolean sameMimeType = mimeType == null ? actualMimeType == null : 617 mimeType.equalsIgnoreCase(actualMimeType); 618 final boolean sameMediaType = (actualMediaType == mediaType); 619 final boolean isSame = sameTime && sameSize && sameMediaType && sameMimeType 620 && !isPendingFromFuse; 621 if (attrs.isDirectory() || isSame) { 622 if (LOGV) Log.v(TAG, "Skipping unchanged " + file); 623 return FileVisitResult.CONTINUE; 624 } 625 } 626 } finally { 627 Trace.endSection(); 628 } 629 630 final ContentProviderOperation.Builder op; 631 Trace.beginSection("scanItem"); 632 try { 633 op = scanItem(existingId, realFile, attrs, actualMimeType, actualMediaType, 634 mVolumeName); 635 } finally { 636 Trace.endSection(); 637 } 638 if (op != null) { 639 // Add owner package name to new insertions when package name is provided. 640 if (op.build().isInsert() && !attrs.isDirectory() && mOwnerPackage != null) { 641 op.withValue(MediaColumns.OWNER_PACKAGE_NAME, mOwnerPackage); 642 } 643 // Force DRM files to be marked as DRM, since the lower level 644 // stack may not set this correctly 645 if (isDrm) { 646 op.withValue(MediaColumns.IS_DRM, 1); 647 } 648 addPending(op.build()); 649 maybeApplyPending(); 650 } 651 return FileVisitResult.CONTINUE; 652 } 653 654 @Override visitFileFailed(Path file, IOException exc)655 public FileVisitResult visitFileFailed(Path file, IOException exc) 656 throws IOException { 657 Log.w(TAG, "Failed to visit " + file + ": " + exc); 658 return FileVisitResult.CONTINUE; 659 } 660 661 @Override postVisitDirectory(Path dir, IOException exc)662 public FileVisitResult postVisitDirectory(Path dir, IOException exc) 663 throws IOException { 664 // We need to drain all pending changes related to this directory 665 // before releasing our lock below 666 applyPending(); 667 668 if (FileUtils.isDirectoryHidden(dir.toFile())) { 669 mHiddenDirCount--; 670 } 671 672 // Now that we're finished scanning this directory, release lock to 673 // allow other parallel scans to proceed 674 releaseDirectoryLock(dir); 675 676 return FileVisitResult.CONTINUE; 677 } 678 addPending(ContentProviderOperation op)679 private void addPending(ContentProviderOperation op) { 680 mPending.add(op); 681 682 if (op.isInsert()) mInsertCount++; 683 if (op.isUpdate()) mUpdateCount++; 684 if (op.isDelete()) mDeleteCount++; 685 } 686 maybeApplyPending()687 private void maybeApplyPending() { 688 if (mPending.size() > BATCH_SIZE) { 689 applyPending(); 690 } 691 } 692 applyPending()693 private void applyPending() { 694 // Bail early when nothing pending 695 if (mPending.isEmpty()) return; 696 697 Trace.beginSection("applyPending"); 698 try { 699 ContentProviderResult[] results = mResolver.applyBatch(AUTHORITY, mPending); 700 for (int index = 0; index < results.length; index++) { 701 ContentProviderResult result = results[index]; 702 ContentProviderOperation operation = mPending.get(index); 703 704 if (result.exception != null) { 705 Log.w(TAG, "Failed to apply " + operation, result.exception); 706 } 707 708 Uri uri = result.uri; 709 if (uri != null) { 710 final long id = ContentUris.parseId(uri); 711 if (mFirstId == -1) { 712 mFirstId = id; 713 } 714 mScannedIds.add(id); 715 } 716 } 717 } catch (RemoteException | OperationApplicationException e) { 718 Log.w(TAG, "Failed to apply", e); 719 } finally { 720 mPending.clear(); 721 Trace.endSection(); 722 } 723 } 724 725 /** 726 * Return the first item encountered by this scan requested. 727 * <p> 728 * Internally resolves to the relevant media collection where this item 729 * exists based on {@link FileColumns#MEDIA_TYPE}. 730 */ getFirstResult()731 public @Nullable Uri getFirstResult() { 732 if (mFirstId == -1) return null; 733 734 final Uri fileUri = MediaStore.Files.getContentUri(mVolumeName, mFirstId); 735 try (Cursor c = mResolver.query(fileUri, 736 new String[] { FileColumns.MEDIA_TYPE }, null, null)) { 737 if (c.moveToFirst()) { 738 switch (c.getInt(0)) { 739 case FileColumns.MEDIA_TYPE_AUDIO: 740 return MediaStore.Audio.Media.getContentUri(mVolumeName, mFirstId); 741 case FileColumns.MEDIA_TYPE_VIDEO: 742 return MediaStore.Video.Media.getContentUri(mVolumeName, mFirstId); 743 case FileColumns.MEDIA_TYPE_IMAGE: 744 return MediaStore.Images.Media.getContentUri(mVolumeName, mFirstId); 745 case FileColumns.MEDIA_TYPE_PLAYLIST: 746 return ContentUris.withAppendedId( 747 MediaStore.Audio.Playlists.getContentUri(mVolumeName), 748 mFirstId); 749 } 750 } 751 } 752 753 // Worst case, we can always use generic collection 754 return fileUri; 755 } 756 } 757 758 /** 759 * Scan the requested file, returning a {@link ContentProviderOperation} 760 * containing all indexed metadata, suitable for passing to a 761 * {@link SQLiteDatabase#replace} operation. 762 */ scanItem(long existingId, File file, BasicFileAttributes attrs, String mimeType, int mediaType, String volumeName)763 private static @Nullable ContentProviderOperation.Builder scanItem(long existingId, File file, 764 BasicFileAttributes attrs, String mimeType, int mediaType, String volumeName) { 765 if (Objects.equals(file.getName(), ".nomedia")) { 766 if (LOGD) Log.d(TAG, "Ignoring .nomedia file: " + file); 767 return null; 768 } 769 770 if (attrs.isDirectory()) { 771 return scanItemDirectory(existingId, file, attrs, mimeType, volumeName); 772 } 773 774 switch (mediaType) { 775 case FileColumns.MEDIA_TYPE_AUDIO: 776 return scanItemAudio(existingId, file, attrs, mimeType, mediaType, volumeName); 777 case FileColumns.MEDIA_TYPE_VIDEO: 778 return scanItemVideo(existingId, file, attrs, mimeType, mediaType, volumeName); 779 case FileColumns.MEDIA_TYPE_IMAGE: 780 return scanItemImage(existingId, file, attrs, mimeType, mediaType, volumeName); 781 case FileColumns.MEDIA_TYPE_PLAYLIST: 782 return scanItemPlaylist(existingId, file, attrs, mimeType, mediaType, volumeName); 783 case FileColumns.MEDIA_TYPE_SUBTITLE: 784 return scanItemSubtitle(existingId, file, attrs, mimeType, mediaType, volumeName); 785 case FileColumns.MEDIA_TYPE_DOCUMENT: 786 return scanItemDocument(existingId, file, attrs, mimeType, mediaType, volumeName); 787 default: 788 return scanItemFile(existingId, file, attrs, mimeType, mediaType, volumeName); 789 } 790 } 791 792 /** 793 * Populate the given {@link ContentProviderOperation} with the generic 794 * {@link MediaColumns} values that can be determined directly from the file 795 * or its attributes. 796 * <p> 797 * This is typically the first set of values defined so that we correctly 798 * clear any values that had been set by a previous scan and which are no 799 * longer present in the media item. 800 */ withGenericValues(ContentProviderOperation.Builder op, File file, BasicFileAttributes attrs, String mimeType, Integer mediaType)801 private static void withGenericValues(ContentProviderOperation.Builder op, 802 File file, BasicFileAttributes attrs, String mimeType, Integer mediaType) { 803 withOptionalMimeTypeAndMediaType(op, Optional.ofNullable(mimeType), 804 Optional.ofNullable(mediaType)); 805 806 op.withValue(MediaColumns.DATA, file.getAbsolutePath()); 807 op.withValue(MediaColumns.SIZE, attrs.size()); 808 op.withValue(MediaColumns.DATE_MODIFIED, lastModifiedTime(file, attrs)); 809 op.withValue(MediaColumns.DATE_TAKEN, null); 810 op.withValue(MediaColumns.IS_DRM, 0); 811 op.withValue(MediaColumns.WIDTH, null); 812 op.withValue(MediaColumns.HEIGHT, null); 813 op.withValue(MediaColumns.RESOLUTION, null); 814 op.withValue(MediaColumns.DOCUMENT_ID, null); 815 op.withValue(MediaColumns.INSTANCE_ID, null); 816 op.withValue(MediaColumns.ORIGINAL_DOCUMENT_ID, null); 817 op.withValue(MediaColumns.ORIENTATION, null); 818 819 op.withValue(MediaColumns.CD_TRACK_NUMBER, null); 820 op.withValue(MediaColumns.ALBUM, null); 821 op.withValue(MediaColumns.ARTIST, null); 822 op.withValue(MediaColumns.AUTHOR, null); 823 op.withValue(MediaColumns.COMPOSER, null); 824 op.withValue(MediaColumns.GENRE, null); 825 op.withValue(MediaColumns.TITLE, FileUtils.extractFileName(file.getName())); 826 op.withValue(MediaColumns.YEAR, null); 827 op.withValue(MediaColumns.DURATION, null); 828 op.withValue(MediaColumns.NUM_TRACKS, null); 829 op.withValue(MediaColumns.WRITER, null); 830 op.withValue(MediaColumns.ALBUM_ARTIST, null); 831 op.withValue(MediaColumns.DISC_NUMBER, null); 832 op.withValue(MediaColumns.COMPILATION, null); 833 op.withValue(MediaColumns.BITRATE, null); 834 op.withValue(MediaColumns.CAPTURE_FRAMERATE, null); 835 } 836 837 /** 838 * Populate the given {@link ContentProviderOperation} with the generic 839 * {@link MediaColumns} values using the given 840 * {@link MediaMetadataRetriever}. 841 */ withRetrieverValues(ContentProviderOperation.Builder op, MediaMetadataRetriever mmr, String mimeType)842 private static void withRetrieverValues(ContentProviderOperation.Builder op, 843 MediaMetadataRetriever mmr, String mimeType) { 844 withOptionalMimeTypeAndMediaType(op, 845 parseOptionalMimeType(mimeType, mmr.extractMetadata(METADATA_KEY_MIMETYPE)), 846 /*optionalMediaType*/ Optional.empty()); 847 848 withOptionalValue(op, MediaColumns.DATE_TAKEN, 849 parseOptionalDate(mmr.extractMetadata(METADATA_KEY_DATE))); 850 withOptionalValue(op, MediaColumns.CD_TRACK_NUMBER, 851 parseOptional(mmr.extractMetadata(METADATA_KEY_CD_TRACK_NUMBER))); 852 withOptionalValue(op, MediaColumns.ALBUM, 853 parseOptional(mmr.extractMetadata(METADATA_KEY_ALBUM))); 854 withOptionalValue(op, MediaColumns.ARTIST, firstPresent( 855 parseOptional(mmr.extractMetadata(METADATA_KEY_ARTIST)), 856 parseOptional(mmr.extractMetadata(METADATA_KEY_ALBUMARTIST)))); 857 withOptionalValue(op, MediaColumns.AUTHOR, 858 parseOptional(mmr.extractMetadata(METADATA_KEY_AUTHOR))); 859 withOptionalValue(op, MediaColumns.COMPOSER, 860 parseOptional(mmr.extractMetadata(METADATA_KEY_COMPOSER))); 861 withOptionalValue(op, MediaColumns.GENRE, 862 parseOptional(mmr.extractMetadata(METADATA_KEY_GENRE))); 863 withOptionalValue(op, MediaColumns.TITLE, 864 parseOptional(mmr.extractMetadata(METADATA_KEY_TITLE))); 865 withOptionalValue(op, MediaColumns.YEAR, 866 parseOptionalYear(mmr.extractMetadata(METADATA_KEY_YEAR))); 867 withOptionalValue(op, MediaColumns.DURATION, 868 parseOptional(mmr.extractMetadata(METADATA_KEY_DURATION))); 869 withOptionalValue(op, MediaColumns.NUM_TRACKS, 870 parseOptional(mmr.extractMetadata(METADATA_KEY_NUM_TRACKS))); 871 withOptionalValue(op, MediaColumns.WRITER, 872 parseOptional(mmr.extractMetadata(METADATA_KEY_WRITER))); 873 withOptionalValue(op, MediaColumns.ALBUM_ARTIST, 874 parseOptional(mmr.extractMetadata(METADATA_KEY_ALBUMARTIST))); 875 withOptionalValue(op, MediaColumns.DISC_NUMBER, 876 parseOptional(mmr.extractMetadata(METADATA_KEY_DISC_NUMBER))); 877 withOptionalValue(op, MediaColumns.COMPILATION, 878 parseOptional(mmr.extractMetadata(METADATA_KEY_COMPILATION))); 879 withOptionalValue(op, MediaColumns.BITRATE, 880 parseOptional(mmr.extractMetadata(METADATA_KEY_BITRATE))); 881 withOptionalValue(op, MediaColumns.CAPTURE_FRAMERATE, 882 parseOptional(mmr.extractMetadata(METADATA_KEY_CAPTURE_FRAMERATE))); 883 } 884 885 /** 886 * Populate the given {@link ContentProviderOperation} with the generic 887 * {@link MediaColumns} values using the given XMP metadata. 888 */ withXmpValues(ContentProviderOperation.Builder op, XmpInterface xmp, String mimeType)889 private static void withXmpValues(ContentProviderOperation.Builder op, 890 XmpInterface xmp, String mimeType) { 891 withOptionalMimeTypeAndMediaType(op, 892 parseOptionalMimeType(mimeType, xmp.getFormat()), 893 /*optionalMediaType*/ Optional.empty()); 894 895 op.withValue(MediaColumns.DOCUMENT_ID, xmp.getDocumentId()); 896 op.withValue(MediaColumns.INSTANCE_ID, xmp.getInstanceId()); 897 op.withValue(MediaColumns.ORIGINAL_DOCUMENT_ID, xmp.getOriginalDocumentId()); 898 op.withValue(MediaColumns.XMP, xmp.getRedactedXmp()); 899 } 900 901 /** 902 * Overwrite a value in the given {@link ContentProviderOperation}, but only 903 * when the given {@link Optional} value is present. 904 */ withOptionalValue(@onNull ContentProviderOperation.Builder op, @NonNull String key, @NonNull Optional<?> value)905 private static void withOptionalValue(@NonNull ContentProviderOperation.Builder op, 906 @NonNull String key, @NonNull Optional<?> value) { 907 if (value.isPresent()) { 908 op.withValue(key, value.get()); 909 } 910 } 911 912 /** 913 * Overwrite the {@link MediaColumns#MIME_TYPE} and 914 * {@link FileColumns#MEDIA_TYPE} values in the given 915 * {@link ContentProviderOperation}, but only when the given 916 * {@link Optional} optionalMimeType is present. 917 * If {@link Optional} optionalMediaType is not present, {@link FileColumns#MEDIA_TYPE} is 918 * resolved from given {@code optionalMimeType} when {@code optionalMimeType} is present. 919 * 920 * @param optionalMimeType An optional MIME type to apply to this operation. 921 * @param optionalMediaType An optional Media type to apply to this operation. 922 */ withOptionalMimeTypeAndMediaType( @onNull ContentProviderOperation.Builder op, @NonNull Optional<String> optionalMimeType, @NonNull Optional<Integer> optionalMediaType)923 private static void withOptionalMimeTypeAndMediaType( 924 @NonNull ContentProviderOperation.Builder op, 925 @NonNull Optional<String> optionalMimeType, 926 @NonNull Optional<Integer> optionalMediaType) { 927 if (optionalMimeType.isPresent()) { 928 final String mimeType = optionalMimeType.get(); 929 op.withValue(MediaColumns.MIME_TYPE, mimeType); 930 if (optionalMediaType.isPresent()) { 931 op.withValue(FileColumns.MEDIA_TYPE, optionalMediaType.get()); 932 } else { 933 op.withValue(FileColumns.MEDIA_TYPE, MimeUtils.resolveMediaType(mimeType)); 934 } 935 } 936 } 937 scanItemDirectory(long existingId, File file, BasicFileAttributes attrs, String mimeType, String volumeName)938 private static @NonNull ContentProviderOperation.Builder scanItemDirectory(long existingId, 939 File file, BasicFileAttributes attrs, String mimeType, String volumeName) { 940 final ContentProviderOperation.Builder op = newUpsert(volumeName, existingId); 941 // Directory doesn't have any MIME type or Media Type. 942 withGenericValues(op, file, attrs, mimeType, /*mediaType*/ null); 943 944 try { 945 op.withValue(FileColumns.FORMAT, MtpConstants.FORMAT_ASSOCIATION); 946 } catch (Exception e) { 947 logTroubleScanning(file, e); 948 } 949 return op; 950 } 951 952 private static ArrayMap<String, String> sAudioTypes = new ArrayMap<>(); 953 954 static { sAudioTypes.put(Environment.DIRECTORY_RINGTONES, AudioColumns.IS_RINGTONE)955 sAudioTypes.put(Environment.DIRECTORY_RINGTONES, AudioColumns.IS_RINGTONE); sAudioTypes.put(Environment.DIRECTORY_NOTIFICATIONS, AudioColumns.IS_NOTIFICATION)956 sAudioTypes.put(Environment.DIRECTORY_NOTIFICATIONS, AudioColumns.IS_NOTIFICATION); sAudioTypes.put(Environment.DIRECTORY_ALARMS, AudioColumns.IS_ALARM)957 sAudioTypes.put(Environment.DIRECTORY_ALARMS, AudioColumns.IS_ALARM); sAudioTypes.put(Environment.DIRECTORY_PODCASTS, AudioColumns.IS_PODCAST)958 sAudioTypes.put(Environment.DIRECTORY_PODCASTS, AudioColumns.IS_PODCAST); sAudioTypes.put(Environment.DIRECTORY_AUDIOBOOKS, AudioColumns.IS_AUDIOBOOK)959 sAudioTypes.put(Environment.DIRECTORY_AUDIOBOOKS, AudioColumns.IS_AUDIOBOOK); sAudioTypes.put(Environment.DIRECTORY_MUSIC, AudioColumns.IS_MUSIC)960 sAudioTypes.put(Environment.DIRECTORY_MUSIC, AudioColumns.IS_MUSIC); 961 } 962 scanItemAudio(long existingId, File file, BasicFileAttributes attrs, String mimeType, int mediaType, String volumeName)963 private static @NonNull ContentProviderOperation.Builder scanItemAudio(long existingId, 964 File file, BasicFileAttributes attrs, String mimeType, int mediaType, 965 String volumeName) { 966 final ContentProviderOperation.Builder op = newUpsert(volumeName, existingId); 967 withGenericValues(op, file, attrs, mimeType, mediaType); 968 969 op.withValue(MediaColumns.ARTIST, UNKNOWN_STRING); 970 op.withValue(MediaColumns.ALBUM, file.getParentFile().getName()); 971 972 final String lowPath = file.getAbsolutePath().toLowerCase(Locale.ROOT); 973 boolean anyMatch = false; 974 for (int i = 0; i < sAudioTypes.size(); i++) { 975 final boolean match = lowPath 976 .contains('/' + sAudioTypes.keyAt(i).toLowerCase(Locale.ROOT) + '/'); 977 op.withValue(sAudioTypes.valueAt(i), match ? 1 : 0); 978 anyMatch |= match; 979 } 980 if (!anyMatch) { 981 op.withValue(AudioColumns.IS_MUSIC, 1); 982 } 983 984 try (FileInputStream is = new FileInputStream(file)) { 985 try (MediaMetadataRetriever mmr = new MediaMetadataRetriever()) { 986 mmr.setDataSource(is.getFD()); 987 988 withRetrieverValues(op, mmr, mimeType); 989 990 withOptionalValue(op, AudioColumns.TRACK, 991 parseOptionalTrack(mmr)); 992 } 993 994 // Also hunt around for XMP metadata 995 final IsoInterface iso = IsoInterface.fromFileDescriptor(is.getFD()); 996 final XmpInterface xmp = XmpInterface.fromContainer(iso); 997 withXmpValues(op, xmp, mimeType); 998 999 } catch (Exception e) { 1000 logTroubleScanning(file, e); 1001 } 1002 return op; 1003 } 1004 scanItemPlaylist(long existingId, File file, BasicFileAttributes attrs, String mimeType, int mediaType, String volumeName)1005 private static @NonNull ContentProviderOperation.Builder scanItemPlaylist(long existingId, 1006 File file, BasicFileAttributes attrs, String mimeType, int mediaType, 1007 String volumeName) { 1008 final ContentProviderOperation.Builder op = newUpsert(volumeName, existingId); 1009 withGenericValues(op, file, attrs, mimeType, mediaType); 1010 1011 try { 1012 op.withValue(PlaylistsColumns.NAME, FileUtils.extractFileName(file.getName())); 1013 } catch (Exception e) { 1014 logTroubleScanning(file, e); 1015 } 1016 return op; 1017 } 1018 scanItemSubtitle(long existingId, File file, BasicFileAttributes attrs, String mimeType, int mediaType, String volumeName)1019 private static @NonNull ContentProviderOperation.Builder scanItemSubtitle(long existingId, 1020 File file, BasicFileAttributes attrs, String mimeType, int mediaType, 1021 String volumeName) { 1022 final ContentProviderOperation.Builder op = newUpsert(volumeName, existingId); 1023 withGenericValues(op, file, attrs, mimeType, mediaType); 1024 1025 return op; 1026 } 1027 scanItemDocument(long existingId, File file, BasicFileAttributes attrs, String mimeType, int mediaType, String volumeName)1028 private static @NonNull ContentProviderOperation.Builder scanItemDocument(long existingId, 1029 File file, BasicFileAttributes attrs, String mimeType, int mediaType, 1030 String volumeName) { 1031 final ContentProviderOperation.Builder op = newUpsert(volumeName, existingId); 1032 withGenericValues(op, file, attrs, mimeType, mediaType); 1033 1034 return op; 1035 } 1036 scanItemVideo(long existingId, File file, BasicFileAttributes attrs, String mimeType, int mediaType, String volumeName)1037 private static @NonNull ContentProviderOperation.Builder scanItemVideo(long existingId, 1038 File file, BasicFileAttributes attrs, String mimeType, int mediaType, 1039 String volumeName) { 1040 final ContentProviderOperation.Builder op = newUpsert(volumeName, existingId); 1041 withGenericValues(op, file, attrs, mimeType, mediaType); 1042 1043 op.withValue(MediaColumns.ARTIST, UNKNOWN_STRING); 1044 op.withValue(MediaColumns.ALBUM, file.getParentFile().getName()); 1045 op.withValue(VideoColumns.COLOR_STANDARD, null); 1046 op.withValue(VideoColumns.COLOR_TRANSFER, null); 1047 op.withValue(VideoColumns.COLOR_RANGE, null); 1048 1049 try (FileInputStream is = new FileInputStream(file)) { 1050 try (MediaMetadataRetriever mmr = new MediaMetadataRetriever()) { 1051 mmr.setDataSource(is.getFD()); 1052 1053 withRetrieverValues(op, mmr, mimeType); 1054 1055 withOptionalValue(op, MediaColumns.WIDTH, 1056 parseOptional(mmr.extractMetadata(METADATA_KEY_VIDEO_WIDTH))); 1057 withOptionalValue(op, MediaColumns.HEIGHT, 1058 parseOptional(mmr.extractMetadata(METADATA_KEY_VIDEO_HEIGHT))); 1059 withOptionalValue(op, MediaColumns.RESOLUTION, 1060 parseOptionalVideoResolution(mmr)); 1061 withOptionalValue(op, MediaColumns.ORIENTATION, 1062 parseOptional(mmr.extractMetadata(METADATA_KEY_VIDEO_ROTATION))); 1063 1064 withOptionalValue(op, VideoColumns.COLOR_STANDARD, 1065 parseOptional(mmr.extractMetadata(METADATA_KEY_COLOR_STANDARD))); 1066 withOptionalValue(op, VideoColumns.COLOR_TRANSFER, 1067 parseOptional(mmr.extractMetadata(METADATA_KEY_COLOR_TRANSFER))); 1068 withOptionalValue(op, VideoColumns.COLOR_RANGE, 1069 parseOptional(mmr.extractMetadata(METADATA_KEY_COLOR_RANGE))); 1070 } 1071 1072 // Also hunt around for XMP metadata 1073 final IsoInterface iso = IsoInterface.fromFileDescriptor(is.getFD()); 1074 final XmpInterface xmp = XmpInterface.fromContainer(iso); 1075 withXmpValues(op, xmp, mimeType); 1076 1077 } catch (Exception e) { 1078 logTroubleScanning(file, e); 1079 } 1080 return op; 1081 } 1082 scanItemImage(long existingId, File file, BasicFileAttributes attrs, String mimeType, int mediaType, String volumeName)1083 private static @NonNull ContentProviderOperation.Builder scanItemImage(long existingId, 1084 File file, BasicFileAttributes attrs, String mimeType, int mediaType, 1085 String volumeName) { 1086 final ContentProviderOperation.Builder op = newUpsert(volumeName, existingId); 1087 withGenericValues(op, file, attrs, mimeType, mediaType); 1088 1089 op.withValue(ImageColumns.DESCRIPTION, null); 1090 1091 try (FileInputStream is = new FileInputStream(file)) { 1092 final ExifInterface exif = new ExifInterface(is); 1093 1094 withOptionalValue(op, MediaColumns.WIDTH, 1095 parseOptionalOrZero(exif.getAttribute(ExifInterface.TAG_IMAGE_WIDTH))); 1096 withOptionalValue(op, MediaColumns.HEIGHT, 1097 parseOptionalOrZero(exif.getAttribute(ExifInterface.TAG_IMAGE_LENGTH))); 1098 withOptionalValue(op, MediaColumns.RESOLUTION, 1099 parseOptionalResolution(exif)); 1100 withOptionalValue(op, MediaColumns.DATE_TAKEN, 1101 parseOptionalDateTaken(exif, lastModifiedTime(file, attrs) * 1000)); 1102 withOptionalValue(op, MediaColumns.ORIENTATION, 1103 parseOptionalOrientation(exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, 1104 ExifInterface.ORIENTATION_UNDEFINED))); 1105 1106 withOptionalValue(op, ImageColumns.DESCRIPTION, 1107 parseOptional(exif.getAttribute(ExifInterface.TAG_IMAGE_DESCRIPTION))); 1108 withOptionalValue(op, ImageColumns.EXPOSURE_TIME, 1109 parseOptional(exif.getAttribute(ExifInterface.TAG_EXPOSURE_TIME))); 1110 withOptionalValue(op, ImageColumns.F_NUMBER, 1111 parseOptional(exif.getAttribute(ExifInterface.TAG_F_NUMBER))); 1112 withOptionalValue(op, ImageColumns.ISO, 1113 parseOptional(exif.getAttribute(ExifInterface.TAG_ISO_SPEED_RATINGS))); 1114 withOptionalValue(op, ImageColumns.SCENE_CAPTURE_TYPE, 1115 parseOptional(exif.getAttribute(ExifInterface.TAG_SCENE_CAPTURE_TYPE))); 1116 1117 // Also hunt around for XMP metadata 1118 final XmpInterface xmp = XmpInterface.fromContainer(exif); 1119 withXmpValues(op, xmp, mimeType); 1120 1121 } catch (Exception e) { 1122 logTroubleScanning(file, e); 1123 } 1124 return op; 1125 } 1126 scanItemFile(long existingId, File file, BasicFileAttributes attrs, String mimeType, int mediaType, String volumeName)1127 private static @NonNull ContentProviderOperation.Builder scanItemFile(long existingId, 1128 File file, BasicFileAttributes attrs, String mimeType, int mediaType, 1129 String volumeName) { 1130 final ContentProviderOperation.Builder op = newUpsert(volumeName, existingId); 1131 withGenericValues(op, file, attrs, mimeType, mediaType); 1132 1133 return op; 1134 } 1135 newUpsert( @onNull String volumeName, long existingId)1136 private static @NonNull ContentProviderOperation.Builder newUpsert( 1137 @NonNull String volumeName, long existingId) { 1138 final Uri uri = MediaStore.Files.getContentUri(volumeName); 1139 if (existingId == -1) { 1140 return ContentProviderOperation.newInsert(uri) 1141 .withExceptionAllowed(true); 1142 } else { 1143 return ContentProviderOperation.newUpdate(ContentUris.withAppendedId(uri, existingId)) 1144 .withExpectedCount(1) 1145 .withExceptionAllowed(true); 1146 } 1147 } 1148 1149 /** 1150 * Pick the first present {@link Optional} value from the given list. 1151 */ 1152 @SafeVarargs firstPresent(@onNull Optional<T>.... options)1153 private static @NonNull <T> Optional<T> firstPresent(@NonNull Optional<T>... options) { 1154 for (Optional<T> option : options) { 1155 if (option.isPresent()) { 1156 return option; 1157 } 1158 } 1159 return Optional.empty(); 1160 } 1161 1162 @VisibleForTesting parseOptional(@ullable T value)1163 static @NonNull <T> Optional<T> parseOptional(@Nullable T value) { 1164 if (value == null) { 1165 return Optional.empty(); 1166 } else if (value instanceof String && ((String) value).length() == 0) { 1167 return Optional.empty(); 1168 } else if (value instanceof String && ((String) value).equals("-1")) { 1169 return Optional.empty(); 1170 } else if (value instanceof String && ((String) value).trim().length() == 0) { 1171 return Optional.empty(); 1172 } else if (value instanceof Number && ((Number) value).intValue() == -1) { 1173 return Optional.empty(); 1174 } else { 1175 return Optional.of(value); 1176 } 1177 } 1178 1179 @VisibleForTesting parseOptionalOrZero(@ullable T value)1180 static @NonNull <T> Optional<T> parseOptionalOrZero(@Nullable T value) { 1181 if (value instanceof String && isZero((String) value)) { 1182 return Optional.empty(); 1183 } else if (value instanceof Number && ((Number) value).intValue() == 0) { 1184 return Optional.empty(); 1185 } else { 1186 return parseOptional(value); 1187 } 1188 } 1189 1190 @VisibleForTesting parseOptionalNumerator(@ullable String value)1191 static @NonNull Optional<Integer> parseOptionalNumerator(@Nullable String value) { 1192 final Optional<String> parsedValue = parseOptional(value); 1193 if (parsedValue.isPresent()) { 1194 value = parsedValue.get(); 1195 final int fractionIndex = value.indexOf('/'); 1196 if (fractionIndex != -1) { 1197 value = value.substring(0, fractionIndex); 1198 } 1199 try { 1200 return Optional.of(Integer.parseInt(value)); 1201 } catch (NumberFormatException ignored) { 1202 return Optional.empty(); 1203 } 1204 } else { 1205 return Optional.empty(); 1206 } 1207 } 1208 1209 /** 1210 * Try our best to calculate {@link MediaColumns#DATE_TAKEN} in reference to 1211 * the epoch, making our best guess from unrelated fields when offset 1212 * information isn't directly available. 1213 */ 1214 @VisibleForTesting parseOptionalDateTaken(@onNull ExifInterface exif, long lastModifiedTime)1215 static @NonNull Optional<Long> parseOptionalDateTaken(@NonNull ExifInterface exif, 1216 long lastModifiedTime) { 1217 final long originalTime = ExifUtils.getDateTimeOriginal(exif); 1218 if (exif.hasAttribute(ExifInterface.TAG_OFFSET_TIME_ORIGINAL)) { 1219 // We have known offset information, return it directly! 1220 return Optional.of(originalTime); 1221 } else { 1222 // Otherwise we need to guess the offset from unrelated fields 1223 final long smallestZone = 15 * MINUTE_IN_MILLIS; 1224 final long gpsTime = ExifUtils.getGpsDateTime(exif); 1225 if (gpsTime > 0) { 1226 final long offset = gpsTime - originalTime; 1227 if (Math.abs(offset) < 24 * HOUR_IN_MILLIS) { 1228 final long rounded = Math.round((float) offset / smallestZone) * smallestZone; 1229 return Optional.of(originalTime + rounded); 1230 } 1231 } 1232 if (lastModifiedTime > 0) { 1233 final long offset = lastModifiedTime - originalTime; 1234 if (Math.abs(offset) < 24 * HOUR_IN_MILLIS) { 1235 final long rounded = Math.round((float) offset / smallestZone) * smallestZone; 1236 return Optional.of(originalTime + rounded); 1237 } 1238 } 1239 return Optional.empty(); 1240 } 1241 } 1242 1243 @VisibleForTesting parseOptionalOrientation(int orientation)1244 static @NonNull Optional<Integer> parseOptionalOrientation(int orientation) { 1245 switch (orientation) { 1246 case ExifInterface.ORIENTATION_NORMAL: return Optional.of(0); 1247 case ExifInterface.ORIENTATION_ROTATE_90: return Optional.of(90); 1248 case ExifInterface.ORIENTATION_ROTATE_180: return Optional.of(180); 1249 case ExifInterface.ORIENTATION_ROTATE_270: return Optional.of(270); 1250 default: return Optional.empty(); 1251 } 1252 } 1253 1254 @VisibleForTesting parseOptionalVideoResolution( @onNull MediaMetadataRetriever mmr)1255 static @NonNull Optional<String> parseOptionalVideoResolution( 1256 @NonNull MediaMetadataRetriever mmr) { 1257 final Optional<?> width = parseOptional(mmr.extractMetadata(METADATA_KEY_VIDEO_WIDTH)); 1258 final Optional<?> height = parseOptional(mmr.extractMetadata(METADATA_KEY_VIDEO_HEIGHT)); 1259 if (width.isPresent() && height.isPresent()) { 1260 return Optional.of(width.get() + "\u00d7" + height.get()); 1261 } else { 1262 return Optional.empty(); 1263 } 1264 } 1265 1266 @VisibleForTesting parseOptionalImageResolution( @onNull MediaMetadataRetriever mmr)1267 static @NonNull Optional<String> parseOptionalImageResolution( 1268 @NonNull MediaMetadataRetriever mmr) { 1269 final Optional<?> width = parseOptional(mmr.extractMetadata(METADATA_KEY_IMAGE_WIDTH)); 1270 final Optional<?> height = parseOptional(mmr.extractMetadata(METADATA_KEY_IMAGE_HEIGHT)); 1271 if (width.isPresent() && height.isPresent()) { 1272 return Optional.of(width.get() + "\u00d7" + height.get()); 1273 } else { 1274 return Optional.empty(); 1275 } 1276 } 1277 1278 @VisibleForTesting parseOptionalResolution( @onNull ExifInterface exif)1279 static @NonNull Optional<String> parseOptionalResolution( 1280 @NonNull ExifInterface exif) { 1281 final Optional<?> width = parseOptionalOrZero( 1282 exif.getAttribute(ExifInterface.TAG_IMAGE_WIDTH)); 1283 final Optional<?> height = parseOptionalOrZero( 1284 exif.getAttribute(ExifInterface.TAG_IMAGE_LENGTH)); 1285 if (width.isPresent() && height.isPresent()) { 1286 return Optional.of(width.get() + "\u00d7" + height.get()); 1287 } else { 1288 return Optional.empty(); 1289 } 1290 } 1291 1292 @VisibleForTesting parseOptionalDate(@ullable String date)1293 static @NonNull Optional<Long> parseOptionalDate(@Nullable String date) { 1294 if (TextUtils.isEmpty(date)) return Optional.empty(); 1295 try { 1296 synchronized (sDateFormat) { 1297 final long value = sDateFormat.parse(date).getTime(); 1298 return (value > 0) ? Optional.of(value) : Optional.empty(); 1299 } 1300 } catch (ParseException e) { 1301 return Optional.empty(); 1302 } 1303 } 1304 1305 @VisibleForTesting parseOptionalYear(@ullable String value)1306 static @NonNull Optional<Integer> parseOptionalYear(@Nullable String value) { 1307 final Optional<String> parsedValue = parseOptional(value); 1308 if (parsedValue.isPresent()) { 1309 final Matcher m = PATTERN_YEAR.matcher(parsedValue.get()); 1310 if (m.find()) { 1311 return Optional.of(Integer.parseInt(m.group(1))); 1312 } else { 1313 return Optional.empty(); 1314 } 1315 } else { 1316 return Optional.empty(); 1317 } 1318 } 1319 1320 @VisibleForTesting parseOptionalTrack( @onNull MediaMetadataRetriever mmr)1321 static @NonNull Optional<Integer> parseOptionalTrack( 1322 @NonNull MediaMetadataRetriever mmr) { 1323 final Optional<Integer> disc = parseOptionalNumerator( 1324 mmr.extractMetadata(METADATA_KEY_DISC_NUMBER)); 1325 final Optional<Integer> track = parseOptionalNumerator( 1326 mmr.extractMetadata(METADATA_KEY_CD_TRACK_NUMBER)); 1327 if (disc.isPresent() && track.isPresent()) { 1328 return Optional.of((disc.get() * 1000) + track.get()); 1329 } else { 1330 return track; 1331 } 1332 } 1333 1334 /** 1335 * Maybe replace the MIME type from extension with the MIME type from the 1336 * refined metadata, but only when the top-level MIME type agrees. 1337 */ 1338 @VisibleForTesting parseOptionalMimeType(@onNull String fileMimeType, @Nullable String refinedMimeType)1339 static @NonNull Optional<String> parseOptionalMimeType(@NonNull String fileMimeType, 1340 @Nullable String refinedMimeType) { 1341 // Ignore when missing 1342 if (TextUtils.isEmpty(refinedMimeType)) return Optional.empty(); 1343 1344 // Ignore when invalid 1345 final int refinedSplit = refinedMimeType.indexOf('/'); 1346 if (refinedSplit == -1) return Optional.empty(); 1347 1348 if (fileMimeType.regionMatches(true, 0, refinedMimeType, 0, refinedSplit + 1)) { 1349 return Optional.of(refinedMimeType); 1350 } else if ("video/mp4".equalsIgnoreCase(fileMimeType) 1351 && "audio/mp4".equalsIgnoreCase(refinedMimeType)) { 1352 // We normally only allow MIME types to be customized when the 1353 // top-level type agrees, but this one very narrow case is added to 1354 // support a music service that was writing "m4a" files as "mp4". 1355 return Optional.of(refinedMimeType); 1356 } else { 1357 return Optional.empty(); 1358 } 1359 } 1360 1361 /** 1362 * Return last modified time of given file. This value is typically read 1363 * from the given {@link BasicFileAttributes}, except in the case of 1364 * read-only partitions, where {@link Build#TIME} is used instead. 1365 */ lastModifiedTime(@onNull File file, @NonNull BasicFileAttributes attrs)1366 public static long lastModifiedTime(@NonNull File file, 1367 @NonNull BasicFileAttributes attrs) { 1368 if (FileUtils.contains(Environment.getStorageDirectory(), file)) { 1369 return attrs.lastModifiedTime().toMillis() / 1000; 1370 } else { 1371 return Build.TIME / 1000; 1372 } 1373 } 1374 1375 /** 1376 * Test if any parents of given path should be scanned and test if any parents of given 1377 * path should be considered hidden. 1378 */ shouldScanPathAndIsPathHidden(@onNull File dir)1379 static Pair<Boolean, Boolean> shouldScanPathAndIsPathHidden(@NonNull File dir) { 1380 Trace.beginSection("shouldScanPathAndIsPathHiodden"); 1381 try { 1382 boolean isPathHidden = false; 1383 while (dir != null) { 1384 if (!shouldScanDirectory(dir)) { 1385 // When the path is not scannable, we don't care if it's hidden or not. 1386 return Pair.create(false, false); 1387 } 1388 isPathHidden = isPathHidden || FileUtils.isDirectoryHidden(dir); 1389 dir = dir.getParentFile(); 1390 } 1391 return Pair.create(true, isPathHidden); 1392 } finally { 1393 Trace.endSection(); 1394 } 1395 } 1396 1397 @VisibleForTesting shouldScanDirectory(@onNull File dir)1398 static boolean shouldScanDirectory(@NonNull File dir) { 1399 final File nomedia = new File(dir, ".nomedia"); 1400 1401 // Handle well-known paths that should always be visible or invisible, 1402 // regardless of .nomedia presence 1403 if (PATTERN_VISIBLE.matcher(dir.getAbsolutePath()).matches()) { 1404 // Well known paths can never be a hidden directory. Delete any non-standard nomedia 1405 // presence in well known path. 1406 nomedia.delete(); 1407 return true; 1408 } 1409 1410 if (PATTERN_INVISIBLE.matcher(dir.getAbsolutePath()).matches()) { 1411 // Create the .nomedia file in paths that are not scannable. This is useful when user 1412 // ejects the SD card and brings it to an older device and its media scanner can 1413 // now correctly identify these paths as not scannable. 1414 try { 1415 nomedia.createNewFile(); 1416 } catch (IOException ignored) { 1417 } 1418 return false; 1419 } 1420 return true; 1421 } 1422 1423 /** 1424 * @return {@link FileColumns#MEDIA_TYPE}, resolved based on the file path and given 1425 * {@code mimeType}. 1426 */ resolveMediaTypeFromFilePath(@onNull File file, @NonNull String mimeType, boolean isHidden)1427 private static int resolveMediaTypeFromFilePath(@NonNull File file, @NonNull String mimeType, 1428 boolean isHidden) { 1429 int mediaType = MimeUtils.resolveMediaType(mimeType); 1430 1431 if (isHidden || FileUtils.isFileHidden(file)) { 1432 mediaType = FileColumns.MEDIA_TYPE_NONE; 1433 } 1434 if (mediaType == FileColumns.MEDIA_TYPE_IMAGE && isFileAlbumArt(file)) { 1435 mediaType = FileColumns.MEDIA_TYPE_NONE; 1436 } 1437 return mediaType; 1438 } 1439 1440 @VisibleForTesting isFileAlbumArt(@onNull File file)1441 static boolean isFileAlbumArt(@NonNull File file) { 1442 return PATTERN_ALBUM_ART.matcher(file.getName()).matches(); 1443 } 1444 isZero(@onNull String value)1445 static boolean isZero(@NonNull String value) { 1446 if (value.length() == 0) { 1447 return false; 1448 } 1449 for (int i = 0; i < value.length(); i++) { 1450 if (value.charAt(i) != '0') { 1451 return false; 1452 } 1453 } 1454 return true; 1455 } 1456 logTroubleScanning(@onNull File file, @NonNull Exception e)1457 static void logTroubleScanning(@NonNull File file, @NonNull Exception e) { 1458 if (LOGW) Log.w(TAG, "Trouble scanning " + file + ": " + e); 1459 } 1460 } 1461