1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media.mediatranscoding.cts; 18 19 import static org.junit.Assert.assertTrue; 20 21 22 import android.content.ContentResolver; 23 import android.content.Context; 24 import android.content.res.AssetFileDescriptor; 25 import android.graphics.ImageFormat; 26 import android.graphics.Rect; 27 import android.media.Image; 28 import android.media.MediaCodec; 29 import android.media.MediaCodecInfo; 30 import android.media.MediaExtractor; 31 import android.media.MediaFormat; 32 import android.media.MediaMetadataRetriever; 33 import android.net.Uri; 34 import android.os.FileUtils; 35 import android.os.ParcelFileDescriptor; 36 import android.util.Log; 37 import android.util.Size; 38 39 import java.io.File; 40 import java.io.FileInputStream; 41 import java.io.FileOutputStream; 42 import java.io.IOException; 43 import java.io.InputStream; 44 import java.nio.ByteBuffer; 45 import java.util.Locale; 46 47 /* package */ class MediaTranscodingTestUtil { 48 private static final String TAG = "MediaTranscodingTestUtil"; 49 50 // Helper class to extract the information from source file and transcoded file. 51 static class VideoFileInfo { 52 String mUri; 53 int mNumVideoFrames = 0; 54 int mWidth = 0; 55 int mHeight = 0; 56 float mVideoFrameRate = 0.0f; 57 boolean mHasAudio = false; 58 int mRotationDegree = 0; 59 toString()60 public String toString() { 61 String str = mUri; 62 str += " Width:" + mWidth; 63 str += " Height:" + mHeight; 64 str += " FrameRate:" + mWidth; 65 str += " FrameCount:" + mNumVideoFrames; 66 str += " HasAudio:" + (mHasAudio ? "Yes" : "No"); 67 return str; 68 } 69 } 70 extractVideoFileInfo(Context ctx, Uri videoUri)71 static VideoFileInfo extractVideoFileInfo(Context ctx, Uri videoUri) throws IOException { 72 VideoFileInfo info = new VideoFileInfo(); 73 AssetFileDescriptor afd = null; 74 MediaMetadataRetriever retriever = null; 75 76 try { 77 afd = ctx.getContentResolver().openAssetFileDescriptor(videoUri, "r"); 78 retriever = new MediaMetadataRetriever(); 79 retriever.setDataSource(afd.getFileDescriptor(), afd.getStartOffset(), afd.getLength()); 80 81 info.mUri = videoUri.getLastPathSegment(); 82 Log.i(TAG, "Trying to transcode to " + info.mUri); 83 String width = retriever.extractMetadata( 84 MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH); 85 String height = retriever.extractMetadata( 86 MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT); 87 if (width != null && height != null) { 88 info.mWidth = Integer.parseInt(width); 89 info.mHeight = Integer.parseInt(height); 90 } 91 92 String frameRate = retriever.extractMetadata( 93 MediaMetadataRetriever.METADATA_KEY_CAPTURE_FRAMERATE); 94 if (frameRate != null) { 95 info.mVideoFrameRate = Float.parseFloat(frameRate); 96 } 97 98 String frameCount = retriever.extractMetadata( 99 MediaMetadataRetriever.METADATA_KEY_VIDEO_FRAME_COUNT); 100 if (frameCount != null) { 101 info.mNumVideoFrames = Integer.parseInt(frameCount); 102 } 103 104 String hasAudio = retriever.extractMetadata( 105 MediaMetadataRetriever.METADATA_KEY_HAS_AUDIO); 106 if (hasAudio != null) { 107 info.mHasAudio = hasAudio.equals("yes"); 108 } 109 110 retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION); 111 String degree = retriever.extractMetadata( 112 MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION); 113 if (degree != null) { 114 info.mRotationDegree = Integer.parseInt(degree); 115 } 116 } finally { 117 if (retriever != null) { 118 retriever.close(); 119 } 120 if (afd != null) { 121 afd.close(); 122 } 123 } 124 return info; 125 } 126 dumpYuvToExternal(final Context ctx, Uri yuvUri)127 static void dumpYuvToExternal(final Context ctx, Uri yuvUri) { 128 Log.i(TAG, "dumping file to external"); 129 try { 130 String filename = + System.nanoTime() + "_" + yuvUri.getLastPathSegment(); 131 String path = "/storage/emulated/0/Download/" + filename; 132 final File file = new File(path); 133 ParcelFileDescriptor pfd = ctx.getContentResolver().openFileDescriptor(yuvUri, "r"); 134 FileInputStream fis = new FileInputStream(pfd.getFileDescriptor()); 135 FileOutputStream fos = new FileOutputStream(file); 136 FileUtils.copy(fis, fos); 137 } catch (IOException e) { 138 Log.e(TAG, "Failed to copy file", e); 139 } 140 } 141 computeStats(final Context ctx, final Uri sourceMp4, final Uri transcodedMp4, boolean debugYuv)142 static VideoTranscodingStatistics computeStats(final Context ctx, final Uri sourceMp4, 143 final Uri transcodedMp4, boolean debugYuv) 144 throws Exception { 145 // First decode the sourceMp4 to a temp yuv in yuv420p format. 146 Uri sourceYUV420PUri = Uri.parse(ContentResolver.SCHEME_FILE + "://" 147 + ctx.getCacheDir().getAbsolutePath() + "/sourceYUV420P.yuv"); 148 decodeMp4ToYuv(ctx, sourceMp4, sourceYUV420PUri); 149 VideoFileInfo srcInfo = extractVideoFileInfo(ctx, sourceMp4); 150 if (debugYuv) { 151 dumpYuvToExternal(ctx, sourceYUV420PUri); 152 } 153 154 // Second decode the transcodedMp4 to a temp yuv in yuv420p format. 155 Uri transcodedYUV420PUri = Uri.parse(ContentResolver.SCHEME_FILE + "://" 156 + ctx.getCacheDir().getAbsolutePath() + "/transcodedYUV420P.yuv"); 157 decodeMp4ToYuv(ctx, transcodedMp4, transcodedYUV420PUri); 158 VideoFileInfo dstInfo = extractVideoFileInfo(ctx, sourceMp4); 159 if (debugYuv) { 160 dumpYuvToExternal(ctx, transcodedYUV420PUri); 161 } 162 163 if ((srcInfo.mWidth != dstInfo.mWidth) || (srcInfo.mHeight != dstInfo.mHeight) || 164 (srcInfo.mNumVideoFrames != dstInfo.mNumVideoFrames) || 165 (srcInfo.mRotationDegree != dstInfo.mRotationDegree)) { 166 throw new UnsupportedOperationException( 167 "Src mp4 and dst mp4 must have same width/height/frames"); 168 } 169 170 // Then Compute the psnr of transcodedYUV420PUri against sourceYUV420PUri. 171 return computePsnr(ctx, sourceYUV420PUri, transcodedYUV420PUri, srcInfo.mWidth, 172 srcInfo.mHeight); 173 } 174 decodeMp4ToYuv(final Context ctx, final Uri fileUri, final Uri yuvUri)175 private static void decodeMp4ToYuv(final Context ctx, final Uri fileUri, final Uri yuvUri) 176 throws Exception { 177 AssetFileDescriptor fileFd = null; 178 MediaExtractor extractor = null; 179 MediaCodec codec = null; 180 AssetFileDescriptor yuvFd = null; 181 FileOutputStream out = null; 182 int width = 0; 183 int height = 0; 184 185 try { 186 fileFd = ctx.getContentResolver().openAssetFileDescriptor(fileUri, "r"); 187 extractor = new MediaExtractor(); 188 extractor.setDataSource(fileFd.getFileDescriptor(), fileFd.getStartOffset(), 189 fileFd.getLength()); 190 191 // Selects the video track. 192 int trackCount = extractor.getTrackCount(); 193 if (trackCount <= 0) { 194 throw new IllegalArgumentException("Invalid mp4 file"); 195 } 196 int videoTrackIndex = -1; 197 for (int i = 0; i < trackCount; i++) { 198 extractor.selectTrack(i); 199 MediaFormat format = extractor.getTrackFormat(i); 200 if (format.getString(MediaFormat.KEY_MIME).startsWith("video/")) { 201 videoTrackIndex = i; 202 break; 203 } 204 extractor.unselectTrack(i); 205 } 206 if (videoTrackIndex == -1) { 207 throw new IllegalArgumentException("Can not find video track"); 208 } 209 210 extractor.selectTrack(videoTrackIndex); 211 MediaFormat format = extractor.getTrackFormat(videoTrackIndex); 212 String mime = format.getString(MediaFormat.KEY_MIME); 213 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, 214 MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar); 215 216 // Opens the yuv file uri. 217 yuvFd = ctx.getContentResolver().openAssetFileDescriptor(yuvUri, 218 "w"); 219 out = new FileOutputStream(yuvFd.getFileDescriptor()); 220 221 codec = MediaCodec.createDecoderByType(mime); 222 codec.configure(format, 223 null, // surface 224 null, // crypto 225 0); // flags 226 codec.start(); 227 228 ByteBuffer[] inputBuffers = codec.getInputBuffers(); 229 ByteBuffer[] outputBuffers = codec.getOutputBuffers(); 230 MediaFormat decoderOutputFormat = codec.getInputFormat(); 231 232 // start decode loop 233 MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); 234 235 final long kTimeOutUs = 1000; // 1ms timeout 236 long lastOutputTimeUs = 0; 237 boolean sawInputEOS = false; 238 boolean sawOutputEOS = false; 239 int inputNum = 0; 240 int outputNum = 0; 241 boolean advanceDone = true; 242 243 long start = System.currentTimeMillis(); 244 while (!sawOutputEOS) { 245 // handle input 246 if (!sawInputEOS) { 247 int inputBufIndex = codec.dequeueInputBuffer(kTimeOutUs); 248 249 if (inputBufIndex >= 0) { 250 ByteBuffer dstBuf = inputBuffers[inputBufIndex]; 251 // sample contains the buffer and the PTS offset normalized to frame index 252 int sampleSize = 253 extractor.readSampleData(dstBuf, 0 /* offset */); 254 long presentationTimeUs = extractor.getSampleTime(); 255 advanceDone = extractor.advance(); 256 257 if (sampleSize < 0) { 258 Log.d(TAG, "saw input EOS."); 259 sawInputEOS = true; 260 sampleSize = 0; 261 } 262 codec.queueInputBuffer( 263 inputBufIndex, 264 0 /* offset */, 265 sampleSize, 266 presentationTimeUs, 267 sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0); 268 } else { 269 Log.d(TAG, "codec.dequeueInputBuffer() unrecognized return value:"); 270 } 271 } 272 273 // handle output 274 int outputBufIndex = codec.dequeueOutputBuffer(info, kTimeOutUs); 275 276 if (outputBufIndex >= 0) { 277 if (info.size > 0) { // Disregard 0-sized buffers at the end. 278 outputNum++; 279 Log.i(TAG, "Output frame numer " + outputNum); 280 Image image = codec.getOutputImage(outputBufIndex); 281 dumpYUV420PToFile(image, out); 282 } 283 284 codec.releaseOutputBuffer(outputBufIndex, false /* render */); 285 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 286 Log.d(TAG, "saw output EOS."); 287 sawOutputEOS = true; 288 } 289 } else if (outputBufIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 290 outputBuffers = codec.getOutputBuffers(); 291 Log.d(TAG, "output buffers have changed."); 292 } else if (outputBufIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 293 decoderOutputFormat = codec.getOutputFormat(); 294 Log.d(TAG, "output resolution " + width + "x" + height); 295 } else { 296 Log.w(TAG, "codec.dequeueOutputBuffer() unrecognized return index"); 297 } 298 } 299 } finally { 300 if (codec != null) { 301 codec.stop(); 302 codec.release(); 303 } 304 if (extractor != null) { 305 extractor.release(); 306 } 307 if (out != null) { 308 out.close(); 309 } 310 if (fileFd != null) { 311 fileFd.close(); 312 } 313 if (yuvFd != null) { 314 yuvFd.close(); 315 } 316 } 317 } 318 dumpYUV420PToFile(Image image, FileOutputStream out)319 private static void dumpYUV420PToFile(Image image, FileOutputStream out) throws IOException { 320 int format = image.getFormat(); 321 322 if (ImageFormat.YUV_420_888 != format) { 323 throw new UnsupportedOperationException("Only supports YUV420P"); 324 } 325 326 Rect crop = image.getCropRect(); 327 int cropLeft = crop.left; 328 int cropRight = crop.right; 329 int cropTop = crop.top; 330 int cropBottom = crop.bottom; 331 int imageWidth = cropRight - cropLeft; 332 int imageHeight = cropBottom - cropTop; 333 byte[] bb = new byte[imageWidth * imageHeight]; 334 byte[] lb = null; 335 Image.Plane[] planes = image.getPlanes(); 336 for (int i = 0; i < planes.length; ++i) { 337 ByteBuffer buf = planes[i].getBuffer(); 338 339 int width, height, rowStride, pixelStride, x, y, top, left; 340 rowStride = planes[i].getRowStride(); 341 pixelStride = planes[i].getPixelStride(); 342 if (i == 0) { 343 width = imageWidth; 344 height = imageHeight; 345 left = cropLeft; 346 top = cropTop; 347 } else { 348 width = imageWidth / 2; 349 height = imageHeight / 2; 350 left = cropLeft / 2; 351 top = cropTop / 2; 352 } 353 354 if (buf.hasArray()) { 355 byte b[] = buf.array(); 356 int offs = buf.arrayOffset(); 357 if (pixelStride == 1) { 358 for (y = 0; y < height; ++y) { 359 System.arraycopy(bb, y * width, b, y * rowStride + offs, width); 360 } 361 } else { 362 // do it pixel-by-pixel 363 for (y = 0; y < height; ++y) { 364 int lineOffset = offs + y * rowStride; 365 for (x = 0; x < width; ++x) { 366 bb[y * width + x] = b[lineOffset + x * pixelStride]; 367 } 368 } 369 } 370 } else { // almost always ends up here due to direct buffers 371 int pos = buf.position(); 372 if (pixelStride == 1) { 373 for (y = 0; y < height; ++y) { 374 buf.position(pos + y * rowStride); 375 buf.get(bb, y * width, width); 376 } 377 } else { 378 // Reallocate linebuffer if necessary. 379 if (lb == null || lb.length < rowStride) { 380 lb = new byte[rowStride]; 381 } 382 // do it pixel-by-pixel 383 for (y = 0; y < height; ++y) { 384 buf.position(pos + left + (top + y) * rowStride); 385 // we're only guaranteed to have pixelStride * (width - 1) + 1 bytes 386 buf.get(lb, 0, pixelStride * (width - 1) + 1); 387 for (x = 0; x < width; ++x) { 388 bb[y * width + x] = lb[x * pixelStride]; 389 } 390 } 391 } 392 buf.position(pos); 393 } 394 // Write out the buffer to the output. 395 out.write(bb, 0, width * height); 396 } 397 } 398 399 //////////////////////////////////////////////////////////////////////////////////////////////// 400 // The following psnr code is leveraged from the following file with minor modification: 401 // cts/tests/tests/media/src/android/media/cts/VideoCodecTestBase.java 402 //////////////////////////////////////////////////////////////////////////////////////////////// 403 // TODO(hkuang): Merge this code with the code in VideoCodecTestBase to use the same one. 404 /** 405 * Calculates PSNR value between two video frames. 406 */ computePSNR(byte[] data0, byte[] data1)407 private static double computePSNR(byte[] data0, byte[] data1) { 408 long squareError = 0; 409 assertTrue(data0.length == data1.length); 410 int length = data0.length; 411 for (int i = 0; i < length; i++) { 412 int diff = ((int) data0[i] & 0xff) - ((int) data1[i] & 0xff); 413 squareError += diff * diff; 414 } 415 double meanSquareError = (double) squareError / length; 416 double psnr = 10 * Math.log10((double) 255 * 255 / meanSquareError); 417 return psnr; 418 } 419 420 /** 421 * Calculates average and minimum PSNR values between 422 * set of reference and decoded video frames. 423 * Runs PSNR calculation for the full duration of the decoded data. 424 */ computePsnr( Context ctx, Uri referenceYuvFileUri, Uri decodedYuvFileUri, int width, int height)425 private static VideoTranscodingStatistics computePsnr( 426 Context ctx, 427 Uri referenceYuvFileUri, 428 Uri decodedYuvFileUri, 429 int width, 430 int height) throws Exception { 431 VideoTranscodingStatistics statistics = new VideoTranscodingStatistics(); 432 AssetFileDescriptor referenceFd = ctx.getContentResolver().openAssetFileDescriptor( 433 referenceYuvFileUri, "r"); 434 InputStream referenceStream = new FileInputStream(referenceFd.getFileDescriptor()); 435 436 AssetFileDescriptor decodedFd = ctx.getContentResolver().openAssetFileDescriptor( 437 decodedYuvFileUri, "r"); 438 InputStream decodedStream = new FileInputStream(decodedFd.getFileDescriptor()); 439 440 int ySize = width * height; 441 int uvSize = width * height / 4; 442 byte[] yRef = new byte[ySize]; 443 byte[] yDec = new byte[ySize]; 444 byte[] uvRef = new byte[uvSize]; 445 byte[] uvDec = new byte[uvSize]; 446 447 int frames = 0; 448 double averageYPSNR = 0; 449 double averageUPSNR = 0; 450 double averageVPSNR = 0; 451 double minimumYPSNR = Integer.MAX_VALUE; 452 double minimumUPSNR = Integer.MAX_VALUE; 453 double minimumVPSNR = Integer.MAX_VALUE; 454 int minimumPSNRFrameIndex = 0; 455 456 while (true) { 457 // Calculate Y PSNR. 458 int bytesReadRef = referenceStream.read(yRef); 459 int bytesReadDec = decodedStream.read(yDec); 460 if (bytesReadDec == -1) { 461 break; 462 } 463 if (bytesReadRef == -1) { 464 break; 465 } 466 double curYPSNR = computePSNR(yRef, yDec); 467 averageYPSNR += curYPSNR; 468 minimumYPSNR = Math.min(minimumYPSNR, curYPSNR); 469 double curMinimumPSNR = curYPSNR; 470 471 // Calculate U PSNR. 472 bytesReadRef = referenceStream.read(uvRef); 473 bytesReadDec = decodedStream.read(uvDec); 474 double curUPSNR = computePSNR(uvRef, uvDec); 475 averageUPSNR += curUPSNR; 476 minimumUPSNR = Math.min(minimumUPSNR, curUPSNR); 477 curMinimumPSNR = Math.min(curMinimumPSNR, curUPSNR); 478 479 // Calculate V PSNR. 480 bytesReadRef = referenceStream.read(uvRef); 481 bytesReadDec = decodedStream.read(uvDec); 482 double curVPSNR = computePSNR(uvRef, uvDec); 483 averageVPSNR += curVPSNR; 484 minimumVPSNR = Math.min(minimumVPSNR, curVPSNR); 485 curMinimumPSNR = Math.min(curMinimumPSNR, curVPSNR); 486 487 // Frame index for minimum PSNR value - help to detect possible distortions 488 if (curMinimumPSNR < statistics.mMinimumPSNR) { 489 statistics.mMinimumPSNR = curMinimumPSNR; 490 minimumPSNRFrameIndex = frames; 491 } 492 493 String logStr = String.format(Locale.US, "PSNR #%d: Y: %.2f. U: %.2f. V: %.2f", 494 frames, curYPSNR, curUPSNR, curVPSNR); 495 Log.v(TAG, logStr); 496 497 frames++; 498 } 499 500 averageYPSNR /= frames; 501 averageUPSNR /= frames; 502 averageVPSNR /= frames; 503 statistics.mAveragePSNR = (4 * averageYPSNR + averageUPSNR + averageVPSNR) / 6; 504 505 Log.d(TAG, "PSNR statistics for " + frames + " frames."); 506 String logStr = String.format(Locale.US, 507 "Average PSNR: Y: %.1f. U: %.1f. V: %.1f. Average: %.1f", 508 averageYPSNR, averageUPSNR, averageVPSNR, statistics.mAveragePSNR); 509 Log.d(TAG, logStr); 510 logStr = String.format(Locale.US, 511 "Minimum PSNR: Y: %.1f. U: %.1f. V: %.1f. Overall: %.1f at frame %d", 512 minimumYPSNR, minimumUPSNR, minimumVPSNR, 513 statistics.mMinimumPSNR, minimumPSNRFrameIndex); 514 Log.d(TAG, logStr); 515 516 referenceStream.close(); 517 decodedStream.close(); 518 referenceFd.close(); 519 decodedFd.close(); 520 return statistics; 521 } 522 523 /** 524 * Transcoding PSNR statistics. 525 */ 526 protected static class VideoTranscodingStatistics { 527 public double mAveragePSNR; 528 public double mMinimumPSNR; 529 VideoTranscodingStatistics()530 VideoTranscodingStatistics() { 531 mMinimumPSNR = Integer.MAX_VALUE; 532 } 533 } 534 } 535