1 /* <lambda>null2 * Copyright (C) 2024 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package platform.test.motion.filmstrip 18 19 import android.media.MediaCodec 20 import android.media.MediaCodecInfo.CodecCapabilities 21 import android.media.MediaFormat 22 import android.media.MediaFormat.KEY_BIT_RATE 23 import android.media.MediaFormat.KEY_COLOR_FORMAT 24 import android.media.MediaFormat.KEY_FRAME_RATE 25 import android.media.MediaMuxer 26 import android.view.Surface 27 import platform.test.motion.golden.TimestampFrameId 28 29 /** Produces an MP4 based on the [screenshots]. */ 30 class VideoRenderer(private val screenshots: List<MotionScreenshot>) { 31 32 private var screenshotWidth = screenshots.maxOf { it.bitmap.width } 33 private var screenshotHeight = screenshots.maxOf { it.bitmap.height } 34 35 /** 36 * Creates an MP4 file at [path], which will contain all screenshots. 37 * 38 * [bitsPerPixel] is used to estimate the bitrate needed. 39 */ 40 fun renderToFile(path: String, bitsPerPixel: Float = 0.25f) { 41 require(screenshots.isNotEmpty()) { "Filmstrip must have at least one screenshot" } 42 val muxer = MediaMuxer(path, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) 43 44 val bitrate = (screenshotWidth * screenshotHeight * bitsPerPixel * FRAME_RATE).toInt() 45 val mime = "video/avc" 46 val format = MediaFormat.createVideoFormat(mime, screenshotWidth, screenshotHeight) 47 format.setInteger(KEY_BIT_RATE, bitrate) 48 format.setFloat(KEY_FRAME_RATE, FRAME_RATE) 49 format.setInteger(KEY_COLOR_FORMAT, CodecCapabilities.COLOR_FormatSurface) 50 format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1) 51 52 val codec = MediaCodec.createEncoderByType(mime) 53 codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) 54 val surface = codec.createInputSurface() 55 codec.start() 56 57 encodeScreenshotsInVideo(codec, muxer, surface) 58 59 codec.stop() 60 codec.release() 61 muxer.stop() 62 muxer.release() 63 } 64 65 private fun encodeScreenshotsInVideo( 66 encoder: MediaCodec, 67 muxer: MediaMuxer, 68 surface: Surface, 69 ) { 70 val bufferInfo = MediaCodec.BufferInfo() 71 val screenshotIterator = screenshots.iterator() 72 var isEndOfStream = false 73 var videoTrackIndex = -1 74 75 // The encoder uses the system clock of the [unlockCanvasAndPost] call as frame time. 76 // However, this is arbitrary, as encoding happens as fast as possible. To avoid the extra 77 // complexity of video bitmap format conversion that would be required when using 78 // [queueInputBuffer] instead (which would allow specifying the presentation time), this 79 // will override the presentation time when muxing instead. 80 val framePresentationTimesUsIterator = 81 buildList { 82 add(0L) 83 var presentationTimeUs = 0L 84 screenshots 85 .zipWithNext { first, second -> 86 if ( 87 first.frameId is TimestampFrameId && 88 second.frameId is TimestampFrameId 89 ) { 90 second.frameId.milliseconds - first.frameId.milliseconds 91 } else { 92 // Exactly one frame for before / after 93 FRAME_DURATION 94 } 95 } 96 .forEach { frameDurationMillis -> 97 presentationTimeUs += frameDurationMillis * 1000L 98 add(presentationTimeUs) 99 } 100 } 101 .iterator() 102 103 while (true) { 104 if (screenshotIterator.hasNext()) { 105 surface.lockCanvas(null).also { canvas -> 106 val screenshot = screenshotIterator.next() 107 canvas.drawBitmap(screenshot.bitmap, 0.0f, 0.0f, null) 108 surface.unlockCanvasAndPost(canvas) 109 } 110 } else if (!isEndOfStream) { 111 encoder.signalEndOfInputStream() 112 isEndOfStream = true 113 } 114 115 val outputBufferIndex = encoder.dequeueOutputBuffer(bufferInfo, DEQUEUE_TIMEOUT_US) 116 if (outputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 117 // No output available yet. 118 continue 119 } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 120 videoTrackIndex = muxer.addTrack(encoder.outputFormat) 121 muxer.start() 122 } else if (outputBufferIndex >= 0) { 123 val encodedDataBuffer = 124 encoder.getOutputBuffer(outputBufferIndex) 125 ?: throw RuntimeException("encoderOutputBuffer $outputBufferIndex was null") 126 127 if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) { 128 // Config was already sent to the muxer in INFO_OUTPUT_FORMAT_CHANGED. 129 bufferInfo.size = 0 130 } 131 132 if (bufferInfo.size != 0) { 133 encodedDataBuffer.position(bufferInfo.offset) 134 encodedDataBuffer.limit(bufferInfo.offset + bufferInfo.size) 135 check(framePresentationTimesUsIterator.hasNext()) { 136 "More than the number of input frames are sent to the output" 137 } 138 bufferInfo.presentationTimeUs = framePresentationTimesUsIterator.next() 139 muxer.writeSampleData(videoTrackIndex, encodedDataBuffer, bufferInfo) 140 } 141 142 encoder.releaseOutputBuffer(outputBufferIndex, false) 143 144 if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) { 145 return 146 } 147 } else { 148 throw AssertionError("Unexpected dequeueOutputBuffer response $outputBufferIndex") 149 } 150 } 151 } 152 153 companion object { 154 // Tests produce a frame every 16ms (62.5fps) 155 const val FRAME_DURATION = 16L 156 const val FRAME_RATE = 1000f / FRAME_DURATION 157 const val DEQUEUE_TIMEOUT_US = 10_000L 158 } 159 } 160