1 /*
2  * Copyright (C) 2023 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.tools.traces.monitors
18 
19 import android.content.Context
20 import android.hardware.display.DisplayManager
21 import android.media.MediaCodec
22 import android.media.MediaCodecInfo
23 import android.media.MediaFormat
24 import android.media.MediaMuxer
25 import android.os.SystemClock
26 import android.tools.traces.deleteIfExists
27 import android.util.DisplayMetrics
28 import android.util.Log
29 import android.view.WindowManager
30 import java.io.File
31 import java.io.FileOutputStream
32 import java.nio.ByteBuffer
33 import java.nio.ByteOrder
34 import java.util.concurrent.TimeUnit
35 
36 /** Runnable to record the screen contents and winscope metadata */
37 class ScreenRecordingRunnable(
38     private val outputFile: File,
39     context: Context,
40     private val width: Int = 720,
41     private val height: Int = 1280
42 ) : Runnable {
43     private val displayManager = context.getSystemService(Context.DISPLAY_SERVICE) as DisplayManager
44     private val windowManager = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager
45     private var finished = false
46     internal var isFrameRecorded = false
47 
48     private val metrics: DisplayMetrics
49         get() {
50             val metrics = DisplayMetrics()
51             windowManager.defaultDisplay.getRealMetrics(metrics)
52             return metrics
53         }
54 
55     private val encoder = createEncoder()
56     private val inputSurface = encoder.createInputSurface()
57     private val virtualDisplay =
58         displayManager.createVirtualDisplay(
59             "Recording Display",
60             width,
61             height,
62             metrics.densityDpi,
63             inputSurface,
64             DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
65             null,
66             null
67         )
68     private val muxer = createMuxer()
69     private var metadataTrackIndex = -1
70     private var videoTrackIndex = -1
71 
stopnull72     internal fun stop() {
73         encoder.signalEndOfInputStream()
74         finished = true
75     }
76 
runnull77     override fun run() {
78         Log.d(LOG_TAG, "Starting screen recording to file $outputFile")
79 
80         val timestampsMonotonicUs = mutableListOf<Long>()
81         try {
82             // Start encoder and muxer
83             encoder.start()
84             val bufferInfo = MediaCodec.BufferInfo()
85 
86             while (true) {
87                 val bufferIndex = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_MS)
88                 if (bufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
89                     prepareMuxer()
90                 } else if (bufferIndex >= 0) {
91                     val timestampMonotonicUs = writeSample(bufferIndex, bufferInfo)
92                     val endOfStream = bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM
93                     // end of the stream samples have 0 timestamp
94                     if (endOfStream > 0) {
95                         break
96                     } else {
97                         timestampsMonotonicUs.add(timestampMonotonicUs)
98                     }
99                 }
100             }
101         } finally {
102             writeMetadata(timestampsMonotonicUs)
103             encoder.stop()
104             muxer.stop()
105             muxer.release()
106             encoder.release()
107             inputSurface.release()
108             virtualDisplay.release()
109         }
110     }
111 
112     /**
113      * Fetches a sample from the encoder and writes it to the video file
114      *
115      * @return sample timestamp (or 0 for invalid buffers)
116      */
writeSamplenull117     private fun writeSample(bufferIndex: Int, bufferInfo: MediaCodec.BufferInfo): Long {
118         val data = encoder.getOutputBuffer(bufferIndex)
119         return if (data != null) {
120             val endOfStream = bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM
121 
122             if (endOfStream == 0) {
123                 val outputBuffer =
124                     encoder.getOutputBuffer(bufferIndex) ?: error("Unable to acquire next frame")
125 
126                 muxer.writeSampleData(videoTrackIndex, outputBuffer, bufferInfo)
127                 isFrameRecorded = true
128             }
129             encoder.releaseOutputBuffer(bufferIndex, false)
130             bufferInfo.presentationTimeUs
131         } else {
132             0
133         }
134     }
135 
prepareMuxernull136     private fun prepareMuxer() {
137         videoTrackIndex = muxer.addTrack(encoder.outputFormat)
138         val metadataFormat = MediaFormat()
139         metadataFormat.setString(MediaFormat.KEY_MIME, MIME_TYPE_METADATA)
140         metadataTrackIndex = muxer.addTrack(metadataFormat)
141         muxer.start()
142     }
143 
144     /**
145      * Saves metadata needed by Winscope to synchronize the screen recording playback with other
146      * traces.
147      *
148      * The metadata (version 2) is written as a binary array with the following format:
149      * - winscope magic string (#VV1NSC0PET1ME2#, 16B).
150      * - the metadata version number (4B little endian).
151      * - Realtime-to-elapsed time offset in nanoseconds (8B little endian).
152      * - the recorded frames count (4B little endian)
153      * - for each recorded frame:
154      * ```
155      *     - System time in elapsed clock timebase in nanoseconds (8B little endian).
156      * ```
157      */
writeMetadatanull158     private fun writeMetadata(timestampsMonotonicUs: List<Long>) {
159         if (timestampsMonotonicUs.isEmpty()) {
160             Log.v(LOG_TAG, "Not writing winscope metadata (no frames/timestamps)")
161             return
162         }
163 
164         Log.v(
165             LOG_TAG,
166             "Writing winscope metadata (size=${timestampsMonotonicUs.size} " +
167                 ", monotonic timestamps range [us] = " +
168                 "${timestampsMonotonicUs.first()}-${timestampsMonotonicUs.last()})"
169         )
170 
171         val monotonicTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.uptimeMillis())
172         val elapsedTimeNs = SystemClock.elapsedRealtimeNanos()
173         val realTimeNs = TimeUnit.MILLISECONDS.toNanos(System.currentTimeMillis())
174 
175         val elapsedToMonotonicTimeOffsetNs = elapsedTimeNs - monotonicTimeNs
176         val realToElapsedTimeOffsetNs = realTimeNs - elapsedTimeNs
177 
178         val bufferSize =
179             WINSCOPE_MAGIC_STRING.toByteArray().size +
180                 Int.SIZE_BYTES +
181                 Long.SIZE_BYTES +
182                 Int.SIZE_BYTES +
183                 (timestampsMonotonicUs.size * Long.SIZE_BYTES)
184 
185         val buffer =
186             ByteBuffer.allocate(bufferSize)
187                 .order(ByteOrder.LITTLE_ENDIAN)
188                 .put(WINSCOPE_MAGIC_STRING.toByteArray())
189                 .putInt(WINSCOPE_METADATA_VERSION)
190                 .putLong(realToElapsedTimeOffsetNs)
191                 .putInt(timestampsMonotonicUs.size)
192                 .apply {
193                     timestampsMonotonicUs.forEach {
194                         putLong(elapsedToMonotonicTimeOffsetNs + TimeUnit.MICROSECONDS.toNanos(it))
195                     }
196                 }
197 
198         val bufferInfo = MediaCodec.BufferInfo()
199         bufferInfo.size = bufferSize
200         bufferInfo.presentationTimeUs = timestampsMonotonicUs[0]
201         muxer.writeSampleData(metadataTrackIndex, buffer, bufferInfo)
202     }
203 
204     /**
205      * Create and configure a MediaCodec encoder with [MIME_TYPE_VIDEO] format.
206      *
207      * @return a Surface that can be used to record
208      */
createEncodernull209     private fun createEncoder(): MediaCodec {
210         val format = MediaFormat.createVideoFormat(MIME_TYPE_VIDEO, width, height)
211         val displayMode = windowManager.defaultDisplay.mode
212         format.setInteger(
213             MediaFormat.KEY_COLOR_FORMAT,
214             MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface
215         )
216         format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE)
217         format.setFloat(MediaFormat.KEY_FRAME_RATE, displayMode.refreshRate)
218         format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL)
219         format.setInteger(MediaFormat.KEY_WIDTH, width)
220         format.setInteger(MediaFormat.KEY_HEIGHT, height)
221         format.setString(MediaFormat.KEY_MIME, MIME_TYPE_VIDEO)
222 
223         val mediaCodec = MediaCodec.createEncoderByType(MIME_TYPE_VIDEO)
224         mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
225         return mediaCodec
226     }
227 
createMuxernull228     private fun createMuxer(): MediaMuxer {
229         outputFile.deleteIfExists()
230         require(!outputFile.exists())
231         outputFile.createNewFile()
232         val inputStream = FileOutputStream(outputFile)
233         return MediaMuxer(inputStream.fd, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
234     }
235 
236     companion object {
237         private const val WINSCOPE_MAGIC_STRING = "#VV1NSC0PET1ME2#"
238         private const val WINSCOPE_METADATA_VERSION = 2
239         private const val MIME_TYPE_VIDEO = MediaFormat.MIMETYPE_VIDEO_AVC
240         private const val MIME_TYPE_METADATA = "application/octet-stream"
241         private const val BIT_RATE = 2000000 // 2Mbps
242         private const val IFRAME_INTERVAL = 2 // 2 second between I-frames
243         private const val TIMEOUT_MS = 100L
244     }
245 }
246