1 /*
2  * Copyright (C) 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.mediapc.cts;
18 
19 import android.media.MediaCodec;
20 import android.media.MediaCodecInfo;
21 import android.media.MediaExtractor;
22 import android.media.MediaFormat;
23 import android.util.Log;
24 import android.util.Pair;
25 import android.view.Surface;
26 
27 import java.io.IOException;
28 import java.nio.ByteBuffer;
29 import java.util.concurrent.Callable;
30 
31 import static org.junit.Assert.assertTrue;
32 import static org.junit.Assert.fail;
33 
34 public class CodecTranscoderTestBase {
35     private static final String LOG_TAG = CodecTranscoderTestBase.class.getSimpleName();
36     private static final boolean ENABLE_LOGS = false;
37     static final String mInpPrefix = WorkDir.getMediaDirString();
38     String mMime;
39     String mTestFile;
40     int mBitrate;
41     int mFrameRate;
42     MediaExtractor mExtractor;
43     int mMaxBFrames;
44     int mLatency;
45 
46     MediaCodec mEncoder;
47     CodecAsyncHandler mAsyncHandleEncoder;
48     MediaCodec mDecoder;
49     CodecAsyncHandler mAsyncHandleDecoder;
50     Surface mSurface;
51 
52     boolean mSawDecInputEOS;
53     boolean mSawDecOutputEOS;
54     boolean mSawEncOutputEOS;
55     boolean mIsCodecInAsyncMode;
56     boolean mSignalEOSWithLastFrame;
57     boolean mReviseLatency;
58     int mDecInputCount;
59     int mDecOutputCount;
60     int mEncOutputCount;
61 
CodecTranscoderTestBase(String mime, String testfile, int bitrate, int frameRate)62     CodecTranscoderTestBase(String mime, String testfile, int bitrate, int frameRate) {
63         mMime = mime;
64         mTestFile = testfile;
65         mBitrate = bitrate;
66         mFrameRate = frameRate;
67         mMaxBFrames = 0;
68         mLatency = mMaxBFrames;
69         mReviseLatency = false;
70         mAsyncHandleDecoder = new CodecAsyncHandler();
71         mAsyncHandleEncoder = new CodecAsyncHandler();
72     }
73 
hasSeenError()74     boolean hasSeenError() {
75         return mAsyncHandleDecoder.hasSeenError() || mAsyncHandleEncoder.hasSeenError();
76     }
77 
setUpSource(String srcFile)78     MediaFormat setUpSource(String srcFile) throws IOException {
79         mExtractor = new MediaExtractor();
80         mExtractor.setDataSource(mInpPrefix + srcFile);
81         for (int trackID = 0; trackID < mExtractor.getTrackCount(); trackID++) {
82             MediaFormat format = mExtractor.getTrackFormat(trackID);
83             String mime = format.getString(MediaFormat.KEY_MIME);
84             if (mime.startsWith("video/")) {
85                 mExtractor.selectTrack(trackID);
86                 format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
87                         MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
88                 format.setInteger(MediaFormat.KEY_PRIORITY, 1); // Best effort
89                 return format;
90             }
91         }
92         mExtractor.release();
93         fail("No video track found in file: " + srcFile);
94         return null;
95     }
96 
resetContext(boolean isAsync, boolean signalEOSWithLastFrame)97     void resetContext(boolean isAsync, boolean signalEOSWithLastFrame) {
98         mAsyncHandleDecoder.resetContext();
99         mAsyncHandleEncoder.resetContext();
100         mIsCodecInAsyncMode = isAsync;
101         mSignalEOSWithLastFrame = signalEOSWithLastFrame;
102         mSawDecInputEOS = false;
103         mSawDecOutputEOS = false;
104         mSawEncOutputEOS = false;
105         mDecInputCount = 0;
106         mDecOutputCount = 0;
107         mEncOutputCount = 0;
108     }
109 
configureCodec(MediaFormat decFormat, MediaFormat encFormat, boolean isAsync, boolean signalEOSWithLastFrame)110     void configureCodec(MediaFormat decFormat, MediaFormat encFormat, boolean isAsync,
111             boolean signalEOSWithLastFrame) {
112         resetContext(isAsync, signalEOSWithLastFrame);
113         mAsyncHandleEncoder.setCallBack(mEncoder, isAsync);
114         mEncoder.configure(encFormat, null, MediaCodec.CONFIGURE_FLAG_ENCODE, null);
115         if (mEncoder.getInputFormat().containsKey(MediaFormat.KEY_LATENCY)) {
116             mReviseLatency = true;
117             mLatency = mEncoder.getInputFormat().getInteger(MediaFormat.KEY_LATENCY);
118         }
119         mSurface = mEncoder.createInputSurface();
120         assertTrue("Surface is not valid", mSurface.isValid());
121         mAsyncHandleDecoder.setCallBack(mDecoder, isAsync);
122         mDecoder.configure(decFormat, mSurface, null, 0);
123         if (ENABLE_LOGS) {
124             Log.v(LOG_TAG, "codec configured");
125         }
126     }
127 
enqueueDecoderEOS(int bufferIndex)128     void enqueueDecoderEOS(int bufferIndex) {
129         if (!mSawDecInputEOS) {
130             mDecoder.queueInputBuffer(bufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
131             mSawDecInputEOS = true;
132             if (ENABLE_LOGS) {
133                 Log.v(LOG_TAG, "Queued End of Stream");
134             }
135         }
136     }
137 
enqueueDecoderInput(int bufferIndex)138     void enqueueDecoderInput(int bufferIndex) {
139         if (mExtractor.getSampleSize() < 0) {
140             enqueueDecoderEOS(bufferIndex);
141         } else {
142             ByteBuffer inputBuffer = mDecoder.getInputBuffer(bufferIndex);
143             int size = mExtractor.readSampleData(inputBuffer, 0);
144             long pts = mExtractor.getSampleTime();
145             int extractorFlags = mExtractor.getSampleFlags();
146             int codecFlags = 0;
147             if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) {
148                 codecFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
149             }
150             if (!mExtractor.advance() && mSignalEOSWithLastFrame) {
151                 codecFlags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
152                 mSawDecInputEOS = true;
153             }
154             mDecoder.queueInputBuffer(bufferIndex, 0, size, pts, codecFlags);
155             if (size > 0 && (codecFlags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
156                 mDecInputCount++;
157             }
158         }
159     }
160 
dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info)161     void dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info) {
162         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
163             mSawDecOutputEOS = true;
164         }
165         if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
166             mDecOutputCount++;
167         }
168         mDecoder.releaseOutputBuffer(bufferIndex, mSurface != null);
169     }
170 
dequeueEncoderOutput(int bufferIndex, MediaCodec.BufferInfo info)171     void dequeueEncoderOutput(int bufferIndex, MediaCodec.BufferInfo info) {
172         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
173             mSawEncOutputEOS = true;
174         }
175         if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
176             mEncOutputCount++;
177         }
178         mEncoder.releaseOutputBuffer(bufferIndex, false);
179     }
180 
tryEncoderOutput(long timeOutUs)181     void tryEncoderOutput(long timeOutUs) throws InterruptedException {
182         if (mIsCodecInAsyncMode) {
183             if (!hasSeenError() && !mSawEncOutputEOS) {
184                 int retry = 0;
185                 while (mReviseLatency) {
186                     if (mAsyncHandleEncoder.hasOutputFormatChanged()) {
187                         mReviseLatency = false;
188                         int actualLatency = mAsyncHandleEncoder.getOutputFormat()
189                                 .getInteger(MediaFormat.KEY_LATENCY, mLatency);
190                         if (mLatency < actualLatency) {
191                             mLatency = actualLatency;
192                             return;
193                         }
194                     } else {
195                         if (retry > CodecTestBase.RETRY_LIMIT) throw new InterruptedException(
196                                 "did not receive output format changed for encoder after " +
197                                         CodecTestBase.Q_DEQ_TIMEOUT_US * CodecTestBase.RETRY_LIMIT +
198                                         " us");
199                         Thread.sleep(CodecTestBase.Q_DEQ_TIMEOUT_US / 1000);
200                         retry ++;
201                     }
202                 }
203                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandleEncoder.getOutput();
204                 if (element != null) {
205                     dequeueEncoderOutput(element.first, element.second);
206                 }
207             }
208         } else {
209             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
210             if (!mSawEncOutputEOS) {
211                 int outputBufferId = mEncoder.dequeueOutputBuffer(outInfo, timeOutUs);
212                 if (outputBufferId >= 0) {
213                     dequeueEncoderOutput(outputBufferId, outInfo);
214                 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
215                     mLatency = mEncoder.getOutputFormat()
216                             .getInteger(MediaFormat.KEY_LATENCY, mLatency);
217                 }
218             }
219         }
220     }
221 
waitForAllEncoderOutputs()222     void waitForAllEncoderOutputs() throws InterruptedException {
223         if (mIsCodecInAsyncMode) {
224             while (!hasSeenError() && !mSawEncOutputEOS) {
225                 tryEncoderOutput(CodecTestBase.Q_DEQ_TIMEOUT_US);
226             }
227         } else {
228             while (!mSawEncOutputEOS) {
229                 tryEncoderOutput(CodecTestBase.Q_DEQ_TIMEOUT_US);
230             }
231         }
232     }
233 
queueEOS()234     void queueEOS() throws InterruptedException {
235         if (mIsCodecInAsyncMode) {
236             while (!mAsyncHandleDecoder.hasSeenError() && !mSawDecInputEOS) {
237                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandleDecoder.getWork();
238                 if (element != null) {
239                     int bufferID = element.first;
240                     MediaCodec.BufferInfo info = element.second;
241                     if (info != null) {
242                         dequeueDecoderOutput(bufferID, info);
243                     } else {
244                         enqueueDecoderEOS(element.first);
245                     }
246                 }
247             }
248         } else {
249             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
250             while (!mSawDecInputEOS) {
251                 int outputBufferId =
252                         mDecoder.dequeueOutputBuffer(outInfo, CodecTestBase.Q_DEQ_TIMEOUT_US);
253                 if (outputBufferId >= 0) {
254                     dequeueDecoderOutput(outputBufferId, outInfo);
255                 }
256                 int inputBufferId = mDecoder.dequeueInputBuffer(CodecTestBase.Q_DEQ_TIMEOUT_US);
257                 if (inputBufferId != -1) {
258                     enqueueDecoderEOS(inputBufferId);
259                 }
260             }
261         }
262         if (mIsCodecInAsyncMode) {
263             while (!hasSeenError() && !mSawDecOutputEOS) {
264                 Pair<Integer, MediaCodec.BufferInfo> decOp = mAsyncHandleDecoder.getOutput();
265                 if (decOp != null) dequeueDecoderOutput(decOp.first, decOp.second);
266                 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream();
267                 if (mDecOutputCount - mEncOutputCount > mLatency) {
268                     tryEncoderOutput(-1);
269                 }
270             }
271         } else {
272             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
273             while (!mSawDecOutputEOS) {
274                 int outputBufferId =
275                         mDecoder.dequeueOutputBuffer(outInfo, CodecTestBase.Q_DEQ_TIMEOUT_US);
276                 if (outputBufferId >= 0) {
277                     dequeueDecoderOutput(outputBufferId, outInfo);
278                 }
279                 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream();
280                 if (mDecOutputCount - mEncOutputCount > mLatency) {
281                     tryEncoderOutput(-1);
282                 }
283             }
284         }
285     }
286 
doWork(int frameLimit)287     void doWork(int frameLimit) throws InterruptedException {
288         int frameCnt = 0;
289         if (mIsCodecInAsyncMode) {
290             // dequeue output after inputEOS is expected to be done in waitForAllOutputs()
291             while (!hasSeenError() && !mSawDecInputEOS && frameCnt < frameLimit) {
292                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandleDecoder.getWork();
293                 if (element != null) {
294                     int bufferID = element.first;
295                     MediaCodec.BufferInfo info = element.second;
296                     if (info != null) {
297                         // <id, info> corresponds to output callback. Handle it accordingly
298                         dequeueDecoderOutput(bufferID, info);
299                     } else {
300                         // <id, null> corresponds to input callback. Handle it accordingly
301                         enqueueDecoderInput(bufferID);
302                         frameCnt++;
303                     }
304                 }
305                 // check decoder EOS
306                 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream();
307                 // encoder output
308                 if (mDecOutputCount - mEncOutputCount > mLatency) {
309                     tryEncoderOutput(-1);
310                 }
311             }
312         } else {
313             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
314             while (!mSawDecInputEOS && frameCnt < frameLimit) {
315                 // decoder input
316                 int inputBufferId = mDecoder.dequeueInputBuffer(CodecTestBase.Q_DEQ_TIMEOUT_US);
317                 if (inputBufferId != -1) {
318                     enqueueDecoderInput(inputBufferId);
319                     frameCnt++;
320                 }
321                 // decoder output
322                 int outputBufferId =
323                         mDecoder.dequeueOutputBuffer(outInfo, CodecTestBase.Q_DEQ_TIMEOUT_US);
324                 if (outputBufferId >= 0) {
325                     dequeueDecoderOutput(outputBufferId, outInfo);
326                 }
327                 // check decoder EOS
328                 if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream();
329                 // encoder output
330                 if (mDecOutputCount - mEncOutputCount > mLatency) {
331                     tryEncoderOutput(-1);
332                 }
333             }
334         }
335     }
336 
setUpEncoderFormat(MediaFormat decoderFormat)337     MediaFormat setUpEncoderFormat(MediaFormat decoderFormat) {
338         MediaFormat encoderFormat = new MediaFormat();
339         encoderFormat.setString(MediaFormat.KEY_MIME, mMime);
340         encoderFormat.setInteger(MediaFormat.KEY_WIDTH,
341                 decoderFormat.getInteger(MediaFormat.KEY_WIDTH));
342         encoderFormat.setInteger(MediaFormat.KEY_HEIGHT,
343                 decoderFormat.getInteger(MediaFormat.KEY_HEIGHT));
344         encoderFormat.setInteger(MediaFormat.KEY_FRAME_RATE, mFrameRate);
345         encoderFormat.setInteger(MediaFormat.KEY_BIT_RATE, mBitrate);
346         encoderFormat.setFloat(MediaFormat.KEY_I_FRAME_INTERVAL, 1.0f);
347         encoderFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT,
348                 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
349         encoderFormat.setInteger(MediaFormat.KEY_MAX_B_FRAMES, mMaxBFrames);
350         encoderFormat.setInteger(MediaFormat.KEY_PRIORITY,
351                 decoderFormat.getInteger(MediaFormat.KEY_PRIORITY));
352         return encoderFormat;
353     }
354 }
355 
356 /**
357  * The following class transcodes the given testFile and returns the achieved fps for transcoding.
358  */
359 class Transcode extends CodecTranscoderTestBase implements Callable<Double> {
360     private static final String LOG_TAG = Transcode.class.getSimpleName();
361 
362     private final String mDecoderName;
363     private final String mEncoderName;
364     private final boolean mIsAsync;
365 
Transcode(String mime, String testFile, String decoderName, String encoderName, boolean isAsync)366     Transcode(String mime, String testFile, String decoderName, String encoderName,
367             boolean isAsync) {
368         super(mime, testFile, 3000000, 30);
369         mDecoderName = decoderName;
370         mEncoderName = encoderName;
371         mIsAsync = isAsync;
372     }
373 
doTranscode()374     public Double doTranscode() throws Exception {
375         MediaFormat decoderFormat = setUpSource(mTestFile);
376         mDecoder = MediaCodec.createByCodecName(mDecoderName);
377         MediaFormat encoderFormat = setUpEncoderFormat(decoderFormat);
378         mEncoder = MediaCodec.createByCodecName(mEncoderName);
379         mExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
380         configureCodec(decoderFormat, encoderFormat, mIsAsync, false);
381         mEncoder.start();
382         mDecoder.start();
383         long start = System.currentTimeMillis();
384         doWork(Integer.MAX_VALUE);
385         queueEOS();
386         waitForAllEncoderOutputs();
387         long end = System.currentTimeMillis();
388         mSurface.release();
389         mDecoder.stop();
390         mDecoder.release();
391         mEncoder.stop();
392         mEncoder.release();
393         mExtractor.release();
394         double fps = mEncOutputCount / ((end - start) / 1000.0);
395         Log.d(LOG_TAG, "Mime: " + mMime + " Decoder: " + mDecoderName + " Encoder: " +
396                 mEncoderName + " Achieved fps: " + fps);
397         return fps;
398     }
399 
400     @Override
call()401     public Double call() throws Exception {
402         return doTranscode();
403     }
404 }
405 
406 /**
407  * The following class transcodes the given testFile until loadStatus is finished.
408  * If input reaches eos, it will rewind the input to start position.
409  */
410 class TranscodeLoad extends Transcode {
411     private final LoadStatus mLoadStatus;
412 
413     private long mMaxPts;
414     private long mBasePts;
415 
TranscodeLoad(String mime, String testFile, String decoderName, String encoderName, LoadStatus loadStatus)416     TranscodeLoad(String mime, String testFile, String decoderName, String encoderName,
417             LoadStatus loadStatus) {
418         super(mime, testFile, decoderName, encoderName, false);
419         mLoadStatus = loadStatus;
420         mMaxPts = 0;
421         mBasePts = 0;
422     }
423 
424     @Override
configureCodec(MediaFormat decFormat, MediaFormat encFormat, boolean isAsync, boolean signalEOSWithLastFrame)425     void configureCodec(MediaFormat decFormat, MediaFormat encFormat, boolean isAsync,
426             boolean signalEOSWithLastFrame) {
427         decFormat.setInteger(MediaFormat.KEY_PRIORITY, 1);
428         encFormat.setInteger(MediaFormat.KEY_PRIORITY, 1);
429         resetContext(isAsync, signalEOSWithLastFrame);
430         mAsyncHandleEncoder.setCallBack(mEncoder, isAsync);
431         mEncoder.configure(encFormat, null, MediaCodec.CONFIGURE_FLAG_ENCODE, null);
432         if (mEncoder.getInputFormat().containsKey(MediaFormat.KEY_LATENCY)) {
433             mReviseLatency = true;
434             mLatency = mEncoder.getInputFormat().getInteger(MediaFormat.KEY_LATENCY);
435         }
436         mSurface = mEncoder.createInputSurface();
437         assertTrue("Surface is not valid", mSurface.isValid());
438         mAsyncHandleDecoder.setCallBack(mDecoder, isAsync);
439         mDecoder.configure(decFormat, mSurface, null, 0);
440     }
441 
442     @Override
enqueueDecoderInput(int bufferIndex)443     void enqueueDecoderInput(int bufferIndex) {
444         if (mExtractor.getSampleSize() < 0 || mLoadStatus.isLoadFinished()) {
445             enqueueDecoderEOS(bufferIndex);
446         } else {
447             ByteBuffer inputBuffer = mDecoder.getInputBuffer(bufferIndex);
448             int size = mExtractor.readSampleData(inputBuffer, 0);
449             long pts = mExtractor.getSampleTime();
450             mMaxPts = Math.max(mMaxPts, mBasePts + pts);
451             int extractorFlags = mExtractor.getSampleFlags();
452             int codecFlags = 0;
453             if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) {
454                 codecFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
455             }
456             mDecoder.queueInputBuffer(bufferIndex, 0, size, mBasePts + pts, codecFlags);
457             if (size > 0 && (codecFlags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
458                 mDecInputCount++;
459             }
460             // If eos is reached, seek to start position.
461             if (!mExtractor.advance()) {
462                 mExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
463                 mBasePts = mMaxPts + 1000000L;
464             }
465         }
466     }
467 }
468 
469 /**
470  * The following class tells the status of the load whether it is finished or not.
471  */
472 class LoadStatus {
473     private boolean mLoadFinished;
474 
LoadStatus()475     public LoadStatus() { mLoadFinished = false; }
476 
setLoadFinished()477     public synchronized void setLoadFinished() { mLoadFinished = true; }
478 
isLoadFinished()479     public synchronized boolean isLoadFinished() { return mLoadFinished; }
480 }
481