1 /*
2  * Copyright (C) 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.video.cts;
18 
19 import android.media.MediaCodec;
20 import android.media.MediaCodecInfo;
21 import android.media.MediaCodecList;
22 import android.media.MediaExtractor;
23 import android.media.MediaFormat;
24 import android.os.Build;
25 import android.os.SystemProperties;
26 import android.util.Range;
27 import android.view.Surface;
28 
29 import java.io.File;
30 import java.io.IOException;
31 import java.nio.ByteBuffer;
32 import java.util.ArrayList;
33 import java.util.List;
34 
35 import org.junit.Before;
36 import static org.junit.Assert.assertTrue;
37 import static org.junit.Assert.fail;
38 import static org.junit.Assume.assumeTrue;
39 
40 class CodecPerformanceTestBase {
41     private static final String LOG_TAG = CodecPerformanceTestBase.class.getSimpleName();
42     static final long Q_DEQ_TIMEOUT_US = 5000; // block at most 5ms while looking for io buffers
43     static final int PER_TEST_TIMEOUT_LARGE_TEST_MS = 300000;
44     static final int MIN_FRAME_COUNT = 500;
45     static final int SELECT_ALL = 0; // Select all codecs
46     static final int SELECT_HARDWARE = 1; // Select Hardware codecs only
47     static final int SELECT_SOFTWARE = 2; // Select Software codecs only
48     // allowed tolerance in measured fps vs expected fps, i.e. codecs achieving fps
49     // that is greater than (FPS_TOLERANCE_FACTOR * expectedFps) will be considered as
50     // passing the test
51     static final double FPS_TOLERANCE_FACTOR;
52     static final boolean IS_AT_LEAST_VNDK_S;
53 
54     static final int DEVICE_INITIAL_SDK;
55 
56     // Some older devices can not support concurrent instances of both decoder and encoder
57     // at max resolution. To handle such cases, this test is limited to test the
58     // resolutions that are less than half of max supported frame sizes of encoder.
59     static final boolean EXCLUDE_ENCODER_MAX_RESOLUTION;
60 
61     static final String mInputPrefix = WorkDir.getMediaDirString();
62 
63     ArrayList<MediaCodec.BufferInfo> mBufferInfos;
64     ByteBuffer mBuff;
65 
66     final String mDecoderName;
67     final String mTestFile;
68     final int mKeyPriority;
69     final float mMaxOpRateScalingFactor;
70 
71     String mDecoderMime;
72     int mWidth;
73     int mHeight;
74     int mFrameRate;
75 
76     boolean mSawDecInputEOS = false;
77     boolean mSawDecOutputEOS = false;
78     int mDecInputNum = 0;
79     int mDecOutputNum = 0;
80     int mSampleIndex = 0;
81 
82     MediaCodec mDecoder;
83     MediaFormat mDecoderFormat;
84     Surface mSurface;
85     double mOperatingRateExpected;
86 
87     static final float[] SCALING_FACTORS_LIST = new float[]{2.5f, 1.25f, 1.0f, 0.75f, 0.0f, -1.0f};
88     static final int[] KEY_PRIORITIES_LIST = new int[]{1, 0};
89 
90     static {
91         // os.Build.VERSION.DEVICE_INITIAL_SDK_INT can be used here, but it was called
92         // os.Build.VERSION.FIRST_SDK_INT in Android R and below. Using DEVICE_INITIAL_SDK_INT
93         // will mean that the tests built in Android S can't be run on Android R and below.
94         DEVICE_INITIAL_SDK = SystemProperties.getInt("ro.product.first_api_level", 0);
95 
96         // fps tolerance factor is kept quite low for devices launched on Android R and lower
97         FPS_TOLERANCE_FACTOR = DEVICE_INITIAL_SDK <= Build.VERSION_CODES.R ? 0.67 : 0.95;
98 
99         IS_AT_LEAST_VNDK_S = SystemProperties.getInt("ro.vndk.version", 0) > Build.VERSION_CODES.R;
100 
101         // Encoders on devices launched on Android Q and lower aren't tested at maximum resolution
102         EXCLUDE_ENCODER_MAX_RESOLUTION = DEVICE_INITIAL_SDK <= Build.VERSION_CODES.Q;
103     }
104 
105     @Before
prologue()106     public void prologue() {
107         assumeTrue("For VNDK R and below, operating rate <= 0 isn't tested",
108                 IS_AT_LEAST_VNDK_S || mMaxOpRateScalingFactor > 0.0);
109 
110         assumeTrue("For devices launched on Android P and below, operating rate tests are disabled",
111                 DEVICE_INITIAL_SDK > Build.VERSION_CODES.P);
112 
113         if (DEVICE_INITIAL_SDK <= Build.VERSION_CODES.Q) {
114             assumeTrue("For devices launched with Android Q and below, operating rate tests are " +
115                             "limited to operating rate scaling factor > 0.0 and <= 1.25",
116                     mMaxOpRateScalingFactor > 0.0 && mMaxOpRateScalingFactor <= 1.25);
117         }
118     }
119 
CodecPerformanceTestBase(String decoderName, String testFile, int keyPriority, float maxOpRateScalingFactor)120     public CodecPerformanceTestBase(String decoderName, String testFile, int keyPriority,
121             float maxOpRateScalingFactor) {
122         mDecoderName = decoderName;
123         mTestFile = testFile;
124         mKeyPriority = keyPriority;
125         mMaxOpRateScalingFactor = maxOpRateScalingFactor;
126         mBufferInfos = new ArrayList<>();
127     }
128 
getVideoFormat(String filePath)129     static MediaFormat getVideoFormat(String filePath) throws IOException {
130         final String input = mInputPrefix + filePath;
131         MediaExtractor extractor = new MediaExtractor();
132         extractor.setDataSource(input);
133         for (int trackID = 0; trackID < extractor.getTrackCount(); trackID++) {
134             MediaFormat format = extractor.getTrackFormat(trackID);
135             if (format.getString(MediaFormat.KEY_MIME).startsWith("video/")) {
136                 extractor.release();
137                 return format;
138             }
139         }
140         extractor.release();
141         return null;
142     }
143 
selectCodecs(String mime, ArrayList<MediaFormat> formats, String[] features, boolean isEncoder)144     static ArrayList<String> selectCodecs(String mime, ArrayList<MediaFormat> formats,
145             String[] features, boolean isEncoder) {
146         return selectCodecs(mime, formats, features, isEncoder, SELECT_ALL);
147     }
148 
selectHardwareCodecs(String mime, ArrayList<MediaFormat> formats, String[] features, boolean isEncoder)149     static ArrayList<String> selectHardwareCodecs(String mime, ArrayList<MediaFormat> formats,
150             String[] features, boolean isEncoder) {
151         return selectCodecs(mime, formats, features, isEncoder, SELECT_HARDWARE);
152     }
153 
selectCodecs(String mime, ArrayList<MediaFormat> formats, String[] features, boolean isEncoder, int selectCodecOption)154     static ArrayList<String> selectCodecs(String mime, ArrayList<MediaFormat> formats,
155             String[] features, boolean isEncoder, int selectCodecOption) {
156         MediaCodecList codecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
157         MediaCodecInfo[] codecInfos = codecList.getCodecInfos();
158         ArrayList<String> listOfCodecs = new ArrayList<>();
159         for (MediaCodecInfo codecInfo : codecInfos) {
160             if (codecInfo.isEncoder() != isEncoder) continue;
161             if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && codecInfo.isAlias()) continue;
162             if (selectCodecOption == SELECT_HARDWARE && !codecInfo.isHardwareAccelerated())
163                 continue;
164             else if (selectCodecOption == SELECT_SOFTWARE && !codecInfo.isSoftwareOnly())
165                 continue;
166             String[] types = codecInfo.getSupportedTypes();
167             for (String type : types) {
168                 if (type.equalsIgnoreCase(mime)) {
169                     boolean isOk = true;
170                     MediaCodecInfo.CodecCapabilities codecCapabilities =
171                             codecInfo.getCapabilitiesForType(type);
172                     if (formats != null) {
173                         for (MediaFormat format : formats) {
174                             if (!codecCapabilities.isFormatSupported(format)) {
175                                 isOk = false;
176                                 break;
177                             }
178                         }
179                     }
180                     if (features != null) {
181                         for (String feature : features) {
182                             if (!codecCapabilities.isFeatureSupported(feature)) {
183                                 isOk = false;
184                                 break;
185                             }
186                         }
187                     }
188                     if (isOk) listOfCodecs.add(codecInfo.getName());
189                 }
190             }
191         }
192         return listOfCodecs;
193     }
194 
setUpDecoderInput()195     MediaFormat setUpDecoderInput() throws IOException {
196         final String input = mInputPrefix + mTestFile;
197         MediaExtractor extractor = new MediaExtractor();
198         extractor.setDataSource(input);
199         for (int trackID = 0; trackID < extractor.getTrackCount(); trackID++) {
200             MediaFormat format = extractor.getTrackFormat(trackID);
201             if (format.getString(MediaFormat.KEY_MIME).startsWith("video/")) {
202                 extractor.selectTrack(trackID);
203                 File file = new File(input);
204                 int bufferSize = (int) file.length();
205                 mBuff = ByteBuffer.allocate(bufferSize);
206                 int offset = 0;
207                 long maxPTS = 0;
208                 while (true) {
209                     MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
210                     bufferInfo.size = extractor.readSampleData(mBuff, offset);
211                     if (bufferInfo.size < 0) break;
212                     bufferInfo.offset = offset;
213                     bufferInfo.presentationTimeUs = extractor.getSampleTime();
214                     maxPTS = Math.max(maxPTS, bufferInfo.presentationTimeUs);
215                     int flags = extractor.getSampleFlags();
216                     bufferInfo.flags = 0;
217                     if ((flags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) {
218                         bufferInfo.flags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
219                     }
220                     mBufferInfos.add(bufferInfo);
221                     extractor.advance();
222                     offset += bufferInfo.size;
223                 }
224 
225                 // If the clip doesn't have sufficient frames, loopback by copying bufferInfos
226                 // from the start of the list and incrementing the timestamp.
227                 int actualBufferInfosCount = mBufferInfos.size();
228                 long ptsOffset;
229                 while (mBufferInfos.size() < MIN_FRAME_COUNT) {
230                     ptsOffset = maxPTS + 1000000L;
231                     for (int i = 0; i < actualBufferInfosCount; i++) {
232                         MediaCodec.BufferInfo tmpBufferInfo = mBufferInfos.get(i);
233                         MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
234                         bufferInfo.set(tmpBufferInfo.offset, tmpBufferInfo.size,
235                                 ptsOffset + tmpBufferInfo.presentationTimeUs,
236                                 tmpBufferInfo.flags);
237                         maxPTS = Math.max(maxPTS, bufferInfo.presentationTimeUs);
238                         mBufferInfos.add(bufferInfo);
239                         if (mBufferInfos.size() >= MIN_FRAME_COUNT) break;
240                     }
241                 }
242                 MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
243                 bufferInfo.set(0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
244                 mBufferInfos.add(bufferInfo);
245                 mDecoderMime = format.getString(MediaFormat.KEY_MIME);
246                 mWidth = format.getInteger(MediaFormat.KEY_WIDTH);
247                 mHeight = format.getInteger(MediaFormat.KEY_HEIGHT);
248                 mFrameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE, 30);
249                 extractor.release();
250                 return format;
251             }
252         }
253         extractor.release();
254         fail("No video track found in file: " + mTestFile);
255         return null;
256     }
257 
258     // TODO (b/193458026) Limit max expected fps
getMaxExpectedFps(int width, int height)259     static int getMaxExpectedFps(int width, int height) {
260         int numSamples = width * height;
261         if (numSamples > 3840 * 2160 * 2) { // 8K
262             return 30;
263         } else if (numSamples > 1920 * 1088 * 2) { // 4K
264             return 120;
265         } else {
266             return 240;
267         }
268     }
269 
getMaxOperatingRate(String codecName, String mime)270     int getMaxOperatingRate(String codecName, String mime) throws IOException {
271         MediaCodec codec = MediaCodec.createByCodecName(codecName);
272         MediaCodecInfo mediaCodecInfo = codec.getCodecInfo();
273         List<MediaCodecInfo.VideoCapabilities.PerformancePoint> pps = mediaCodecInfo
274                 .getCapabilitiesForType(mime).getVideoCapabilities()
275                 .getSupportedPerformancePoints();
276         assertTrue(pps.size() > 0);
277         MediaCodecInfo.VideoCapabilities.PerformancePoint cpp =
278                 new MediaCodecInfo.VideoCapabilities.PerformancePoint(mWidth, mHeight, mFrameRate);
279         int macroblocks = cpp.getMaxMacroBlocks();
280         int maxOperatingRate = -1;
281         for (MediaCodecInfo.VideoCapabilities.PerformancePoint pp : pps) {
282             if (pp.covers(cpp)) {
283                 maxOperatingRate = Math.max(Math.min(pp.getMaxFrameRate(),
284                         (int) pp.getMaxMacroBlockRate() / macroblocks), maxOperatingRate);
285             }
286         }
287         codec.release();
288         assertTrue(maxOperatingRate != -1);
289         return maxOperatingRate;
290     }
291 
getEncoderMinComplexity(String codecName, String mime)292     int getEncoderMinComplexity(String codecName, String mime) throws IOException {
293         MediaCodec codec = MediaCodec.createByCodecName(codecName);
294         MediaCodecInfo mediaCodecInfo = codec.getCodecInfo();
295         int minComplexity = -1;
296         if (mediaCodecInfo.isEncoder()) {
297             Range<Integer> complexityRange = mediaCodecInfo
298                     .getCapabilitiesForType(mime).getEncoderCapabilities()
299                     .getComplexityRange();
300             minComplexity = complexityRange.getLower();
301         }
302         codec.release();
303         return minComplexity;
304     }
305 
getMaxFrameSize(String codecName, String mime)306     static int getMaxFrameSize(String codecName, String mime) throws IOException {
307         MediaCodec codec = MediaCodec.createByCodecName(codecName);
308         MediaCodecInfo.CodecCapabilities codecCapabilities =
309                 codec.getCodecInfo().getCapabilitiesForType(mime);
310         MediaCodecInfo.VideoCapabilities vc = codecCapabilities.getVideoCapabilities();
311         Range<Integer> heights = vc.getSupportedHeights();
312         Range<Integer> widths = vc.getSupportedWidthsFor(heights.getUpper());
313         int maxFrameSize = heights.getUpper() * widths.getUpper();
314         codec.release();
315         return maxFrameSize;
316     }
317 
enqueueDecoderInput(int bufferIndex)318     void enqueueDecoderInput(int bufferIndex) {
319         MediaCodec.BufferInfo info = mBufferInfos.get(mSampleIndex++);
320         if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
321             ByteBuffer dstBuf = mDecoder.getInputBuffer(bufferIndex);
322             dstBuf.put(mBuff.array(), info.offset, info.size);
323             mDecInputNum++;
324         }
325         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
326             mSawDecInputEOS = true;
327         }
328         mDecoder.queueInputBuffer(bufferIndex, 0, info.size, info.presentationTimeUs, info.flags);
329     }
330 
dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info, boolean render)331     void dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info, boolean render) {
332         if (info.size > 0) {
333             mDecOutputNum++;
334         }
335         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
336             mSawDecOutputEOS = true;
337         }
338         mDecoder.releaseOutputBuffer(bufferIndex, render);
339     }
340 }
341