1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.media.samplevideoencoder;
18 
19 import android.content.Context;
20 import android.content.res.AssetFileDescriptor;
21 import android.media.MediaCodec;
22 import android.media.MediaCodecInfo;
23 import android.media.MediaExtractor;
24 import android.media.MediaFormat;
25 import android.media.MediaMuxer;
26 import android.os.Build;
27 import android.util.Log;
28 import android.util.Pair;
29 import android.view.Surface;
30 
31 import java.io.IOException;
32 import java.nio.ByteBuffer;
33 import java.util.ArrayList;
34 import java.util.Arrays;
35 
36 import static com.android.media.samplevideoencoder.MainActivity.FRAME_TYPE_B;
37 import static com.android.media.samplevideoencoder.MainActivity.FRAME_TYPE_I;
38 import static com.android.media.samplevideoencoder.MainActivity.FRAME_TYPE_P;
39 
40 public class MediaCodecSurfaceEncoder {
41     private static final String TAG = MediaCodecSurfaceEncoder.class.getSimpleName();
42     private static final boolean DEBUG = false;
43     private static final int VIDEO_BITRATE = 8000000  /*8 Mbps*/;
44     private static final int VIDEO_FRAMERATE = 30;
45     private final Context mActivityContext;
46     private final int mResID;
47     private final int mMaxBFrames;
48     private final String mMime;
49     private final String mOutputPath;
50     private int mTrackID = -1;
51     private int mFrameNum = 0;
52     private int[] mFrameTypeOccurrences = {0, 0, 0};
53 
54     private Surface mSurface;
55     private MediaExtractor mExtractor;
56     private MediaCodec mDecoder;
57     private MediaCodec mEncoder;
58     private MediaMuxer mMuxer;
59 
60     private final boolean mIsCodecSoftware;
61     private boolean mSawDecInputEOS;
62     private boolean mSawDecOutputEOS;
63     private boolean mSawEncOutputEOS;
64     private int mDecOutputCount;
65     private int mEncOutputCount;
66 
67     private final CodecAsyncHandler mAsyncHandleEncoder = new CodecAsyncHandler();
68     private final CodecAsyncHandler mAsyncHandleDecoder = new CodecAsyncHandler();
69 
MediaCodecSurfaceEncoder(Context context, int resId, String mime, boolean isSoftware, String outputPath, int maxBFrames)70     public MediaCodecSurfaceEncoder(Context context, int resId, String mime, boolean isSoftware,
71                                     String outputPath, int maxBFrames) {
72         mActivityContext = context;
73         mResID = resId;
74         mMime = mime;
75         mIsCodecSoftware = isSoftware;
76         mOutputPath = outputPath;
77         mMaxBFrames = maxBFrames;
78     }
79 
MediaCodecSurfaceEncoder(Context context, int resId, String mime, boolean isSoftware, String outputPath)80     public MediaCodecSurfaceEncoder(Context context, int resId, String mime, boolean isSoftware,
81                                     String outputPath) {
82         // Default value of MediaFormat.KEY_MAX_B_FRAMES is set to 1, if not passed as a parameter.
83         this(context, resId, mime, isSoftware, outputPath, 1);
84     }
85 
startEncodingSurface()86     public int startEncodingSurface() throws IOException, InterruptedException {
87         MediaFormat decoderFormat = setUpSource();
88         if (decoderFormat == null) {
89             return -1;
90         }
91 
92         String decoderMime = decoderFormat.getString(MediaFormat.KEY_MIME);
93         ArrayList<String> decoders =
94                 MediaCodecBase.selectCodecs(decoderMime, null, null, false, mIsCodecSoftware);
95         if (decoders.isEmpty()) {
96             Log.e(TAG, "No suitable decoder found for mime: " + decoderMime);
97             return -1;
98         }
99         mDecoder = MediaCodec.createByCodecName(decoders.get(0));
100 
101         MediaFormat encoderFormat = setUpEncoderFormat(decoderFormat);
102         ArrayList<String> listOfEncoders =
103                 MediaCodecBase.selectCodecs(mMime, null, null, true, mIsCodecSoftware);
104         if (listOfEncoders.isEmpty()) {
105             Log.e(TAG, "No suitable encoder found for mime: " + mMime);
106             return -1;
107         }
108 
109         boolean muxOutput = true;
110         for (String encoder : listOfEncoders) {
111             mEncoder = MediaCodec.createByCodecName(encoder);
112             mExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
113             if (muxOutput) {
114                 int muxerFormat = MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4;
115                 mMuxer = new MediaMuxer(mOutputPath, muxerFormat);
116             }
117             configureCodec(decoderFormat, encoderFormat);
118             mEncoder.start();
119             mDecoder.start();
120             doWork(Integer.MAX_VALUE);
121             queueEOS();
122             waitForAllEncoderOutputs();
123             if (muxOutput) {
124                 if (mTrackID != -1) {
125                     mMuxer.stop();
126                     mTrackID = -1;
127                 }
128                 if (mMuxer != null) {
129                     mMuxer.release();
130                     mMuxer = null;
131                 }
132             }
133             mDecoder.reset();
134             mEncoder.reset();
135             mSurface.release();
136             mSurface = null;
137             Log.i(TAG, "Number of I-frames = " + mFrameTypeOccurrences[FRAME_TYPE_I]);
138             Log.i(TAG, "Number of P-frames = " + mFrameTypeOccurrences[FRAME_TYPE_P]);
139             Log.i(TAG, "Number of B-frames = " + mFrameTypeOccurrences[FRAME_TYPE_B]);
140         }
141         mEncoder.release();
142         mDecoder.release();
143         mExtractor.release();
144         return 0;
145     }
146 
setUpSource()147     private MediaFormat setUpSource() throws IOException {
148         mExtractor = new MediaExtractor();
149         AssetFileDescriptor fd = mActivityContext.getResources().openRawResourceFd(mResID);
150         mExtractor.setDataSource(fd.getFileDescriptor(), fd.getStartOffset(), fd.getLength());
151         for (int trackID = 0; trackID < mExtractor.getTrackCount(); trackID++) {
152             MediaFormat format = mExtractor.getTrackFormat(trackID);
153             String mime = format.getString(MediaFormat.KEY_MIME);
154             if (mime.startsWith("video/")) {
155                 mExtractor.selectTrack(trackID);
156                 format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
157                         MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible);
158                 return format;
159             }
160         }
161         mExtractor.release();
162         return null;
163     }
164 
setUpEncoderFormat(MediaFormat decoderFormat)165     private MediaFormat setUpEncoderFormat(MediaFormat decoderFormat) {
166         MediaFormat encoderFormat = new MediaFormat();
167         encoderFormat.setString(MediaFormat.KEY_MIME, mMime);
168         encoderFormat
169                 .setInteger(MediaFormat.KEY_WIDTH, decoderFormat.getInteger(MediaFormat.KEY_WIDTH));
170         encoderFormat.setInteger(MediaFormat.KEY_HEIGHT,
171                 decoderFormat.getInteger(MediaFormat.KEY_HEIGHT));
172         encoderFormat.setInteger(MediaFormat.KEY_FRAME_RATE, VIDEO_FRAMERATE);
173         encoderFormat.setInteger(MediaFormat.KEY_BIT_RATE, VIDEO_BITRATE);
174         encoderFormat.setFloat(MediaFormat.KEY_I_FRAME_INTERVAL, 1.0f);
175         encoderFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT,
176                 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
177         if (mMime.equals(MediaFormat.MIMETYPE_VIDEO_HEVC)) {
178             encoderFormat.setInteger(MediaFormat.KEY_PROFILE,
179                     MediaCodecInfo.CodecProfileLevel.HEVCProfileMain);
180             encoderFormat.setInteger(MediaFormat.KEY_LEVEL,
181                     MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel4);
182         } else {
183             encoderFormat.setInteger(MediaFormat.KEY_PROFILE,
184                     MediaCodecInfo.CodecProfileLevel.AVCProfileMain);
185             encoderFormat
186                     .setInteger(MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel4);
187         }
188         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
189             encoderFormat.setInteger(MediaFormat.KEY_MAX_B_FRAMES, mMaxBFrames);
190         } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
191             encoderFormat.setInteger(MediaFormat.KEY_LATENCY, 1);
192         }
193         return encoderFormat;
194     }
195 
resetContext()196     private void resetContext() {
197         mAsyncHandleDecoder.resetContext();
198         mAsyncHandleEncoder.resetContext();
199         mSawDecInputEOS = false;
200         mSawDecOutputEOS = false;
201         mSawEncOutputEOS = false;
202         mDecOutputCount = 0;
203         mEncOutputCount = 0;
204         mFrameNum = 0;
205         Arrays.fill(mFrameTypeOccurrences, 0);
206     }
207 
configureCodec(MediaFormat decFormat, MediaFormat encFormat)208     private void configureCodec(MediaFormat decFormat, MediaFormat encFormat) {
209         resetContext();
210         mAsyncHandleEncoder.setCallBack(mEncoder, true);
211         mEncoder.configure(encFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
212         mSurface = mEncoder.createInputSurface();
213         if (!mSurface.isValid()) {
214             Log.e(TAG, "Surface is not valid");
215             return;
216         }
217         mAsyncHandleDecoder.setCallBack(mDecoder, true);
218         mDecoder.configure(decFormat, mSurface, null, 0);
219         Log.d(TAG, "Codec configured");
220         if (DEBUG) {
221             Log.d(TAG, "Encoder Output format: " + mEncoder.getOutputFormat());
222         }
223     }
224 
dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info)225     private void dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info) {
226         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
227             mSawDecOutputEOS = true;
228         }
229         if (DEBUG) {
230             Log.d(TAG,
231                     "output: id: " + bufferIndex + " flags: " + info.flags + " size: " + info.size +
232                             " timestamp: " + info.presentationTimeUs);
233         }
234         if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
235             mDecOutputCount++;
236         }
237         mDecoder.releaseOutputBuffer(bufferIndex, mSurface != null);
238     }
239 
enqueueDecoderInput(int bufferIndex)240     private void enqueueDecoderInput(int bufferIndex) {
241         ByteBuffer inputBuffer = mDecoder.getInputBuffer(bufferIndex);
242         int size = mExtractor.readSampleData(inputBuffer, 0);
243         if (size < 0) {
244             enqueueDecoderEOS(bufferIndex);
245         } else {
246             long pts = mExtractor.getSampleTime();
247             int extractorFlags = mExtractor.getSampleFlags();
248             int codecFlags = 0;
249             if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) {
250                 codecFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
251             }
252             if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_PARTIAL_FRAME) != 0) {
253                 codecFlags |= MediaCodec.BUFFER_FLAG_PARTIAL_FRAME;
254             }
255             if (!mExtractor.advance()) {
256                 codecFlags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
257                 mSawDecInputEOS = true;
258             }
259             if (DEBUG) {
260                 Log.d(TAG, "input: id: " + bufferIndex + " size: " + size + " pts: " + pts +
261                         " flags: " + codecFlags);
262             }
263             mDecoder.queueInputBuffer(bufferIndex, 0, size, pts, codecFlags);
264         }
265     }
266 
doWork(int frameLimit)267     private void doWork(int frameLimit) throws InterruptedException {
268         int frameCount = 0;
269         while (!hasSeenError() && !mSawDecInputEOS && frameCount < frameLimit) {
270             Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandleDecoder.getWork();
271             if (element != null) {
272                 int bufferID = element.first;
273                 MediaCodec.BufferInfo info = element.second;
274                 if (info != null) {
275                     // <id, info> corresponds to output callback.
276                     dequeueDecoderOutput(bufferID, info);
277                 } else {
278                     // <id, null> corresponds to input callback.
279                     enqueueDecoderInput(bufferID);
280                     frameCount++;
281                 }
282             }
283             // check decoder EOS
284             if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream();
285             // encoder output
286             if (mDecOutputCount - mEncOutputCount > mMaxBFrames) {
287                 tryEncoderOutput();
288             }
289         }
290     }
291 
queueEOS()292     private void queueEOS() throws InterruptedException {
293         while (!mAsyncHandleDecoder.hasSeenError() && !mSawDecInputEOS) {
294             Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandleDecoder.getWork();
295             if (element != null) {
296                 int bufferID = element.first;
297                 MediaCodec.BufferInfo info = element.second;
298                 if (info != null) {
299                     dequeueDecoderOutput(bufferID, info);
300                 } else {
301                     enqueueDecoderEOS(element.first);
302                 }
303             }
304         }
305 
306         while (!hasSeenError() && !mSawDecOutputEOS) {
307             Pair<Integer, MediaCodec.BufferInfo> decOp = mAsyncHandleDecoder.getOutput();
308             if (decOp != null) dequeueDecoderOutput(decOp.first, decOp.second);
309             if (mSawDecOutputEOS) mEncoder.signalEndOfInputStream();
310             if (mDecOutputCount - mEncOutputCount > mMaxBFrames) {
311                 tryEncoderOutput();
312             }
313         }
314     }
315 
tryEncoderOutput()316     private void tryEncoderOutput() throws InterruptedException {
317         if (!hasSeenError() && !mSawEncOutputEOS) {
318             Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandleEncoder.getOutput();
319             if (element != null) {
320                 dequeueEncoderOutput(element.first, element.second);
321             }
322         }
323     }
324 
waitForAllEncoderOutputs()325     private void waitForAllEncoderOutputs() throws InterruptedException {
326         while (!hasSeenError() && !mSawEncOutputEOS) {
327             tryEncoderOutput();
328         }
329     }
330 
enqueueDecoderEOS(int bufferIndex)331     private void enqueueDecoderEOS(int bufferIndex) {
332         if (!mSawDecInputEOS) {
333             mDecoder.queueInputBuffer(bufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
334             mSawDecInputEOS = true;
335             Log.d(TAG, "Queued End of Stream");
336         }
337     }
338 
dequeueEncoderOutput(int bufferIndex, MediaCodec.BufferInfo info)339     private void dequeueEncoderOutput(int bufferIndex, MediaCodec.BufferInfo info) {
340         if (DEBUG) {
341             Log.d(TAG, "encoder output: id: " + bufferIndex + " flags: " + info.flags + " size: " +
342                     info.size + " timestamp: " + info.presentationTimeUs);
343         }
344         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
345             mSawEncOutputEOS = true;
346         }
347         if (info.size > 0) {
348             ByteBuffer buf = mEncoder.getOutputBuffer(bufferIndex);
349             // Parse the buffer to get the frame type
350             if (DEBUG) Log.d(TAG, "[ Frame : " + (mFrameNum++) + " ]");
351             int frameTypeResult = -1;
352             if (mMime == MediaFormat.MIMETYPE_VIDEO_AVC) {
353                 frameTypeResult = NalUnitUtil.getStandardizedFrameTypesFromAVC(buf);
354             } else if (mMime == MediaFormat.MIMETYPE_VIDEO_HEVC){
355                 frameTypeResult = NalUnitUtil.getStandardizedFrameTypesFromHEVC(buf);
356             } else {
357                 Log.e(TAG, "Mime type " + mMime + " is not supported.");
358                 return;
359             }
360             if (frameTypeResult != -1) {
361                 mFrameTypeOccurrences[frameTypeResult]++;
362             }
363 
364             if (mMuxer != null) {
365                 if (mTrackID == -1) {
366                     mTrackID = mMuxer.addTrack(mEncoder.getOutputFormat());
367                     mMuxer.start();
368                 }
369                 mMuxer.writeSampleData(mTrackID, buf, info);
370             }
371             if ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
372                 mEncOutputCount++;
373             }
374         }
375         mEncoder.releaseOutputBuffer(bufferIndex, false);
376     }
377 
hasSeenError()378     private boolean hasSeenError() {
379         return mAsyncHandleDecoder.hasSeenError() || mAsyncHandleEncoder.hasSeenError();
380     }
381 
getFrameTypes()382     public int[] getFrameTypes() {
383         return mFrameTypeOccurrences;
384     }
385 }
386