1 /*
2  *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 package org.webrtc;
12 
13 import static com.google.common.truth.Truth.assertThat;
14 import static org.mockito.ArgumentMatchers.any;
15 import static org.mockito.ArgumentMatchers.anyInt;
16 import static org.mockito.ArgumentMatchers.anyLong;
17 import static org.mockito.Mockito.spy;
18 import static org.mockito.Mockito.verify;
19 
20 import android.media.MediaCodec;
21 import android.media.MediaCodecInfo;
22 import android.media.MediaFormat;
23 import java.nio.ByteBuffer;
24 import java.util.HashMap;
25 import java.util.Map;
26 import org.chromium.testing.local.LocalRobolectricTestRunner;
27 import org.junit.Before;
28 import org.junit.Test;
29 import org.junit.runner.RunWith;
30 import org.mockito.ArgumentCaptor;
31 import org.mockito.Mock;
32 import org.mockito.MockitoAnnotations;
33 import org.robolectric.annotation.Config;
34 import org.webrtc.EncodedImage;
35 import org.webrtc.EncodedImage.FrameType;
36 import org.webrtc.FakeMediaCodecWrapper.State;
37 import org.webrtc.VideoCodecStatus;
38 import org.webrtc.VideoEncoder;
39 import org.webrtc.VideoEncoder.CodecSpecificInfo;
40 import org.webrtc.VideoEncoder.EncodeInfo;
41 import org.webrtc.VideoEncoder.Settings;
42 import org.webrtc.VideoFrame;
43 import org.webrtc.VideoFrame.Buffer;
44 import org.webrtc.VideoFrame.I420Buffer;
45 
46 @RunWith(LocalRobolectricTestRunner.class)
47 @Config(manifest = Config.NONE)
48 public class HardwareVideoEncoderTest {
49   private static final VideoEncoder.Settings TEST_ENCODER_SETTINGS = new Settings(
50       /* numberOfCores= */ 1,
51       /* width= */ 640,
52       /* height= */ 480,
53       /* startBitrate= */ 10000,
54       /* maxFramerate= */ 30,
55       /* numberOfSimulcastStreams= */ 1,
56       /* automaticResizeOn= */ true,
57       /* capabilities= */ new VideoEncoder.Capabilities(false /* lossNotification */));
58   private static final long POLL_DELAY_MS = 10;
59   private static final long DELIVER_ENCODED_IMAGE_DELAY_MS = 10;
60 
61   private static class TestEncoder extends HardwareVideoEncoder {
62     private final Object deliverEncodedImageLock = new Object();
63     private boolean deliverEncodedImageDone = true;
64 
TestEncoder(MediaCodecWrapperFactory mediaCodecWrapperFactory, String codecName, VideoCodecMimeType codecType, Integer surfaceColorFormat, Integer yuvColorFormat, Map<String, String> params, int keyFrameIntervalSec, int forceKeyFrameIntervalMs, BitrateAdjuster bitrateAdjuster, EglBase14.Context sharedContext)65     TestEncoder(MediaCodecWrapperFactory mediaCodecWrapperFactory, String codecName,
66         VideoCodecMimeType codecType, Integer surfaceColorFormat, Integer yuvColorFormat,
67         Map<String, String> params, int keyFrameIntervalSec, int forceKeyFrameIntervalMs,
68         BitrateAdjuster bitrateAdjuster, EglBase14.Context sharedContext) {
69       super(mediaCodecWrapperFactory, codecName, codecType, surfaceColorFormat, yuvColorFormat,
70           params, keyFrameIntervalSec, forceKeyFrameIntervalMs, bitrateAdjuster, sharedContext);
71     }
72 
waitDeliverEncodedImage()73     public void waitDeliverEncodedImage() throws InterruptedException {
74       synchronized (deliverEncodedImageLock) {
75         deliverEncodedImageDone = false;
76         deliverEncodedImageLock.notifyAll();
77         while (!deliverEncodedImageDone) {
78           deliverEncodedImageLock.wait();
79         }
80       }
81     }
82 
83     @SuppressWarnings("WaitNotInLoop") // This method is called inside a loop.
84     @Override
deliverEncodedImage()85     protected void deliverEncodedImage() {
86       synchronized (deliverEncodedImageLock) {
87         if (deliverEncodedImageDone) {
88           try {
89             deliverEncodedImageLock.wait(DELIVER_ENCODED_IMAGE_DELAY_MS);
90           } catch (InterruptedException e) {
91             Thread.currentThread().interrupt();
92             return;
93           }
94         }
95         if (deliverEncodedImageDone) {
96           return;
97         }
98         super.deliverEncodedImage();
99         deliverEncodedImageDone = true;
100         deliverEncodedImageLock.notifyAll();
101       }
102     }
103 
104     @Override
fillInputBuffer(ByteBuffer buffer, Buffer videoFrameBuffer)105     protected void fillInputBuffer(ByteBuffer buffer, Buffer videoFrameBuffer) {
106       I420Buffer i420Buffer = videoFrameBuffer.toI420();
107       buffer.put(i420Buffer.getDataY());
108       buffer.put(i420Buffer.getDataU());
109       buffer.put(i420Buffer.getDataV());
110       buffer.flip();
111       i420Buffer.release();
112     }
113   }
114 
115   private class TestEncoderBuilder {
116     private VideoCodecMimeType codecType = VideoCodecMimeType.VP8;
117 
setCodecType(VideoCodecMimeType codecType)118     public TestEncoderBuilder setCodecType(VideoCodecMimeType codecType) {
119       this.codecType = codecType;
120       return this;
121     }
122 
build()123     public TestEncoder build() {
124       return new TestEncoder((String name)
125                                  -> fakeMediaCodecWrapper,
126           "org.webrtc.testencoder", codecType,
127           /* surfaceColorFormat= */ null,
128           /* yuvColorFormat= */ MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar,
129           /* params= */ new HashMap<>(),
130           /* keyFrameIntervalSec= */ 0,
131           /* forceKeyFrameIntervalMs= */ 0,
132           /* bitrateAdjuster= */ new BaseBitrateAdjuster(),
133           /* sharedContext= */ null);
134     }
135   }
136 
137   @Mock VideoEncoder.Callback mockEncoderCallback;
138   private FakeMediaCodecWrapper fakeMediaCodecWrapper;
139 
140   @Before
setUp()141   public void setUp() {
142     MockitoAnnotations.initMocks(this);
143     MediaFormat outputFormat = new MediaFormat();
144     // TODO(sakal): Add more details to output format as needed.
145     fakeMediaCodecWrapper = spy(new FakeMediaCodecWrapper(outputFormat));
146   }
147 
148   @Test
testInit()149   public void testInit() {
150     // Set-up.
151     HardwareVideoEncoder encoder =
152         new TestEncoderBuilder().setCodecType(VideoCodecMimeType.VP8).build();
153 
154     // Test.
155     assertThat(encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback))
156         .isEqualTo(VideoCodecStatus.OK);
157 
158     // Verify.
159     assertThat(fakeMediaCodecWrapper.getState()).isEqualTo(State.EXECUTING_RUNNING);
160 
161     MediaFormat mediaFormat = fakeMediaCodecWrapper.getConfiguredFormat();
162     assertThat(mediaFormat).isNotNull();
163     assertThat(mediaFormat.getInteger(MediaFormat.KEY_WIDTH))
164         .isEqualTo(TEST_ENCODER_SETTINGS.width);
165     assertThat(mediaFormat.getInteger(MediaFormat.KEY_HEIGHT))
166         .isEqualTo(TEST_ENCODER_SETTINGS.height);
167     assertThat(mediaFormat.getString(MediaFormat.KEY_MIME))
168         .isEqualTo(VideoCodecMimeType.VP8.mimeType());
169 
170     assertThat(fakeMediaCodecWrapper.getConfiguredFlags())
171         .isEqualTo(MediaCodec.CONFIGURE_FLAG_ENCODE);
172   }
173 
174   @Test
testEncodeByteBuffer()175   public void testEncodeByteBuffer() {
176     // Set-up.
177     HardwareVideoEncoder encoder = new TestEncoderBuilder().build();
178     encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback);
179 
180     // Test.
181     byte[] i420 = CodecTestHelper.generateRandomData(
182         TEST_ENCODER_SETTINGS.width * TEST_ENCODER_SETTINGS.height * 3 / 2);
183     final VideoFrame.I420Buffer testBuffer =
184         CodecTestHelper.wrapI420(TEST_ENCODER_SETTINGS.width, TEST_ENCODER_SETTINGS.height, i420);
185     final VideoFrame testFrame =
186         new VideoFrame(testBuffer, /* rotation= */ 0, /* timestampNs= */ 0);
187     assertThat(encoder.encode(testFrame, new EncodeInfo(new FrameType[] {FrameType.VideoFrameKey})))
188         .isEqualTo(VideoCodecStatus.OK);
189 
190     // Verify.
191     ArgumentCaptor<Integer> indexCaptor = ArgumentCaptor.forClass(Integer.class);
192     ArgumentCaptor<Integer> offsetCaptor = ArgumentCaptor.forClass(Integer.class);
193     ArgumentCaptor<Integer> sizeCaptor = ArgumentCaptor.forClass(Integer.class);
194     verify(fakeMediaCodecWrapper)
195         .queueInputBuffer(indexCaptor.capture(), offsetCaptor.capture(), sizeCaptor.capture(),
196             anyLong(), anyInt());
197     ByteBuffer buffer = fakeMediaCodecWrapper.getInputBuffers()[indexCaptor.getValue()];
198     CodecTestHelper.assertEqualContents(
199         i420, buffer, offsetCaptor.getValue(), sizeCaptor.getValue());
200   }
201 
202   @Test
testDeliversOutputData()203   public void testDeliversOutputData() throws InterruptedException {
204     final int outputDataLength = 100;
205 
206     // Set-up.
207     TestEncoder encoder = new TestEncoderBuilder().build();
208     encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback);
209     byte[] i420 = CodecTestHelper.generateRandomData(
210         TEST_ENCODER_SETTINGS.width * TEST_ENCODER_SETTINGS.height * 3 / 2);
211     final VideoFrame.I420Buffer testBuffer =
212         CodecTestHelper.wrapI420(TEST_ENCODER_SETTINGS.width, TEST_ENCODER_SETTINGS.height, i420);
213     final VideoFrame testFrame =
214         new VideoFrame(testBuffer, /* rotation= */ 0, /* timestampNs= */ 42);
215     encoder.encode(testFrame, new EncodeInfo(new FrameType[] {FrameType.VideoFrameKey}));
216 
217     // Test.
218     byte[] outputData = CodecTestHelper.generateRandomData(outputDataLength);
219     fakeMediaCodecWrapper.addOutputData(outputData,
220         /* presentationTimestampUs= */ 0,
221         /* flags= */ MediaCodec.BUFFER_FLAG_SYNC_FRAME);
222 
223     encoder.waitDeliverEncodedImage();
224 
225     // Verify.
226     ArgumentCaptor<EncodedImage> videoFrameCaptor = ArgumentCaptor.forClass(EncodedImage.class);
227     verify(mockEncoderCallback)
228         .onEncodedFrame(videoFrameCaptor.capture(), any(CodecSpecificInfo.class));
229 
230     EncodedImage videoFrame = videoFrameCaptor.getValue();
231     assertThat(videoFrame).isNotNull();
232     assertThat(videoFrame.encodedWidth).isEqualTo(TEST_ENCODER_SETTINGS.width);
233     assertThat(videoFrame.encodedHeight).isEqualTo(TEST_ENCODER_SETTINGS.height);
234     assertThat(videoFrame.rotation).isEqualTo(0);
235     assertThat(videoFrame.captureTimeNs).isEqualTo(42);
236     assertThat(videoFrame.completeFrame).isTrue();
237     assertThat(videoFrame.frameType).isEqualTo(FrameType.VideoFrameKey);
238     CodecTestHelper.assertEqualContents(
239         outputData, videoFrame.buffer, /* offset= */ 0, videoFrame.buffer.capacity());
240   }
241 
242   @Test
testRelease()243   public void testRelease() {
244     // Set-up.
245     HardwareVideoEncoder encoder = new TestEncoderBuilder().build();
246     encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback);
247 
248     // Test.
249     assertThat(encoder.release()).isEqualTo(VideoCodecStatus.OK);
250 
251     // Verify.
252     assertThat(fakeMediaCodecWrapper.getState()).isEqualTo(State.RELEASED);
253   }
254 
255   @Test
testReleaseMultipleTimes()256   public void testReleaseMultipleTimes() {
257     // Set-up.
258     HardwareVideoEncoder encoder = new TestEncoderBuilder().build();
259     encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback);
260 
261     // Test.
262     assertThat(encoder.release()).isEqualTo(VideoCodecStatus.OK);
263     assertThat(encoder.release()).isEqualTo(VideoCodecStatus.OK);
264 
265     // Verify.
266     assertThat(fakeMediaCodecWrapper.getState()).isEqualTo(State.RELEASED);
267   }
268 }
269