1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 *
10 */
11
12 // Everything declared/defined in this header is only required when WebRTC is
13 // build with H264 support, please do not move anything out of the
14 // #ifdef unless needed and tested.
15 #ifdef WEBRTC_USE_H264
16
17 #include "modules/video_coding/codecs/h264/h264_decoder_impl.h"
18
19 #include <algorithm>
20 #include <limits>
21 #include <memory>
22
23 extern "C" {
24 #include "third_party/ffmpeg/libavcodec/avcodec.h"
25 #include "third_party/ffmpeg/libavformat/avformat.h"
26 #include "third_party/ffmpeg/libavutil/imgutils.h"
27 } // extern "C"
28
29 #include "api/video/color_space.h"
30 #include "api/video/i010_buffer.h"
31 #include "api/video/i420_buffer.h"
32 #include "common_video/include/video_frame_buffer.h"
33 #include "modules/video_coding/codecs/h264/h264_color_space.h"
34 #include "rtc_base/checks.h"
35 #include "rtc_base/keep_ref_until_done.h"
36 #include "rtc_base/logging.h"
37 #include "system_wrappers/include/field_trial.h"
38 #include "system_wrappers/include/metrics.h"
39
40 namespace webrtc {
41
42 namespace {
43
44 const AVPixelFormat kPixelFormatDefault = AV_PIX_FMT_YUV420P;
45 const AVPixelFormat kPixelFormatFullRange = AV_PIX_FMT_YUVJ420P;
46 const size_t kYPlaneIndex = 0;
47 const size_t kUPlaneIndex = 1;
48 const size_t kVPlaneIndex = 2;
49
50 // Used by histograms. Values of entries should not be changed.
51 enum H264DecoderImplEvent {
52 kH264DecoderEventInit = 0,
53 kH264DecoderEventError = 1,
54 kH264DecoderEventMax = 16,
55 };
56
57 } // namespace
58
AVGetBuffer2(AVCodecContext * context,AVFrame * av_frame,int flags)59 int H264DecoderImpl::AVGetBuffer2(AVCodecContext* context,
60 AVFrame* av_frame,
61 int flags) {
62 // Set in |InitDecode|.
63 H264DecoderImpl* decoder = static_cast<H264DecoderImpl*>(context->opaque);
64 // DCHECK values set in |InitDecode|.
65 RTC_DCHECK(decoder);
66 // Necessary capability to be allowed to provide our own buffers.
67 RTC_DCHECK(context->codec->capabilities | AV_CODEC_CAP_DR1);
68
69 // Limited or full range YUV420 is expected.
70 RTC_CHECK(context->pix_fmt == kPixelFormatDefault ||
71 context->pix_fmt == kPixelFormatFullRange);
72
73 // |av_frame->width| and |av_frame->height| are set by FFmpeg. These are the
74 // actual image's dimensions and may be different from |context->width| and
75 // |context->coded_width| due to reordering.
76 int width = av_frame->width;
77 int height = av_frame->height;
78 // See |lowres|, if used the decoder scales the image by 1/2^(lowres). This
79 // has implications on which resolutions are valid, but we don't use it.
80 RTC_CHECK_EQ(context->lowres, 0);
81 // Adjust the |width| and |height| to values acceptable by the decoder.
82 // Without this, FFmpeg may overflow the buffer. If modified, |width| and/or
83 // |height| are larger than the actual image and the image has to be cropped
84 // (top-left corner) after decoding to avoid visible borders to the right and
85 // bottom of the actual image.
86 avcodec_align_dimensions(context, &width, &height);
87
88 RTC_CHECK_GE(width, 0);
89 RTC_CHECK_GE(height, 0);
90 int ret = av_image_check_size(static_cast<unsigned int>(width),
91 static_cast<unsigned int>(height), 0, nullptr);
92 if (ret < 0) {
93 RTC_LOG(LS_ERROR) << "Invalid picture size " << width << "x" << height;
94 decoder->ReportError();
95 return ret;
96 }
97
98 // The video frame is stored in |frame_buffer|. |av_frame| is FFmpeg's version
99 // of a video frame and will be set up to reference |frame_buffer|'s data.
100
101 // FFmpeg expects the initial allocation to be zero-initialized according to
102 // http://crbug.com/390941. Our pool is set up to zero-initialize new buffers.
103 // TODO(nisse): Delete that feature from the video pool, instead add
104 // an explicit call to InitializeData here.
105 rtc::scoped_refptr<I420Buffer> frame_buffer =
106 decoder->pool_.CreateBuffer(width, height);
107
108 int y_size = width * height;
109 int uv_size = frame_buffer->ChromaWidth() * frame_buffer->ChromaHeight();
110 // DCHECK that we have a continuous buffer as is required.
111 RTC_DCHECK_EQ(frame_buffer->DataU(), frame_buffer->DataY() + y_size);
112 RTC_DCHECK_EQ(frame_buffer->DataV(), frame_buffer->DataU() + uv_size);
113 int total_size = y_size + 2 * uv_size;
114
115 av_frame->format = context->pix_fmt;
116 av_frame->reordered_opaque = context->reordered_opaque;
117
118 // Set |av_frame| members as required by FFmpeg.
119 av_frame->data[kYPlaneIndex] = frame_buffer->MutableDataY();
120 av_frame->linesize[kYPlaneIndex] = frame_buffer->StrideY();
121 av_frame->data[kUPlaneIndex] = frame_buffer->MutableDataU();
122 av_frame->linesize[kUPlaneIndex] = frame_buffer->StrideU();
123 av_frame->data[kVPlaneIndex] = frame_buffer->MutableDataV();
124 av_frame->linesize[kVPlaneIndex] = frame_buffer->StrideV();
125 RTC_DCHECK_EQ(av_frame->extended_data, av_frame->data);
126
127 // Create a VideoFrame object, to keep a reference to the buffer.
128 // TODO(nisse): The VideoFrame's timestamp and rotation info is not used.
129 // Refactor to do not use a VideoFrame object at all.
130 av_frame->buf[0] = av_buffer_create(
131 av_frame->data[kYPlaneIndex], total_size, AVFreeBuffer2,
132 static_cast<void*>(
133 std::make_unique<VideoFrame>(VideoFrame::Builder()
134 .set_video_frame_buffer(frame_buffer)
135 .set_rotation(kVideoRotation_0)
136 .set_timestamp_us(0)
137 .build())
138 .release()),
139 0);
140 RTC_CHECK(av_frame->buf[0]);
141 return 0;
142 }
143
AVFreeBuffer2(void * opaque,uint8_t * data)144 void H264DecoderImpl::AVFreeBuffer2(void* opaque, uint8_t* data) {
145 // The buffer pool recycles the buffer used by |video_frame| when there are no
146 // more references to it. |video_frame| is a thin buffer holder and is not
147 // recycled.
148 VideoFrame* video_frame = static_cast<VideoFrame*>(opaque);
149 delete video_frame;
150 }
151
H264DecoderImpl()152 H264DecoderImpl::H264DecoderImpl()
153 : pool_(true),
154 decoded_image_callback_(nullptr),
155 has_reported_init_(false),
156 has_reported_error_(false) {}
157
~H264DecoderImpl()158 H264DecoderImpl::~H264DecoderImpl() {
159 Release();
160 }
161
InitDecode(const VideoCodec * codec_settings,int32_t number_of_cores)162 int32_t H264DecoderImpl::InitDecode(const VideoCodec* codec_settings,
163 int32_t number_of_cores) {
164 ReportInit();
165 if (codec_settings && codec_settings->codecType != kVideoCodecH264) {
166 ReportError();
167 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
168 }
169
170 // Release necessary in case of re-initializing.
171 int32_t ret = Release();
172 if (ret != WEBRTC_VIDEO_CODEC_OK) {
173 ReportError();
174 return ret;
175 }
176 RTC_DCHECK(!av_context_);
177
178 // Initialize AVCodecContext.
179 av_context_.reset(avcodec_alloc_context3(nullptr));
180
181 av_context_->codec_type = AVMEDIA_TYPE_VIDEO;
182 av_context_->codec_id = AV_CODEC_ID_H264;
183 if (codec_settings) {
184 av_context_->coded_width = codec_settings->width;
185 av_context_->coded_height = codec_settings->height;
186 }
187 av_context_->pix_fmt = kPixelFormatDefault;
188 av_context_->extradata = nullptr;
189 av_context_->extradata_size = 0;
190
191 // If this is ever increased, look at |av_context_->thread_safe_callbacks| and
192 // make it possible to disable the thread checker in the frame buffer pool.
193 av_context_->thread_count = 1;
194 av_context_->thread_type = FF_THREAD_SLICE;
195
196 // Function used by FFmpeg to get buffers to store decoded frames in.
197 av_context_->get_buffer2 = AVGetBuffer2;
198 // |get_buffer2| is called with the context, there |opaque| can be used to get
199 // a pointer |this|.
200 av_context_->opaque = this;
201
202 AVCodec* codec = avcodec_find_decoder(av_context_->codec_id);
203 if (!codec) {
204 // This is an indication that FFmpeg has not been initialized or it has not
205 // been compiled/initialized with the correct set of codecs.
206 RTC_LOG(LS_ERROR) << "FFmpeg H.264 decoder not found.";
207 Release();
208 ReportError();
209 return WEBRTC_VIDEO_CODEC_ERROR;
210 }
211 int res = avcodec_open2(av_context_.get(), codec, nullptr);
212 if (res < 0) {
213 RTC_LOG(LS_ERROR) << "avcodec_open2 error: " << res;
214 Release();
215 ReportError();
216 return WEBRTC_VIDEO_CODEC_ERROR;
217 }
218
219 av_frame_.reset(av_frame_alloc());
220
221 if (codec_settings && codec_settings->buffer_pool_size) {
222 if (!pool_.Resize(*codec_settings->buffer_pool_size)) {
223 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
224 }
225 }
226 return WEBRTC_VIDEO_CODEC_OK;
227 }
228
Release()229 int32_t H264DecoderImpl::Release() {
230 av_context_.reset();
231 av_frame_.reset();
232 return WEBRTC_VIDEO_CODEC_OK;
233 }
234
RegisterDecodeCompleteCallback(DecodedImageCallback * callback)235 int32_t H264DecoderImpl::RegisterDecodeCompleteCallback(
236 DecodedImageCallback* callback) {
237 decoded_image_callback_ = callback;
238 return WEBRTC_VIDEO_CODEC_OK;
239 }
240
Decode(const EncodedImage & input_image,bool,int64_t)241 int32_t H264DecoderImpl::Decode(const EncodedImage& input_image,
242 bool /*missing_frames*/,
243 int64_t /*render_time_ms*/) {
244 if (!IsInitialized()) {
245 ReportError();
246 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
247 }
248 if (!decoded_image_callback_) {
249 RTC_LOG(LS_WARNING)
250 << "InitDecode() has been called, but a callback function "
251 "has not been set with RegisterDecodeCompleteCallback()";
252 ReportError();
253 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
254 }
255 if (!input_image.data() || !input_image.size()) {
256 ReportError();
257 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
258 }
259
260 AVPacket packet;
261 av_init_packet(&packet);
262 // packet.data has a non-const type, but isn't modified by
263 // avcodec_send_packet.
264 packet.data = const_cast<uint8_t*>(input_image.data());
265 if (input_image.size() >
266 static_cast<size_t>(std::numeric_limits<int>::max())) {
267 ReportError();
268 return WEBRTC_VIDEO_CODEC_ERROR;
269 }
270 packet.size = static_cast<int>(input_image.size());
271 int64_t frame_timestamp_us = input_image.ntp_time_ms_ * 1000; // ms -> μs
272 av_context_->reordered_opaque = frame_timestamp_us;
273
274 int result = avcodec_send_packet(av_context_.get(), &packet);
275 if (result < 0) {
276 RTC_LOG(LS_ERROR) << "avcodec_send_packet error: " << result;
277 ReportError();
278 return WEBRTC_VIDEO_CODEC_ERROR;
279 }
280
281 result = avcodec_receive_frame(av_context_.get(), av_frame_.get());
282 if (result < 0) {
283 RTC_LOG(LS_ERROR) << "avcodec_receive_frame error: " << result;
284 ReportError();
285 return WEBRTC_VIDEO_CODEC_ERROR;
286 }
287
288 // We don't expect reordering. Decoded frame tamestamp should match
289 // the input one.
290 RTC_DCHECK_EQ(av_frame_->reordered_opaque, frame_timestamp_us);
291
292 absl::optional<uint8_t> qp;
293 // TODO(sakal): Maybe it is possible to get QP directly from FFmpeg.
294 h264_bitstream_parser_.ParseBitstream(input_image.data(), input_image.size());
295 int qp_int;
296 if (h264_bitstream_parser_.GetLastSliceQp(&qp_int)) {
297 qp.emplace(qp_int);
298 }
299
300 // Obtain the |video_frame| containing the decoded image.
301 VideoFrame* input_frame =
302 static_cast<VideoFrame*>(av_buffer_get_opaque(av_frame_->buf[0]));
303 RTC_DCHECK(input_frame);
304 const webrtc::I420BufferInterface* i420_buffer =
305 input_frame->video_frame_buffer()->GetI420();
306
307 // When needed, FFmpeg applies cropping by moving plane pointers and adjusting
308 // frame width/height. Ensure that cropped buffers lie within the allocated
309 // memory.
310 RTC_DCHECK_LE(av_frame_->width, i420_buffer->width());
311 RTC_DCHECK_LE(av_frame_->height, i420_buffer->height());
312 RTC_DCHECK_GE(av_frame_->data[kYPlaneIndex], i420_buffer->DataY());
313 RTC_DCHECK_LE(
314 av_frame_->data[kYPlaneIndex] +
315 av_frame_->linesize[kYPlaneIndex] * av_frame_->height,
316 i420_buffer->DataY() + i420_buffer->StrideY() * i420_buffer->height());
317 RTC_DCHECK_GE(av_frame_->data[kUPlaneIndex], i420_buffer->DataU());
318 RTC_DCHECK_LE(av_frame_->data[kUPlaneIndex] +
319 av_frame_->linesize[kUPlaneIndex] * av_frame_->height / 2,
320 i420_buffer->DataU() +
321 i420_buffer->StrideU() * i420_buffer->height() / 2);
322 RTC_DCHECK_GE(av_frame_->data[kVPlaneIndex], i420_buffer->DataV());
323 RTC_DCHECK_LE(av_frame_->data[kVPlaneIndex] +
324 av_frame_->linesize[kVPlaneIndex] * av_frame_->height / 2,
325 i420_buffer->DataV() +
326 i420_buffer->StrideV() * i420_buffer->height() / 2);
327
328 auto cropped_buffer = WrapI420Buffer(
329 av_frame_->width, av_frame_->height, av_frame_->data[kYPlaneIndex],
330 av_frame_->linesize[kYPlaneIndex], av_frame_->data[kUPlaneIndex],
331 av_frame_->linesize[kUPlaneIndex], av_frame_->data[kVPlaneIndex],
332 av_frame_->linesize[kVPlaneIndex], rtc::KeepRefUntilDone(i420_buffer));
333
334 // Pass on color space from input frame if explicitly specified.
335 const ColorSpace& color_space =
336 input_image.ColorSpace() ? *input_image.ColorSpace()
337 : ExtractH264ColorSpace(av_context_.get());
338
339 VideoFrame decoded_frame = VideoFrame::Builder()
340 .set_video_frame_buffer(cropped_buffer)
341 .set_timestamp_rtp(input_image.Timestamp())
342 .set_color_space(color_space)
343 .build();
344
345 // Return decoded frame.
346 // TODO(nisse): Timestamp and rotation are all zero here. Change decoder
347 // interface to pass a VideoFrameBuffer instead of a VideoFrame?
348 decoded_image_callback_->Decoded(decoded_frame, absl::nullopt, qp);
349
350 // Stop referencing it, possibly freeing |input_frame|.
351 av_frame_unref(av_frame_.get());
352 input_frame = nullptr;
353
354 return WEBRTC_VIDEO_CODEC_OK;
355 }
356
ImplementationName() const357 const char* H264DecoderImpl::ImplementationName() const {
358 return "FFmpeg";
359 }
360
IsInitialized() const361 bool H264DecoderImpl::IsInitialized() const {
362 return av_context_ != nullptr;
363 }
364
ReportInit()365 void H264DecoderImpl::ReportInit() {
366 if (has_reported_init_)
367 return;
368 RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264DecoderImpl.Event",
369 kH264DecoderEventInit, kH264DecoderEventMax);
370 has_reported_init_ = true;
371 }
372
ReportError()373 void H264DecoderImpl::ReportError() {
374 if (has_reported_error_)
375 return;
376 RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264DecoderImpl.Event",
377 kH264DecoderEventError, kH264DecoderEventMax);
378 has_reported_error_ = true;
379 }
380
381 } // namespace webrtc
382
383 #endif // WEBRTC_USE_H264
384