1 // Copyright 2019 The Chromium OS Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 // #define LOG_NDEBUG 0
6 #define LOG_TAG "MediaCodecDecoder"
7 
8 #include "mediacodec_decoder.h"
9 
10 #include <assert.h>
11 #include <inttypes.h>
12 
13 #include <utility>
14 #include <vector>
15 
16 #include <log/log.h>
17 #include <media/NdkMediaFormat.h>
18 
19 namespace android {
20 namespace {
21 constexpr int64_t kSecToNs = 1000000000;
22 // The timeout of AMediaCodec_dequeueOutputBuffer function calls.
23 constexpr int kTimeoutWaitForOutputUs = 1000;  // 1 millisecond
24 
25 // The timeout of AMediaCodec_dequeueInputBuffer function calls.
26 constexpr int kTimeoutWaitForInputUs = 1000;  // 1 millisecond
27 
28 // The maximal retry times of doDecode routine.
29 // The maximal tolerable interval between two dequeued outputs will be:
30 //   kTimeoutWaitForOutputUs * kTimeoutMaxRetries = 500 milliseconds
31 constexpr size_t kTimeoutMaxRetries = 500;
32 
33 // Helper function to get possible C2 hardware decoder names from |type|.
34 // Note: A single test APK is built for both ARC++ and ARCVM, so both the VDA decoder and the new
35 // V4L2 decoder names need to be specified here.
GetC2VideoDecoderNames(VideoCodecType type)36 std::vector<const char*> GetC2VideoDecoderNames(VideoCodecType type) {
37     switch (type) {
38     case VideoCodecType::H264:
39         return {"c2.v4l2.avc.decoder", "c2.vda.avc.decoder"};
40     case VideoCodecType::VP8:
41         return {"c2.v4l2.vp8.decoder", "c2.vda.vp8.decoder"};
42     case VideoCodecType::VP9:
43         return {"c2.v4l2.vp9.decoder", "c2.vda.vp9.decoder"};
44     default:  // unknown type
45         return {};
46     }
47 }
48 
49 // Helper function to get possible software decoder names from |type|.
50 // Note: A single test APK is built for both ARC++ and ARCVM, so both the OMX decoder used on
51 // Android P and the c2.android decoder used on Android R need to be specified here.
GetSwVideoDecoderNames(VideoCodecType type)52 std::vector<const char*> GetSwVideoDecoderNames(VideoCodecType type) {
53     switch (type) {
54     case VideoCodecType::H264:
55         return {"c2.android.avc.decoder", "OMX.google.h264.decoder"};
56     case VideoCodecType::VP8:
57         return {"c2.android.vp8.decoder", "OMX.google.vp8.decoder"};
58     case VideoCodecType::VP9:
59         return {"c2.android.vp9.decoder", "OMX.google.vp9.decoder"};
60     default:  // unknown type
61         return {};
62     }
63 }
64 
65 const uint32_t BUFFER_FLAG_CODEC_CONFIG = AMEDIACODEC_BUFFER_FLAG_CODEC_CONFIG;
66 const char* FORMAT_KEY_SLICE_HEIGHT = AMEDIAFORMAT_KEY_SLICE_HEIGHT;
67 
GetCurrentTimeNs()68 int64_t GetCurrentTimeNs() {
69     timespec now;
70     clock_gettime(CLOCK_MONOTONIC, &now);
71     return now.tv_sec * UINT64_C(1000000000) + now.tv_nsec;
72 }
73 
RoundUp(int64_t n,int64_t multiple)74 int64_t RoundUp(int64_t n, int64_t multiple) {
75     return ((n + (multiple - 1)) / multiple) * multiple;
76 }
77 
78 }  // namespace
79 
80 // static
Create(const std::string & input_path,VideoCodecProfile profile,bool use_sw_decoder,const Size & video_size,int frame_rate,ANativeWindow * surface,bool render_on_release,bool loop,bool use_fake_renderer)81 std::unique_ptr<MediaCodecDecoder> MediaCodecDecoder::Create(
82         const std::string& input_path, VideoCodecProfile profile, bool use_sw_decoder,
83         const Size& video_size, int frame_rate, ANativeWindow* surface, bool render_on_release,
84         bool loop, bool use_fake_renderer) {
85     if (video_size.IsEmpty()) {
86         ALOGE("Size is not valid: %dx%d", video_size.width, video_size.height);
87         return nullptr;
88     }
89 
90     VideoCodecType type = VideoCodecProfileToType(profile);
91 
92     std::unique_ptr<EncodedDataHelper> encoded_data_helper(new EncodedDataHelper(input_path, type));
93     if (!encoded_data_helper->IsValid()) {
94         ALOGE("EncodedDataHelper is not created for file: %s", input_path.c_str());
95         return nullptr;
96     }
97 
98     AMediaCodec* codec = nullptr;
99     auto decoder_names =
100             use_sw_decoder ? GetSwVideoDecoderNames(type) : GetC2VideoDecoderNames(type);
101     for (const auto& decoder_name : decoder_names) {
102         codec = AMediaCodec_createCodecByName(decoder_name);
103         if (codec) {
104             ALOGD("Created mediacodec decoder by name: %s", decoder_name);
105             break;
106         }
107     }
108     if (!codec) {
109         ALOGE("Failed to create mediacodec decoder.");
110         return nullptr;
111     }
112 
113     auto ret = std::unique_ptr<MediaCodecDecoder>(
114             new MediaCodecDecoder(codec, std::move(encoded_data_helper), type, video_size,
115                                   frame_rate, surface, render_on_release, loop, use_fake_renderer));
116 
117     AMediaCodecOnAsyncNotifyCallback cb{
118             .onAsyncInputAvailable =
119                     [](AMediaCodec* codec, void* userdata, int32_t index) {
120                         reinterpret_cast<MediaCodecDecoder*>(userdata)->OnAsyncInputAvailable(
121                                 index);
122                     },
123             .onAsyncOutputAvailable =
124                     [](AMediaCodec* codec, void* userdata, int32_t index,
125                        AMediaCodecBufferInfo* buffer_info) {
126                         reinterpret_cast<MediaCodecDecoder*>(userdata)->OnAsyncOutputAvailable(
127                                 index, buffer_info);
128                     },
129             .onAsyncFormatChanged =
130                     [](AMediaCodec* codec, void* userdata, AMediaFormat* format) {
131                         reinterpret_cast<MediaCodecDecoder*>(userdata)->OnAsyncFormatChanged(
132                                 format);
133                     },
134             .onAsyncError =
135                     [](AMediaCodec* codec, void* userdata, media_status_t error, int32_t code,
136                        const char* detail) {
137                         ALOGE("Error %d (%d) %s", error, code, detail);
138                         assert(false);
139                     }};
140 
141     auto status = AMediaCodec_setAsyncNotifyCallback(codec, cb, ret.get());
142     if (status != AMEDIA_OK) {
143         ALOGE("Failed to set async callback.");
144         return nullptr;
145     }
146 
147     return ret;
148 }
149 
MediaCodecDecoder(AMediaCodec * codec,std::unique_ptr<EncodedDataHelper> encoded_data_helper,VideoCodecType type,const Size & size,int frame_rate,ANativeWindow * surface,bool render_on_release,bool loop,bool use_fake_renderer)150 MediaCodecDecoder::MediaCodecDecoder(AMediaCodec* codec,
151                                      std::unique_ptr<EncodedDataHelper> encoded_data_helper,
152                                      VideoCodecType type, const Size& size, int frame_rate,
153                                      ANativeWindow* surface, bool render_on_release, bool loop,
154                                      bool use_fake_renderer)
155       : codec_(codec),
156         encoded_data_helper_(std::move(encoded_data_helper)),
157         type_(type),
158         input_visible_size_(size),
159         frame_rate_(frame_rate),
160         surface_(surface),
161         render_on_release_(render_on_release),
162         looping_(loop),
163         fake_renderer_running_(use_fake_renderer),
164         fake_render_thread_([](MediaCodecDecoder* dec) { dec->FakeRenderLoop(); }, this) {}
165 
~MediaCodecDecoder()166 MediaCodecDecoder::~MediaCodecDecoder() {
167     if (codec_ != nullptr) {
168         AMediaCodec_delete(codec_);
169     }
170     fake_renderer_running_ = false;
171     fake_render_cv_.notify_one();
172     fake_render_thread_.join();
173 }
174 
AddOutputBufferReadyCb(const OutputBufferReadyCb & cb)175 void MediaCodecDecoder::AddOutputBufferReadyCb(const OutputBufferReadyCb& cb) {
176     output_buffer_ready_cbs_.push_back(cb);
177 }
178 
AddOutputFormatChangedCb(const OutputFormatChangedCb & cb)179 void MediaCodecDecoder::AddOutputFormatChangedCb(const OutputFormatChangedCb& cb) {
180     output_format_changed_cbs_.push_back(cb);
181 }
182 
OnAsyncInputAvailable(int32_t idx)183 void MediaCodecDecoder::OnAsyncInputAvailable(int32_t idx) {
184     std::lock_guard<std::mutex> lock(event_queue_mut_);
185     event_queue_.push({.type = INPUT_AVAILABLE, .idx = idx});
186     event_queue_cv_.notify_one();
187 }
188 
OnAsyncOutputAvailable(int32_t idx,AMediaCodecBufferInfo * info)189 void MediaCodecDecoder::OnAsyncOutputAvailable(int32_t idx, AMediaCodecBufferInfo* info) {
190     std::lock_guard<std::mutex> lock(event_queue_mut_);
191     event_queue_.push({.type = OUTPUT_AVAILABLE, .idx = idx, .info = *info});
192     event_queue_cv_.notify_one();
193 }
194 
OnAsyncFormatChanged(AMediaFormat * format)195 void MediaCodecDecoder::OnAsyncFormatChanged(AMediaFormat* format) {
196     std::lock_guard<std::mutex> lock(event_queue_mut_);
197     event_queue_.push({.type = FORMAT_CHANGED});
198     event_queue_cv_.notify_one();
199 }
200 
Rewind()201 void MediaCodecDecoder::Rewind() {
202     encoded_data_helper_->Rewind();
203     input_fragment_index_ = 0;
204 }
205 
Configure()206 bool MediaCodecDecoder::Configure() {
207     ALOGD("configure: mime=%s, width=%d, height=%d", GetMimeType(type_), input_visible_size_.width,
208           input_visible_size_.height);
209     AMediaFormat* format = AMediaFormat_new();
210     AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, GetMimeType(type_));
211     AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, input_visible_size_.width);
212     AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, input_visible_size_.height);
213     media_status_t ret =
214             AMediaCodec_configure(codec_, format, surface_, nullptr /* crtpto */, 0 /* flag */);
215     AMediaFormat_delete(format);
216     if (ret != AMEDIA_OK) {
217         ALOGE("Configure return error: %d", ret);
218         return false;
219     }
220     return true;
221 }
222 
Start()223 bool MediaCodecDecoder::Start() {
224     media_status_t ret = AMediaCodec_start(codec_);
225     if (ret != AMEDIA_OK) {
226         ALOGE("Start return error: %d", ret);
227         return false;
228     }
229     return true;
230 }
231 
Decode()232 bool MediaCodecDecoder::Decode() {
233     while (!output_done_) {
234         CodecEvent evt;
235         {
236             std::unique_lock<std::mutex> lock(event_queue_mut_);
237             while (event_queue_.empty()) {
238                 event_queue_cv_.wait(lock);
239             }
240             evt = event_queue_.front();
241             event_queue_.pop();
242         }
243 
244         bool success;
245         switch (evt.type) {
246         case INPUT_AVAILABLE:
247             success = EnqueueInputBuffers(evt.idx);
248             break;
249         case OUTPUT_AVAILABLE:
250             success = DequeueOutputBuffer(evt.idx, evt.info);
251             break;
252         case FORMAT_CHANGED:
253             success = GetOutputFormat();
254             break;
255         case FAKE_FRAME_RENDERED:
256             media_status_t status = AMediaCodec_releaseOutputBuffer(codec_, evt.idx, false);
257             if (status != AMEDIA_OK) {
258                 ALOGE("Failed to releaseOutputBuffer(index=%zu): %d", evt.idx, status);
259                 success = false;
260             }
261             break;
262         }
263         assert(success);
264     }
265     return true;
266 }
267 
EnqueueInputBuffers(int32_t index)268 bool MediaCodecDecoder::EnqueueInputBuffers(int32_t index) {
269     if (index < 0) {
270         ALOGE("Unknown error while dequeueInputBuffer: %zd", index);
271         return false;
272     }
273 
274     if (looping_ && encoded_data_helper_->ReachEndOfStream()) {
275         encoded_data_helper_->Rewind();
276     }
277 
278     if (encoded_data_helper_->ReachEndOfStream()) {
279         if (!FeedEOSInputBuffer(index)) return false;
280         input_done_ = true;
281     } else {
282         if (!FeedInputBuffer(index)) return false;
283     }
284     return true;
285 }
286 
DequeueOutputBuffer(int32_t index,AMediaCodecBufferInfo info)287 bool MediaCodecDecoder::DequeueOutputBuffer(int32_t index, AMediaCodecBufferInfo info) {
288     if (index < 0) {
289         ALOGE("Unknown error while dequeueOutputBuffer: %zd", index);
290         return false;
291     }
292 
293     if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) output_done_ = true;
294 
295     const uint64_t now = GetCurrentTimeNs();
296     bool render_frame = render_on_release_;
297     if (base_timestamp_ns_ == 0) {
298         assert(received_outputs_ == 0);
299         // The first output buffer is dequeued. Set the base timestamp.
300         base_timestamp_ns_ = now;
301     } else if (now > GetReleaseTimestampNs(received_outputs_)) {
302         drop_frame_count_++;
303         ALOGD("Drop frame #%d: frame %d deadline %" PRIu64 "us, actual %" PRIu64 "us",
304               drop_frame_count_, received_outputs_, (received_outputs_ * 1000000ull / frame_rate_),
305               (now - base_timestamp_ns_) / 1000);
306         render_frame = false;  // We don't render the dropped frame.
307     }
308 
309     if (!ReceiveOutputBuffer(index, info, render_frame)) return false;
310 
311     return true;
312 }
313 
Stop()314 bool MediaCodecDecoder::Stop() {
315     return AMediaCodec_stop(codec_) == AMEDIA_OK;
316 }
317 
FeedInputBuffer(size_t index)318 bool MediaCodecDecoder::FeedInputBuffer(size_t index) {
319     assert(!encoded_data_helper_->ReachEndOfStream());
320 
321     size_t buf_size = 0;
322     uint8_t* buf = AMediaCodec_getInputBuffer(codec_, index, &buf_size);
323     if (!buf) {
324         ALOGE("Failed to getInputBuffer: index=%zu", index);
325         return false;
326     }
327 
328     auto fragment = encoded_data_helper_->GetNextFragment();
329     assert(fragment);
330 
331     if (buf_size < fragment->data.size()) {
332         ALOGE("Input buffer size is not enough: buf_size=%zu, data_size=%zu", buf_size,
333               fragment->data.size());
334         return false;
335     }
336 
337     memcpy(reinterpret_cast<char*>(buf), fragment->data.data(), fragment->data.size());
338 
339     uint32_t input_flag = 0;
340     if (fragment->csd_flag) input_flag |= BUFFER_FLAG_CODEC_CONFIG;
341 
342     // We don't parse the display order of each bitstream buffer. Let's trust the order of received
343     // output buffers from |codec_|.
344     uint64_t timestamp_us = 0;
345 
346     ALOGV("queueInputBuffer(index=%zu, offset=0, size=%zu, time=%" PRIu64 ", flags=%u) #%d", index,
347           fragment->data.size(), timestamp_us, input_flag, input_fragment_index_);
348     media_status_t status = AMediaCodec_queueInputBuffer(
349             codec_, index, 0 /* offset */, fragment->data.size(), timestamp_us, input_flag);
350     if (status != AMEDIA_OK) {
351         ALOGE("Failed to queueInputBuffer: %d", status);
352         return false;
353     }
354     ++input_fragment_index_;
355     return true;
356 }
357 
FeedEOSInputBuffer(size_t index)358 bool MediaCodecDecoder::FeedEOSInputBuffer(size_t index) {
359     // Timestamp of EOS input buffer is undefined, use 0 here to test decoder
360     // robustness.
361     uint64_t timestamp_us = 0;
362 
363     ALOGV("queueInputBuffer(index=%zu) EOS", index);
364     media_status_t status =
365             AMediaCodec_queueInputBuffer(codec_, index, 0 /* offset */, 0 /* size */, timestamp_us,
366                                          AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM);
367     if (status != AMEDIA_OK) {
368         ALOGE("Failed to queueInputBuffer EOS: %d", status);
369         return false;
370     }
371     return true;
372 }
373 
ReceiveOutputBuffer(int32_t index,const AMediaCodecBufferInfo & info,bool render_buffer)374 bool MediaCodecDecoder::ReceiveOutputBuffer(int32_t index, const AMediaCodecBufferInfo& info,
375                                             bool render_buffer) {
376     size_t out_size = 0;
377     uint8_t* buf = nullptr;
378     if (!surface_) {
379         buf = AMediaCodec_getOutputBuffer(codec_, index, &out_size);
380         if (!buf) {
381             ALOGE("Failed to getOutputBuffer(index=%zu)", index);
382             return false;
383         }
384     }
385 
386     received_outputs_++;
387     ALOGV("ReceiveOutputBuffer(index=%zu, size=%d, flags=%u) #%d", index, info.size, info.flags,
388           received_outputs_);
389 
390     // Do not callback for dummy EOS output (info.size == 0)
391     if (info.size > 0) {
392         for (const auto& callback : output_buffer_ready_cbs_)
393             callback(buf, info.size, received_outputs_);
394     }
395 
396     if (fake_renderer_running_) {
397         std::lock_guard<std::mutex> lock(fake_render_mut_);
398         fake_render_frames_.emplace(index, GetReleaseTimestampNs(received_outputs_));
399         fake_render_cv_.notify_one();
400     } else {
401         media_status_t status =
402                 render_buffer ? AMediaCodec_releaseOutputBufferAtTime(
403                                         codec_, index, GetReleaseTimestampNs(received_outputs_))
404                               : AMediaCodec_releaseOutputBuffer(codec_, index, false /* render */);
405         if (status != AMEDIA_OK) {
406             ALOGE("Failed to releaseOutputBuffer(index=%zu): %d", index, status);
407             return false;
408         }
409     }
410     return true;
411 }
412 
FakeRenderLoop()413 void MediaCodecDecoder::FakeRenderLoop() {
414     while (fake_renderer_running_) {
415         std::pair<int32_t, int64_t> next_frame;
416         {
417             std::unique_lock<std::mutex> lock(fake_render_mut_);
418             fake_render_cv_.wait(lock, [&]() {
419                 return !fake_renderer_running_ || !fake_render_frames_.empty();
420             });
421             if (!fake_renderer_running_) {
422                 break;
423             }
424 
425             next_frame = fake_render_frames_.front();
426             fake_render_frames_.pop();
427         }
428 
429         const uint64_t now = GetCurrentTimeNs();
430         if (now < next_frame.second) {
431             usleep((next_frame.second - now) / 1000);
432         }
433 
434         std::lock_guard<std::mutex> lock(event_queue_mut_);
435         event_queue_.push({.type = FAKE_FRAME_RENDERED, .idx = next_frame.first});
436         event_queue_cv_.notify_one();
437     }
438 }
439 
GetOutputFormat()440 bool MediaCodecDecoder::GetOutputFormat() {
441     AMediaFormat* format = AMediaCodec_getOutputFormat(codec_);
442     bool success = true;
443 
444     // Required formats
445     int32_t width = 0;
446     if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_WIDTH, &width)) {
447         ALOGE("Cannot find width in format.");
448         success = false;
449     }
450 
451     int32_t height = 0;
452     if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_HEIGHT, &height)) {
453         ALOGE("Cannot find height in format.");
454         success = false;
455     }
456 
457     int32_t color_format = 0;
458     if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, &color_format)) {
459         ALOGE("Cannot find color-format in format.");
460         success = false;
461     }
462 
463     // Optional formats
464     int32_t crop_left = 0;
465     int32_t crop_top = 0;
466     int32_t crop_right = width - 1;
467     int32_t crop_bottom = height - 1;
468     // Crop info is only avaiable on NDK version >= Pie.
469     if (!AMediaFormat_getRect(format, AMEDIAFORMAT_KEY_DISPLAY_CROP, &crop_left, &crop_top,
470                               &crop_right, &crop_bottom)) {
471         ALOGD("Cannot find crop window in format. Set as large as frame size.");
472         crop_left = 0;
473         crop_top = 0;
474         crop_right = width - 1;
475         crop_bottom = height - 1;
476     }
477 
478     // In current exiting ARC video decoder crop origin is always at (0,0)
479     if (crop_left != 0 || crop_top != 0) {
480         ALOGE("Crop origin is not (0,0): (%d,%d)", crop_left, crop_top);
481         success = false;
482     }
483 
484     int32_t stride = 0;
485     if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_STRIDE, &stride)) {
486         ALOGD("Cannot find stride in format. Set as frame width.");
487         stride = width;
488     }
489 
490     int32_t slice_height = 0;
491     if (!AMediaFormat_getInt32(format, FORMAT_KEY_SLICE_HEIGHT, &slice_height)) {
492         ALOGD("Cannot find slice-height in format. Set as frame height.");
493         slice_height = height;
494     }
495 
496     for (const auto& callback : output_format_changed_cbs_) {
497         callback(Size(stride, slice_height),
498                  Size(crop_right - crop_left + 1, crop_bottom - crop_top + 1), color_format);
499     }
500     return success;
501 }
502 
GetReleaseTimestampNs(size_t frame_order)503 int64_t MediaCodecDecoder::GetReleaseTimestampNs(size_t frame_order) {
504     assert(base_timestamp_ns_ != 0);
505 
506     return base_timestamp_ns_ + frame_order * kSecToNs / frame_rate_;
507 }
508 
dropped_frame_rate() const509 double MediaCodecDecoder::dropped_frame_rate() const {
510     assert(received_outputs_ > 0);
511 
512     return (double)drop_frame_count_ / (double)received_outputs_;
513 }
514 
515 }  // namespace android
516