1 /*
2  * Copyright 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "NuPlayerDecoder"
19 #include <utils/Log.h>
20 #include <inttypes.h>
21 
22 #include <algorithm>
23 
24 #include "NuPlayerCCDecoder.h"
25 #include "NuPlayerDecoder.h"
26 #include "NuPlayerDrm.h"
27 #include "NuPlayerRenderer.h"
28 #include "NuPlayerSource.h"
29 
30 #include <cutils/properties.h>
31 #include <media/ICrypto.h>
32 #include <media/MediaCodecBuffer.h>
33 #include <media/stagefright/foundation/ABuffer.h>
34 #include <media/stagefright/foundation/ADebug.h>
35 #include <media/stagefright/foundation/AMessage.h>
36 #include <media/stagefright/MediaBuffer.h>
37 #include <media/stagefright/MediaCodec.h>
38 #include <media/stagefright/MediaDefs.h>
39 #include <media/stagefright/MediaErrors.h>
40 #include <media/stagefright/SurfaceUtils.h>
41 #include <gui/Surface.h>
42 
43 #include "avc_utils.h"
44 #include "ATSParser.h"
45 
46 namespace android {
47 
48 static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display
49 
50 // The default total video frame rate of a stream when that info is not available from
51 // the source.
52 static float kDefaultVideoFrameRateTotal = 30.f;
53 
getAudioDeepBufferSetting()54 static inline bool getAudioDeepBufferSetting() {
55     return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
56 }
57 
Decoder(const sp<AMessage> & notify,const sp<Source> & source,pid_t pid,uid_t uid,const sp<Renderer> & renderer,const sp<Surface> & surface,const sp<CCDecoder> & ccDecoder)58 NuPlayer::Decoder::Decoder(
59         const sp<AMessage> &notify,
60         const sp<Source> &source,
61         pid_t pid,
62         uid_t uid,
63         const sp<Renderer> &renderer,
64         const sp<Surface> &surface,
65         const sp<CCDecoder> &ccDecoder)
66     : DecoderBase(notify),
67       mSurface(surface),
68       mSource(source),
69       mRenderer(renderer),
70       mCCDecoder(ccDecoder),
71       mPid(pid),
72       mUid(uid),
73       mSkipRenderingUntilMediaTimeUs(-1ll),
74       mNumFramesTotal(0ll),
75       mNumInputFramesDropped(0ll),
76       mNumOutputFramesDropped(0ll),
77       mVideoWidth(0),
78       mVideoHeight(0),
79       mIsAudio(true),
80       mIsVideoAVC(false),
81       mIsSecure(false),
82       mIsEncrypted(false),
83       mIsEncryptedObservedEarlier(false),
84       mFormatChangePending(false),
85       mTimeChangePending(false),
86       mFrameRateTotal(kDefaultVideoFrameRateTotal),
87       mPlaybackSpeed(1.0f),
88       mNumVideoTemporalLayerTotal(1), // decode all layers
89       mNumVideoTemporalLayerAllowed(1),
90       mCurrentMaxVideoTemporalLayerId(0),
91       mResumePending(false),
92       mComponentName("decoder") {
93     mCodecLooper = new ALooper;
94     mCodecLooper->setName("NPDecoder-CL");
95     mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
96     mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;
97 }
98 
~Decoder()99 NuPlayer::Decoder::~Decoder() {
100     // Need to stop looper first since mCodec could be accessed on the mDecoderLooper.
101     stopLooper();
102     if (mCodec != NULL) {
103         mCodec->release();
104     }
105     releaseAndResetMediaBuffers();
106 }
107 
getStats() const108 sp<AMessage> NuPlayer::Decoder::getStats() const {
109     mStats->setInt64("frames-total", mNumFramesTotal);
110     mStats->setInt64("frames-dropped-input", mNumInputFramesDropped);
111     mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped);
112     return mStats;
113 }
114 
setVideoSurface(const sp<Surface> & surface)115 status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) {
116     if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) {
117         return BAD_VALUE;
118     }
119 
120     sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
121 
122     msg->setObject("surface", surface);
123     sp<AMessage> response;
124     status_t err = msg->postAndAwaitResponse(&response);
125     if (err == OK && response != NULL) {
126         CHECK(response->findInt32("err", &err));
127     }
128     return err;
129 }
130 
onMessageReceived(const sp<AMessage> & msg)131 void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
132     ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
133 
134     switch (msg->what()) {
135         case kWhatCodecNotify:
136         {
137             int32_t cbID;
138             CHECK(msg->findInt32("callbackID", &cbID));
139 
140             ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d",
141                     mIsAudio ? "audio" : "video", cbID, mPaused);
142 
143             if (mPaused) {
144                 break;
145             }
146 
147             switch (cbID) {
148                 case MediaCodec::CB_INPUT_AVAILABLE:
149                 {
150                     int32_t index;
151                     CHECK(msg->findInt32("index", &index));
152 
153                     handleAnInputBuffer(index);
154                     break;
155                 }
156 
157                 case MediaCodec::CB_OUTPUT_AVAILABLE:
158                 {
159                     int32_t index;
160                     size_t offset;
161                     size_t size;
162                     int64_t timeUs;
163                     int32_t flags;
164 
165                     CHECK(msg->findInt32("index", &index));
166                     CHECK(msg->findSize("offset", &offset));
167                     CHECK(msg->findSize("size", &size));
168                     CHECK(msg->findInt64("timeUs", &timeUs));
169                     CHECK(msg->findInt32("flags", &flags));
170 
171                     handleAnOutputBuffer(index, offset, size, timeUs, flags);
172                     break;
173                 }
174 
175                 case MediaCodec::CB_OUTPUT_FORMAT_CHANGED:
176                 {
177                     sp<AMessage> format;
178                     CHECK(msg->findMessage("format", &format));
179 
180                     handleOutputFormatChange(format);
181                     break;
182                 }
183 
184                 case MediaCodec::CB_ERROR:
185                 {
186                     status_t err;
187                     CHECK(msg->findInt32("err", &err));
188                     ALOGE("Decoder (%s) reported error : 0x%x",
189                             mIsAudio ? "audio" : "video", err);
190 
191                     handleError(err);
192                     break;
193                 }
194 
195                 default:
196                 {
197                     TRESPASS();
198                     break;
199                 }
200             }
201 
202             break;
203         }
204 
205         case kWhatRenderBuffer:
206         {
207             if (!isStaleReply(msg)) {
208                 onRenderBuffer(msg);
209             }
210             break;
211         }
212 
213         case kWhatAudioOutputFormatChanged:
214         {
215             if (!isStaleReply(msg)) {
216                 status_t err;
217                 if (msg->findInt32("err", &err) && err != OK) {
218                     ALOGE("Renderer reported 0x%x when changing audio output format", err);
219                     handleError(err);
220                 }
221             }
222             break;
223         }
224 
225         case kWhatSetVideoSurface:
226         {
227             sp<AReplyToken> replyID;
228             CHECK(msg->senderAwaitsResponse(&replyID));
229 
230             sp<RefBase> obj;
231             CHECK(msg->findObject("surface", &obj));
232             sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null
233             int32_t err = INVALID_OPERATION;
234             // NOTE: in practice mSurface is always non-null, but checking here for completeness
235             if (mCodec != NULL && mSurface != NULL) {
236                 // TODO: once AwesomePlayer is removed, remove this automatic connecting
237                 // to the surface by MediaPlayerService.
238                 //
239                 // at this point MediaPlayerService::client has already connected to the
240                 // surface, which MediaCodec does not expect
241                 err = nativeWindowDisconnect(surface.get(), "kWhatSetVideoSurface(surface)");
242                 if (err == OK) {
243                     err = mCodec->setSurface(surface);
244                     ALOGI_IF(err, "codec setSurface returned: %d", err);
245                     if (err == OK) {
246                         // reconnect to the old surface as MPS::Client will expect to
247                         // be able to disconnect from it.
248                         (void)nativeWindowConnect(mSurface.get(), "kWhatSetVideoSurface(mSurface)");
249                         mSurface = surface;
250                     }
251                 }
252                 if (err != OK) {
253                     // reconnect to the new surface on error as MPS::Client will expect to
254                     // be able to disconnect from it.
255                     (void)nativeWindowConnect(surface.get(), "kWhatSetVideoSurface(err)");
256                 }
257             }
258 
259             sp<AMessage> response = new AMessage;
260             response->setInt32("err", err);
261             response->postReply(replyID);
262             break;
263         }
264 
265         case kWhatDrmReleaseCrypto:
266         {
267             ALOGV("kWhatDrmReleaseCrypto");
268             onReleaseCrypto(msg);
269             break;
270         }
271 
272         default:
273             DecoderBase::onMessageReceived(msg);
274             break;
275     }
276 }
277 
onConfigure(const sp<AMessage> & format)278 void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
279     CHECK(mCodec == NULL);
280 
281     mFormatChangePending = false;
282     mTimeChangePending = false;
283 
284     ++mBufferGeneration;
285 
286     AString mime;
287     CHECK(format->findString("mime", &mime));
288 
289     mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
290     mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
291 
292     mComponentName = mime;
293     mComponentName.append(" decoder");
294     ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
295 
296     mCodec = MediaCodec::CreateByType(
297             mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid);
298     int32_t secure = 0;
299     if (format->findInt32("secure", &secure) && secure != 0) {
300         if (mCodec != NULL) {
301             mCodec->getName(&mComponentName);
302             mComponentName.append(".secure");
303             mCodec->release();
304             ALOGI("[%s] creating", mComponentName.c_str());
305             mCodec = MediaCodec::CreateByComponentName(
306                     mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid);
307         }
308     }
309     if (mCodec == NULL) {
310         ALOGE("Failed to create %s%s decoder",
311                 (secure ? "secure " : ""), mime.c_str());
312         handleError(UNKNOWN_ERROR);
313         return;
314     }
315     mIsSecure = secure;
316 
317     mCodec->getName(&mComponentName);
318 
319     status_t err;
320     if (mSurface != NULL) {
321         // disconnect from surface as MediaCodec will reconnect
322         err = nativeWindowDisconnect(mSurface.get(), "onConfigure");
323         // We treat this as a warning, as this is a preparatory step.
324         // Codec will try to connect to the surface, which is where
325         // any error signaling will occur.
326         ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
327     }
328 
329     // Modular DRM
330     void *pCrypto;
331     if (!format->findPointer("crypto", &pCrypto)) {
332         pCrypto = NULL;
333     }
334     sp<ICrypto> crypto = (ICrypto*)pCrypto;
335     // non-encrypted source won't have a crypto
336     mIsEncrypted = (crypto != NULL);
337     // configure is called once; still using OR in case the behavior changes.
338     mIsEncryptedObservedEarlier = mIsEncryptedObservedEarlier || mIsEncrypted;
339     ALOGV("onConfigure mCrypto: %p (%d)  mIsSecure: %d",
340             crypto.get(), (crypto != NULL ? crypto->getStrongCount() : 0), mIsSecure);
341 
342     err = mCodec->configure(
343             format, mSurface, crypto, 0 /* flags */);
344 
345     if (err != OK) {
346         ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
347         mCodec->release();
348         mCodec.clear();
349         handleError(err);
350         return;
351     }
352     rememberCodecSpecificData(format);
353 
354     // the following should work in configured state
355     CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
356     CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
357 
358     mStats->setString("mime", mime.c_str());
359     mStats->setString("component-name", mComponentName.c_str());
360 
361     if (!mIsAudio) {
362         int32_t width, height;
363         if (mOutputFormat->findInt32("width", &width)
364                 && mOutputFormat->findInt32("height", &height)) {
365             mStats->setInt32("width", width);
366             mStats->setInt32("height", height);
367         }
368     }
369 
370     sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
371     mCodec->setCallback(reply);
372 
373     err = mCodec->start();
374     if (err != OK) {
375         ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
376         mCodec->release();
377         mCodec.clear();
378         handleError(err);
379         return;
380     }
381 
382     releaseAndResetMediaBuffers();
383 
384     mPaused = false;
385     mResumePending = false;
386 }
387 
onSetParameters(const sp<AMessage> & params)388 void NuPlayer::Decoder::onSetParameters(const sp<AMessage> &params) {
389     bool needAdjustLayers = false;
390     float frameRateTotal;
391     if (params->findFloat("frame-rate-total", &frameRateTotal)
392             && mFrameRateTotal != frameRateTotal) {
393         needAdjustLayers = true;
394         mFrameRateTotal = frameRateTotal;
395     }
396 
397     int32_t numVideoTemporalLayerTotal;
398     if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal)
399             && numVideoTemporalLayerTotal >= 0
400             && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers
401             && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) {
402         needAdjustLayers = true;
403         mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1);
404     }
405 
406     if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) {
407         // TODO: For now, layer fps is calculated for some specific architectures.
408         // But it really should be extracted from the stream.
409         mVideoTemporalLayerAggregateFps[0] =
410             mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1));
411         for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) {
412             mVideoTemporalLayerAggregateFps[i] =
413                 mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i))
414                 + mVideoTemporalLayerAggregateFps[i - 1];
415         }
416     }
417 
418     float playbackSpeed;
419     if (params->findFloat("playback-speed", &playbackSpeed)
420             && mPlaybackSpeed != playbackSpeed) {
421         needAdjustLayers = true;
422         mPlaybackSpeed = playbackSpeed;
423     }
424 
425     if (needAdjustLayers) {
426         float decodeFrameRate = mFrameRateTotal;
427         // enable temporal layering optimization only if we know the layering depth
428         if (mNumVideoTemporalLayerTotal > 1) {
429             int32_t layerId;
430             for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) {
431                 if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed
432                         >= kDisplayRefreshingRate * 0.9) {
433                     break;
434                 }
435             }
436             mNumVideoTemporalLayerAllowed = layerId + 1;
437             decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId];
438         }
439         ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g",
440                 mNumVideoTemporalLayerAllowed, decodeFrameRate);
441 
442         if (mCodec == NULL) {
443             ALOGW("onSetParameters called before codec is created.");
444             return;
445         }
446 
447         sp<AMessage> codecParams = new AMessage();
448         codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed);
449         mCodec->setParameters(codecParams);
450     }
451 }
452 
onSetRenderer(const sp<Renderer> & renderer)453 void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
454     mRenderer = renderer;
455 }
456 
onResume(bool notifyComplete)457 void NuPlayer::Decoder::onResume(bool notifyComplete) {
458     mPaused = false;
459 
460     if (notifyComplete) {
461         mResumePending = true;
462     }
463     mCodec->start();
464 }
465 
doFlush(bool notifyComplete)466 void NuPlayer::Decoder::doFlush(bool notifyComplete) {
467     if (mCCDecoder != NULL) {
468         mCCDecoder->flush();
469     }
470 
471     if (mRenderer != NULL) {
472         mRenderer->flush(mIsAudio, notifyComplete);
473         mRenderer->signalTimeDiscontinuity();
474     }
475 
476     status_t err = OK;
477     if (mCodec != NULL) {
478         err = mCodec->flush();
479         mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
480         ++mBufferGeneration;
481     }
482 
483     if (err != OK) {
484         ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
485         handleError(err);
486         // finish with posting kWhatFlushCompleted.
487         // we attempt to release the buffers even if flush fails.
488     }
489     releaseAndResetMediaBuffers();
490     mPaused = true;
491 }
492 
493 
onFlush()494 void NuPlayer::Decoder::onFlush() {
495     doFlush(true);
496 
497     if (isDiscontinuityPending()) {
498         // This could happen if the client starts seeking/shutdown
499         // after we queued an EOS for discontinuities.
500         // We can consider discontinuity handled.
501         finishHandleDiscontinuity(false /* flushOnTimeChange */);
502     }
503 
504     sp<AMessage> notify = mNotify->dup();
505     notify->setInt32("what", kWhatFlushCompleted);
506     notify->post();
507 }
508 
onShutdown(bool notifyComplete)509 void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
510     status_t err = OK;
511 
512     // if there is a pending resume request, notify complete now
513     notifyResumeCompleteIfNecessary();
514 
515     if (mCodec != NULL) {
516         err = mCodec->release();
517         mCodec = NULL;
518         ++mBufferGeneration;
519 
520         if (mSurface != NULL) {
521             // reconnect to surface as MediaCodec disconnected from it
522             status_t error = nativeWindowConnect(mSurface.get(), "onShutdown");
523             ALOGW_IF(error != NO_ERROR,
524                     "[%s] failed to connect to native window, error=%d",
525                     mComponentName.c_str(), error);
526         }
527         mComponentName = "decoder";
528     }
529 
530     releaseAndResetMediaBuffers();
531 
532     if (err != OK) {
533         ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
534         handleError(err);
535         // finish with posting kWhatShutdownCompleted.
536     }
537 
538     if (notifyComplete) {
539         sp<AMessage> notify = mNotify->dup();
540         notify->setInt32("what", kWhatShutdownCompleted);
541         notify->post();
542         mPaused = true;
543     }
544 }
545 
546 /*
547  * returns true if we should request more data
548  */
doRequestBuffers()549 bool NuPlayer::Decoder::doRequestBuffers() {
550     if (isDiscontinuityPending()) {
551         return false;
552     }
553     status_t err = OK;
554     while (err == OK && !mDequeuedInputBuffers.empty()) {
555         size_t bufferIx = *mDequeuedInputBuffers.begin();
556         sp<AMessage> msg = new AMessage();
557         msg->setSize("buffer-ix", bufferIx);
558         err = fetchInputData(msg);
559         if (err != OK && err != ERROR_END_OF_STREAM) {
560             // if EOS, need to queue EOS buffer
561             break;
562         }
563         mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
564 
565         if (!mPendingInputMessages.empty()
566                 || !onInputBufferFetched(msg)) {
567             mPendingInputMessages.push_back(msg);
568         }
569     }
570 
571     return err == -EWOULDBLOCK
572             && mSource->feedMoreTSData() == OK;
573 }
574 
handleError(int32_t err)575 void NuPlayer::Decoder::handleError(int32_t err)
576 {
577     // We cannot immediately release the codec due to buffers still outstanding
578     // in the renderer.  We signal to the player the error so it can shutdown/release the
579     // decoder after flushing and increment the generation to discard unnecessary messages.
580 
581     ++mBufferGeneration;
582 
583     sp<AMessage> notify = mNotify->dup();
584     notify->setInt32("what", kWhatError);
585     notify->setInt32("err", err);
586     notify->post();
587 }
588 
releaseCrypto()589 status_t NuPlayer::Decoder::releaseCrypto()
590 {
591     ALOGV("releaseCrypto");
592 
593     sp<AMessage> msg = new AMessage(kWhatDrmReleaseCrypto, this);
594 
595     sp<AMessage> response;
596     status_t status = msg->postAndAwaitResponse(&response);
597     if (status == OK && response != NULL) {
598         CHECK(response->findInt32("status", &status));
599         ALOGV("releaseCrypto ret: %d ", status);
600     } else {
601         ALOGE("releaseCrypto err: %d", status);
602     }
603 
604     return status;
605 }
606 
onReleaseCrypto(const sp<AMessage> & msg)607 void NuPlayer::Decoder::onReleaseCrypto(const sp<AMessage>& msg)
608 {
609     status_t status = INVALID_OPERATION;
610     if (mCodec != NULL) {
611         status = mCodec->releaseCrypto();
612     } else {
613         // returning OK if the codec has been already released
614         status = OK;
615         ALOGE("onReleaseCrypto No mCodec. err: %d", status);
616     }
617 
618     sp<AMessage> response = new AMessage;
619     response->setInt32("status", status);
620     // Clearing the state as it's tied to crypto. mIsEncryptedObservedEarlier is sticky though
621     // and lasts for the lifetime of this codec. See its use in fetchInputData.
622     mIsEncrypted = false;
623 
624     sp<AReplyToken> replyID;
625     CHECK(msg->senderAwaitsResponse(&replyID));
626     response->postReply(replyID);
627 }
628 
handleAnInputBuffer(size_t index)629 bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
630     if (isDiscontinuityPending()) {
631         return false;
632     }
633 
634     sp<MediaCodecBuffer> buffer;
635     mCodec->getInputBuffer(index, &buffer);
636 
637     if (buffer == NULL) {
638         handleError(UNKNOWN_ERROR);
639         return false;
640     }
641 
642     if (index >= mInputBuffers.size()) {
643         for (size_t i = mInputBuffers.size(); i <= index; ++i) {
644             mInputBuffers.add();
645             mMediaBuffers.add();
646             mInputBufferIsDequeued.add();
647             mMediaBuffers.editItemAt(i) = NULL;
648             mInputBufferIsDequeued.editItemAt(i) = false;
649         }
650     }
651     mInputBuffers.editItemAt(index) = buffer;
652 
653     //CHECK_LT(bufferIx, mInputBuffers.size());
654 
655     if (mMediaBuffers[index] != NULL) {
656         mMediaBuffers[index]->release();
657         mMediaBuffers.editItemAt(index) = NULL;
658     }
659     mInputBufferIsDequeued.editItemAt(index) = true;
660 
661     if (!mCSDsToSubmit.isEmpty()) {
662         sp<AMessage> msg = new AMessage();
663         msg->setSize("buffer-ix", index);
664 
665         sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
666         ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
667         msg->setBuffer("buffer", buffer);
668         mCSDsToSubmit.removeAt(0);
669         if (!onInputBufferFetched(msg)) {
670             handleError(UNKNOWN_ERROR);
671             return false;
672         }
673         return true;
674     }
675 
676     while (!mPendingInputMessages.empty()) {
677         sp<AMessage> msg = *mPendingInputMessages.begin();
678         if (!onInputBufferFetched(msg)) {
679             break;
680         }
681         mPendingInputMessages.erase(mPendingInputMessages.begin());
682     }
683 
684     if (!mInputBufferIsDequeued.editItemAt(index)) {
685         return true;
686     }
687 
688     mDequeuedInputBuffers.push_back(index);
689 
690     onRequestInputBuffers();
691     return true;
692 }
693 
handleAnOutputBuffer(size_t index,size_t offset,size_t size,int64_t timeUs,int32_t flags)694 bool NuPlayer::Decoder::handleAnOutputBuffer(
695         size_t index,
696         size_t offset,
697         size_t size,
698         int64_t timeUs,
699         int32_t flags) {
700 //    CHECK_LT(bufferIx, mOutputBuffers.size());
701     sp<MediaCodecBuffer> buffer;
702     mCodec->getOutputBuffer(index, &buffer);
703 
704     if (buffer == NULL) {
705         handleError(UNKNOWN_ERROR);
706         return false;
707     }
708 
709     if (index >= mOutputBuffers.size()) {
710         for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
711             mOutputBuffers.add();
712         }
713     }
714 
715     mOutputBuffers.editItemAt(index) = buffer;
716 
717     buffer->setRange(offset, size);
718     buffer->meta()->clear();
719     buffer->meta()->setInt64("timeUs", timeUs);
720 
721     bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
722     // we do not expect CODECCONFIG or SYNCFRAME for decoder
723 
724     sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
725     reply->setSize("buffer-ix", index);
726     reply->setInt32("generation", mBufferGeneration);
727 
728     if (eos) {
729         ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video");
730 
731         buffer->meta()->setInt32("eos", true);
732         reply->setInt32("eos", true);
733     } else if (mSkipRenderingUntilMediaTimeUs >= 0) {
734         if (timeUs < mSkipRenderingUntilMediaTimeUs) {
735             ALOGV("[%s] dropping buffer at time %lld as requested.",
736                      mComponentName.c_str(), (long long)timeUs);
737 
738             reply->post();
739             return true;
740         }
741 
742         mSkipRenderingUntilMediaTimeUs = -1;
743     }
744 
745     mNumFramesTotal += !mIsAudio;
746 
747     // wait until 1st frame comes out to signal resume complete
748     notifyResumeCompleteIfNecessary();
749 
750     if (mRenderer != NULL) {
751         // send the buffer to renderer.
752         mRenderer->queueBuffer(mIsAudio, buffer, reply);
753         if (eos && !isDiscontinuityPending()) {
754             mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
755         }
756     }
757 
758     return true;
759 }
760 
handleOutputFormatChange(const sp<AMessage> & format)761 void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) {
762     if (!mIsAudio) {
763         int32_t width, height;
764         if (format->findInt32("width", &width)
765                 && format->findInt32("height", &height)) {
766             mStats->setInt32("width", width);
767             mStats->setInt32("height", height);
768         }
769         sp<AMessage> notify = mNotify->dup();
770         notify->setInt32("what", kWhatVideoSizeChanged);
771         notify->setMessage("format", format);
772         notify->post();
773     } else if (mRenderer != NULL) {
774         uint32_t flags;
775         int64_t durationUs;
776         bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
777         if (getAudioDeepBufferSetting() // override regardless of source duration
778                 || (mSource->getDuration(&durationUs) == OK
779                         && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) {
780             flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
781         } else {
782             flags = AUDIO_OUTPUT_FLAG_NONE;
783         }
784 
785         sp<AMessage> reply = new AMessage(kWhatAudioOutputFormatChanged, this);
786         reply->setInt32("generation", mBufferGeneration);
787         mRenderer->changeAudioFormat(
788                 format, false /* offloadOnly */, hasVideo,
789                 flags, mSource->isStreaming(), reply);
790     }
791 }
792 
releaseAndResetMediaBuffers()793 void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
794     for (size_t i = 0; i < mMediaBuffers.size(); i++) {
795         if (mMediaBuffers[i] != NULL) {
796             mMediaBuffers[i]->release();
797             mMediaBuffers.editItemAt(i) = NULL;
798         }
799     }
800     mMediaBuffers.resize(mInputBuffers.size());
801     for (size_t i = 0; i < mMediaBuffers.size(); i++) {
802         mMediaBuffers.editItemAt(i) = NULL;
803     }
804     mInputBufferIsDequeued.clear();
805     mInputBufferIsDequeued.resize(mInputBuffers.size());
806     for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
807         mInputBufferIsDequeued.editItemAt(i) = false;
808     }
809 
810     mPendingInputMessages.clear();
811     mDequeuedInputBuffers.clear();
812     mSkipRenderingUntilMediaTimeUs = -1;
813 }
814 
requestCodecNotification()815 void NuPlayer::Decoder::requestCodecNotification() {
816     if (mCodec != NULL) {
817         sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
818         reply->setInt32("generation", mBufferGeneration);
819         mCodec->requestActivityNotification(reply);
820     }
821 }
822 
isStaleReply(const sp<AMessage> & msg)823 bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
824     int32_t generation;
825     CHECK(msg->findInt32("generation", &generation));
826     return generation != mBufferGeneration;
827 }
828 
fetchInputData(sp<AMessage> & reply)829 status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
830     sp<ABuffer> accessUnit;
831     bool dropAccessUnit = true;
832     do {
833         status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
834 
835         if (err == -EWOULDBLOCK) {
836             return err;
837         } else if (err != OK) {
838             if (err == INFO_DISCONTINUITY) {
839                 int32_t type;
840                 CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
841 
842                 bool formatChange =
843                     (mIsAudio &&
844                      (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
845                     || (!mIsAudio &&
846                             (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
847 
848                 bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
849 
850                 ALOGI("%s discontinuity (format=%d, time=%d)",
851                         mIsAudio ? "audio" : "video", formatChange, timeChange);
852 
853                 bool seamlessFormatChange = false;
854                 sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
855                 if (formatChange) {
856                     seamlessFormatChange =
857                         supportsSeamlessFormatChange(newFormat);
858                     // treat seamless format change separately
859                     formatChange = !seamlessFormatChange;
860                 }
861 
862                 // For format or time change, return EOS to queue EOS input,
863                 // then wait for EOS on output.
864                 if (formatChange /* not seamless */) {
865                     mFormatChangePending = true;
866                     err = ERROR_END_OF_STREAM;
867                 } else if (timeChange) {
868                     rememberCodecSpecificData(newFormat);
869                     mTimeChangePending = true;
870                     err = ERROR_END_OF_STREAM;
871                 } else if (seamlessFormatChange) {
872                     // reuse existing decoder and don't flush
873                     rememberCodecSpecificData(newFormat);
874                     continue;
875                 } else {
876                     // This stream is unaffected by the discontinuity
877                     return -EWOULDBLOCK;
878                 }
879             }
880 
881             // reply should only be returned without a buffer set
882             // when there is an error (including EOS)
883             CHECK(err != OK);
884 
885             reply->setInt32("err", err);
886             return ERROR_END_OF_STREAM;
887         }
888 
889         dropAccessUnit = false;
890         if (!mIsAudio && !mIsEncrypted) {
891             // Extra safeguard if higher-level behavior changes. Otherwise, not required now.
892             // Preventing the buffer from being processed (and sent to codec) if this is a later
893             // round of playback but this time without prepareDrm. Or if there is a race between
894             // stop (which is not blocking) and releaseDrm allowing buffers being processed after
895             // Crypto has been released (GenericSource currently prevents this race though).
896             // Particularly doing this check before IsAVCReferenceFrame call to prevent parsing
897             // of encrypted data.
898             if (mIsEncryptedObservedEarlier) {
899                 ALOGE("fetchInputData: mismatched mIsEncrypted/mIsEncryptedObservedEarlier (0/1)");
900 
901                 return INVALID_OPERATION;
902             }
903 
904             int32_t layerId = 0;
905             bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId);
906             if (mRenderer->getVideoLateByUs() > 100000ll
907                     && mIsVideoAVC
908                     && !IsAVCReferenceFrame(accessUnit)) {
909                 dropAccessUnit = true;
910             } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) {
911                 // Add only one layer each time.
912                 if (layerId > mCurrentMaxVideoTemporalLayerId + 1
913                         || layerId >= mNumVideoTemporalLayerAllowed) {
914                     dropAccessUnit = true;
915                     ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d",
916                             layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed,
917                             mCurrentMaxVideoTemporalLayerId);
918                 } else if (layerId > mCurrentMaxVideoTemporalLayerId) {
919                     mCurrentMaxVideoTemporalLayerId = layerId;
920                 } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1 && IsIDR(accessUnit)) {
921                     mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1;
922                 }
923             }
924             if (dropAccessUnit) {
925                 if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) {
926                     mCurrentMaxVideoTemporalLayerId = layerId - 1;
927                 }
928                 ++mNumInputFramesDropped;
929             }
930         }
931     } while (dropAccessUnit);
932 
933     // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
934 #if 0
935     int64_t mediaTimeUs;
936     CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
937     ALOGV("[%s] feeding input buffer at media time %.3f",
938          mIsAudio ? "audio" : "video",
939          mediaTimeUs / 1E6);
940 #endif
941 
942     if (mCCDecoder != NULL) {
943         mCCDecoder->decode(accessUnit);
944     }
945 
946     reply->setBuffer("buffer", accessUnit);
947 
948     return OK;
949 }
950 
onInputBufferFetched(const sp<AMessage> & msg)951 bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
952     size_t bufferIx;
953     CHECK(msg->findSize("buffer-ix", &bufferIx));
954     CHECK_LT(bufferIx, mInputBuffers.size());
955     sp<MediaCodecBuffer> codecBuffer = mInputBuffers[bufferIx];
956 
957     sp<ABuffer> buffer;
958     bool hasBuffer = msg->findBuffer("buffer", &buffer);
959     bool needsCopy = true;
960 
961     if (buffer == NULL /* includes !hasBuffer */) {
962         int32_t streamErr = ERROR_END_OF_STREAM;
963         CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
964 
965         CHECK(streamErr != OK);
966 
967         // attempt to queue EOS
968         status_t err = mCodec->queueInputBuffer(
969                 bufferIx,
970                 0,
971                 0,
972                 0,
973                 MediaCodec::BUFFER_FLAG_EOS);
974         if (err == OK) {
975             mInputBufferIsDequeued.editItemAt(bufferIx) = false;
976         } else if (streamErr == ERROR_END_OF_STREAM) {
977             streamErr = err;
978             // err will not be ERROR_END_OF_STREAM
979         }
980 
981         if (streamErr != ERROR_END_OF_STREAM) {
982             ALOGE("Stream error for %s (err=%d), EOS %s queued",
983                     mComponentName.c_str(),
984                     streamErr,
985                     err == OK ? "successfully" : "unsuccessfully");
986             handleError(streamErr);
987         }
988     } else {
989         sp<AMessage> extra;
990         if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
991             int64_t resumeAtMediaTimeUs;
992             if (extra->findInt64(
993                         "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
994                 ALOGI("[%s] suppressing rendering until %lld us",
995                         mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
996                 mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
997             }
998         }
999 
1000         int64_t timeUs = 0;
1001         uint32_t flags = 0;
1002         CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1003 
1004         int32_t eos, csd;
1005         // we do not expect SYNCFRAME for decoder
1006         if (buffer->meta()->findInt32("eos", &eos) && eos) {
1007             flags |= MediaCodec::BUFFER_FLAG_EOS;
1008         } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
1009             flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
1010         }
1011 
1012         // Modular DRM
1013         MediaBuffer *mediaBuf = NULL;
1014         NuPlayerDrm::CryptoInfo *cryptInfo = NULL;
1015 
1016         // copy into codec buffer
1017         if (needsCopy) {
1018             if (buffer->size() > codecBuffer->capacity()) {
1019                 handleError(ERROR_BUFFER_TOO_SMALL);
1020                 mDequeuedInputBuffers.push_back(bufferIx);
1021                 return false;
1022             }
1023 
1024             if (buffer->data() != NULL) {
1025                 codecBuffer->setRange(0, buffer->size());
1026                 memcpy(codecBuffer->data(), buffer->data(), buffer->size());
1027             } else { // No buffer->data()
1028                 //Modular DRM
1029                 mediaBuf = (MediaBuffer*)buffer->getMediaBufferBase();
1030                 if (mediaBuf != NULL) {
1031                     codecBuffer->setRange(0, mediaBuf->size());
1032                     memcpy(codecBuffer->data(), mediaBuf->data(), mediaBuf->size());
1033 
1034                     sp<MetaData> meta_data = mediaBuf->meta_data();
1035                     cryptInfo = NuPlayerDrm::getSampleCryptoInfo(meta_data);
1036 
1037                     // since getMediaBuffer() has incremented the refCount
1038                     mediaBuf->release();
1039                 } else { // No mediaBuf
1040                     ALOGE("onInputBufferFetched: buffer->data()/mediaBuf are NULL for %p",
1041                             buffer.get());
1042                     handleError(UNKNOWN_ERROR);
1043                     return false;
1044                 }
1045             } // buffer->data()
1046         } // needsCopy
1047 
1048         status_t err;
1049         AString errorDetailMsg;
1050         if (cryptInfo != NULL) {
1051             err = mCodec->queueSecureInputBuffer(
1052                     bufferIx,
1053                     codecBuffer->offset(),
1054                     cryptInfo->subSamples,
1055                     cryptInfo->numSubSamples,
1056                     cryptInfo->key,
1057                     cryptInfo->iv,
1058                     cryptInfo->mode,
1059                     cryptInfo->pattern,
1060                     timeUs,
1061                     flags,
1062                     &errorDetailMsg);
1063             // synchronous call so done with cryptInfo here
1064             free(cryptInfo);
1065         } else {
1066             err = mCodec->queueInputBuffer(
1067                     bufferIx,
1068                     codecBuffer->offset(),
1069                     codecBuffer->size(),
1070                     timeUs,
1071                     flags,
1072                     &errorDetailMsg);
1073         } // no cryptInfo
1074 
1075         if (err != OK) {
1076             ALOGE("onInputBufferFetched: queue%sInputBuffer failed for %s (err=%d, %s)",
1077                     (cryptInfo != NULL ? "Secure" : ""),
1078                     mComponentName.c_str(), err, errorDetailMsg.c_str());
1079             handleError(err);
1080         } else {
1081             mInputBufferIsDequeued.editItemAt(bufferIx) = false;
1082         }
1083 
1084     }   // buffer != NULL
1085     return true;
1086 }
1087 
onRenderBuffer(const sp<AMessage> & msg)1088 void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
1089     status_t err;
1090     int32_t render;
1091     size_t bufferIx;
1092     int32_t eos;
1093     CHECK(msg->findSize("buffer-ix", &bufferIx));
1094 
1095     if (!mIsAudio) {
1096         int64_t timeUs;
1097         sp<MediaCodecBuffer> buffer = mOutputBuffers[bufferIx];
1098         buffer->meta()->findInt64("timeUs", &timeUs);
1099 
1100         if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
1101             mCCDecoder->display(timeUs);
1102         }
1103     }
1104 
1105     if (msg->findInt32("render", &render) && render) {
1106         int64_t timestampNs;
1107         CHECK(msg->findInt64("timestampNs", &timestampNs));
1108         err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
1109     } else {
1110         mNumOutputFramesDropped += !mIsAudio;
1111         err = mCodec->releaseOutputBuffer(bufferIx);
1112     }
1113     if (err != OK) {
1114         ALOGE("failed to release output buffer for %s (err=%d)",
1115                 mComponentName.c_str(), err);
1116         handleError(err);
1117     }
1118     if (msg->findInt32("eos", &eos) && eos
1119             && isDiscontinuityPending()) {
1120         finishHandleDiscontinuity(true /* flushOnTimeChange */);
1121     }
1122 }
1123 
isDiscontinuityPending() const1124 bool NuPlayer::Decoder::isDiscontinuityPending() const {
1125     return mFormatChangePending || mTimeChangePending;
1126 }
1127 
finishHandleDiscontinuity(bool flushOnTimeChange)1128 void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) {
1129     ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d",
1130             mFormatChangePending, mTimeChangePending, flushOnTimeChange);
1131 
1132     // If we have format change, pause and wait to be killed;
1133     // If we have time change only, flush and restart fetching.
1134 
1135     if (mFormatChangePending) {
1136         mPaused = true;
1137     } else if (mTimeChangePending) {
1138         if (flushOnTimeChange) {
1139             doFlush(false /* notifyComplete */);
1140             signalResume(false /* notifyComplete */);
1141         }
1142     }
1143 
1144     // Notify NuPlayer to either shutdown decoder, or rescan sources
1145     sp<AMessage> msg = mNotify->dup();
1146     msg->setInt32("what", kWhatInputDiscontinuity);
1147     msg->setInt32("formatChange", mFormatChangePending);
1148     msg->post();
1149 
1150     mFormatChangePending = false;
1151     mTimeChangePending = false;
1152 }
1153 
supportsSeamlessAudioFormatChange(const sp<AMessage> & targetFormat) const1154 bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
1155         const sp<AMessage> &targetFormat) const {
1156     if (targetFormat == NULL) {
1157         return true;
1158     }
1159 
1160     AString mime;
1161     if (!targetFormat->findString("mime", &mime)) {
1162         return false;
1163     }
1164 
1165     if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
1166         // field-by-field comparison
1167         const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
1168         for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
1169             int32_t oldVal, newVal;
1170             if (!mInputFormat->findInt32(keys[i], &oldVal) ||
1171                     !targetFormat->findInt32(keys[i], &newVal) ||
1172                     oldVal != newVal) {
1173                 return false;
1174             }
1175         }
1176 
1177         sp<ABuffer> oldBuf, newBuf;
1178         if (mInputFormat->findBuffer("csd-0", &oldBuf) &&
1179                 targetFormat->findBuffer("csd-0", &newBuf)) {
1180             if (oldBuf->size() != newBuf->size()) {
1181                 return false;
1182             }
1183             return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
1184         }
1185     }
1186     return false;
1187 }
1188 
supportsSeamlessFormatChange(const sp<AMessage> & targetFormat) const1189 bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
1190     if (mInputFormat == NULL) {
1191         return false;
1192     }
1193 
1194     if (targetFormat == NULL) {
1195         return true;
1196     }
1197 
1198     AString oldMime, newMime;
1199     if (!mInputFormat->findString("mime", &oldMime)
1200             || !targetFormat->findString("mime", &newMime)
1201             || !(oldMime == newMime)) {
1202         return false;
1203     }
1204 
1205     bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
1206     bool seamless;
1207     if (audio) {
1208         seamless = supportsSeamlessAudioFormatChange(targetFormat);
1209     } else {
1210         int32_t isAdaptive;
1211         seamless = (mCodec != NULL &&
1212                 mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
1213                 isAdaptive);
1214     }
1215 
1216     ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
1217     return seamless;
1218 }
1219 
rememberCodecSpecificData(const sp<AMessage> & format)1220 void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
1221     if (format == NULL) {
1222         return;
1223     }
1224     mCSDsForCurrentFormat.clear();
1225     for (int32_t i = 0; ; ++i) {
1226         AString tag = "csd-";
1227         tag.append(i);
1228         sp<ABuffer> buffer;
1229         if (!format->findBuffer(tag.c_str(), &buffer)) {
1230             break;
1231         }
1232         mCSDsForCurrentFormat.push(buffer);
1233     }
1234 }
1235 
notifyResumeCompleteIfNecessary()1236 void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() {
1237     if (mResumePending) {
1238         mResumePending = false;
1239 
1240         sp<AMessage> notify = mNotify->dup();
1241         notify->setInt32("what", kWhatResumeCompleted);
1242         notify->post();
1243     }
1244 }
1245 
1246 }  // namespace android
1247 
1248