1 /*
2  * Copyright 2015 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "MediaSync"
19 #include <inttypes.h>
20 
21 #include <gui/BufferQueue.h>
22 #include <gui/IGraphicBufferConsumer.h>
23 #include <gui/IGraphicBufferProducer.h>
24 
25 #include <media/AudioTrack.h>
26 #include <media/stagefright/MediaClock.h>
27 #include <media/stagefright/MediaSync.h>
28 #include <media/stagefright/VideoFrameScheduler.h>
29 #include <media/stagefright/foundation/ADebug.h>
30 #include <media/stagefright/foundation/ALooper.h>
31 #include <media/stagefright/foundation/AMessage.h>
32 
33 #include <ui/GraphicBuffer.h>
34 
35 // Maximum late time allowed for a video frame to be rendered. When a video
36 // frame arrives later than this number, it will be discarded without rendering.
37 static const int64_t kMaxAllowedVideoLateTimeUs = 40000ll;
38 
39 namespace android {
40 
41 // static
create()42 sp<MediaSync> MediaSync::create() {
43     sp<MediaSync> sync = new MediaSync();
44     sync->mLooper->registerHandler(sync);
45     return sync;
46 }
47 
MediaSync()48 MediaSync::MediaSync()
49       : mIsAbandoned(false),
50         mMutex(),
51         mReleaseCondition(),
52         mNumOutstandingBuffers(0),
53         mUsageFlagsFromOutput(0),
54         mMaxAcquiredBufferCount(1),
55         mReturnPendingInputFrame(false),
56         mNativeSampleRateInHz(0),
57         mNumFramesWritten(0),
58         mHasAudio(false),
59         mNextBufferItemMediaUs(-1),
60         mPlaybackRate(0.0) {
61     mMediaClock = new MediaClock;
62 
63     // initialize settings
64     mPlaybackSettings = AUDIO_PLAYBACK_RATE_DEFAULT;
65     mPlaybackSettings.mSpeed = mPlaybackRate;
66 
67     mLooper = new ALooper;
68     mLooper->setName("MediaSync");
69     mLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
70 }
71 
~MediaSync()72 MediaSync::~MediaSync() {
73     if (mInput != NULL) {
74         mInput->consumerDisconnect();
75     }
76     if (mOutput != NULL) {
77         mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
78     }
79 
80     if (mLooper != NULL) {
81         mLooper->unregisterHandler(id());
82         mLooper->stop();
83     }
84 }
85 
setSurface(const sp<IGraphicBufferProducer> & output)86 status_t MediaSync::setSurface(const sp<IGraphicBufferProducer> &output) {
87     Mutex::Autolock lock(mMutex);
88 
89     if (output == mOutput) {
90         return NO_ERROR;  // same output surface.
91     }
92 
93     if (output == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_VSYNC) {
94         ALOGE("setSurface: output surface is used as sync source and cannot be removed.");
95         return INVALID_OPERATION;
96     }
97 
98     if (output != NULL) {
99         int newUsage = 0;
100         output->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &newUsage);
101 
102         // Check usage flags only when current output surface has been used to create input surface.
103         if (mOutput != NULL && mInput != NULL) {
104             int ignoredFlags = (GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_COMPOSER
105                     | GRALLOC_USAGE_EXTERNAL_DISP);
106             // New output surface is not allowed to add new usage flag except ignored ones.
107             if ((newUsage & ~(mUsageFlagsFromOutput | ignoredFlags)) != 0) {
108                 ALOGE("setSurface: new output surface has new usage flag not used by current one.");
109                 return BAD_VALUE;
110             }
111         }
112 
113         // Try to connect to new output surface. If failed, current output surface will not
114         // be changed.
115         IGraphicBufferProducer::QueueBufferOutput queueBufferOutput;
116         sp<OutputListener> listener(new OutputListener(this, output));
117         IInterface::asBinder(output)->linkToDeath(listener);
118         status_t status =
119             output->connect(listener,
120                             NATIVE_WINDOW_API_MEDIA,
121                             true /* producerControlledByApp */,
122                             &queueBufferOutput);
123         if (status != NO_ERROR) {
124             ALOGE("setSurface: failed to connect (%d)", status);
125             return status;
126         }
127 
128         if (mFrameScheduler == NULL) {
129             mFrameScheduler = new VideoFrameScheduler();
130             mFrameScheduler->init();
131         }
132     }
133 
134     if (mOutput != NULL) {
135         mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
136         while (!mBuffersSentToOutput.isEmpty()) {
137             returnBufferToInput_l(mBuffersSentToOutput.valueAt(0), Fence::NO_FENCE);
138             mBuffersSentToOutput.removeItemsAt(0);
139         }
140     }
141 
142     mOutput = output;
143 
144     return NO_ERROR;
145 }
146 
147 // |audioTrack| is used only for querying information.
setAudioTrack(const sp<AudioTrack> & audioTrack)148 status_t MediaSync::setAudioTrack(const sp<AudioTrack> &audioTrack) {
149     Mutex::Autolock lock(mMutex);
150 
151     // TODO: support audio track change.
152     if (mAudioTrack != NULL) {
153         ALOGE("setAudioTrack: audioTrack has already been configured.");
154         return INVALID_OPERATION;
155     }
156 
157     if (audioTrack == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_AUDIO) {
158         ALOGE("setAudioTrack: audioTrack is used as sync source and cannot be removed.");
159         return INVALID_OPERATION;
160     }
161 
162     if (audioTrack != NULL) {
163         // check if audio track supports the playback settings
164         if (mPlaybackSettings.mSpeed != 0.f
165                 && audioTrack->setPlaybackRate(mPlaybackSettings) != OK) {
166             ALOGE("playback settings are not supported by the audio track");
167             return INVALID_OPERATION;
168         }
169         uint32_t nativeSampleRateInHz = audioTrack->getOriginalSampleRate();
170         if (nativeSampleRateInHz <= 0) {
171             ALOGE("setAudioTrack: native sample rate should be positive.");
172             return BAD_VALUE;
173         }
174         mAudioTrack = audioTrack;
175         mNativeSampleRateInHz = nativeSampleRateInHz;
176         (void)setPlaybackSettings_l(mPlaybackSettings);
177     }
178     else {
179         mAudioTrack = NULL;
180         mNativeSampleRateInHz = 0;
181     }
182 
183     // potentially resync to new source
184     resync_l();
185     return OK;
186 }
187 
createInputSurface(sp<IGraphicBufferProducer> * outBufferProducer)188 status_t MediaSync::createInputSurface(
189         sp<IGraphicBufferProducer> *outBufferProducer) {
190     if (outBufferProducer == NULL) {
191         return BAD_VALUE;
192     }
193 
194     Mutex::Autolock lock(mMutex);
195 
196     if (mOutput == NULL) {
197         return NO_INIT;
198     }
199 
200     if (mInput != NULL) {
201         return INVALID_OPERATION;
202     }
203 
204     sp<IGraphicBufferProducer> bufferProducer;
205     sp<IGraphicBufferConsumer> bufferConsumer;
206     BufferQueue::createBufferQueue(&bufferProducer, &bufferConsumer);
207 
208     sp<InputListener> listener(new InputListener(this));
209     IInterface::asBinder(bufferConsumer)->linkToDeath(listener);
210     status_t status =
211         bufferConsumer->consumerConnect(listener, false /* controlledByApp */);
212     if (status == NO_ERROR) {
213         bufferConsumer->setConsumerName(String8("MediaSync"));
214         // propagate usage bits from output surface
215         mUsageFlagsFromOutput = 0;
216         mOutput->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &mUsageFlagsFromOutput);
217         bufferConsumer->setConsumerUsageBits(mUsageFlagsFromOutput);
218         *outBufferProducer = bufferProducer;
219         mInput = bufferConsumer;
220 
221         // set undequeued buffer count
222         int minUndequeuedBuffers;
223         mOutput->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBuffers);
224         mMaxAcquiredBufferCount = minUndequeuedBuffers;
225         bufferConsumer->setMaxAcquiredBufferCount(mMaxAcquiredBufferCount);
226     }
227     return status;
228 }
229 
resync_l()230 void MediaSync::resync_l() {
231     AVSyncSource src = mSyncSettings.mSource;
232     if (src == AVSYNC_SOURCE_DEFAULT) {
233         if (mAudioTrack != NULL) {
234             src = AVSYNC_SOURCE_AUDIO;
235         } else {
236             src = AVSYNC_SOURCE_SYSTEM_CLOCK;
237         }
238     }
239 
240     // TODO: resync ourselves to the current clock (e.g. on sync source change)
241     updatePlaybackRate_l(mPlaybackRate);
242 }
243 
updatePlaybackRate_l(float rate)244 void MediaSync::updatePlaybackRate_l(float rate) {
245     if (rate > mPlaybackRate) {
246         mNextBufferItemMediaUs = -1;
247     }
248     mPlaybackRate = rate;
249     // TODO: update frame scheduler with this info
250     mMediaClock->setPlaybackRate(rate);
251     onDrainVideo_l();
252 }
253 
getMediaClock()254 sp<const MediaClock> MediaSync::getMediaClock() {
255     return mMediaClock;
256 }
257 
getPlayTimeForPendingAudioFrames(int64_t * outTimeUs)258 status_t MediaSync::getPlayTimeForPendingAudioFrames(int64_t *outTimeUs) {
259     Mutex::Autolock lock(mMutex);
260     // User should check the playback rate if it doesn't want to receive a
261     // huge number for play time.
262     if (mPlaybackRate == 0.0f) {
263         *outTimeUs = INT64_MAX;
264         return OK;
265     }
266 
267     uint32_t numFramesPlayed = 0;
268     if (mAudioTrack != NULL) {
269         status_t res = mAudioTrack->getPosition(&numFramesPlayed);
270         if (res != OK) {
271             return res;
272         }
273     }
274 
275     int64_t numPendingFrames = mNumFramesWritten - numFramesPlayed;
276     if (numPendingFrames < 0) {
277         numPendingFrames = 0;
278         ALOGW("getPlayTimeForPendingAudioFrames: pending frame count is negative.");
279     }
280     double timeUs = numPendingFrames * 1000000.0
281             / (mNativeSampleRateInHz * (double)mPlaybackRate);
282     if (timeUs > (double)INT64_MAX) {
283         // Overflow.
284         *outTimeUs = INT64_MAX;
285         ALOGW("getPlayTimeForPendingAudioFrames: play time for pending audio frames "
286               "is too high, possibly due to super low playback rate(%f)", mPlaybackRate);
287     } else {
288         *outTimeUs = (int64_t)timeUs;
289     }
290 
291     return OK;
292 }
293 
updateQueuedAudioData(size_t sizeInBytes,int64_t presentationTimeUs)294 status_t MediaSync::updateQueuedAudioData(
295         size_t sizeInBytes, int64_t presentationTimeUs) {
296     if (sizeInBytes == 0) {
297         return OK;
298     }
299 
300     Mutex::Autolock lock(mMutex);
301 
302     if (mAudioTrack == NULL) {
303         ALOGW("updateQueuedAudioData: audioTrack has NOT been configured.");
304         return INVALID_OPERATION;
305     }
306 
307     int64_t numFrames = sizeInBytes / mAudioTrack->frameSize();
308     int64_t maxMediaTimeUs = presentationTimeUs
309             + getDurationIfPlayedAtNativeSampleRate_l(numFrames);
310 
311     int64_t nowUs = ALooper::GetNowUs();
312     int64_t nowMediaUs = presentationTimeUs
313             - getDurationIfPlayedAtNativeSampleRate_l(mNumFramesWritten)
314             + getPlayedOutAudioDurationMedia_l(nowUs);
315 
316     mNumFramesWritten += numFrames;
317 
318     int64_t oldRealTime = -1;
319     if (mNextBufferItemMediaUs != -1) {
320         oldRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
321     }
322 
323     mMediaClock->updateAnchor(nowMediaUs, nowUs, maxMediaTimeUs);
324     mHasAudio = true;
325 
326     if (oldRealTime != -1) {
327         int64_t newRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
328         if (newRealTime >= oldRealTime) {
329             return OK;
330         }
331     }
332 
333     mNextBufferItemMediaUs = -1;
334     onDrainVideo_l();
335     return OK;
336 }
337 
setName(const AString & name)338 void MediaSync::setName(const AString &name) {
339     Mutex::Autolock lock(mMutex);
340     mInput->setConsumerName(String8(name.c_str()));
341 }
342 
flush()343 void MediaSync::flush() {
344     Mutex::Autolock lock(mMutex);
345     if (mFrameScheduler != NULL) {
346         mFrameScheduler->restart();
347     }
348     while (!mBufferItems.empty()) {
349         BufferItem *bufferItem = &*mBufferItems.begin();
350         returnBufferToInput_l(bufferItem->mGraphicBuffer, bufferItem->mFence);
351         mBufferItems.erase(mBufferItems.begin());
352     }
353     mNextBufferItemMediaUs = -1;
354     mNumFramesWritten = 0;
355     mReturnPendingInputFrame = true;
356     mReleaseCondition.signal();
357     mMediaClock->clearAnchor();
358 }
359 
setVideoFrameRateHint(float rate)360 status_t MediaSync::setVideoFrameRateHint(float rate) {
361     Mutex::Autolock lock(mMutex);
362     if (rate < 0.f) {
363         return BAD_VALUE;
364     }
365     if (mFrameScheduler != NULL) {
366         mFrameScheduler->init(rate);
367     }
368     return OK;
369 }
370 
getVideoFrameRate()371 float MediaSync::getVideoFrameRate() {
372     Mutex::Autolock lock(mMutex);
373     if (mFrameScheduler != NULL) {
374         float fps = mFrameScheduler->getFrameRate();
375         if (fps > 0.f) {
376             return fps;
377         }
378     }
379 
380     // we don't have or know the frame rate
381     return -1.f;
382 }
383 
setSyncSettings(const AVSyncSettings & syncSettings)384 status_t MediaSync::setSyncSettings(const AVSyncSettings &syncSettings) {
385     // validate settings
386     if (syncSettings.mSource >= AVSYNC_SOURCE_MAX
387             || syncSettings.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX
388             || syncSettings.mTolerance < 0.f
389             || syncSettings.mTolerance >= AVSYNC_TOLERANCE_MAX) {
390         return BAD_VALUE;
391     }
392 
393     Mutex::Autolock lock(mMutex);
394 
395     // verify that we have the sync source
396     switch (syncSettings.mSource) {
397         case AVSYNC_SOURCE_AUDIO:
398             if (mAudioTrack == NULL) {
399                 ALOGE("setSyncSettings: audio sync source requires an audio track");
400                 return BAD_VALUE;
401             }
402             break;
403         case AVSYNC_SOURCE_VSYNC:
404             if (mOutput == NULL) {
405                 ALOGE("setSyncSettings: vsync sync source requires an output surface");
406                 return BAD_VALUE;
407             }
408             break;
409         default:
410             break;
411     }
412 
413     mSyncSettings = syncSettings;
414     resync_l();
415     return OK;
416 }
417 
getSyncSettings(AVSyncSettings * syncSettings)418 void MediaSync::getSyncSettings(AVSyncSettings *syncSettings) {
419     Mutex::Autolock lock(mMutex);
420     *syncSettings = mSyncSettings;
421 }
422 
setPlaybackSettings(const AudioPlaybackRate & rate)423 status_t MediaSync::setPlaybackSettings(const AudioPlaybackRate &rate) {
424     Mutex::Autolock lock(mMutex);
425 
426     status_t err = setPlaybackSettings_l(rate);
427     if (err == OK) {
428         // TODO: adjust rate if using VSYNC as source
429         updatePlaybackRate_l(rate.mSpeed);
430     }
431     return err;
432 }
433 
setPlaybackSettings_l(const AudioPlaybackRate & rate)434 status_t MediaSync::setPlaybackSettings_l(const AudioPlaybackRate &rate) {
435     if (rate.mSpeed < 0.f || rate.mPitch < 0.f) {
436         // We don't validate other audio settings.
437         // They will be validated when/if audiotrack is set.
438         return BAD_VALUE;
439     }
440 
441     if (mAudioTrack != NULL) {
442         if (rate.mSpeed == 0.f) {
443             mAudioTrack->pause();
444         } else {
445             status_t err = mAudioTrack->setPlaybackRate(rate);
446             if (err != OK) {
447                 return BAD_VALUE;
448             }
449 
450             // ignore errors
451             (void)mAudioTrack->start();
452         }
453     }
454     mPlaybackSettings = rate;
455     return OK;
456 }
457 
getPlaybackSettings(AudioPlaybackRate * rate)458 void MediaSync::getPlaybackSettings(AudioPlaybackRate *rate) {
459     Mutex::Autolock lock(mMutex);
460     *rate = mPlaybackSettings;
461 }
462 
getRealTime(int64_t mediaTimeUs,int64_t nowUs)463 int64_t MediaSync::getRealTime(int64_t mediaTimeUs, int64_t nowUs) {
464     int64_t realUs;
465     if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
466         // If failed to get current position, e.g. due to audio clock is
467         // not ready, then just play out video immediately without delay.
468         return nowUs;
469     }
470     return realUs;
471 }
472 
getDurationIfPlayedAtNativeSampleRate_l(int64_t numFrames)473 int64_t MediaSync::getDurationIfPlayedAtNativeSampleRate_l(int64_t numFrames) {
474     return (numFrames * 1000000LL / mNativeSampleRateInHz);
475 }
476 
getPlayedOutAudioDurationMedia_l(int64_t nowUs)477 int64_t MediaSync::getPlayedOutAudioDurationMedia_l(int64_t nowUs) {
478     CHECK(mAudioTrack != NULL);
479 
480     uint32_t numFramesPlayed;
481     int64_t numFramesPlayedAt;
482     AudioTimestamp ts;
483     static const int64_t kStaleTimestamp100ms = 100000;
484 
485     status_t res = mAudioTrack->getTimestamp(ts);
486     if (res == OK) {
487         // case 1: mixing audio tracks.
488         numFramesPlayed = ts.mPosition;
489         numFramesPlayedAt =
490             ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
491         const int64_t timestampAge = nowUs - numFramesPlayedAt;
492         if (timestampAge > kStaleTimestamp100ms) {
493             // This is an audio FIXME.
494             // getTimestamp returns a timestamp which may come from audio
495             // mixing threads. After pausing, the MixerThread may go idle,
496             // thus the mTime estimate may become stale. Assuming that the
497             // MixerThread runs 20ms, with FastMixer at 5ms, the max latency
498             // should be about 25ms with an average around 12ms (to be
499             // verified). For safety we use 100ms.
500             ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) "
501                   "numFramesPlayedAt(%lld)",
502                   (long long)nowUs, (long long)numFramesPlayedAt);
503             numFramesPlayedAt = nowUs - kStaleTimestamp100ms;
504         }
505         //ALOGD("getTimestamp: OK %d %lld",
506         //      numFramesPlayed, (long long)numFramesPlayedAt);
507     } else if (res == WOULD_BLOCK) {
508         // case 2: transitory state on start of a new track
509         numFramesPlayed = 0;
510         numFramesPlayedAt = nowUs;
511         //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
512         //      numFramesPlayed, (long long)numFramesPlayedAt);
513     } else {
514         // case 3: transitory at new track or audio fast tracks.
515         res = mAudioTrack->getPosition(&numFramesPlayed);
516         CHECK_EQ(res, (status_t)OK);
517         numFramesPlayedAt = nowUs;
518         numFramesPlayedAt += 1000LL * mAudioTrack->latency() / 2; /* XXX */
519         //ALOGD("getPosition: %d %lld", numFramesPlayed, (long long)numFramesPlayedAt);
520     }
521 
522     //can't be negative until 12.4 hrs, test.
523     //CHECK_EQ(numFramesPlayed & (1 << 31), 0);
524     int64_t durationUs =
525         getDurationIfPlayedAtNativeSampleRate_l(numFramesPlayed)
526             + nowUs - numFramesPlayedAt;
527     if (durationUs < 0) {
528         // Occurs when numFramesPlayed position is very small and the following:
529         // (1) In case 1, the time nowUs is computed before getTimestamp() is
530         //     called and numFramesPlayedAt is greater than nowUs by time more
531         //     than numFramesPlayed.
532         // (2) In case 3, using getPosition and adding mAudioTrack->latency()
533         //     to numFramesPlayedAt, by a time amount greater than
534         //     numFramesPlayed.
535         //
536         // Both of these are transitory conditions.
537         ALOGV("getPlayedOutAudioDurationMedia_l: negative duration %lld "
538               "set to zero", (long long)durationUs);
539         durationUs = 0;
540     }
541     ALOGV("getPlayedOutAudioDurationMedia_l(%lld) nowUs(%lld) frames(%u) "
542           "framesAt(%lld)",
543           (long long)durationUs, (long long)nowUs, numFramesPlayed,
544           (long long)numFramesPlayedAt);
545     return durationUs;
546 }
547 
onDrainVideo_l()548 void MediaSync::onDrainVideo_l() {
549     if (!isPlaying()) {
550         return;
551     }
552 
553     while (!mBufferItems.empty()) {
554         int64_t nowUs = ALooper::GetNowUs();
555         BufferItem *bufferItem = &*mBufferItems.begin();
556         int64_t itemMediaUs = bufferItem->mTimestamp / 1000;
557         int64_t itemRealUs = getRealTime(itemMediaUs, nowUs);
558 
559         // adjust video frame PTS based on vsync
560         itemRealUs = mFrameScheduler->schedule(itemRealUs * 1000) / 1000;
561         int64_t twoVsyncsUs = 2 * (mFrameScheduler->getVsyncPeriod() / 1000);
562 
563         // post 2 display refreshes before rendering is due
564         if (itemRealUs <= nowUs + twoVsyncsUs) {
565             ALOGV("adjusting PTS from %lld to %lld",
566                     (long long)bufferItem->mTimestamp / 1000, (long long)itemRealUs);
567             bufferItem->mTimestamp = itemRealUs * 1000;
568             bufferItem->mIsAutoTimestamp = false;
569 
570             if (mHasAudio) {
571                 if (nowUs - itemRealUs <= kMaxAllowedVideoLateTimeUs) {
572                     renderOneBufferItem_l(*bufferItem);
573                 } else {
574                     // too late.
575                     returnBufferToInput_l(
576                             bufferItem->mGraphicBuffer, bufferItem->mFence);
577                     mFrameScheduler->restart();
578                 }
579             } else {
580                 // always render video buffer in video-only mode.
581                 renderOneBufferItem_l(*bufferItem);
582 
583                 // smooth out videos >= 10fps
584                 mMediaClock->updateAnchor(
585                         itemMediaUs, nowUs, itemMediaUs + 100000);
586             }
587 
588             mBufferItems.erase(mBufferItems.begin());
589             mNextBufferItemMediaUs = -1;
590         } else {
591             if (mNextBufferItemMediaUs == -1
592                     || mNextBufferItemMediaUs > itemMediaUs) {
593                 sp<AMessage> msg = new AMessage(kWhatDrainVideo, this);
594                 msg->post(itemRealUs - nowUs - twoVsyncsUs);
595                 mNextBufferItemMediaUs = itemMediaUs;
596             }
597             break;
598         }
599     }
600 }
601 
onFrameAvailableFromInput()602 void MediaSync::onFrameAvailableFromInput() {
603     Mutex::Autolock lock(mMutex);
604 
605     const static nsecs_t kAcquireWaitTimeout = 2000000000; // 2 seconds
606 
607     mReturnPendingInputFrame = false;
608 
609     // If there are too many outstanding buffers, wait until a buffer is
610     // released back to the input in onBufferReleased.
611     // NOTE: BufferQueue allows dequeuing maxAcquiredBufferCount + 1 buffers
612     while (mNumOutstandingBuffers > mMaxAcquiredBufferCount
613             && !mIsAbandoned && !mReturnPendingInputFrame) {
614         if (mReleaseCondition.waitRelative(mMutex, kAcquireWaitTimeout) != OK) {
615             ALOGI_IF(mPlaybackRate != 0.f, "still waiting to release a buffer before acquire");
616         }
617 
618         // If the sync is abandoned while we are waiting, the release
619         // condition variable will be broadcast, and we should just return
620         // without attempting to do anything more (since the input queue will
621         // also be abandoned).
622         if (mIsAbandoned) {
623             return;
624         }
625     }
626 
627     // Acquire and detach the buffer from the input.
628     BufferItem bufferItem;
629     status_t status = mInput->acquireBuffer(&bufferItem, 0 /* presentWhen */);
630     if (status != NO_ERROR) {
631         ALOGE("acquiring buffer from input failed (%d)", status);
632         return;
633     }
634     ++mNumOutstandingBuffers;
635 
636     ALOGV("acquired buffer %#llx from input", (long long)bufferItem.mGraphicBuffer->getId());
637 
638     status = mInput->detachBuffer(bufferItem.mSlot);
639     if (status != NO_ERROR) {
640         ALOGE("detaching buffer from input failed (%d)", status);
641         if (status == NO_INIT) {
642             // If the input has been abandoned, move on.
643             onAbandoned_l(true /* isInput */);
644         }
645         return;
646     }
647 
648     if (mBuffersFromInput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
649         // Something is wrong since this buffer should be at our hands, bail.
650         ALOGE("received buffer multiple times from input");
651         mInput->consumerDisconnect();
652         onAbandoned_l(true /* isInput */);
653         return;
654     }
655     mBuffersFromInput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
656 
657     // If flush happened while waiting for a buffer to be released, simply return it
658     // TRICKY: do it here after it is detached so that we don't have to cache mGraphicBuffer.
659     if (mReturnPendingInputFrame) {
660         mReturnPendingInputFrame = false;
661         returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence);
662         return;
663     }
664 
665     mBufferItems.push_back(bufferItem);
666 
667     if (mBufferItems.size() == 1) {
668         onDrainVideo_l();
669     }
670 }
671 
renderOneBufferItem_l(const BufferItem & bufferItem)672 void MediaSync::renderOneBufferItem_l(const BufferItem &bufferItem) {
673     IGraphicBufferProducer::QueueBufferInput queueInput(
674             bufferItem.mTimestamp,
675             bufferItem.mIsAutoTimestamp,
676             bufferItem.mDataSpace,
677             bufferItem.mCrop,
678             static_cast<int32_t>(bufferItem.mScalingMode),
679             bufferItem.mTransform,
680             bufferItem.mFence);
681 
682     // Attach and queue the buffer to the output.
683     int slot;
684     mOutput->setGenerationNumber(bufferItem.mGraphicBuffer->getGenerationNumber());
685     status_t status = mOutput->attachBuffer(&slot, bufferItem.mGraphicBuffer);
686     ALOGE_IF(status != NO_ERROR, "attaching buffer to output failed (%d)", status);
687     if (status == NO_ERROR) {
688         IGraphicBufferProducer::QueueBufferOutput queueOutput;
689         status = mOutput->queueBuffer(slot, queueInput, &queueOutput);
690         ALOGE_IF(status != NO_ERROR, "queueing buffer to output failed (%d)", status);
691     }
692 
693     if (status != NO_ERROR) {
694         returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence);
695         if (status == NO_INIT) {
696             // If the output has been abandoned, move on.
697             onAbandoned_l(false /* isInput */);
698         }
699         return;
700     }
701 
702     if (mBuffersSentToOutput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
703         // Something is wrong since this buffer should be held by output now, bail.
704         mInput->consumerDisconnect();
705         onAbandoned_l(true /* isInput */);
706         return;
707     }
708     mBuffersSentToOutput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
709 
710     ALOGV("queued buffer %#llx to output", (long long)bufferItem.mGraphicBuffer->getId());
711 }
712 
onBufferReleasedByOutput(sp<IGraphicBufferProducer> & output)713 void MediaSync::onBufferReleasedByOutput(sp<IGraphicBufferProducer> &output) {
714     Mutex::Autolock lock(mMutex);
715 
716     if (output != mOutput) {
717         return;  // This is not the current output, ignore.
718     }
719 
720     sp<GraphicBuffer> buffer;
721     sp<Fence> fence;
722     status_t status = mOutput->detachNextBuffer(&buffer, &fence);
723     ALOGE_IF(status != NO_ERROR, "detaching buffer from output failed (%d)", status);
724 
725     if (status == NO_INIT) {
726         // If the output has been abandoned, we can't do anything else,
727         // since buffer is invalid.
728         onAbandoned_l(false /* isInput */);
729         return;
730     }
731 
732     ALOGV("detached buffer %#llx from output", (long long)buffer->getId());
733 
734     // If we've been abandoned, we can't return the buffer to the input, so just
735     // move on.
736     if (mIsAbandoned) {
737         return;
738     }
739 
740     ssize_t ix = mBuffersSentToOutput.indexOfKey(buffer->getId());
741     if (ix < 0) {
742         // The buffer is unknown, maybe leftover, ignore.
743         return;
744     }
745     mBuffersSentToOutput.removeItemsAt(ix);
746 
747     returnBufferToInput_l(buffer, fence);
748 }
749 
returnBufferToInput_l(const sp<GraphicBuffer> & buffer,const sp<Fence> & fence)750 void MediaSync::returnBufferToInput_l(
751         const sp<GraphicBuffer> &buffer, const sp<Fence> &fence) {
752     ssize_t ix = mBuffersFromInput.indexOfKey(buffer->getId());
753     if (ix < 0) {
754         // The buffer is unknown, something is wrong, bail.
755         ALOGE("output returned unknown buffer");
756         mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
757         onAbandoned_l(false /* isInput */);
758         return;
759     }
760     sp<GraphicBuffer> oldBuffer = mBuffersFromInput.valueAt(ix);
761     mBuffersFromInput.removeItemsAt(ix);
762 
763     // Attach and release the buffer back to the input.
764     int consumerSlot;
765     status_t status = mInput->attachBuffer(&consumerSlot, oldBuffer);
766     ALOGE_IF(status != NO_ERROR, "attaching buffer to input failed (%d)", status);
767     if (status == NO_ERROR) {
768         status = mInput->releaseBuffer(consumerSlot, 0 /* frameNumber */,
769                 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
770         ALOGE_IF(status != NO_ERROR, "releasing buffer to input failed (%d)", status);
771     }
772 
773     // Notify any waiting onFrameAvailable calls.
774     --mNumOutstandingBuffers;
775     mReleaseCondition.signal();
776 
777     if (status == NO_ERROR) {
778         ALOGV("released buffer %#llx to input", (long long)oldBuffer->getId());
779     }
780 }
781 
onAbandoned_l(bool isInput)782 void MediaSync::onAbandoned_l(bool isInput) {
783     ALOGE("the %s has abandoned me", (isInput ? "input" : "output"));
784     if (!mIsAbandoned) {
785         if (isInput) {
786             mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
787         } else {
788             mInput->consumerDisconnect();
789         }
790         mIsAbandoned = true;
791     }
792     mReleaseCondition.broadcast();
793 }
794 
onMessageReceived(const sp<AMessage> & msg)795 void MediaSync::onMessageReceived(const sp<AMessage> &msg) {
796     switch (msg->what()) {
797         case kWhatDrainVideo:
798         {
799             Mutex::Autolock lock(mMutex);
800             if (mNextBufferItemMediaUs != -1) {
801                 int64_t nowUs = ALooper::GetNowUs();
802                 int64_t itemRealUs = getRealTime(mNextBufferItemMediaUs, nowUs);
803 
804                 // The message could arrive earlier than expected due to
805                 // various reasons, e.g., media clock has been changed because
806                 // of new anchor time or playback rate. In such cases, the
807                 // message needs to be re-posted.
808                 if (itemRealUs > nowUs) {
809                     msg->post(itemRealUs - nowUs);
810                     break;
811                 }
812             }
813 
814             onDrainVideo_l();
815             break;
816         }
817 
818         default:
819             TRESPASS();
820             break;
821     }
822 }
823 
InputListener(const sp<MediaSync> & sync)824 MediaSync::InputListener::InputListener(const sp<MediaSync> &sync)
825       : mSync(sync) {}
826 
~InputListener()827 MediaSync::InputListener::~InputListener() {}
828 
onFrameAvailable(const BufferItem &)829 void MediaSync::InputListener::onFrameAvailable(const BufferItem &/* item */) {
830     mSync->onFrameAvailableFromInput();
831 }
832 
833 // We don't care about sideband streams, since we won't relay them.
onSidebandStreamChanged()834 void MediaSync::InputListener::onSidebandStreamChanged() {
835     ALOGE("onSidebandStreamChanged: got sideband stream unexpectedly.");
836 }
837 
838 
binderDied(const wp<IBinder> &)839 void MediaSync::InputListener::binderDied(const wp<IBinder> &/* who */) {
840     Mutex::Autolock lock(mSync->mMutex);
841     mSync->onAbandoned_l(true /* isInput */);
842 }
843 
OutputListener(const sp<MediaSync> & sync,const sp<IGraphicBufferProducer> & output)844 MediaSync::OutputListener::OutputListener(const sp<MediaSync> &sync,
845         const sp<IGraphicBufferProducer> &output)
846       : mSync(sync),
847         mOutput(output) {}
848 
~OutputListener()849 MediaSync::OutputListener::~OutputListener() {}
850 
onBufferReleased()851 void MediaSync::OutputListener::onBufferReleased() {
852     mSync->onBufferReleasedByOutput(mOutput);
853 }
854 
binderDied(const wp<IBinder> &)855 void MediaSync::OutputListener::binderDied(const wp<IBinder> &/* who */) {
856     Mutex::Autolock lock(mSync->mMutex);
857     mSync->onAbandoned_l(false /* isInput */);
858 }
859 
860 } // namespace android
861