• Home
  • History
  • Annotate
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2013 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <inttypes.h>
18 
19 #define LOG_TAG "GraphicBufferSource"
20 //#define LOG_NDEBUG 0
21 #include <utils/Log.h>
22 
23 #include "GraphicBufferSource.h"
24 
25 #include <OMX_Core.h>
26 #include <media/stagefright/foundation/ADebug.h>
27 #include <media/stagefright/foundation/AMessage.h>
28 
29 #include <media/hardware/MetadataBufferType.h>
30 #include <ui/GraphicBuffer.h>
31 
32 #include <inttypes.h>
33 
34 namespace android {
35 
36 static const bool EXTRA_CHECK = true;
37 
38 
GraphicBufferSource(OMXNodeInstance * nodeInstance,uint32_t bufferWidth,uint32_t bufferHeight,uint32_t bufferCount,bool useGraphicBufferInMeta)39 GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance,
40         uint32_t bufferWidth, uint32_t bufferHeight, uint32_t bufferCount,
41         bool useGraphicBufferInMeta) :
42     mInitCheck(UNKNOWN_ERROR),
43     mNodeInstance(nodeInstance),
44     mExecuting(false),
45     mSuspended(false),
46     mNumFramesAvailable(0),
47     mEndOfStream(false),
48     mEndOfStreamSent(false),
49     mMaxTimestampGapUs(-1ll),
50     mPrevOriginalTimeUs(-1ll),
51     mPrevModifiedTimeUs(-1ll),
52     mSkipFramesBeforeNs(-1ll),
53     mRepeatAfterUs(-1ll),
54     mRepeatLastFrameGeneration(0),
55     mRepeatLastFrameTimestamp(-1ll),
56     mLatestSubmittedBufferId(-1),
57     mLatestSubmittedBufferFrameNum(0),
58     mLatestSubmittedBufferUseCount(0),
59     mRepeatBufferDeferred(false),
60     mTimePerCaptureUs(-1ll),
61     mTimePerFrameUs(-1ll),
62     mPrevCaptureUs(-1ll),
63     mPrevFrameUs(-1ll),
64     mUseGraphicBufferInMeta(useGraphicBufferInMeta) {
65 
66     ALOGV("GraphicBufferSource w=%u h=%u c=%u",
67             bufferWidth, bufferHeight, bufferCount);
68 
69     if (bufferWidth == 0 || bufferHeight == 0) {
70         ALOGE("Invalid dimensions %ux%u", bufferWidth, bufferHeight);
71         mInitCheck = BAD_VALUE;
72         return;
73     }
74 
75     String8 name("GraphicBufferSource");
76 
77     BufferQueue::createBufferQueue(&mProducer, &mConsumer);
78     mConsumer->setConsumerName(name);
79     mConsumer->setDefaultBufferSize(bufferWidth, bufferHeight);
80     mConsumer->setConsumerUsageBits(GRALLOC_USAGE_HW_VIDEO_ENCODER);
81 
82     mInitCheck = mConsumer->setMaxAcquiredBufferCount(bufferCount);
83     if (mInitCheck != NO_ERROR) {
84         ALOGE("Unable to set BQ max acquired buffer count to %u: %d",
85                 bufferCount, mInitCheck);
86         return;
87     }
88 
89     // Note that we can't create an sp<...>(this) in a ctor that will not keep a
90     // reference once the ctor ends, as that would cause the refcount of 'this'
91     // dropping to 0 at the end of the ctor.  Since all we need is a wp<...>
92     // that's what we create.
93     wp<BufferQueue::ConsumerListener> listener = static_cast<BufferQueue::ConsumerListener*>(this);
94     sp<BufferQueue::ProxyConsumerListener> proxy = new BufferQueue::ProxyConsumerListener(listener);
95 
96     mInitCheck = mConsumer->consumerConnect(proxy, false);
97     if (mInitCheck != NO_ERROR) {
98         ALOGE("Error connecting to BufferQueue: %s (%d)",
99                 strerror(-mInitCheck), mInitCheck);
100         return;
101     }
102 
103     CHECK(mInitCheck == NO_ERROR);
104 }
105 
~GraphicBufferSource()106 GraphicBufferSource::~GraphicBufferSource() {
107     ALOGV("~GraphicBufferSource");
108     if (mConsumer != NULL) {
109         status_t err = mConsumer->consumerDisconnect();
110         if (err != NO_ERROR) {
111             ALOGW("consumerDisconnect failed: %d", err);
112         }
113     }
114 }
115 
omxExecuting()116 void GraphicBufferSource::omxExecuting() {
117     Mutex::Autolock autoLock(mMutex);
118     ALOGV("--> executing; avail=%zu, codec vec size=%zd",
119             mNumFramesAvailable, mCodecBuffers.size());
120     CHECK(!mExecuting);
121     mExecuting = true;
122 
123     // Start by loading up as many buffers as possible.  We want to do this,
124     // rather than just submit the first buffer, to avoid a degenerate case:
125     // if all BQ buffers arrive before we start executing, and we only submit
126     // one here, the other BQ buffers will just sit until we get notified
127     // that the codec buffer has been released.  We'd then acquire and
128     // submit a single additional buffer, repeatedly, never using more than
129     // one codec buffer simultaneously.  (We could instead try to submit
130     // all BQ buffers whenever any codec buffer is freed, but if we get the
131     // initial conditions right that will never be useful.)
132     while (mNumFramesAvailable) {
133         if (!fillCodecBuffer_l()) {
134             ALOGV("stop load with frames available (codecAvail=%d)",
135                     isCodecBufferAvailable_l());
136             break;
137         }
138     }
139 
140     ALOGV("done loading initial frames, avail=%zu", mNumFramesAvailable);
141 
142     // If EOS has already been signaled, and there are no more frames to
143     // submit, try to send EOS now as well.
144     if (mEndOfStream && mNumFramesAvailable == 0) {
145         submitEndOfInputStream_l();
146     }
147 
148     if (mRepeatAfterUs > 0ll && mLooper == NULL) {
149         mReflector = new AHandlerReflector<GraphicBufferSource>(this);
150 
151         mLooper = new ALooper;
152         mLooper->registerHandler(mReflector);
153         mLooper->start();
154 
155         if (mLatestSubmittedBufferId >= 0) {
156             sp<AMessage> msg =
157                 new AMessage(kWhatRepeatLastFrame, mReflector->id());
158 
159             msg->setInt32("generation", ++mRepeatLastFrameGeneration);
160             msg->post(mRepeatAfterUs);
161         }
162     }
163 }
164 
omxIdle()165 void GraphicBufferSource::omxIdle() {
166     ALOGV("omxIdle");
167 
168     Mutex::Autolock autoLock(mMutex);
169 
170     if (mExecuting) {
171         // We are only interested in the transition from executing->idle,
172         // not loaded->idle.
173         mExecuting = false;
174     }
175 }
176 
omxLoaded()177 void GraphicBufferSource::omxLoaded(){
178     Mutex::Autolock autoLock(mMutex);
179     if (!mExecuting) {
180         // This can happen if something failed very early.
181         ALOGW("Dropped back down to Loaded without Executing");
182     }
183 
184     if (mLooper != NULL) {
185         mLooper->unregisterHandler(mReflector->id());
186         mReflector.clear();
187 
188         mLooper->stop();
189         mLooper.clear();
190     }
191 
192     ALOGV("--> loaded; avail=%zu eos=%d eosSent=%d",
193             mNumFramesAvailable, mEndOfStream, mEndOfStreamSent);
194 
195     // Codec is no longer executing.  Discard all codec-related state.
196     mCodecBuffers.clear();
197     // TODO: scan mCodecBuffers to verify that all mGraphicBuffer entries
198     //       are null; complain if not
199 
200     mExecuting = false;
201 }
202 
addCodecBuffer(OMX_BUFFERHEADERTYPE * header)203 void GraphicBufferSource::addCodecBuffer(OMX_BUFFERHEADERTYPE* header) {
204     Mutex::Autolock autoLock(mMutex);
205 
206     if (mExecuting) {
207         // This should never happen -- buffers can only be allocated when
208         // transitioning from "loaded" to "idle".
209         ALOGE("addCodecBuffer: buffer added while executing");
210         return;
211     }
212 
213     ALOGV("addCodecBuffer h=%p size=%" PRIu32 " p=%p",
214             header, header->nAllocLen, header->pBuffer);
215     CodecBuffer codecBuffer;
216     codecBuffer.mHeader = header;
217     mCodecBuffers.add(codecBuffer);
218 }
219 
codecBufferEmptied(OMX_BUFFERHEADERTYPE * header)220 void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) {
221     Mutex::Autolock autoLock(mMutex);
222 
223     if (!mExecuting) {
224         return;
225     }
226 
227     int cbi = findMatchingCodecBuffer_l(header);
228     if (cbi < 0) {
229         // This should never happen.
230         ALOGE("codecBufferEmptied: buffer not recognized (h=%p)", header);
231         return;
232     }
233 
234     ALOGV("codecBufferEmptied h=%p size=%" PRIu32 " filled=%" PRIu32 " p=%p",
235             header, header->nAllocLen, header->nFilledLen,
236             header->pBuffer);
237     CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
238 
239     // header->nFilledLen may not be the original value, so we can't compare
240     // that to zero to see of this was the EOS buffer.  Instead we just
241     // see if the GraphicBuffer reference was null, which should only ever
242     // happen for EOS.
243     if (codecBuffer.mGraphicBuffer == NULL) {
244         if (!(mEndOfStream && mEndOfStreamSent)) {
245             // This can happen when broken code sends us the same buffer
246             // twice in a row.
247             ALOGE("ERROR: codecBufferEmptied on non-EOS null buffer "
248                     "(buffer emptied twice?)");
249         }
250         // No GraphicBuffer to deal with, no additional input or output is
251         // expected, so just return.
252         return;
253     }
254 
255     if (EXTRA_CHECK) {
256         // Pull the graphic buffer handle back out of the buffer, and confirm
257         // that it matches expectations.
258         OMX_U8* data = header->pBuffer;
259         MetadataBufferType type = *(MetadataBufferType *)data;
260         if (type == kMetadataBufferTypeGrallocSource) {
261             buffer_handle_t bufferHandle;
262             memcpy(&bufferHandle, data + 4, sizeof(buffer_handle_t));
263             if (bufferHandle != codecBuffer.mGraphicBuffer->handle) {
264                 // should never happen
265                 ALOGE("codecBufferEmptied: buffer's handle is %p, expected %p",
266                         bufferHandle, codecBuffer.mGraphicBuffer->handle);
267                 CHECK(!"codecBufferEmptied: mismatched buffer");
268             }
269         } else if (type == kMetadataBufferTypeGraphicBuffer) {
270             GraphicBuffer *buffer;
271             memcpy(&buffer, data + 4, sizeof(buffer));
272             if (buffer != codecBuffer.mGraphicBuffer.get()) {
273                 // should never happen
274                 ALOGE("codecBufferEmptied: buffer is %p, expected %p",
275                         buffer, codecBuffer.mGraphicBuffer.get());
276                 CHECK(!"codecBufferEmptied: mismatched buffer");
277             }
278         }
279     }
280 
281     // Find matching entry in our cached copy of the BufferQueue slots.
282     // If we find a match, release that slot.  If we don't, the BufferQueue
283     // has dropped that GraphicBuffer, and there's nothing for us to release.
284     int id = codecBuffer.mBuf;
285     if (mBufferSlot[id] != NULL &&
286         mBufferSlot[id]->handle == codecBuffer.mGraphicBuffer->handle) {
287         ALOGV("cbi %d matches bq slot %d, handle=%p",
288                 cbi, id, mBufferSlot[id]->handle);
289 
290         if (id == mLatestSubmittedBufferId) {
291             CHECK_GT(mLatestSubmittedBufferUseCount--, 0);
292         } else {
293             mConsumer->releaseBuffer(id, codecBuffer.mFrameNumber,
294                     EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE);
295         }
296     } else {
297         ALOGV("codecBufferEmptied: no match for emptied buffer in cbi %d",
298                 cbi);
299     }
300 
301     // Mark the codec buffer as available by clearing the GraphicBuffer ref.
302     codecBuffer.mGraphicBuffer = NULL;
303 
304     if (mNumFramesAvailable) {
305         // Fill this codec buffer.
306         CHECK(!mEndOfStreamSent);
307         ALOGV("buffer freed, %zu frames avail (eos=%d)",
308                 mNumFramesAvailable, mEndOfStream);
309         fillCodecBuffer_l();
310     } else if (mEndOfStream) {
311         // No frames available, but EOS is pending, so use this buffer to
312         // send that.
313         ALOGV("buffer freed, EOS pending");
314         submitEndOfInputStream_l();
315     } else if (mRepeatBufferDeferred) {
316         bool success = repeatLatestSubmittedBuffer_l();
317         if (success) {
318             ALOGV("deferred repeatLatestSubmittedBuffer_l SUCCESS");
319         } else {
320             ALOGV("deferred repeatLatestSubmittedBuffer_l FAILURE");
321         }
322         mRepeatBufferDeferred = false;
323     }
324 
325     return;
326 }
327 
codecBufferFilled(OMX_BUFFERHEADERTYPE * header)328 void GraphicBufferSource::codecBufferFilled(OMX_BUFFERHEADERTYPE* header) {
329     Mutex::Autolock autoLock(mMutex);
330 
331     if (mMaxTimestampGapUs > 0ll
332             && !(header->nFlags & OMX_BUFFERFLAG_CODECCONFIG)) {
333         ssize_t index = mOriginalTimeUs.indexOfKey(header->nTimeStamp);
334         if (index >= 0) {
335             ALOGV("OUT timestamp: %lld -> %lld",
336                     static_cast<long long>(header->nTimeStamp),
337                     static_cast<long long>(mOriginalTimeUs[index]));
338             header->nTimeStamp = mOriginalTimeUs[index];
339             mOriginalTimeUs.removeItemsAt(index);
340         } else {
341             // giving up the effort as encoder doesn't appear to preserve pts
342             ALOGW("giving up limiting timestamp gap (pts = %lld)",
343                     header->nTimeStamp);
344             mMaxTimestampGapUs = -1ll;
345         }
346         if (mOriginalTimeUs.size() > BufferQueue::NUM_BUFFER_SLOTS) {
347             // something terribly wrong must have happened, giving up...
348             ALOGE("mOriginalTimeUs has too many entries (%zu)",
349                     mOriginalTimeUs.size());
350             mMaxTimestampGapUs = -1ll;
351         }
352     }
353 }
354 
suspend(bool suspend)355 void GraphicBufferSource::suspend(bool suspend) {
356     Mutex::Autolock autoLock(mMutex);
357 
358     if (suspend) {
359         mSuspended = true;
360 
361         while (mNumFramesAvailable > 0) {
362             BufferQueue::BufferItem item;
363             status_t err = mConsumer->acquireBuffer(&item, 0);
364 
365             if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
366                 // shouldn't happen.
367                 ALOGW("suspend: frame was not available");
368                 break;
369             } else if (err != OK) {
370                 ALOGW("suspend: acquireBuffer returned err=%d", err);
371                 break;
372             }
373 
374             --mNumFramesAvailable;
375 
376             mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber,
377                     EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, item.mFence);
378         }
379         return;
380     }
381 
382     mSuspended = false;
383 
384     if (mExecuting && mNumFramesAvailable == 0 && mRepeatBufferDeferred) {
385         if (repeatLatestSubmittedBuffer_l()) {
386             ALOGV("suspend/deferred repeatLatestSubmittedBuffer_l SUCCESS");
387 
388             mRepeatBufferDeferred = false;
389         } else {
390             ALOGV("suspend/deferred repeatLatestSubmittedBuffer_l FAILURE");
391         }
392     }
393 }
394 
fillCodecBuffer_l()395 bool GraphicBufferSource::fillCodecBuffer_l() {
396     CHECK(mExecuting && mNumFramesAvailable > 0);
397 
398     if (mSuspended) {
399         return false;
400     }
401 
402     int cbi = findAvailableCodecBuffer_l();
403     if (cbi < 0) {
404         // No buffers available, bail.
405         ALOGV("fillCodecBuffer_l: no codec buffers, avail now %zu",
406                 mNumFramesAvailable);
407         return false;
408     }
409 
410     ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%zu",
411             mNumFramesAvailable);
412     BufferQueue::BufferItem item;
413     status_t err = mConsumer->acquireBuffer(&item, 0);
414     if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
415         // shouldn't happen
416         ALOGW("fillCodecBuffer_l: frame was not available");
417         return false;
418     } else if (err != OK) {
419         // now what? fake end-of-stream?
420         ALOGW("fillCodecBuffer_l: acquireBuffer returned err=%d", err);
421         return false;
422     }
423 
424     mNumFramesAvailable--;
425 
426     // Wait for it to become available.
427     err = item.mFence->waitForever("GraphicBufferSource::fillCodecBuffer_l");
428     if (err != OK) {
429         ALOGW("failed to wait for buffer fence: %d", err);
430         // keep going
431     }
432 
433     // If this is the first time we're seeing this buffer, add it to our
434     // slot table.
435     if (item.mGraphicBuffer != NULL) {
436         ALOGV("fillCodecBuffer_l: setting mBufferSlot %d", item.mBuf);
437         mBufferSlot[item.mBuf] = item.mGraphicBuffer;
438     }
439 
440     err = UNKNOWN_ERROR;
441 
442     // only submit sample if start time is unspecified, or sample
443     // is queued after the specified start time
444     if (mSkipFramesBeforeNs < 0ll || item.mTimestamp >= mSkipFramesBeforeNs) {
445         // if start time is set, offset time stamp by start time
446         if (mSkipFramesBeforeNs > 0) {
447             item.mTimestamp -= mSkipFramesBeforeNs;
448         }
449         err = submitBuffer_l(item, cbi);
450     }
451 
452     if (err != OK) {
453         ALOGV("submitBuffer_l failed, releasing bq buf %d", item.mBuf);
454         mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber,
455                 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE);
456     } else {
457         ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi);
458         setLatestSubmittedBuffer_l(item);
459     }
460 
461     return true;
462 }
463 
repeatLatestSubmittedBuffer_l()464 bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() {
465     CHECK(mExecuting && mNumFramesAvailable == 0);
466 
467     if (mLatestSubmittedBufferId < 0 || mSuspended) {
468         return false;
469     }
470     if (mBufferSlot[mLatestSubmittedBufferId] == NULL) {
471         // This can happen if the remote side disconnects, causing
472         // onBuffersReleased() to NULL out our copy of the slots.  The
473         // buffer is gone, so we have nothing to show.
474         //
475         // To be on the safe side we try to release the buffer.
476         ALOGD("repeatLatestSubmittedBuffer_l: slot was NULL");
477         mConsumer->releaseBuffer(
478                 mLatestSubmittedBufferId,
479                 mLatestSubmittedBufferFrameNum,
480                 EGL_NO_DISPLAY,
481                 EGL_NO_SYNC_KHR,
482                 Fence::NO_FENCE);
483         mLatestSubmittedBufferId = -1;
484         mLatestSubmittedBufferFrameNum = 0;
485         return false;
486     }
487 
488     int cbi = findAvailableCodecBuffer_l();
489     if (cbi < 0) {
490         // No buffers available, bail.
491         ALOGV("repeatLatestSubmittedBuffer_l: no codec buffers.");
492         return false;
493     }
494 
495     BufferQueue::BufferItem item;
496     item.mBuf = mLatestSubmittedBufferId;
497     item.mFrameNumber = mLatestSubmittedBufferFrameNum;
498     item.mTimestamp = mRepeatLastFrameTimestamp;
499 
500     status_t err = submitBuffer_l(item, cbi);
501 
502     if (err != OK) {
503         return false;
504     }
505 
506     ++mLatestSubmittedBufferUseCount;
507 
508     /* repeat last frame up to kRepeatLastFrameCount times.
509      * in case of static scene, a single repeat might not get rid of encoder
510      * ghosting completely, refresh a couple more times to get better quality
511      */
512     if (--mRepeatLastFrameCount > 0) {
513         mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
514 
515         if (mReflector != NULL) {
516             sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id());
517             msg->setInt32("generation", ++mRepeatLastFrameGeneration);
518             msg->post(mRepeatAfterUs);
519         }
520     }
521 
522     return true;
523 }
524 
setLatestSubmittedBuffer_l(const BufferQueue::BufferItem & item)525 void GraphicBufferSource::setLatestSubmittedBuffer_l(
526         const BufferQueue::BufferItem &item) {
527     ALOGV("setLatestSubmittedBuffer_l");
528 
529     if (mLatestSubmittedBufferId >= 0) {
530         if (mLatestSubmittedBufferUseCount == 0) {
531             mConsumer->releaseBuffer(
532                     mLatestSubmittedBufferId,
533                     mLatestSubmittedBufferFrameNum,
534                     EGL_NO_DISPLAY,
535                     EGL_NO_SYNC_KHR,
536                     Fence::NO_FENCE);
537         }
538     }
539 
540     mLatestSubmittedBufferId = item.mBuf;
541     mLatestSubmittedBufferFrameNum = item.mFrameNumber;
542     mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
543 
544     mLatestSubmittedBufferUseCount = 1;
545     mRepeatBufferDeferred = false;
546     mRepeatLastFrameCount = kRepeatLastFrameCount;
547 
548     if (mReflector != NULL) {
549         sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id());
550         msg->setInt32("generation", ++mRepeatLastFrameGeneration);
551         msg->post(mRepeatAfterUs);
552     }
553 }
554 
signalEndOfInputStream()555 status_t GraphicBufferSource::signalEndOfInputStream() {
556     Mutex::Autolock autoLock(mMutex);
557     ALOGV("signalEndOfInputStream: exec=%d avail=%zu eos=%d",
558             mExecuting, mNumFramesAvailable, mEndOfStream);
559 
560     if (mEndOfStream) {
561         ALOGE("EOS was already signaled");
562         return INVALID_OPERATION;
563     }
564 
565     // Set the end-of-stream flag.  If no frames are pending from the
566     // BufferQueue, and a codec buffer is available, and we're executing,
567     // we initiate the EOS from here.  Otherwise, we'll let
568     // codecBufferEmptied() (or omxExecuting) do it.
569     //
570     // Note: if there are no pending frames and all codec buffers are
571     // available, we *must* submit the EOS from here or we'll just
572     // stall since no future events are expected.
573     mEndOfStream = true;
574 
575     if (mExecuting && mNumFramesAvailable == 0) {
576         submitEndOfInputStream_l();
577     }
578 
579     return OK;
580 }
581 
getTimestamp(const BufferQueue::BufferItem & item)582 int64_t GraphicBufferSource::getTimestamp(const BufferQueue::BufferItem &item) {
583     int64_t timeUs = item.mTimestamp / 1000;
584 
585     if (mTimePerCaptureUs > 0ll) {
586         // Time lapse or slow motion mode
587         if (mPrevCaptureUs < 0ll) {
588             // first capture
589             mPrevCaptureUs = timeUs;
590             mPrevFrameUs = timeUs;
591         } else {
592             // snap to nearest capture point
593             int64_t nFrames = (timeUs + mTimePerCaptureUs / 2 - mPrevCaptureUs)
594                     / mTimePerCaptureUs;
595             if (nFrames <= 0) {
596                 // skip this frame as it's too close to previous capture
597                 ALOGV("skipping frame, timeUs %lld", static_cast<long long>(timeUs));
598                 return -1;
599             }
600             mPrevCaptureUs = mPrevCaptureUs + nFrames * mTimePerCaptureUs;
601             mPrevFrameUs += mTimePerFrameUs * nFrames;
602         }
603 
604         ALOGV("timeUs %lld, captureUs %lld, frameUs %lld",
605                 static_cast<long long>(timeUs),
606                 static_cast<long long>(mPrevCaptureUs),
607                 static_cast<long long>(mPrevFrameUs));
608 
609         return mPrevFrameUs;
610     } else if (mMaxTimestampGapUs > 0ll) {
611         /* Cap timestamp gap between adjacent frames to specified max
612          *
613          * In the scenario of cast mirroring, encoding could be suspended for
614          * prolonged periods. Limiting the pts gap to workaround the problem
615          * where encoder's rate control logic produces huge frames after a
616          * long period of suspension.
617          */
618 
619         int64_t originalTimeUs = timeUs;
620         if (mPrevOriginalTimeUs >= 0ll) {
621             if (originalTimeUs < mPrevOriginalTimeUs) {
622                 // Drop the frame if it's going backward in time. Bad timestamp
623                 // could disrupt encoder's rate control completely.
624                 ALOGW("Dropping frame that's going backward in time");
625                 return -1;
626             }
627             int64_t timestampGapUs = originalTimeUs - mPrevOriginalTimeUs;
628             timeUs = (timestampGapUs < mMaxTimestampGapUs ?
629                     timestampGapUs : mMaxTimestampGapUs) + mPrevModifiedTimeUs;
630         }
631         mPrevOriginalTimeUs = originalTimeUs;
632         mPrevModifiedTimeUs = timeUs;
633         mOriginalTimeUs.add(timeUs, originalTimeUs);
634         ALOGV("IN  timestamp: %lld -> %lld",
635             static_cast<long long>(originalTimeUs),
636             static_cast<long long>(timeUs));
637     }
638 
639     return timeUs;
640 }
641 
submitBuffer_l(const BufferQueue::BufferItem & item,int cbi)642 status_t GraphicBufferSource::submitBuffer_l(
643         const BufferQueue::BufferItem &item, int cbi) {
644     ALOGV("submitBuffer_l cbi=%d", cbi);
645 
646     int64_t timeUs = getTimestamp(item);
647     if (timeUs < 0ll) {
648         return UNKNOWN_ERROR;
649     }
650 
651     CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
652     codecBuffer.mGraphicBuffer = mBufferSlot[item.mBuf];
653     codecBuffer.mBuf = item.mBuf;
654     codecBuffer.mFrameNumber = item.mFrameNumber;
655 
656     OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
657     CHECK(header->nAllocLen >= 4 + sizeof(buffer_handle_t));
658     OMX_U8* data = header->pBuffer;
659     buffer_handle_t handle;
660     if (!mUseGraphicBufferInMeta) {
661         const OMX_U32 type = kMetadataBufferTypeGrallocSource;
662         handle = codecBuffer.mGraphicBuffer->handle;
663         memcpy(data, &type, 4);
664         memcpy(data + 4, &handle, sizeof(buffer_handle_t));
665     } else {
666         // codecBuffer holds a reference to the GraphicBuffer, so
667         // it is valid while it is with the OMX component
668         const OMX_U32 type = kMetadataBufferTypeGraphicBuffer;
669         memcpy(data, &type, 4);
670         // passing a non-reference-counted graphicBuffer
671         GraphicBuffer *buffer = codecBuffer.mGraphicBuffer.get();
672         handle = buffer->handle;
673         memcpy(data + 4, &buffer, sizeof(buffer));
674     }
675 
676     status_t err = mNodeInstance->emptyDirectBuffer(header, 0,
677             4 + sizeof(buffer_handle_t), OMX_BUFFERFLAG_ENDOFFRAME,
678             timeUs);
679     if (err != OK) {
680         ALOGW("WARNING: emptyDirectBuffer failed: 0x%x", err);
681         codecBuffer.mGraphicBuffer = NULL;
682         return err;
683     }
684 
685     ALOGV("emptyDirectBuffer succeeded, h=%p p=%p bufhandle=%p",
686             header, header->pBuffer, handle);
687     return OK;
688 }
689 
submitEndOfInputStream_l()690 void GraphicBufferSource::submitEndOfInputStream_l() {
691     CHECK(mEndOfStream);
692     if (mEndOfStreamSent) {
693         ALOGV("EOS already sent");
694         return;
695     }
696 
697     int cbi = findAvailableCodecBuffer_l();
698     if (cbi < 0) {
699         ALOGV("submitEndOfInputStream_l: no codec buffers available");
700         return;
701     }
702 
703     // We reject any additional incoming graphic buffers, so there's no need
704     // to stick a placeholder into codecBuffer.mGraphicBuffer to mark it as
705     // in-use.
706     CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
707 
708     OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
709     if (EXTRA_CHECK) {
710         // Guard against implementations that don't check nFilledLen.
711         size_t fillLen = 4 + sizeof(buffer_handle_t);
712         CHECK(header->nAllocLen >= fillLen);
713         OMX_U8* data = header->pBuffer;
714         memset(data, 0xcd, fillLen);
715     }
716 
717     uint64_t timestamp = 0; // does this matter?
718 
719     status_t err = mNodeInstance->emptyDirectBuffer(header, /*offset*/ 0,
720             /*length*/ 0, OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_EOS,
721             timestamp);
722     if (err != OK) {
723         ALOGW("emptyDirectBuffer EOS failed: 0x%x", err);
724     } else {
725         ALOGV("submitEndOfInputStream_l: buffer submitted, header=%p cbi=%d",
726                 header, cbi);
727         mEndOfStreamSent = true;
728     }
729 }
730 
findAvailableCodecBuffer_l()731 int GraphicBufferSource::findAvailableCodecBuffer_l() {
732     CHECK(mCodecBuffers.size() > 0);
733 
734     for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
735         if (mCodecBuffers[i].mGraphicBuffer == NULL) {
736             return i;
737         }
738     }
739     return -1;
740 }
741 
findMatchingCodecBuffer_l(const OMX_BUFFERHEADERTYPE * header)742 int GraphicBufferSource::findMatchingCodecBuffer_l(
743         const OMX_BUFFERHEADERTYPE* header) {
744     for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
745         if (mCodecBuffers[i].mHeader == header) {
746             return i;
747         }
748     }
749     return -1;
750 }
751 
752 // BufferQueue::ConsumerListener callback
onFrameAvailable(const BufferItem &)753 void GraphicBufferSource::onFrameAvailable(const BufferItem& /*item*/) {
754     Mutex::Autolock autoLock(mMutex);
755 
756     ALOGV("onFrameAvailable exec=%d avail=%zu",
757             mExecuting, mNumFramesAvailable);
758 
759     if (mEndOfStream || mSuspended) {
760         if (mEndOfStream) {
761             // This should only be possible if a new buffer was queued after
762             // EOS was signaled, i.e. the app is misbehaving.
763 
764             ALOGW("onFrameAvailable: EOS is set, ignoring frame");
765         } else {
766             ALOGV("onFrameAvailable: suspended, ignoring frame");
767         }
768 
769         BufferQueue::BufferItem item;
770         status_t err = mConsumer->acquireBuffer(&item, 0);
771         if (err == OK) {
772             // If this is the first time we're seeing this buffer, add it to our
773             // slot table.
774             if (item.mGraphicBuffer != NULL) {
775                 ALOGV("onFrameAvailable: setting mBufferSlot %d", item.mBuf);
776                 mBufferSlot[item.mBuf] = item.mGraphicBuffer;
777             }
778             mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber,
779                     EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, item.mFence);
780         }
781         return;
782     }
783 
784     mNumFramesAvailable++;
785 
786     mRepeatBufferDeferred = false;
787     ++mRepeatLastFrameGeneration;
788 
789     if (mExecuting) {
790         fillCodecBuffer_l();
791     }
792 }
793 
794 // BufferQueue::ConsumerListener callback
onBuffersReleased()795 void GraphicBufferSource::onBuffersReleased() {
796     Mutex::Autolock lock(mMutex);
797 
798     uint64_t slotMask;
799     if (mConsumer->getReleasedBuffers(&slotMask) != NO_ERROR) {
800         ALOGW("onBuffersReleased: unable to get released buffer set");
801         slotMask = 0xffffffffffffffffULL;
802     }
803 
804     ALOGV("onBuffersReleased: 0x%016" PRIx64, slotMask);
805 
806     for (int i = 0; i < BufferQueue::NUM_BUFFER_SLOTS; i++) {
807         if ((slotMask & 0x01) != 0) {
808             mBufferSlot[i] = NULL;
809         }
810         slotMask >>= 1;
811     }
812 }
813 
814 // BufferQueue::ConsumerListener callback
onSidebandStreamChanged()815 void GraphicBufferSource::onSidebandStreamChanged() {
816     ALOG_ASSERT(false, "GraphicBufferSource can't consume sideband streams");
817 }
818 
setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs)819 status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs(
820         int64_t repeatAfterUs) {
821     Mutex::Autolock autoLock(mMutex);
822 
823     if (mExecuting || repeatAfterUs <= 0ll) {
824         return INVALID_OPERATION;
825     }
826 
827     mRepeatAfterUs = repeatAfterUs;
828 
829     return OK;
830 }
831 
setMaxTimestampGapUs(int64_t maxGapUs)832 status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) {
833     Mutex::Autolock autoLock(mMutex);
834 
835     if (mExecuting || maxGapUs <= 0ll) {
836         return INVALID_OPERATION;
837     }
838 
839     mMaxTimestampGapUs = maxGapUs;
840 
841     return OK;
842 }
843 
setSkipFramesBeforeUs(int64_t skipFramesBeforeUs)844 void GraphicBufferSource::setSkipFramesBeforeUs(int64_t skipFramesBeforeUs) {
845     Mutex::Autolock autoLock(mMutex);
846 
847     mSkipFramesBeforeNs =
848             (skipFramesBeforeUs > 0) ? (skipFramesBeforeUs * 1000) : -1ll;
849 }
850 
setTimeLapseUs(int64_t * data)851 status_t GraphicBufferSource::setTimeLapseUs(int64_t* data) {
852     Mutex::Autolock autoLock(mMutex);
853 
854     if (mExecuting || data[0] <= 0ll || data[1] <= 0ll) {
855         return INVALID_OPERATION;
856     }
857 
858     mTimePerFrameUs = data[0];
859     mTimePerCaptureUs = data[1];
860 
861     return OK;
862 }
863 
onMessageReceived(const sp<AMessage> & msg)864 void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) {
865     switch (msg->what()) {
866         case kWhatRepeatLastFrame:
867         {
868             Mutex::Autolock autoLock(mMutex);
869 
870             int32_t generation;
871             CHECK(msg->findInt32("generation", &generation));
872 
873             if (generation != mRepeatLastFrameGeneration) {
874                 // stale
875                 break;
876             }
877 
878             if (!mExecuting || mNumFramesAvailable > 0) {
879                 break;
880             }
881 
882             bool success = repeatLatestSubmittedBuffer_l();
883 
884             if (success) {
885                 ALOGV("repeatLatestSubmittedBuffer_l SUCCESS");
886             } else {
887                 ALOGV("repeatLatestSubmittedBuffer_l FAILURE");
888                 mRepeatBufferDeferred = true;
889             }
890             break;
891         }
892 
893         default:
894             TRESPASS();
895     }
896 }
897 
898 }  // namespace android
899