1 /*
2 * Copyright (C) 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <inttypes.h>
18
19 #define LOG_TAG "GraphicBufferSource"
20 //#define LOG_NDEBUG 0
21 #include <utils/Log.h>
22
23 #include "GraphicBufferSource.h"
24
25 #include <OMX_Core.h>
26 #include <media/stagefright/foundation/ADebug.h>
27 #include <media/stagefright/foundation/AMessage.h>
28
29 #include <media/hardware/MetadataBufferType.h>
30 #include <ui/GraphicBuffer.h>
31
32 #include <inttypes.h>
33
34 namespace android {
35
36 static const bool EXTRA_CHECK = true;
37
38
GraphicBufferSource(OMXNodeInstance * nodeInstance,uint32_t bufferWidth,uint32_t bufferHeight,uint32_t bufferCount,bool useGraphicBufferInMeta)39 GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance,
40 uint32_t bufferWidth, uint32_t bufferHeight, uint32_t bufferCount,
41 bool useGraphicBufferInMeta) :
42 mInitCheck(UNKNOWN_ERROR),
43 mNodeInstance(nodeInstance),
44 mExecuting(false),
45 mSuspended(false),
46 mNumFramesAvailable(0),
47 mEndOfStream(false),
48 mEndOfStreamSent(false),
49 mMaxTimestampGapUs(-1ll),
50 mPrevOriginalTimeUs(-1ll),
51 mPrevModifiedTimeUs(-1ll),
52 mSkipFramesBeforeNs(-1ll),
53 mRepeatAfterUs(-1ll),
54 mRepeatLastFrameGeneration(0),
55 mRepeatLastFrameTimestamp(-1ll),
56 mLatestSubmittedBufferId(-1),
57 mLatestSubmittedBufferFrameNum(0),
58 mLatestSubmittedBufferUseCount(0),
59 mRepeatBufferDeferred(false),
60 mTimePerCaptureUs(-1ll),
61 mTimePerFrameUs(-1ll),
62 mPrevCaptureUs(-1ll),
63 mPrevFrameUs(-1ll),
64 mUseGraphicBufferInMeta(useGraphicBufferInMeta) {
65
66 ALOGV("GraphicBufferSource w=%u h=%u c=%u",
67 bufferWidth, bufferHeight, bufferCount);
68
69 if (bufferWidth == 0 || bufferHeight == 0) {
70 ALOGE("Invalid dimensions %ux%u", bufferWidth, bufferHeight);
71 mInitCheck = BAD_VALUE;
72 return;
73 }
74
75 String8 name("GraphicBufferSource");
76
77 BufferQueue::createBufferQueue(&mProducer, &mConsumer);
78 mConsumer->setConsumerName(name);
79 mConsumer->setDefaultBufferSize(bufferWidth, bufferHeight);
80 mConsumer->setConsumerUsageBits(GRALLOC_USAGE_HW_VIDEO_ENCODER);
81
82 mInitCheck = mConsumer->setMaxAcquiredBufferCount(bufferCount);
83 if (mInitCheck != NO_ERROR) {
84 ALOGE("Unable to set BQ max acquired buffer count to %u: %d",
85 bufferCount, mInitCheck);
86 return;
87 }
88
89 // Note that we can't create an sp<...>(this) in a ctor that will not keep a
90 // reference once the ctor ends, as that would cause the refcount of 'this'
91 // dropping to 0 at the end of the ctor. Since all we need is a wp<...>
92 // that's what we create.
93 wp<BufferQueue::ConsumerListener> listener = static_cast<BufferQueue::ConsumerListener*>(this);
94 sp<BufferQueue::ProxyConsumerListener> proxy = new BufferQueue::ProxyConsumerListener(listener);
95
96 mInitCheck = mConsumer->consumerConnect(proxy, false);
97 if (mInitCheck != NO_ERROR) {
98 ALOGE("Error connecting to BufferQueue: %s (%d)",
99 strerror(-mInitCheck), mInitCheck);
100 return;
101 }
102
103 CHECK(mInitCheck == NO_ERROR);
104 }
105
~GraphicBufferSource()106 GraphicBufferSource::~GraphicBufferSource() {
107 ALOGV("~GraphicBufferSource");
108 if (mConsumer != NULL) {
109 status_t err = mConsumer->consumerDisconnect();
110 if (err != NO_ERROR) {
111 ALOGW("consumerDisconnect failed: %d", err);
112 }
113 }
114 }
115
omxExecuting()116 void GraphicBufferSource::omxExecuting() {
117 Mutex::Autolock autoLock(mMutex);
118 ALOGV("--> executing; avail=%zu, codec vec size=%zd",
119 mNumFramesAvailable, mCodecBuffers.size());
120 CHECK(!mExecuting);
121 mExecuting = true;
122
123 // Start by loading up as many buffers as possible. We want to do this,
124 // rather than just submit the first buffer, to avoid a degenerate case:
125 // if all BQ buffers arrive before we start executing, and we only submit
126 // one here, the other BQ buffers will just sit until we get notified
127 // that the codec buffer has been released. We'd then acquire and
128 // submit a single additional buffer, repeatedly, never using more than
129 // one codec buffer simultaneously. (We could instead try to submit
130 // all BQ buffers whenever any codec buffer is freed, but if we get the
131 // initial conditions right that will never be useful.)
132 while (mNumFramesAvailable) {
133 if (!fillCodecBuffer_l()) {
134 ALOGV("stop load with frames available (codecAvail=%d)",
135 isCodecBufferAvailable_l());
136 break;
137 }
138 }
139
140 ALOGV("done loading initial frames, avail=%zu", mNumFramesAvailable);
141
142 // If EOS has already been signaled, and there are no more frames to
143 // submit, try to send EOS now as well.
144 if (mEndOfStream && mNumFramesAvailable == 0) {
145 submitEndOfInputStream_l();
146 }
147
148 if (mRepeatAfterUs > 0ll && mLooper == NULL) {
149 mReflector = new AHandlerReflector<GraphicBufferSource>(this);
150
151 mLooper = new ALooper;
152 mLooper->registerHandler(mReflector);
153 mLooper->start();
154
155 if (mLatestSubmittedBufferId >= 0) {
156 sp<AMessage> msg =
157 new AMessage(kWhatRepeatLastFrame, mReflector->id());
158
159 msg->setInt32("generation", ++mRepeatLastFrameGeneration);
160 msg->post(mRepeatAfterUs);
161 }
162 }
163 }
164
omxIdle()165 void GraphicBufferSource::omxIdle() {
166 ALOGV("omxIdle");
167
168 Mutex::Autolock autoLock(mMutex);
169
170 if (mExecuting) {
171 // We are only interested in the transition from executing->idle,
172 // not loaded->idle.
173 mExecuting = false;
174 }
175 }
176
omxLoaded()177 void GraphicBufferSource::omxLoaded(){
178 Mutex::Autolock autoLock(mMutex);
179 if (!mExecuting) {
180 // This can happen if something failed very early.
181 ALOGW("Dropped back down to Loaded without Executing");
182 }
183
184 if (mLooper != NULL) {
185 mLooper->unregisterHandler(mReflector->id());
186 mReflector.clear();
187
188 mLooper->stop();
189 mLooper.clear();
190 }
191
192 ALOGV("--> loaded; avail=%zu eos=%d eosSent=%d",
193 mNumFramesAvailable, mEndOfStream, mEndOfStreamSent);
194
195 // Codec is no longer executing. Discard all codec-related state.
196 mCodecBuffers.clear();
197 // TODO: scan mCodecBuffers to verify that all mGraphicBuffer entries
198 // are null; complain if not
199
200 mExecuting = false;
201 }
202
addCodecBuffer(OMX_BUFFERHEADERTYPE * header)203 void GraphicBufferSource::addCodecBuffer(OMX_BUFFERHEADERTYPE* header) {
204 Mutex::Autolock autoLock(mMutex);
205
206 if (mExecuting) {
207 // This should never happen -- buffers can only be allocated when
208 // transitioning from "loaded" to "idle".
209 ALOGE("addCodecBuffer: buffer added while executing");
210 return;
211 }
212
213 ALOGV("addCodecBuffer h=%p size=%" PRIu32 " p=%p",
214 header, header->nAllocLen, header->pBuffer);
215 CodecBuffer codecBuffer;
216 codecBuffer.mHeader = header;
217 mCodecBuffers.add(codecBuffer);
218 }
219
codecBufferEmptied(OMX_BUFFERHEADERTYPE * header)220 void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) {
221 Mutex::Autolock autoLock(mMutex);
222
223 if (!mExecuting) {
224 return;
225 }
226
227 int cbi = findMatchingCodecBuffer_l(header);
228 if (cbi < 0) {
229 // This should never happen.
230 ALOGE("codecBufferEmptied: buffer not recognized (h=%p)", header);
231 return;
232 }
233
234 ALOGV("codecBufferEmptied h=%p size=%" PRIu32 " filled=%" PRIu32 " p=%p",
235 header, header->nAllocLen, header->nFilledLen,
236 header->pBuffer);
237 CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
238
239 // header->nFilledLen may not be the original value, so we can't compare
240 // that to zero to see of this was the EOS buffer. Instead we just
241 // see if the GraphicBuffer reference was null, which should only ever
242 // happen for EOS.
243 if (codecBuffer.mGraphicBuffer == NULL) {
244 if (!(mEndOfStream && mEndOfStreamSent)) {
245 // This can happen when broken code sends us the same buffer
246 // twice in a row.
247 ALOGE("ERROR: codecBufferEmptied on non-EOS null buffer "
248 "(buffer emptied twice?)");
249 }
250 // No GraphicBuffer to deal with, no additional input or output is
251 // expected, so just return.
252 return;
253 }
254
255 if (EXTRA_CHECK) {
256 // Pull the graphic buffer handle back out of the buffer, and confirm
257 // that it matches expectations.
258 OMX_U8* data = header->pBuffer;
259 MetadataBufferType type = *(MetadataBufferType *)data;
260 if (type == kMetadataBufferTypeGrallocSource) {
261 buffer_handle_t bufferHandle;
262 memcpy(&bufferHandle, data + 4, sizeof(buffer_handle_t));
263 if (bufferHandle != codecBuffer.mGraphicBuffer->handle) {
264 // should never happen
265 ALOGE("codecBufferEmptied: buffer's handle is %p, expected %p",
266 bufferHandle, codecBuffer.mGraphicBuffer->handle);
267 CHECK(!"codecBufferEmptied: mismatched buffer");
268 }
269 } else if (type == kMetadataBufferTypeGraphicBuffer) {
270 GraphicBuffer *buffer;
271 memcpy(&buffer, data + 4, sizeof(buffer));
272 if (buffer != codecBuffer.mGraphicBuffer.get()) {
273 // should never happen
274 ALOGE("codecBufferEmptied: buffer is %p, expected %p",
275 buffer, codecBuffer.mGraphicBuffer.get());
276 CHECK(!"codecBufferEmptied: mismatched buffer");
277 }
278 }
279 }
280
281 // Find matching entry in our cached copy of the BufferQueue slots.
282 // If we find a match, release that slot. If we don't, the BufferQueue
283 // has dropped that GraphicBuffer, and there's nothing for us to release.
284 int id = codecBuffer.mBuf;
285 if (mBufferSlot[id] != NULL &&
286 mBufferSlot[id]->handle == codecBuffer.mGraphicBuffer->handle) {
287 ALOGV("cbi %d matches bq slot %d, handle=%p",
288 cbi, id, mBufferSlot[id]->handle);
289
290 if (id == mLatestSubmittedBufferId) {
291 CHECK_GT(mLatestSubmittedBufferUseCount--, 0);
292 } else {
293 mConsumer->releaseBuffer(id, codecBuffer.mFrameNumber,
294 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE);
295 }
296 } else {
297 ALOGV("codecBufferEmptied: no match for emptied buffer in cbi %d",
298 cbi);
299 }
300
301 // Mark the codec buffer as available by clearing the GraphicBuffer ref.
302 codecBuffer.mGraphicBuffer = NULL;
303
304 if (mNumFramesAvailable) {
305 // Fill this codec buffer.
306 CHECK(!mEndOfStreamSent);
307 ALOGV("buffer freed, %zu frames avail (eos=%d)",
308 mNumFramesAvailable, mEndOfStream);
309 fillCodecBuffer_l();
310 } else if (mEndOfStream) {
311 // No frames available, but EOS is pending, so use this buffer to
312 // send that.
313 ALOGV("buffer freed, EOS pending");
314 submitEndOfInputStream_l();
315 } else if (mRepeatBufferDeferred) {
316 bool success = repeatLatestSubmittedBuffer_l();
317 if (success) {
318 ALOGV("deferred repeatLatestSubmittedBuffer_l SUCCESS");
319 } else {
320 ALOGV("deferred repeatLatestSubmittedBuffer_l FAILURE");
321 }
322 mRepeatBufferDeferred = false;
323 }
324
325 return;
326 }
327
codecBufferFilled(OMX_BUFFERHEADERTYPE * header)328 void GraphicBufferSource::codecBufferFilled(OMX_BUFFERHEADERTYPE* header) {
329 Mutex::Autolock autoLock(mMutex);
330
331 if (mMaxTimestampGapUs > 0ll
332 && !(header->nFlags & OMX_BUFFERFLAG_CODECCONFIG)) {
333 ssize_t index = mOriginalTimeUs.indexOfKey(header->nTimeStamp);
334 if (index >= 0) {
335 ALOGV("OUT timestamp: %lld -> %lld",
336 static_cast<long long>(header->nTimeStamp),
337 static_cast<long long>(mOriginalTimeUs[index]));
338 header->nTimeStamp = mOriginalTimeUs[index];
339 mOriginalTimeUs.removeItemsAt(index);
340 } else {
341 // giving up the effort as encoder doesn't appear to preserve pts
342 ALOGW("giving up limiting timestamp gap (pts = %lld)",
343 header->nTimeStamp);
344 mMaxTimestampGapUs = -1ll;
345 }
346 if (mOriginalTimeUs.size() > BufferQueue::NUM_BUFFER_SLOTS) {
347 // something terribly wrong must have happened, giving up...
348 ALOGE("mOriginalTimeUs has too many entries (%zu)",
349 mOriginalTimeUs.size());
350 mMaxTimestampGapUs = -1ll;
351 }
352 }
353 }
354
suspend(bool suspend)355 void GraphicBufferSource::suspend(bool suspend) {
356 Mutex::Autolock autoLock(mMutex);
357
358 if (suspend) {
359 mSuspended = true;
360
361 while (mNumFramesAvailable > 0) {
362 BufferQueue::BufferItem item;
363 status_t err = mConsumer->acquireBuffer(&item, 0);
364
365 if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
366 // shouldn't happen.
367 ALOGW("suspend: frame was not available");
368 break;
369 } else if (err != OK) {
370 ALOGW("suspend: acquireBuffer returned err=%d", err);
371 break;
372 }
373
374 --mNumFramesAvailable;
375
376 mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber,
377 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, item.mFence);
378 }
379 return;
380 }
381
382 mSuspended = false;
383
384 if (mExecuting && mNumFramesAvailable == 0 && mRepeatBufferDeferred) {
385 if (repeatLatestSubmittedBuffer_l()) {
386 ALOGV("suspend/deferred repeatLatestSubmittedBuffer_l SUCCESS");
387
388 mRepeatBufferDeferred = false;
389 } else {
390 ALOGV("suspend/deferred repeatLatestSubmittedBuffer_l FAILURE");
391 }
392 }
393 }
394
fillCodecBuffer_l()395 bool GraphicBufferSource::fillCodecBuffer_l() {
396 CHECK(mExecuting && mNumFramesAvailable > 0);
397
398 if (mSuspended) {
399 return false;
400 }
401
402 int cbi = findAvailableCodecBuffer_l();
403 if (cbi < 0) {
404 // No buffers available, bail.
405 ALOGV("fillCodecBuffer_l: no codec buffers, avail now %zu",
406 mNumFramesAvailable);
407 return false;
408 }
409
410 ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%zu",
411 mNumFramesAvailable);
412 BufferQueue::BufferItem item;
413 status_t err = mConsumer->acquireBuffer(&item, 0);
414 if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
415 // shouldn't happen
416 ALOGW("fillCodecBuffer_l: frame was not available");
417 return false;
418 } else if (err != OK) {
419 // now what? fake end-of-stream?
420 ALOGW("fillCodecBuffer_l: acquireBuffer returned err=%d", err);
421 return false;
422 }
423
424 mNumFramesAvailable--;
425
426 // Wait for it to become available.
427 err = item.mFence->waitForever("GraphicBufferSource::fillCodecBuffer_l");
428 if (err != OK) {
429 ALOGW("failed to wait for buffer fence: %d", err);
430 // keep going
431 }
432
433 // If this is the first time we're seeing this buffer, add it to our
434 // slot table.
435 if (item.mGraphicBuffer != NULL) {
436 ALOGV("fillCodecBuffer_l: setting mBufferSlot %d", item.mBuf);
437 mBufferSlot[item.mBuf] = item.mGraphicBuffer;
438 }
439
440 err = UNKNOWN_ERROR;
441
442 // only submit sample if start time is unspecified, or sample
443 // is queued after the specified start time
444 if (mSkipFramesBeforeNs < 0ll || item.mTimestamp >= mSkipFramesBeforeNs) {
445 // if start time is set, offset time stamp by start time
446 if (mSkipFramesBeforeNs > 0) {
447 item.mTimestamp -= mSkipFramesBeforeNs;
448 }
449 err = submitBuffer_l(item, cbi);
450 }
451
452 if (err != OK) {
453 ALOGV("submitBuffer_l failed, releasing bq buf %d", item.mBuf);
454 mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber,
455 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE);
456 } else {
457 ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi);
458 setLatestSubmittedBuffer_l(item);
459 }
460
461 return true;
462 }
463
repeatLatestSubmittedBuffer_l()464 bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() {
465 CHECK(mExecuting && mNumFramesAvailable == 0);
466
467 if (mLatestSubmittedBufferId < 0 || mSuspended) {
468 return false;
469 }
470 if (mBufferSlot[mLatestSubmittedBufferId] == NULL) {
471 // This can happen if the remote side disconnects, causing
472 // onBuffersReleased() to NULL out our copy of the slots. The
473 // buffer is gone, so we have nothing to show.
474 //
475 // To be on the safe side we try to release the buffer.
476 ALOGD("repeatLatestSubmittedBuffer_l: slot was NULL");
477 mConsumer->releaseBuffer(
478 mLatestSubmittedBufferId,
479 mLatestSubmittedBufferFrameNum,
480 EGL_NO_DISPLAY,
481 EGL_NO_SYNC_KHR,
482 Fence::NO_FENCE);
483 mLatestSubmittedBufferId = -1;
484 mLatestSubmittedBufferFrameNum = 0;
485 return false;
486 }
487
488 int cbi = findAvailableCodecBuffer_l();
489 if (cbi < 0) {
490 // No buffers available, bail.
491 ALOGV("repeatLatestSubmittedBuffer_l: no codec buffers.");
492 return false;
493 }
494
495 BufferQueue::BufferItem item;
496 item.mBuf = mLatestSubmittedBufferId;
497 item.mFrameNumber = mLatestSubmittedBufferFrameNum;
498 item.mTimestamp = mRepeatLastFrameTimestamp;
499
500 status_t err = submitBuffer_l(item, cbi);
501
502 if (err != OK) {
503 return false;
504 }
505
506 ++mLatestSubmittedBufferUseCount;
507
508 /* repeat last frame up to kRepeatLastFrameCount times.
509 * in case of static scene, a single repeat might not get rid of encoder
510 * ghosting completely, refresh a couple more times to get better quality
511 */
512 if (--mRepeatLastFrameCount > 0) {
513 mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
514
515 if (mReflector != NULL) {
516 sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id());
517 msg->setInt32("generation", ++mRepeatLastFrameGeneration);
518 msg->post(mRepeatAfterUs);
519 }
520 }
521
522 return true;
523 }
524
setLatestSubmittedBuffer_l(const BufferQueue::BufferItem & item)525 void GraphicBufferSource::setLatestSubmittedBuffer_l(
526 const BufferQueue::BufferItem &item) {
527 ALOGV("setLatestSubmittedBuffer_l");
528
529 if (mLatestSubmittedBufferId >= 0) {
530 if (mLatestSubmittedBufferUseCount == 0) {
531 mConsumer->releaseBuffer(
532 mLatestSubmittedBufferId,
533 mLatestSubmittedBufferFrameNum,
534 EGL_NO_DISPLAY,
535 EGL_NO_SYNC_KHR,
536 Fence::NO_FENCE);
537 }
538 }
539
540 mLatestSubmittedBufferId = item.mBuf;
541 mLatestSubmittedBufferFrameNum = item.mFrameNumber;
542 mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
543
544 mLatestSubmittedBufferUseCount = 1;
545 mRepeatBufferDeferred = false;
546 mRepeatLastFrameCount = kRepeatLastFrameCount;
547
548 if (mReflector != NULL) {
549 sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id());
550 msg->setInt32("generation", ++mRepeatLastFrameGeneration);
551 msg->post(mRepeatAfterUs);
552 }
553 }
554
signalEndOfInputStream()555 status_t GraphicBufferSource::signalEndOfInputStream() {
556 Mutex::Autolock autoLock(mMutex);
557 ALOGV("signalEndOfInputStream: exec=%d avail=%zu eos=%d",
558 mExecuting, mNumFramesAvailable, mEndOfStream);
559
560 if (mEndOfStream) {
561 ALOGE("EOS was already signaled");
562 return INVALID_OPERATION;
563 }
564
565 // Set the end-of-stream flag. If no frames are pending from the
566 // BufferQueue, and a codec buffer is available, and we're executing,
567 // we initiate the EOS from here. Otherwise, we'll let
568 // codecBufferEmptied() (or omxExecuting) do it.
569 //
570 // Note: if there are no pending frames and all codec buffers are
571 // available, we *must* submit the EOS from here or we'll just
572 // stall since no future events are expected.
573 mEndOfStream = true;
574
575 if (mExecuting && mNumFramesAvailable == 0) {
576 submitEndOfInputStream_l();
577 }
578
579 return OK;
580 }
581
getTimestamp(const BufferQueue::BufferItem & item)582 int64_t GraphicBufferSource::getTimestamp(const BufferQueue::BufferItem &item) {
583 int64_t timeUs = item.mTimestamp / 1000;
584
585 if (mTimePerCaptureUs > 0ll) {
586 // Time lapse or slow motion mode
587 if (mPrevCaptureUs < 0ll) {
588 // first capture
589 mPrevCaptureUs = timeUs;
590 mPrevFrameUs = timeUs;
591 } else {
592 // snap to nearest capture point
593 int64_t nFrames = (timeUs + mTimePerCaptureUs / 2 - mPrevCaptureUs)
594 / mTimePerCaptureUs;
595 if (nFrames <= 0) {
596 // skip this frame as it's too close to previous capture
597 ALOGV("skipping frame, timeUs %lld", static_cast<long long>(timeUs));
598 return -1;
599 }
600 mPrevCaptureUs = mPrevCaptureUs + nFrames * mTimePerCaptureUs;
601 mPrevFrameUs += mTimePerFrameUs * nFrames;
602 }
603
604 ALOGV("timeUs %lld, captureUs %lld, frameUs %lld",
605 static_cast<long long>(timeUs),
606 static_cast<long long>(mPrevCaptureUs),
607 static_cast<long long>(mPrevFrameUs));
608
609 return mPrevFrameUs;
610 } else if (mMaxTimestampGapUs > 0ll) {
611 /* Cap timestamp gap between adjacent frames to specified max
612 *
613 * In the scenario of cast mirroring, encoding could be suspended for
614 * prolonged periods. Limiting the pts gap to workaround the problem
615 * where encoder's rate control logic produces huge frames after a
616 * long period of suspension.
617 */
618
619 int64_t originalTimeUs = timeUs;
620 if (mPrevOriginalTimeUs >= 0ll) {
621 if (originalTimeUs < mPrevOriginalTimeUs) {
622 // Drop the frame if it's going backward in time. Bad timestamp
623 // could disrupt encoder's rate control completely.
624 ALOGW("Dropping frame that's going backward in time");
625 return -1;
626 }
627 int64_t timestampGapUs = originalTimeUs - mPrevOriginalTimeUs;
628 timeUs = (timestampGapUs < mMaxTimestampGapUs ?
629 timestampGapUs : mMaxTimestampGapUs) + mPrevModifiedTimeUs;
630 }
631 mPrevOriginalTimeUs = originalTimeUs;
632 mPrevModifiedTimeUs = timeUs;
633 mOriginalTimeUs.add(timeUs, originalTimeUs);
634 ALOGV("IN timestamp: %lld -> %lld",
635 static_cast<long long>(originalTimeUs),
636 static_cast<long long>(timeUs));
637 }
638
639 return timeUs;
640 }
641
submitBuffer_l(const BufferQueue::BufferItem & item,int cbi)642 status_t GraphicBufferSource::submitBuffer_l(
643 const BufferQueue::BufferItem &item, int cbi) {
644 ALOGV("submitBuffer_l cbi=%d", cbi);
645
646 int64_t timeUs = getTimestamp(item);
647 if (timeUs < 0ll) {
648 return UNKNOWN_ERROR;
649 }
650
651 CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
652 codecBuffer.mGraphicBuffer = mBufferSlot[item.mBuf];
653 codecBuffer.mBuf = item.mBuf;
654 codecBuffer.mFrameNumber = item.mFrameNumber;
655
656 OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
657 CHECK(header->nAllocLen >= 4 + sizeof(buffer_handle_t));
658 OMX_U8* data = header->pBuffer;
659 buffer_handle_t handle;
660 if (!mUseGraphicBufferInMeta) {
661 const OMX_U32 type = kMetadataBufferTypeGrallocSource;
662 handle = codecBuffer.mGraphicBuffer->handle;
663 memcpy(data, &type, 4);
664 memcpy(data + 4, &handle, sizeof(buffer_handle_t));
665 } else {
666 // codecBuffer holds a reference to the GraphicBuffer, so
667 // it is valid while it is with the OMX component
668 const OMX_U32 type = kMetadataBufferTypeGraphicBuffer;
669 memcpy(data, &type, 4);
670 // passing a non-reference-counted graphicBuffer
671 GraphicBuffer *buffer = codecBuffer.mGraphicBuffer.get();
672 handle = buffer->handle;
673 memcpy(data + 4, &buffer, sizeof(buffer));
674 }
675
676 status_t err = mNodeInstance->emptyDirectBuffer(header, 0,
677 4 + sizeof(buffer_handle_t), OMX_BUFFERFLAG_ENDOFFRAME,
678 timeUs);
679 if (err != OK) {
680 ALOGW("WARNING: emptyDirectBuffer failed: 0x%x", err);
681 codecBuffer.mGraphicBuffer = NULL;
682 return err;
683 }
684
685 ALOGV("emptyDirectBuffer succeeded, h=%p p=%p bufhandle=%p",
686 header, header->pBuffer, handle);
687 return OK;
688 }
689
submitEndOfInputStream_l()690 void GraphicBufferSource::submitEndOfInputStream_l() {
691 CHECK(mEndOfStream);
692 if (mEndOfStreamSent) {
693 ALOGV("EOS already sent");
694 return;
695 }
696
697 int cbi = findAvailableCodecBuffer_l();
698 if (cbi < 0) {
699 ALOGV("submitEndOfInputStream_l: no codec buffers available");
700 return;
701 }
702
703 // We reject any additional incoming graphic buffers, so there's no need
704 // to stick a placeholder into codecBuffer.mGraphicBuffer to mark it as
705 // in-use.
706 CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
707
708 OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
709 if (EXTRA_CHECK) {
710 // Guard against implementations that don't check nFilledLen.
711 size_t fillLen = 4 + sizeof(buffer_handle_t);
712 CHECK(header->nAllocLen >= fillLen);
713 OMX_U8* data = header->pBuffer;
714 memset(data, 0xcd, fillLen);
715 }
716
717 uint64_t timestamp = 0; // does this matter?
718
719 status_t err = mNodeInstance->emptyDirectBuffer(header, /*offset*/ 0,
720 /*length*/ 0, OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_EOS,
721 timestamp);
722 if (err != OK) {
723 ALOGW("emptyDirectBuffer EOS failed: 0x%x", err);
724 } else {
725 ALOGV("submitEndOfInputStream_l: buffer submitted, header=%p cbi=%d",
726 header, cbi);
727 mEndOfStreamSent = true;
728 }
729 }
730
findAvailableCodecBuffer_l()731 int GraphicBufferSource::findAvailableCodecBuffer_l() {
732 CHECK(mCodecBuffers.size() > 0);
733
734 for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
735 if (mCodecBuffers[i].mGraphicBuffer == NULL) {
736 return i;
737 }
738 }
739 return -1;
740 }
741
findMatchingCodecBuffer_l(const OMX_BUFFERHEADERTYPE * header)742 int GraphicBufferSource::findMatchingCodecBuffer_l(
743 const OMX_BUFFERHEADERTYPE* header) {
744 for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
745 if (mCodecBuffers[i].mHeader == header) {
746 return i;
747 }
748 }
749 return -1;
750 }
751
752 // BufferQueue::ConsumerListener callback
onFrameAvailable(const BufferItem &)753 void GraphicBufferSource::onFrameAvailable(const BufferItem& /*item*/) {
754 Mutex::Autolock autoLock(mMutex);
755
756 ALOGV("onFrameAvailable exec=%d avail=%zu",
757 mExecuting, mNumFramesAvailable);
758
759 if (mEndOfStream || mSuspended) {
760 if (mEndOfStream) {
761 // This should only be possible if a new buffer was queued after
762 // EOS was signaled, i.e. the app is misbehaving.
763
764 ALOGW("onFrameAvailable: EOS is set, ignoring frame");
765 } else {
766 ALOGV("onFrameAvailable: suspended, ignoring frame");
767 }
768
769 BufferQueue::BufferItem item;
770 status_t err = mConsumer->acquireBuffer(&item, 0);
771 if (err == OK) {
772 // If this is the first time we're seeing this buffer, add it to our
773 // slot table.
774 if (item.mGraphicBuffer != NULL) {
775 ALOGV("onFrameAvailable: setting mBufferSlot %d", item.mBuf);
776 mBufferSlot[item.mBuf] = item.mGraphicBuffer;
777 }
778 mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber,
779 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, item.mFence);
780 }
781 return;
782 }
783
784 mNumFramesAvailable++;
785
786 mRepeatBufferDeferred = false;
787 ++mRepeatLastFrameGeneration;
788
789 if (mExecuting) {
790 fillCodecBuffer_l();
791 }
792 }
793
794 // BufferQueue::ConsumerListener callback
onBuffersReleased()795 void GraphicBufferSource::onBuffersReleased() {
796 Mutex::Autolock lock(mMutex);
797
798 uint64_t slotMask;
799 if (mConsumer->getReleasedBuffers(&slotMask) != NO_ERROR) {
800 ALOGW("onBuffersReleased: unable to get released buffer set");
801 slotMask = 0xffffffffffffffffULL;
802 }
803
804 ALOGV("onBuffersReleased: 0x%016" PRIx64, slotMask);
805
806 for (int i = 0; i < BufferQueue::NUM_BUFFER_SLOTS; i++) {
807 if ((slotMask & 0x01) != 0) {
808 mBufferSlot[i] = NULL;
809 }
810 slotMask >>= 1;
811 }
812 }
813
814 // BufferQueue::ConsumerListener callback
onSidebandStreamChanged()815 void GraphicBufferSource::onSidebandStreamChanged() {
816 ALOG_ASSERT(false, "GraphicBufferSource can't consume sideband streams");
817 }
818
setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs)819 status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs(
820 int64_t repeatAfterUs) {
821 Mutex::Autolock autoLock(mMutex);
822
823 if (mExecuting || repeatAfterUs <= 0ll) {
824 return INVALID_OPERATION;
825 }
826
827 mRepeatAfterUs = repeatAfterUs;
828
829 return OK;
830 }
831
setMaxTimestampGapUs(int64_t maxGapUs)832 status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) {
833 Mutex::Autolock autoLock(mMutex);
834
835 if (mExecuting || maxGapUs <= 0ll) {
836 return INVALID_OPERATION;
837 }
838
839 mMaxTimestampGapUs = maxGapUs;
840
841 return OK;
842 }
843
setSkipFramesBeforeUs(int64_t skipFramesBeforeUs)844 void GraphicBufferSource::setSkipFramesBeforeUs(int64_t skipFramesBeforeUs) {
845 Mutex::Autolock autoLock(mMutex);
846
847 mSkipFramesBeforeNs =
848 (skipFramesBeforeUs > 0) ? (skipFramesBeforeUs * 1000) : -1ll;
849 }
850
setTimeLapseUs(int64_t * data)851 status_t GraphicBufferSource::setTimeLapseUs(int64_t* data) {
852 Mutex::Autolock autoLock(mMutex);
853
854 if (mExecuting || data[0] <= 0ll || data[1] <= 0ll) {
855 return INVALID_OPERATION;
856 }
857
858 mTimePerFrameUs = data[0];
859 mTimePerCaptureUs = data[1];
860
861 return OK;
862 }
863
onMessageReceived(const sp<AMessage> & msg)864 void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) {
865 switch (msg->what()) {
866 case kWhatRepeatLastFrame:
867 {
868 Mutex::Autolock autoLock(mMutex);
869
870 int32_t generation;
871 CHECK(msg->findInt32("generation", &generation));
872
873 if (generation != mRepeatLastFrameGeneration) {
874 // stale
875 break;
876 }
877
878 if (!mExecuting || mNumFramesAvailable > 0) {
879 break;
880 }
881
882 bool success = repeatLatestSubmittedBuffer_l();
883
884 if (success) {
885 ALOGV("repeatLatestSubmittedBuffer_l SUCCESS");
886 } else {
887 ALOGV("repeatLatestSubmittedBuffer_l FAILURE");
888 mRepeatBufferDeferred = true;
889 }
890 break;
891 }
892
893 default:
894 TRESPASS();
895 }
896 }
897
898 } // namespace android
899