1 /*
2 * Copyright 2017, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "CCodecBufferChannel"
19 #include <utils/Log.h>
20
21 #include <numeric>
22
23 #include <C2AllocatorGralloc.h>
24 #include <C2PlatformSupport.h>
25 #include <C2BlockInternal.h>
26 #include <C2Config.h>
27 #include <C2Debug.h>
28
29 #include <android/hardware/cas/native/1.0/IDescrambler.h>
30 #include <android-base/stringprintf.h>
31 #include <binder/MemoryDealer.h>
32 #include <gui/Surface.h>
33 #include <media/openmax/OMX_Core.h>
34 #include <media/stagefright/foundation/ABuffer.h>
35 #include <media/stagefright/foundation/ALookup.h>
36 #include <media/stagefright/foundation/AMessage.h>
37 #include <media/stagefright/foundation/AUtils.h>
38 #include <media/stagefright/foundation/hexdump.h>
39 #include <media/stagefright/MediaCodec.h>
40 #include <media/stagefright/MediaCodecConstants.h>
41 #include <media/MediaCodecBuffer.h>
42 #include <system/window.h>
43
44 #include "CCodecBufferChannel.h"
45 #include "Codec2Buffer.h"
46 #include "SkipCutBuffer.h"
47
48 namespace android {
49
50 using android::base::StringPrintf;
51 using hardware::hidl_handle;
52 using hardware::hidl_string;
53 using hardware::hidl_vec;
54 using namespace hardware::cas::V1_0;
55 using namespace hardware::cas::native::V1_0;
56
57 using CasStatus = hardware::cas::V1_0::Status;
58
59 namespace {
60
61 constexpr size_t kSmoothnessFactor = 4;
62 constexpr size_t kRenderingDepth = 3;
63
64 // This is for keeping IGBP's buffer dropping logic in legacy mode other
65 // than making it non-blocking. Do not change this value.
66 const static size_t kDequeueTimeoutNs = 0;
67
68 } // namespace
69
QueueGuard(CCodecBufferChannel::QueueSync & sync)70 CCodecBufferChannel::QueueGuard::QueueGuard(
71 CCodecBufferChannel::QueueSync &sync) : mSync(sync) {
72 Mutex::Autolock l(mSync.mGuardLock);
73 // At this point it's guaranteed that mSync is not under state transition,
74 // as we are holding its mutex.
75
76 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
77 if (count->value == -1) {
78 mRunning = false;
79 } else {
80 ++count->value;
81 mRunning = true;
82 }
83 }
84
~QueueGuard()85 CCodecBufferChannel::QueueGuard::~QueueGuard() {
86 if (mRunning) {
87 // We are not holding mGuardLock at this point so that QueueSync::stop() can
88 // keep holding the lock until mCount reaches zero.
89 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
90 --count->value;
91 count->cond.broadcast();
92 }
93 }
94
start()95 void CCodecBufferChannel::QueueSync::start() {
96 Mutex::Autolock l(mGuardLock);
97 // If stopped, it goes to running state; otherwise no-op.
98 Mutexed<Counter>::Locked count(mCount);
99 if (count->value == -1) {
100 count->value = 0;
101 }
102 }
103
stop()104 void CCodecBufferChannel::QueueSync::stop() {
105 Mutex::Autolock l(mGuardLock);
106 Mutexed<Counter>::Locked count(mCount);
107 if (count->value == -1) {
108 // no-op
109 return;
110 }
111 // Holding mGuardLock here blocks creation of additional QueueGuard objects, so
112 // mCount can only decrement. In other words, threads that acquired the lock
113 // are allowed to finish execution but additional threads trying to acquire
114 // the lock at this point will block, and then get QueueGuard at STOPPED
115 // state.
116 while (count->value != 0) {
117 count.waitForCondition(count->cond);
118 }
119 count->value = -1;
120 }
121
122 // CCodecBufferChannel::ReorderStash
123
ReorderStash()124 CCodecBufferChannel::ReorderStash::ReorderStash() {
125 clear();
126 }
127
clear()128 void CCodecBufferChannel::ReorderStash::clear() {
129 mPending.clear();
130 mStash.clear();
131 mDepth = 0;
132 mKey = C2Config::ORDINAL;
133 }
134
flush()135 void CCodecBufferChannel::ReorderStash::flush() {
136 mPending.clear();
137 mStash.clear();
138 }
139
setDepth(uint32_t depth)140 void CCodecBufferChannel::ReorderStash::setDepth(uint32_t depth) {
141 mPending.splice(mPending.end(), mStash);
142 mDepth = depth;
143 }
144
setKey(C2Config::ordinal_key_t key)145 void CCodecBufferChannel::ReorderStash::setKey(C2Config::ordinal_key_t key) {
146 mPending.splice(mPending.end(), mStash);
147 mKey = key;
148 }
149
pop(Entry * entry)150 bool CCodecBufferChannel::ReorderStash::pop(Entry *entry) {
151 if (mPending.empty()) {
152 return false;
153 }
154 entry->buffer = mPending.front().buffer;
155 entry->timestamp = mPending.front().timestamp;
156 entry->flags = mPending.front().flags;
157 entry->ordinal = mPending.front().ordinal;
158 mPending.pop_front();
159 return true;
160 }
161
emplace(const std::shared_ptr<C2Buffer> & buffer,int64_t timestamp,int32_t flags,const C2WorkOrdinalStruct & ordinal)162 void CCodecBufferChannel::ReorderStash::emplace(
163 const std::shared_ptr<C2Buffer> &buffer,
164 int64_t timestamp,
165 int32_t flags,
166 const C2WorkOrdinalStruct &ordinal) {
167 bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
168 if (!buffer && eos) {
169 // TRICKY: we may be violating ordering of the stash here. Because we
170 // don't expect any more emplace() calls after this, the ordering should
171 // not matter.
172 mStash.emplace_back(buffer, timestamp, flags, ordinal);
173 } else {
174 flags = flags & ~MediaCodec::BUFFER_FLAG_EOS;
175 auto it = mStash.begin();
176 for (; it != mStash.end(); ++it) {
177 if (less(ordinal, it->ordinal)) {
178 break;
179 }
180 }
181 mStash.emplace(it, buffer, timestamp, flags, ordinal);
182 if (eos) {
183 mStash.back().flags = mStash.back().flags | MediaCodec::BUFFER_FLAG_EOS;
184 }
185 }
186 while (!mStash.empty() && mStash.size() > mDepth) {
187 mPending.push_back(mStash.front());
188 mStash.pop_front();
189 }
190 }
191
defer(const CCodecBufferChannel::ReorderStash::Entry & entry)192 void CCodecBufferChannel::ReorderStash::defer(
193 const CCodecBufferChannel::ReorderStash::Entry &entry) {
194 mPending.push_front(entry);
195 }
196
hasPending() const197 bool CCodecBufferChannel::ReorderStash::hasPending() const {
198 return !mPending.empty();
199 }
200
less(const C2WorkOrdinalStruct & o1,const C2WorkOrdinalStruct & o2)201 bool CCodecBufferChannel::ReorderStash::less(
202 const C2WorkOrdinalStruct &o1, const C2WorkOrdinalStruct &o2) {
203 switch (mKey) {
204 case C2Config::ORDINAL: return o1.frameIndex < o2.frameIndex;
205 case C2Config::TIMESTAMP: return o1.timestamp < o2.timestamp;
206 case C2Config::CUSTOM: return o1.customOrdinal < o2.customOrdinal;
207 default:
208 ALOGD("Unrecognized key; default to timestamp");
209 return o1.frameIndex < o2.frameIndex;
210 }
211 }
212
213 // Input
214
Input()215 CCodecBufferChannel::Input::Input() : extraBuffers("extra") {}
216
217 // CCodecBufferChannel
218
CCodecBufferChannel(const std::shared_ptr<CCodecCallback> & callback)219 CCodecBufferChannel::CCodecBufferChannel(
220 const std::shared_ptr<CCodecCallback> &callback)
221 : mHeapSeqNum(-1),
222 mCCodecCallback(callback),
223 mFrameIndex(0u),
224 mFirstValidFrameIndex(0u),
225 mMetaMode(MODE_NONE),
226 mInputMetEos(false) {
227 mOutputSurface.lock()->maxDequeueBuffers = kSmoothnessFactor + kRenderingDepth;
228 {
229 Mutexed<Input>::Locked input(mInput);
230 input->buffers.reset(new DummyInputBuffers(""));
231 input->extraBuffers.flush();
232 input->inputDelay = 0u;
233 input->pipelineDelay = 0u;
234 input->numSlots = kSmoothnessFactor;
235 input->numExtraSlots = 0u;
236 }
237 {
238 Mutexed<Output>::Locked output(mOutput);
239 output->outputDelay = 0u;
240 output->numSlots = kSmoothnessFactor;
241 }
242 }
243
~CCodecBufferChannel()244 CCodecBufferChannel::~CCodecBufferChannel() {
245 if (mCrypto != nullptr && mDealer != nullptr && mHeapSeqNum >= 0) {
246 mCrypto->unsetHeap(mHeapSeqNum);
247 }
248 }
249
setComponent(const std::shared_ptr<Codec2Client::Component> & component)250 void CCodecBufferChannel::setComponent(
251 const std::shared_ptr<Codec2Client::Component> &component) {
252 mComponent = component;
253 mComponentName = component->getName() + StringPrintf("#%d", int(uintptr_t(component.get()) % 997));
254 mName = mComponentName.c_str();
255 }
256
setInputSurface(const std::shared_ptr<InputSurfaceWrapper> & surface)257 status_t CCodecBufferChannel::setInputSurface(
258 const std::shared_ptr<InputSurfaceWrapper> &surface) {
259 ALOGV("[%s] setInputSurface", mName);
260 mInputSurface = surface;
261 return mInputSurface->connect(mComponent);
262 }
263
signalEndOfInputStream()264 status_t CCodecBufferChannel::signalEndOfInputStream() {
265 if (mInputSurface == nullptr) {
266 return INVALID_OPERATION;
267 }
268 return mInputSurface->signalEndOfInputStream();
269 }
270
queueInputBufferInternal(sp<MediaCodecBuffer> buffer)271 status_t CCodecBufferChannel::queueInputBufferInternal(sp<MediaCodecBuffer> buffer) {
272 int64_t timeUs;
273 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
274
275 if (mInputMetEos) {
276 ALOGD("[%s] buffers after EOS ignored (%lld us)", mName, (long long)timeUs);
277 return OK;
278 }
279
280 int32_t flags = 0;
281 int32_t tmp = 0;
282 bool eos = false;
283 if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
284 eos = true;
285 mInputMetEos = true;
286 ALOGV("[%s] input EOS", mName);
287 }
288 if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
289 flags |= C2FrameData::FLAG_CODEC_CONFIG;
290 }
291 ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
292 std::unique_ptr<C2Work> work(new C2Work);
293 work->input.ordinal.timestamp = timeUs;
294 work->input.ordinal.frameIndex = mFrameIndex++;
295 // WORKAROUND: until codecs support handling work after EOS and max output sizing, use timestamp
296 // manipulation to achieve image encoding via video codec, and to constrain encoded output.
297 // Keep client timestamp in customOrdinal
298 work->input.ordinal.customOrdinal = timeUs;
299 work->input.buffers.clear();
300
301 uint64_t queuedFrameIndex = work->input.ordinal.frameIndex.peeku();
302 std::vector<std::shared_ptr<C2Buffer>> queuedBuffers;
303 sp<Codec2Buffer> copy;
304
305 if (buffer->size() > 0u) {
306 Mutexed<Input>::Locked input(mInput);
307 std::shared_ptr<C2Buffer> c2buffer;
308 if (!input->buffers->releaseBuffer(buffer, &c2buffer, false)) {
309 return -ENOENT;
310 }
311 // TODO: we want to delay copying buffers.
312 if (input->extraBuffers.numComponentBuffers() < input->numExtraSlots) {
313 copy = input->buffers->cloneAndReleaseBuffer(buffer);
314 if (copy != nullptr) {
315 (void)input->extraBuffers.assignSlot(copy);
316 if (!input->extraBuffers.releaseSlot(copy, &c2buffer, false)) {
317 return UNKNOWN_ERROR;
318 }
319 bool released = input->buffers->releaseBuffer(buffer, nullptr, true);
320 ALOGV("[%s] queueInputBuffer: buffer copied; %sreleased",
321 mName, released ? "" : "not ");
322 buffer.clear();
323 } else {
324 ALOGW("[%s] queueInputBuffer: failed to copy a buffer; this may cause input "
325 "buffer starvation on component.", mName);
326 }
327 }
328 work->input.buffers.push_back(c2buffer);
329 queuedBuffers.push_back(c2buffer);
330 } else if (eos) {
331 flags |= C2FrameData::FLAG_END_OF_STREAM;
332 }
333 work->input.flags = (C2FrameData::flags_t)flags;
334 // TODO: fill info's
335
336 work->input.configUpdate = std::move(mParamsToBeSet);
337 work->worklets.clear();
338 work->worklets.emplace_back(new C2Worklet);
339
340 std::list<std::unique_ptr<C2Work>> items;
341 items.push_back(std::move(work));
342 mPipelineWatcher.lock()->onWorkQueued(
343 queuedFrameIndex,
344 std::move(queuedBuffers),
345 PipelineWatcher::Clock::now());
346 c2_status_t err = mComponent->queue(&items);
347 if (err != C2_OK) {
348 mPipelineWatcher.lock()->onWorkDone(queuedFrameIndex);
349 }
350
351 if (err == C2_OK && eos && buffer->size() > 0u) {
352 work.reset(new C2Work);
353 work->input.ordinal.timestamp = timeUs;
354 work->input.ordinal.frameIndex = mFrameIndex++;
355 // WORKAROUND: keep client timestamp in customOrdinal
356 work->input.ordinal.customOrdinal = timeUs;
357 work->input.buffers.clear();
358 work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
359 work->worklets.emplace_back(new C2Worklet);
360
361 queuedFrameIndex = work->input.ordinal.frameIndex.peeku();
362 queuedBuffers.clear();
363
364 items.clear();
365 items.push_back(std::move(work));
366
367 mPipelineWatcher.lock()->onWorkQueued(
368 queuedFrameIndex,
369 std::move(queuedBuffers),
370 PipelineWatcher::Clock::now());
371 err = mComponent->queue(&items);
372 if (err != C2_OK) {
373 mPipelineWatcher.lock()->onWorkDone(queuedFrameIndex);
374 }
375 }
376 if (err == C2_OK) {
377 Mutexed<Input>::Locked input(mInput);
378 bool released = false;
379 if (buffer) {
380 released = input->buffers->releaseBuffer(buffer, nullptr, true);
381 } else if (copy) {
382 released = input->extraBuffers.releaseSlot(copy, nullptr, true);
383 }
384 ALOGV("[%s] queueInputBuffer: buffer%s %sreleased",
385 mName, (buffer == nullptr) ? "(copy)" : "", released ? "" : "not ");
386 }
387
388 feedInputBufferIfAvailableInternal();
389 return err;
390 }
391
setParameters(std::vector<std::unique_ptr<C2Param>> & params)392 status_t CCodecBufferChannel::setParameters(std::vector<std::unique_ptr<C2Param>> ¶ms) {
393 QueueGuard guard(mSync);
394 if (!guard.isRunning()) {
395 ALOGD("[%s] setParameters is only supported in the running state.", mName);
396 return -ENOSYS;
397 }
398 mParamsToBeSet.insert(mParamsToBeSet.end(),
399 std::make_move_iterator(params.begin()),
400 std::make_move_iterator(params.end()));
401 params.clear();
402 return OK;
403 }
404
queueInputBuffer(const sp<MediaCodecBuffer> & buffer)405 status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
406 QueueGuard guard(mSync);
407 if (!guard.isRunning()) {
408 ALOGD("[%s] No more buffers should be queued at current state.", mName);
409 return -ENOSYS;
410 }
411 return queueInputBufferInternal(buffer);
412 }
413
queueSecureInputBuffer(const sp<MediaCodecBuffer> & buffer,bool secure,const uint8_t * key,const uint8_t * iv,CryptoPlugin::Mode mode,CryptoPlugin::Pattern pattern,const CryptoPlugin::SubSample * subSamples,size_t numSubSamples,AString * errorDetailMsg)414 status_t CCodecBufferChannel::queueSecureInputBuffer(
415 const sp<MediaCodecBuffer> &buffer, bool secure, const uint8_t *key,
416 const uint8_t *iv, CryptoPlugin::Mode mode, CryptoPlugin::Pattern pattern,
417 const CryptoPlugin::SubSample *subSamples, size_t numSubSamples,
418 AString *errorDetailMsg) {
419 QueueGuard guard(mSync);
420 if (!guard.isRunning()) {
421 ALOGD("[%s] No more buffers should be queued at current state.", mName);
422 return -ENOSYS;
423 }
424
425 if (!hasCryptoOrDescrambler()) {
426 return -ENOSYS;
427 }
428 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get());
429
430 ssize_t result = -1;
431 ssize_t codecDataOffset = 0;
432 if (mCrypto != nullptr) {
433 ICrypto::DestinationBuffer destination;
434 if (secure) {
435 destination.mType = ICrypto::kDestinationTypeNativeHandle;
436 destination.mHandle = encryptedBuffer->handle();
437 } else {
438 destination.mType = ICrypto::kDestinationTypeSharedMemory;
439 destination.mSharedMemory = mDecryptDestination;
440 }
441 ICrypto::SourceBuffer source;
442 encryptedBuffer->fillSourceBuffer(&source);
443 result = mCrypto->decrypt(
444 key, iv, mode, pattern, source, buffer->offset(),
445 subSamples, numSubSamples, destination, errorDetailMsg);
446 if (result < 0) {
447 return result;
448 }
449 if (destination.mType == ICrypto::kDestinationTypeSharedMemory) {
450 encryptedBuffer->copyDecryptedContent(mDecryptDestination, result);
451 }
452 } else {
453 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
454 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
455 hidl_vec<SubSample> hidlSubSamples;
456 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
457
458 hardware::cas::native::V1_0::SharedBuffer srcBuffer;
459 encryptedBuffer->fillSourceBuffer(&srcBuffer);
460
461 DestinationBuffer dstBuffer;
462 if (secure) {
463 dstBuffer.type = BufferType::NATIVE_HANDLE;
464 dstBuffer.secureMemory = hidl_handle(encryptedBuffer->handle());
465 } else {
466 dstBuffer.type = BufferType::SHARED_MEMORY;
467 dstBuffer.nonsecureMemory = srcBuffer;
468 }
469
470 CasStatus status = CasStatus::OK;
471 hidl_string detailedError;
472 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
473
474 if (key != nullptr) {
475 sctrl = (ScramblingControl)key[0];
476 // Adjust for the PES offset
477 codecDataOffset = key[2] | (key[3] << 8);
478 }
479
480 auto returnVoid = mDescrambler->descramble(
481 sctrl,
482 hidlSubSamples,
483 srcBuffer,
484 0,
485 dstBuffer,
486 0,
487 [&status, &result, &detailedError] (
488 CasStatus _status, uint32_t _bytesWritten,
489 const hidl_string& _detailedError) {
490 status = _status;
491 result = (ssize_t)_bytesWritten;
492 detailedError = _detailedError;
493 });
494
495 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
496 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
497 mName, returnVoid.description().c_str(), status, result);
498 return UNKNOWN_ERROR;
499 }
500
501 if (result < codecDataOffset) {
502 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
503 return BAD_VALUE;
504 }
505
506 ALOGV("[%s] descramble succeeded, %zd bytes", mName, result);
507
508 if (dstBuffer.type == BufferType::SHARED_MEMORY) {
509 encryptedBuffer->copyDecryptedContentFromMemory(result);
510 }
511 }
512
513 buffer->setRange(codecDataOffset, result - codecDataOffset);
514 return queueInputBufferInternal(buffer);
515 }
516
feedInputBufferIfAvailable()517 void CCodecBufferChannel::feedInputBufferIfAvailable() {
518 QueueGuard guard(mSync);
519 if (!guard.isRunning()) {
520 ALOGV("[%s] We're not running --- no input buffer reported", mName);
521 return;
522 }
523 feedInputBufferIfAvailableInternal();
524 }
525
feedInputBufferIfAvailableInternal()526 void CCodecBufferChannel::feedInputBufferIfAvailableInternal() {
527 if (mInputMetEos ||
528 mReorderStash.lock()->hasPending() ||
529 mPipelineWatcher.lock()->pipelineFull()) {
530 return;
531 } else {
532 Mutexed<Output>::Locked output(mOutput);
533 if (output->buffers->numClientBuffers() >= output->numSlots) {
534 return;
535 }
536 }
537 size_t numInputSlots = mInput.lock()->numSlots;
538 for (size_t i = 0; i < numInputSlots; ++i) {
539 sp<MediaCodecBuffer> inBuffer;
540 size_t index;
541 {
542 Mutexed<Input>::Locked input(mInput);
543 if (input->buffers->numClientBuffers() >= input->numSlots) {
544 return;
545 }
546 if (!input->buffers->requestNewBuffer(&index, &inBuffer)) {
547 ALOGV("[%s] no new buffer available", mName);
548 break;
549 }
550 }
551 ALOGV("[%s] new input index = %zu [%p]", mName, index, inBuffer.get());
552 mCallback->onInputBufferAvailable(index, inBuffer);
553 }
554 }
555
renderOutputBuffer(const sp<MediaCodecBuffer> & buffer,int64_t timestampNs)556 status_t CCodecBufferChannel::renderOutputBuffer(
557 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) {
558 ALOGV("[%s] renderOutputBuffer: %p", mName, buffer.get());
559 std::shared_ptr<C2Buffer> c2Buffer;
560 bool released = false;
561 {
562 Mutexed<Output>::Locked output(mOutput);
563 if (output->buffers) {
564 released = output->buffers->releaseBuffer(buffer, &c2Buffer);
565 }
566 }
567 // NOTE: some apps try to releaseOutputBuffer() with timestamp and/or render
568 // set to true.
569 sendOutputBuffers();
570 // input buffer feeding may have been gated by pending output buffers
571 feedInputBufferIfAvailable();
572 if (!c2Buffer) {
573 if (released) {
574 std::call_once(mRenderWarningFlag, [this] {
575 ALOGW("[%s] The app is calling releaseOutputBuffer() with "
576 "timestamp or render=true with non-video buffers. Apps should "
577 "call releaseOutputBuffer() with render=false for those.",
578 mName);
579 });
580 }
581 return INVALID_OPERATION;
582 }
583
584 #if 0
585 const std::vector<std::shared_ptr<const C2Info>> infoParams = c2Buffer->info();
586 ALOGV("[%s] queuing gfx buffer with %zu infos", mName, infoParams.size());
587 for (const std::shared_ptr<const C2Info> &info : infoParams) {
588 AString res;
589 for (size_t ix = 0; ix + 3 < info->size(); ix += 4) {
590 if (ix) res.append(", ");
591 res.append(*((int32_t*)info.get() + (ix / 4)));
592 }
593 ALOGV(" [%s]", res.c_str());
594 }
595 #endif
596 std::shared_ptr<const C2StreamRotationInfo::output> rotation =
597 std::static_pointer_cast<const C2StreamRotationInfo::output>(
598 c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE));
599 bool flip = rotation && (rotation->flip & 1);
600 uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3;
601 uint32_t transform = 0;
602 switch (quarters) {
603 case 0: // no rotation
604 transform = flip ? HAL_TRANSFORM_FLIP_H : 0;
605 break;
606 case 1: // 90 degrees counter-clockwise
607 transform = flip ? (HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_ROT_90)
608 : HAL_TRANSFORM_ROT_270;
609 break;
610 case 2: // 180 degrees
611 transform = flip ? HAL_TRANSFORM_FLIP_V : HAL_TRANSFORM_ROT_180;
612 break;
613 case 3: // 90 degrees clockwise
614 transform = flip ? (HAL_TRANSFORM_FLIP_H | HAL_TRANSFORM_ROT_90)
615 : HAL_TRANSFORM_ROT_90;
616 break;
617 }
618
619 std::shared_ptr<const C2StreamSurfaceScalingInfo::output> surfaceScaling =
620 std::static_pointer_cast<const C2StreamSurfaceScalingInfo::output>(
621 c2Buffer->getInfo(C2StreamSurfaceScalingInfo::output::PARAM_TYPE));
622 uint32_t videoScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW;
623 if (surfaceScaling) {
624 videoScalingMode = surfaceScaling->value;
625 }
626
627 // Use dataspace from format as it has the default aspects already applied
628 android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
629 (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace);
630
631 // HDR static info
632 std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo =
633 std::static_pointer_cast<const C2StreamHdrStaticInfo::output>(
634 c2Buffer->getInfo(C2StreamHdrStaticInfo::output::PARAM_TYPE));
635
636 // HDR10 plus info
637 std::shared_ptr<const C2StreamHdr10PlusInfo::output> hdr10PlusInfo =
638 std::static_pointer_cast<const C2StreamHdr10PlusInfo::output>(
639 c2Buffer->getInfo(C2StreamHdr10PlusInfo::output::PARAM_TYPE));
640
641 {
642 Mutexed<OutputSurface>::Locked output(mOutputSurface);
643 if (output->surface == nullptr) {
644 ALOGI("[%s] cannot render buffer without surface", mName);
645 return OK;
646 }
647 }
648
649 std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks();
650 if (blocks.size() != 1u) {
651 ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size());
652 return UNKNOWN_ERROR;
653 }
654 const C2ConstGraphicBlock &block = blocks.front();
655
656 // TODO: revisit this after C2Fence implementation.
657 android::IGraphicBufferProducer::QueueBufferInput qbi(
658 timestampNs,
659 false, // droppable
660 dataSpace,
661 Rect(blocks.front().crop().left,
662 blocks.front().crop().top,
663 blocks.front().crop().right(),
664 blocks.front().crop().bottom()),
665 videoScalingMode,
666 transform,
667 Fence::NO_FENCE, 0);
668 if (hdrStaticInfo || hdr10PlusInfo) {
669 HdrMetadata hdr;
670 if (hdrStaticInfo) {
671 struct android_smpte2086_metadata smpte2086_meta = {
672 .displayPrimaryRed = {
673 hdrStaticInfo->mastering.red.x, hdrStaticInfo->mastering.red.y
674 },
675 .displayPrimaryGreen = {
676 hdrStaticInfo->mastering.green.x, hdrStaticInfo->mastering.green.y
677 },
678 .displayPrimaryBlue = {
679 hdrStaticInfo->mastering.blue.x, hdrStaticInfo->mastering.blue.y
680 },
681 .whitePoint = {
682 hdrStaticInfo->mastering.white.x, hdrStaticInfo->mastering.white.y
683 },
684 .maxLuminance = hdrStaticInfo->mastering.maxLuminance,
685 .minLuminance = hdrStaticInfo->mastering.minLuminance,
686 };
687
688 struct android_cta861_3_metadata cta861_meta = {
689 .maxContentLightLevel = hdrStaticInfo->maxCll,
690 .maxFrameAverageLightLevel = hdrStaticInfo->maxFall,
691 };
692
693 hdr.validTypes = HdrMetadata::SMPTE2086 | HdrMetadata::CTA861_3;
694 hdr.smpte2086 = smpte2086_meta;
695 hdr.cta8613 = cta861_meta;
696 }
697 if (hdr10PlusInfo) {
698 hdr.validTypes |= HdrMetadata::HDR10PLUS;
699 hdr.hdr10plus.assign(
700 hdr10PlusInfo->m.value,
701 hdr10PlusInfo->m.value + hdr10PlusInfo->flexCount());
702 }
703 qbi.setHdrMetadata(hdr);
704 }
705 // we don't have dirty regions
706 qbi.setSurfaceDamage(Region::INVALID_REGION);
707 android::IGraphicBufferProducer::QueueBufferOutput qbo;
708 status_t result = mComponent->queueToOutputSurface(block, qbi, &qbo);
709 if (result != OK) {
710 ALOGI("[%s] queueBuffer failed: %d", mName, result);
711 return result;
712 }
713 ALOGV("[%s] queue buffer successful", mName);
714
715 int64_t mediaTimeUs = 0;
716 (void)buffer->meta()->findInt64("timeUs", &mediaTimeUs);
717 mCCodecCallback->onOutputFramesRendered(mediaTimeUs, timestampNs);
718
719 return OK;
720 }
721
discardBuffer(const sp<MediaCodecBuffer> & buffer)722 status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
723 ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
724 bool released = false;
725 {
726 Mutexed<Input>::Locked input(mInput);
727 if (input->buffers && input->buffers->releaseBuffer(buffer, nullptr, true)) {
728 released = true;
729 }
730 }
731 {
732 Mutexed<Output>::Locked output(mOutput);
733 if (output->buffers && output->buffers->releaseBuffer(buffer, nullptr)) {
734 released = true;
735 }
736 }
737 if (released) {
738 sendOutputBuffers();
739 feedInputBufferIfAvailable();
740 } else {
741 ALOGD("[%s] MediaCodec discarded an unknown buffer", mName);
742 }
743 return OK;
744 }
745
getInputBufferArray(Vector<sp<MediaCodecBuffer>> * array)746 void CCodecBufferChannel::getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
747 array->clear();
748 Mutexed<Input>::Locked input(mInput);
749
750 if (!input->buffers->isArrayMode()) {
751 input->buffers = input->buffers->toArrayMode(input->numSlots);
752 }
753
754 input->buffers->getArray(array);
755 }
756
getOutputBufferArray(Vector<sp<MediaCodecBuffer>> * array)757 void CCodecBufferChannel::getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
758 array->clear();
759 Mutexed<Output>::Locked output(mOutput);
760
761 if (!output->buffers->isArrayMode()) {
762 output->buffers = output->buffers->toArrayMode(output->numSlots);
763 }
764
765 output->buffers->getArray(array);
766 }
767
start(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat)768 status_t CCodecBufferChannel::start(
769 const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) {
770 C2StreamBufferTypeSetting::input iStreamFormat(0u);
771 C2StreamBufferTypeSetting::output oStreamFormat(0u);
772 C2PortReorderBufferDepthTuning::output reorderDepth;
773 C2PortReorderKeySetting::output reorderKey;
774 C2PortActualDelayTuning::input inputDelay(0);
775 C2PortActualDelayTuning::output outputDelay(0);
776 C2ActualPipelineDelayTuning pipelineDelay(0);
777
778 c2_status_t err = mComponent->query(
779 {
780 &iStreamFormat,
781 &oStreamFormat,
782 &reorderDepth,
783 &reorderKey,
784 &inputDelay,
785 &pipelineDelay,
786 &outputDelay,
787 },
788 {},
789 C2_DONT_BLOCK,
790 nullptr);
791 if (err == C2_BAD_INDEX) {
792 if (!iStreamFormat || !oStreamFormat) {
793 return UNKNOWN_ERROR;
794 }
795 } else if (err != C2_OK) {
796 return UNKNOWN_ERROR;
797 }
798
799 {
800 Mutexed<ReorderStash>::Locked reorder(mReorderStash);
801 reorder->clear();
802 if (reorderDepth) {
803 reorder->setDepth(reorderDepth.value);
804 }
805 if (reorderKey) {
806 reorder->setKey(reorderKey.value);
807 }
808 }
809
810 uint32_t inputDelayValue = inputDelay ? inputDelay.value : 0;
811 uint32_t pipelineDelayValue = pipelineDelay ? pipelineDelay.value : 0;
812 uint32_t outputDelayValue = outputDelay ? outputDelay.value : 0;
813
814 size_t numInputSlots = inputDelayValue + pipelineDelayValue + kSmoothnessFactor;
815 size_t numOutputSlots = outputDelayValue + kSmoothnessFactor;
816
817 // TODO: get this from input format
818 bool secure = mComponent->getName().find(".secure") != std::string::npos;
819
820 std::shared_ptr<C2AllocatorStore> allocatorStore = GetCodec2PlatformAllocatorStore();
821 int poolMask = property_get_int32(
822 "debug.stagefright.c2-poolmask",
823 1 << C2PlatformAllocatorStore::ION |
824 1 << C2PlatformAllocatorStore::BUFFERQUEUE);
825
826 if (inputFormat != nullptr) {
827 bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);
828 std::shared_ptr<C2BlockPool> pool;
829 {
830 Mutexed<BlockPools>::Locked pools(mBlockPools);
831
832 // set default allocator ID.
833 pools->inputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
834 : C2PlatformAllocatorStore::ION;
835
836 // query C2PortAllocatorsTuning::input from component. If an allocator ID is obtained
837 // from component, create the input block pool with given ID. Otherwise, use default IDs.
838 std::vector<std::unique_ptr<C2Param>> params;
839 err = mComponent->query({ },
840 { C2PortAllocatorsTuning::input::PARAM_TYPE },
841 C2_DONT_BLOCK,
842 ¶ms);
843 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
844 ALOGD("[%s] Query input allocators returned %zu params => %s (%u)",
845 mName, params.size(), asString(err), err);
846 } else if (err == C2_OK && params.size() == 1) {
847 C2PortAllocatorsTuning::input *inputAllocators =
848 C2PortAllocatorsTuning::input::From(params[0].get());
849 if (inputAllocators && inputAllocators->flexCount() > 0) {
850 std::shared_ptr<C2Allocator> allocator;
851 // verify allocator IDs and resolve default allocator
852 allocatorStore->fetchAllocator(inputAllocators->m.values[0], &allocator);
853 if (allocator) {
854 pools->inputAllocatorId = allocator->getId();
855 } else {
856 ALOGD("[%s] component requested invalid input allocator ID %u",
857 mName, inputAllocators->m.values[0]);
858 }
859 }
860 }
861
862 // TODO: use C2Component wrapper to associate this pool with ourselves
863 if ((poolMask >> pools->inputAllocatorId) & 1) {
864 err = CreateCodec2BlockPool(pools->inputAllocatorId, nullptr, &pool);
865 ALOGD("[%s] Created input block pool with allocatorID %u => poolID %llu - %s (%d)",
866 mName, pools->inputAllocatorId,
867 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
868 asString(err), err);
869 } else {
870 err = C2_NOT_FOUND;
871 }
872 if (err != C2_OK) {
873 C2BlockPool::local_id_t inputPoolId =
874 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
875 err = GetCodec2BlockPool(inputPoolId, nullptr, &pool);
876 ALOGD("[%s] Using basic input block pool with poolID %llu => got %llu - %s (%d)",
877 mName, (unsigned long long)inputPoolId,
878 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
879 asString(err), err);
880 if (err != C2_OK) {
881 return NO_MEMORY;
882 }
883 }
884 pools->inputPool = pool;
885 }
886
887 bool forceArrayMode = false;
888 Mutexed<Input>::Locked input(mInput);
889 input->inputDelay = inputDelayValue;
890 input->pipelineDelay = pipelineDelayValue;
891 input->numSlots = numInputSlots;
892 input->extraBuffers.flush();
893 input->numExtraSlots = 0u;
894 if (graphic) {
895 if (mInputSurface) {
896 input->buffers.reset(new DummyInputBuffers(mName));
897 } else if (mMetaMode == MODE_ANW) {
898 input->buffers.reset(new GraphicMetadataInputBuffers(mName));
899 // This is to ensure buffers do not get released prematurely.
900 // TODO: handle this without going into array mode
901 forceArrayMode = true;
902 } else {
903 input->buffers.reset(new GraphicInputBuffers(numInputSlots, mName));
904 }
905 } else {
906 if (hasCryptoOrDescrambler()) {
907 int32_t capacity = kLinearBufferSize;
908 (void)inputFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
909 if ((size_t)capacity > kMaxLinearBufferSize) {
910 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
911 capacity = kMaxLinearBufferSize;
912 }
913 if (mDealer == nullptr) {
914 mDealer = new MemoryDealer(
915 align(capacity, MemoryDealer::getAllocationAlignment())
916 * (numInputSlots + 1),
917 "EncryptedLinearInputBuffers");
918 mDecryptDestination = mDealer->allocate((size_t)capacity);
919 }
920 if (mCrypto != nullptr && mHeapSeqNum < 0) {
921 mHeapSeqNum = mCrypto->setHeap(mDealer->getMemoryHeap());
922 } else {
923 mHeapSeqNum = -1;
924 }
925 input->buffers.reset(new EncryptedLinearInputBuffers(
926 secure, mDealer, mCrypto, mHeapSeqNum, (size_t)capacity,
927 numInputSlots, mName));
928 forceArrayMode = true;
929 } else {
930 input->buffers.reset(new LinearInputBuffers(mName));
931 }
932 }
933 input->buffers->setFormat(inputFormat);
934
935 if (err == C2_OK) {
936 input->buffers->setPool(pool);
937 } else {
938 // TODO: error
939 }
940
941 if (forceArrayMode) {
942 input->buffers = input->buffers->toArrayMode(numInputSlots);
943 }
944 }
945
946 if (outputFormat != nullptr) {
947 sp<IGraphicBufferProducer> outputSurface;
948 uint32_t outputGeneration;
949 {
950 Mutexed<OutputSurface>::Locked output(mOutputSurface);
951 output->maxDequeueBuffers = numOutputSlots + reorderDepth.value + kRenderingDepth;
952 outputSurface = output->surface ?
953 output->surface->getIGraphicBufferProducer() : nullptr;
954 if (outputSurface) {
955 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
956 }
957 outputGeneration = output->generation;
958 }
959
960 bool graphic = (oStreamFormat.value == C2BufferData::GRAPHIC);
961 C2BlockPool::local_id_t outputPoolId_;
962
963 {
964 Mutexed<BlockPools>::Locked pools(mBlockPools);
965
966 // set default allocator ID.
967 pools->outputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
968 : C2PlatformAllocatorStore::ION;
969
970 // query C2PortAllocatorsTuning::output from component, or use default allocator if
971 // unsuccessful.
972 std::vector<std::unique_ptr<C2Param>> params;
973 err = mComponent->query({ },
974 { C2PortAllocatorsTuning::output::PARAM_TYPE },
975 C2_DONT_BLOCK,
976 ¶ms);
977 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
978 ALOGD("[%s] Query output allocators returned %zu params => %s (%u)",
979 mName, params.size(), asString(err), err);
980 } else if (err == C2_OK && params.size() == 1) {
981 C2PortAllocatorsTuning::output *outputAllocators =
982 C2PortAllocatorsTuning::output::From(params[0].get());
983 if (outputAllocators && outputAllocators->flexCount() > 0) {
984 std::shared_ptr<C2Allocator> allocator;
985 // verify allocator IDs and resolve default allocator
986 allocatorStore->fetchAllocator(outputAllocators->m.values[0], &allocator);
987 if (allocator) {
988 pools->outputAllocatorId = allocator->getId();
989 } else {
990 ALOGD("[%s] component requested invalid output allocator ID %u",
991 mName, outputAllocators->m.values[0]);
992 }
993 }
994 }
995
996 // use bufferqueue if outputting to a surface.
997 // query C2PortSurfaceAllocatorTuning::output from component, or use default allocator
998 // if unsuccessful.
999 if (outputSurface) {
1000 params.clear();
1001 err = mComponent->query({ },
1002 { C2PortSurfaceAllocatorTuning::output::PARAM_TYPE },
1003 C2_DONT_BLOCK,
1004 ¶ms);
1005 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1006 ALOGD("[%s] Query output surface allocator returned %zu params => %s (%u)",
1007 mName, params.size(), asString(err), err);
1008 } else if (err == C2_OK && params.size() == 1) {
1009 C2PortSurfaceAllocatorTuning::output *surfaceAllocator =
1010 C2PortSurfaceAllocatorTuning::output::From(params[0].get());
1011 if (surfaceAllocator) {
1012 std::shared_ptr<C2Allocator> allocator;
1013 // verify allocator IDs and resolve default allocator
1014 allocatorStore->fetchAllocator(surfaceAllocator->value, &allocator);
1015 if (allocator) {
1016 pools->outputAllocatorId = allocator->getId();
1017 } else {
1018 ALOGD("[%s] component requested invalid surface output allocator ID %u",
1019 mName, surfaceAllocator->value);
1020 err = C2_BAD_VALUE;
1021 }
1022 }
1023 }
1024 if (pools->outputAllocatorId == C2PlatformAllocatorStore::GRALLOC
1025 && err != C2_OK
1026 && ((poolMask >> C2PlatformAllocatorStore::BUFFERQUEUE) & 1)) {
1027 pools->outputAllocatorId = C2PlatformAllocatorStore::BUFFERQUEUE;
1028 }
1029 }
1030
1031 if ((poolMask >> pools->outputAllocatorId) & 1) {
1032 err = mComponent->createBlockPool(
1033 pools->outputAllocatorId, &pools->outputPoolId, &pools->outputPoolIntf);
1034 ALOGI("[%s] Created output block pool with allocatorID %u => poolID %llu - %s",
1035 mName, pools->outputAllocatorId,
1036 (unsigned long long)pools->outputPoolId,
1037 asString(err));
1038 } else {
1039 err = C2_NOT_FOUND;
1040 }
1041 if (err != C2_OK) {
1042 // use basic pool instead
1043 pools->outputPoolId =
1044 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1045 }
1046
1047 // Configure output block pool ID as parameter C2PortBlockPoolsTuning::output to
1048 // component.
1049 std::unique_ptr<C2PortBlockPoolsTuning::output> poolIdsTuning =
1050 C2PortBlockPoolsTuning::output::AllocUnique({ pools->outputPoolId });
1051
1052 std::vector<std::unique_ptr<C2SettingResult>> failures;
1053 err = mComponent->config({ poolIdsTuning.get() }, C2_MAY_BLOCK, &failures);
1054 ALOGD("[%s] Configured output block pool ids %llu => %s",
1055 mName, (unsigned long long)poolIdsTuning->m.values[0], asString(err));
1056 outputPoolId_ = pools->outputPoolId;
1057 }
1058
1059 Mutexed<Output>::Locked output(mOutput);
1060 output->outputDelay = outputDelayValue;
1061 output->numSlots = numOutputSlots;
1062 if (graphic) {
1063 if (outputSurface) {
1064 output->buffers.reset(new GraphicOutputBuffers(mName));
1065 } else {
1066 output->buffers.reset(new RawGraphicOutputBuffers(numOutputSlots, mName));
1067 }
1068 } else {
1069 output->buffers.reset(new LinearOutputBuffers(mName));
1070 }
1071 output->buffers->setFormat(outputFormat->dup());
1072
1073
1074 // Try to set output surface to created block pool if given.
1075 if (outputSurface) {
1076 mComponent->setOutputSurface(
1077 outputPoolId_,
1078 outputSurface,
1079 outputGeneration);
1080 }
1081
1082 if (oStreamFormat.value == C2BufferData::LINEAR
1083 && mComponentName.find("c2.qti.") == std::string::npos) {
1084 // WORKAROUND: if we're using early CSD workaround we convert to
1085 // array mode, to appease apps assuming the output
1086 // buffers to be of the same size.
1087 output->buffers = output->buffers->toArrayMode(numOutputSlots);
1088
1089 int32_t channelCount;
1090 int32_t sampleRate;
1091 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
1092 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
1093 int32_t delay = 0;
1094 int32_t padding = 0;;
1095 if (!outputFormat->findInt32("encoder-delay", &delay)) {
1096 delay = 0;
1097 }
1098 if (!outputFormat->findInt32("encoder-padding", &padding)) {
1099 padding = 0;
1100 }
1101 if (delay || padding) {
1102 // We need write access to the buffers, and we're already in
1103 // array mode.
1104 output->buffers->initSkipCutBuffer(delay, padding, sampleRate, channelCount);
1105 }
1106 }
1107 }
1108 }
1109
1110 // Set up pipeline control. This has to be done after mInputBuffers and
1111 // mOutputBuffers are initialized to make sure that lingering callbacks
1112 // about buffers from the previous generation do not interfere with the
1113 // newly initialized pipeline capacity.
1114
1115 {
1116 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
1117 watcher->inputDelay(inputDelayValue)
1118 .pipelineDelay(pipelineDelayValue)
1119 .outputDelay(outputDelayValue)
1120 .smoothnessFactor(kSmoothnessFactor);
1121 watcher->flush();
1122 }
1123
1124 mInputMetEos = false;
1125 mSync.start();
1126 return OK;
1127 }
1128
requestInitialInputBuffers()1129 status_t CCodecBufferChannel::requestInitialInputBuffers() {
1130 if (mInputSurface) {
1131 return OK;
1132 }
1133
1134 C2StreamBufferTypeSetting::output oStreamFormat(0u);
1135 c2_status_t err = mComponent->query({ &oStreamFormat }, {}, C2_DONT_BLOCK, nullptr);
1136 if (err != C2_OK) {
1137 return UNKNOWN_ERROR;
1138 }
1139 size_t numInputSlots = mInput.lock()->numSlots;
1140 std::vector<sp<MediaCodecBuffer>> toBeQueued;
1141 for (size_t i = 0; i < numInputSlots; ++i) {
1142 size_t index;
1143 sp<MediaCodecBuffer> buffer;
1144 {
1145 Mutexed<Input>::Locked input(mInput);
1146 if (!input->buffers->requestNewBuffer(&index, &buffer)) {
1147 if (i == 0) {
1148 ALOGW("[%s] start: cannot allocate memory at all", mName);
1149 return NO_MEMORY;
1150 } else {
1151 ALOGV("[%s] start: cannot allocate memory, only %zu buffers allocated",
1152 mName, i);
1153 }
1154 break;
1155 }
1156 }
1157 if (buffer) {
1158 Mutexed<std::list<sp<ABuffer>>>::Locked configs(mFlushedConfigs);
1159 ALOGV("[%s] input buffer %zu available", mName, index);
1160 bool post = true;
1161 if (!configs->empty()) {
1162 sp<ABuffer> config = configs->front();
1163 configs->pop_front();
1164 if (buffer->capacity() >= config->size()) {
1165 memcpy(buffer->base(), config->data(), config->size());
1166 buffer->setRange(0, config->size());
1167 buffer->meta()->clear();
1168 buffer->meta()->setInt64("timeUs", 0);
1169 buffer->meta()->setInt32("csd", 1);
1170 post = false;
1171 } else {
1172 ALOGD("[%s] buffer capacity too small for the config (%zu < %zu)",
1173 mName, buffer->capacity(), config->size());
1174 }
1175 } else if (oStreamFormat.value == C2BufferData::LINEAR && i == 0
1176 && mComponentName.find("c2.qti.") == std::string::npos) {
1177 // WORKAROUND: Some apps expect CSD available without queueing
1178 // any input. Queue an empty buffer to get the CSD.
1179 buffer->setRange(0, 0);
1180 buffer->meta()->clear();
1181 buffer->meta()->setInt64("timeUs", 0);
1182 post = false;
1183 }
1184 if (post) {
1185 mCallback->onInputBufferAvailable(index, buffer);
1186 } else {
1187 toBeQueued.emplace_back(buffer);
1188 }
1189 }
1190 }
1191 for (const sp<MediaCodecBuffer> &buffer : toBeQueued) {
1192 if (queueInputBufferInternal(buffer) != OK) {
1193 ALOGV("[%s] Error while queueing initial buffers", mName);
1194 }
1195 }
1196 return OK;
1197 }
1198
stop()1199 void CCodecBufferChannel::stop() {
1200 mSync.stop();
1201 mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed);
1202 if (mInputSurface != nullptr) {
1203 mInputSurface.reset();
1204 }
1205 }
1206
flush(const std::list<std::unique_ptr<C2Work>> & flushedWork)1207 void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) {
1208 ALOGV("[%s] flush", mName);
1209 {
1210 Mutexed<std::list<sp<ABuffer>>>::Locked configs(mFlushedConfigs);
1211 for (const std::unique_ptr<C2Work> &work : flushedWork) {
1212 if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) {
1213 continue;
1214 }
1215 if (work->input.buffers.empty()
1216 || work->input.buffers.front()->data().linearBlocks().empty()) {
1217 ALOGD("[%s] no linear codec config data found", mName);
1218 continue;
1219 }
1220 C2ReadView view =
1221 work->input.buffers.front()->data().linearBlocks().front().map().get();
1222 if (view.error() != C2_OK) {
1223 ALOGD("[%s] failed to map flushed codec config data: %d", mName, view.error());
1224 continue;
1225 }
1226 configs->push_back(ABuffer::CreateAsCopy(view.data(), view.capacity()));
1227 ALOGV("[%s] stashed flushed codec config data (size=%u)", mName, view.capacity());
1228 }
1229 }
1230 {
1231 Mutexed<Input>::Locked input(mInput);
1232 input->buffers->flush();
1233 input->extraBuffers.flush();
1234 }
1235 {
1236 Mutexed<Output>::Locked output(mOutput);
1237 output->buffers->flush(flushedWork);
1238 }
1239 mReorderStash.lock()->flush();
1240 mPipelineWatcher.lock()->flush();
1241 }
1242
onWorkDone(std::unique_ptr<C2Work> work,const sp<AMessage> & outputFormat,const C2StreamInitDataInfo::output * initData)1243 void CCodecBufferChannel::onWorkDone(
1244 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat,
1245 const C2StreamInitDataInfo::output *initData) {
1246 if (handleWork(std::move(work), outputFormat, initData)) {
1247 feedInputBufferIfAvailable();
1248 }
1249 }
1250
onInputBufferDone(uint64_t frameIndex,size_t arrayIndex)1251 void CCodecBufferChannel::onInputBufferDone(
1252 uint64_t frameIndex, size_t arrayIndex) {
1253 if (mInputSurface) {
1254 return;
1255 }
1256 std::shared_ptr<C2Buffer> buffer =
1257 mPipelineWatcher.lock()->onInputBufferReleased(frameIndex, arrayIndex);
1258 bool newInputSlotAvailable;
1259 {
1260 Mutexed<Input>::Locked input(mInput);
1261 newInputSlotAvailable = input->buffers->expireComponentBuffer(buffer);
1262 if (!newInputSlotAvailable) {
1263 (void)input->extraBuffers.expireComponentBuffer(buffer);
1264 }
1265 }
1266 if (newInputSlotAvailable) {
1267 feedInputBufferIfAvailable();
1268 }
1269 }
1270
handleWork(std::unique_ptr<C2Work> work,const sp<AMessage> & outputFormat,const C2StreamInitDataInfo::output * initData)1271 bool CCodecBufferChannel::handleWork(
1272 std::unique_ptr<C2Work> work,
1273 const sp<AMessage> &outputFormat,
1274 const C2StreamInitDataInfo::output *initData) {
1275 if ((work->input.ordinal.frameIndex - mFirstValidFrameIndex.load()).peek() < 0) {
1276 // Discard frames from previous generation.
1277 ALOGD("[%s] Discard frames from previous generation.", mName);
1278 return false;
1279 }
1280
1281 if (mInputSurface == nullptr && (work->worklets.size() != 1u
1282 || !work->worklets.front()
1283 || !(work->worklets.front()->output.flags & C2FrameData::FLAG_INCOMPLETE))) {
1284 mPipelineWatcher.lock()->onWorkDone(work->input.ordinal.frameIndex.peeku());
1285 }
1286
1287 if (work->result == C2_NOT_FOUND) {
1288 ALOGD("[%s] flushed work; ignored.", mName);
1289 return true;
1290 }
1291
1292 if (work->result != C2_OK) {
1293 ALOGD("[%s] work failed to complete: %d", mName, work->result);
1294 mCCodecCallback->onError(work->result, ACTION_CODE_FATAL);
1295 return false;
1296 }
1297
1298 // NOTE: MediaCodec usage supposedly have only one worklet
1299 if (work->worklets.size() != 1u) {
1300 ALOGI("[%s] onWorkDone: incorrect number of worklets: %zu",
1301 mName, work->worklets.size());
1302 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1303 return false;
1304 }
1305
1306 const std::unique_ptr<C2Worklet> &worklet = work->worklets.front();
1307
1308 std::shared_ptr<C2Buffer> buffer;
1309 // NOTE: MediaCodec usage supposedly have only one output stream.
1310 if (worklet->output.buffers.size() > 1u) {
1311 ALOGI("[%s] onWorkDone: incorrect number of output buffers: %zu",
1312 mName, worklet->output.buffers.size());
1313 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1314 return false;
1315 } else if (worklet->output.buffers.size() == 1u) {
1316 buffer = worklet->output.buffers[0];
1317 if (!buffer) {
1318 ALOGD("[%s] onWorkDone: nullptr found in buffers; ignored.", mName);
1319 }
1320 }
1321
1322 std::optional<uint32_t> newInputDelay, newPipelineDelay;
1323 while (!worklet->output.configUpdate.empty()) {
1324 std::unique_ptr<C2Param> param;
1325 worklet->output.configUpdate.back().swap(param);
1326 worklet->output.configUpdate.pop_back();
1327 switch (param->coreIndex().coreIndex()) {
1328 case C2PortReorderBufferDepthTuning::CORE_INDEX: {
1329 C2PortReorderBufferDepthTuning::output reorderDepth;
1330 if (reorderDepth.updateFrom(*param)) {
1331 mReorderStash.lock()->setDepth(reorderDepth.value);
1332 ALOGV("[%s] onWorkDone: updated reorder depth to %u",
1333 mName, reorderDepth.value);
1334 size_t numOutputSlots = mOutput.lock()->numSlots;
1335 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1336 output->maxDequeueBuffers =
1337 numOutputSlots + reorderDepth.value + kRenderingDepth;
1338 if (output->surface) {
1339 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1340 }
1341 } else {
1342 ALOGD("[%s] onWorkDone: failed to read reorder depth", mName);
1343 }
1344 break;
1345 }
1346 case C2PortReorderKeySetting::CORE_INDEX: {
1347 C2PortReorderKeySetting::output reorderKey;
1348 if (reorderKey.updateFrom(*param)) {
1349 mReorderStash.lock()->setKey(reorderKey.value);
1350 ALOGV("[%s] onWorkDone: updated reorder key to %u",
1351 mName, reorderKey.value);
1352 } else {
1353 ALOGD("[%s] onWorkDone: failed to read reorder key", mName);
1354 }
1355 break;
1356 }
1357 case C2PortActualDelayTuning::CORE_INDEX: {
1358 if (param->isGlobal()) {
1359 C2ActualPipelineDelayTuning pipelineDelay;
1360 if (pipelineDelay.updateFrom(*param)) {
1361 ALOGV("[%s] onWorkDone: updating pipeline delay %u",
1362 mName, pipelineDelay.value);
1363 newPipelineDelay = pipelineDelay.value;
1364 (void)mPipelineWatcher.lock()->pipelineDelay(pipelineDelay.value);
1365 }
1366 }
1367 if (param->forInput()) {
1368 C2PortActualDelayTuning::input inputDelay;
1369 if (inputDelay.updateFrom(*param)) {
1370 ALOGV("[%s] onWorkDone: updating input delay %u",
1371 mName, inputDelay.value);
1372 newInputDelay = inputDelay.value;
1373 (void)mPipelineWatcher.lock()->inputDelay(inputDelay.value);
1374 }
1375 }
1376 if (param->forOutput()) {
1377 C2PortActualDelayTuning::output outputDelay;
1378 if (outputDelay.updateFrom(*param)) {
1379 ALOGV("[%s] onWorkDone: updating output delay %u",
1380 mName, outputDelay.value);
1381 (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value);
1382
1383 bool outputBuffersChanged = false;
1384 size_t numOutputSlots = 0;
1385 {
1386 Mutexed<Output>::Locked output(mOutput);
1387 output->outputDelay = outputDelay.value;
1388 numOutputSlots = outputDelay.value + kSmoothnessFactor;
1389 if (output->numSlots < numOutputSlots) {
1390 output->numSlots = numOutputSlots;
1391 if (output->buffers->isArrayMode()) {
1392 OutputBuffersArray *array =
1393 (OutputBuffersArray *)output->buffers.get();
1394 ALOGV("[%s] onWorkDone: growing output buffer array to %zu",
1395 mName, numOutputSlots);
1396 array->grow(numOutputSlots);
1397 outputBuffersChanged = true;
1398 }
1399 }
1400 numOutputSlots = output->numSlots;
1401 }
1402
1403 if (outputBuffersChanged) {
1404 mCCodecCallback->onOutputBuffersChanged();
1405 }
1406
1407 uint32_t depth = mReorderStash.lock()->depth();
1408 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1409 output->maxDequeueBuffers = numOutputSlots + depth + kRenderingDepth;
1410 if (output->surface) {
1411 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1412 }
1413 }
1414 }
1415 break;
1416 }
1417 default:
1418 ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
1419 mName, param->index());
1420 break;
1421 }
1422 }
1423 if (newInputDelay || newPipelineDelay) {
1424 Mutexed<Input>::Locked input(mInput);
1425 size_t newNumSlots =
1426 newInputDelay.value_or(input->inputDelay) +
1427 newPipelineDelay.value_or(input->pipelineDelay) +
1428 kSmoothnessFactor;
1429 if (input->buffers->isArrayMode()) {
1430 if (input->numSlots >= newNumSlots) {
1431 input->numExtraSlots = 0;
1432 } else {
1433 input->numExtraSlots = newNumSlots - input->numSlots;
1434 }
1435 ALOGV("[%s] onWorkDone: updated number of extra slots to %zu (input array mode)",
1436 mName, input->numExtraSlots);
1437 } else {
1438 input->numSlots = newNumSlots;
1439 }
1440 }
1441
1442 if (outputFormat != nullptr) {
1443 Mutexed<Output>::Locked output(mOutput);
1444 ALOGD("[%s] onWorkDone: output format changed to %s",
1445 mName, outputFormat->debugString().c_str());
1446 output->buffers->setFormat(outputFormat);
1447
1448 AString mediaType;
1449 if (outputFormat->findString(KEY_MIME, &mediaType)
1450 && mediaType == MIMETYPE_AUDIO_RAW) {
1451 int32_t channelCount;
1452 int32_t sampleRate;
1453 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
1454 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
1455 output->buffers->updateSkipCutBuffer(sampleRate, channelCount);
1456 }
1457 }
1458 }
1459
1460 int32_t flags = 0;
1461 if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) {
1462 flags |= MediaCodec::BUFFER_FLAG_EOS;
1463 ALOGV("[%s] onWorkDone: output EOS", mName);
1464 }
1465
1466 sp<MediaCodecBuffer> outBuffer;
1467 size_t index;
1468
1469 // WORKAROUND: adjust output timestamp based on client input timestamp and codec
1470 // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to
1471 // the codec input timestamp, but client output timestamp should (reported in timeUs)
1472 // shall correspond to the client input timesamp (in customOrdinal). By using the
1473 // delta between the two, this allows for some timestamp deviation - e.g. if one input
1474 // produces multiple output.
1475 c2_cntr64_t timestamp =
1476 worklet->output.ordinal.timestamp + work->input.ordinal.customOrdinal
1477 - work->input.ordinal.timestamp;
1478 if (mInputSurface != nullptr) {
1479 // When using input surface we need to restore the original input timestamp.
1480 timestamp = work->input.ordinal.customOrdinal;
1481 }
1482 ALOGV("[%s] onWorkDone: input %lld, codec %lld => output %lld => %lld",
1483 mName,
1484 work->input.ordinal.customOrdinal.peekll(),
1485 work->input.ordinal.timestamp.peekll(),
1486 worklet->output.ordinal.timestamp.peekll(),
1487 timestamp.peekll());
1488
1489 if (initData != nullptr) {
1490 Mutexed<Output>::Locked output(mOutput);
1491 if (output->buffers->registerCsd(initData, &index, &outBuffer) == OK) {
1492 outBuffer->meta()->setInt64("timeUs", timestamp.peek());
1493 outBuffer->meta()->setInt32("flags", MediaCodec::BUFFER_FLAG_CODECCONFIG);
1494 ALOGV("[%s] onWorkDone: csd index = %zu [%p]", mName, index, outBuffer.get());
1495
1496 output.unlock();
1497 mCallback->onOutputBufferAvailable(index, outBuffer);
1498 } else {
1499 ALOGD("[%s] onWorkDone: unable to register csd", mName);
1500 output.unlock();
1501 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1502 return false;
1503 }
1504 }
1505
1506 if (!buffer && !flags) {
1507 ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)",
1508 mName, work->input.ordinal.frameIndex.peekull());
1509 return true;
1510 }
1511
1512 if (buffer) {
1513 for (const std::shared_ptr<const C2Info> &info : buffer->info()) {
1514 // TODO: properly translate these to metadata
1515 switch (info->coreIndex().coreIndex()) {
1516 case C2StreamPictureTypeMaskInfo::CORE_INDEX:
1517 if (((C2StreamPictureTypeMaskInfo *)info.get())->value & C2Config::SYNC_FRAME) {
1518 flags |= MediaCodec::BUFFER_FLAG_SYNCFRAME;
1519 }
1520 break;
1521 default:
1522 break;
1523 }
1524 }
1525 }
1526
1527 {
1528 Mutexed<ReorderStash>::Locked reorder(mReorderStash);
1529 reorder->emplace(buffer, timestamp.peek(), flags, worklet->output.ordinal);
1530 if (flags & MediaCodec::BUFFER_FLAG_EOS) {
1531 // Flush reorder stash
1532 reorder->setDepth(0);
1533 }
1534 }
1535 sendOutputBuffers();
1536 return true;
1537 }
1538
sendOutputBuffers()1539 void CCodecBufferChannel::sendOutputBuffers() {
1540 ReorderStash::Entry entry;
1541 sp<MediaCodecBuffer> outBuffer;
1542 size_t index;
1543
1544 while (true) {
1545 Mutexed<ReorderStash>::Locked reorder(mReorderStash);
1546 if (!reorder->hasPending()) {
1547 break;
1548 }
1549 if (!reorder->pop(&entry)) {
1550 break;
1551 }
1552
1553 Mutexed<Output>::Locked output(mOutput);
1554 status_t err = output->buffers->registerBuffer(entry.buffer, &index, &outBuffer);
1555 if (err != OK) {
1556 bool outputBuffersChanged = false;
1557 if (err != WOULD_BLOCK) {
1558 if (!output->buffers->isArrayMode()) {
1559 output->buffers = output->buffers->toArrayMode(output->numSlots);
1560 }
1561 OutputBuffersArray *array = (OutputBuffersArray *)output->buffers.get();
1562 array->realloc(entry.buffer);
1563 outputBuffersChanged = true;
1564 }
1565 ALOGV("[%s] sendOutputBuffers: unable to register output buffer", mName);
1566 reorder->defer(entry);
1567
1568 output.unlock();
1569 reorder.unlock();
1570
1571 if (outputBuffersChanged) {
1572 mCCodecCallback->onOutputBuffersChanged();
1573 }
1574 return;
1575 }
1576 output.unlock();
1577 reorder.unlock();
1578
1579 outBuffer->meta()->setInt64("timeUs", entry.timestamp);
1580 outBuffer->meta()->setInt32("flags", entry.flags);
1581 ALOGV("[%s] sendOutputBuffers: out buffer index = %zu [%p] => %p + %zu (%lld)",
1582 mName, index, outBuffer.get(), outBuffer->data(), outBuffer->size(),
1583 (long long)entry.timestamp);
1584 mCallback->onOutputBufferAvailable(index, outBuffer);
1585 }
1586 }
1587
setSurface(const sp<Surface> & newSurface)1588 status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface) {
1589 static std::atomic_uint32_t surfaceGeneration{0};
1590 uint32_t generation = (getpid() << 10) |
1591 ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
1592 & ((1 << 10) - 1));
1593
1594 sp<IGraphicBufferProducer> producer;
1595 if (newSurface) {
1596 newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW);
1597 newSurface->setDequeueTimeout(kDequeueTimeoutNs);
1598 newSurface->setMaxDequeuedBufferCount(mOutputSurface.lock()->maxDequeueBuffers);
1599 producer = newSurface->getIGraphicBufferProducer();
1600 producer->setGenerationNumber(generation);
1601 } else {
1602 ALOGE("[%s] setting output surface to null", mName);
1603 return INVALID_OPERATION;
1604 }
1605
1606 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf;
1607 C2BlockPool::local_id_t outputPoolId;
1608 {
1609 Mutexed<BlockPools>::Locked pools(mBlockPools);
1610 outputPoolId = pools->outputPoolId;
1611 outputPoolIntf = pools->outputPoolIntf;
1612 }
1613
1614 if (outputPoolIntf) {
1615 if (mComponent->setOutputSurface(
1616 outputPoolId,
1617 producer,
1618 generation) != C2_OK) {
1619 ALOGI("[%s] setSurface: component setOutputSurface failed", mName);
1620 return INVALID_OPERATION;
1621 }
1622 }
1623
1624 {
1625 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1626 output->surface = newSurface;
1627 output->generation = generation;
1628 }
1629
1630 return OK;
1631 }
1632
elapsed()1633 PipelineWatcher::Clock::duration CCodecBufferChannel::elapsed() {
1634 // When client pushed EOS, we want all the work to be done quickly.
1635 // Otherwise, component may have stalled work due to input starvation up to
1636 // the sum of the delay in the pipeline.
1637 size_t n = 0;
1638 if (!mInputMetEos) {
1639 size_t outputDelay = mOutput.lock()->outputDelay;
1640 Mutexed<Input>::Locked input(mInput);
1641 n = input->inputDelay + input->pipelineDelay + outputDelay;
1642 }
1643 return mPipelineWatcher.lock()->elapsed(PipelineWatcher::Clock::now(), n);
1644 }
1645
setMetaMode(MetaMode mode)1646 void CCodecBufferChannel::setMetaMode(MetaMode mode) {
1647 mMetaMode = mode;
1648 }
1649
toStatusT(c2_status_t c2s,c2_operation_t c2op)1650 status_t toStatusT(c2_status_t c2s, c2_operation_t c2op) {
1651 // C2_OK is always translated to OK.
1652 if (c2s == C2_OK) {
1653 return OK;
1654 }
1655
1656 // Operation-dependent translation
1657 // TODO: Add as necessary
1658 switch (c2op) {
1659 case C2_OPERATION_Component_start:
1660 switch (c2s) {
1661 case C2_NO_MEMORY:
1662 return NO_MEMORY;
1663 default:
1664 return UNKNOWN_ERROR;
1665 }
1666 default:
1667 break;
1668 }
1669
1670 // Backup operation-agnostic translation
1671 switch (c2s) {
1672 case C2_BAD_INDEX:
1673 return BAD_INDEX;
1674 case C2_BAD_VALUE:
1675 return BAD_VALUE;
1676 case C2_BLOCKING:
1677 return WOULD_BLOCK;
1678 case C2_DUPLICATE:
1679 return ALREADY_EXISTS;
1680 case C2_NO_INIT:
1681 return NO_INIT;
1682 case C2_NO_MEMORY:
1683 return NO_MEMORY;
1684 case C2_NOT_FOUND:
1685 return NAME_NOT_FOUND;
1686 case C2_TIMED_OUT:
1687 return TIMED_OUT;
1688 case C2_BAD_STATE:
1689 case C2_CANCELED:
1690 case C2_CANNOT_DO:
1691 case C2_CORRUPTED:
1692 case C2_OMITTED:
1693 case C2_REFUSED:
1694 return UNKNOWN_ERROR;
1695 default:
1696 return -static_cast<status_t>(c2s);
1697 }
1698 }
1699
1700 } // namespace android
1701