1 /*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "CCodec"
19 #include <utils/Log.h>
20
21 #include <sstream>
22 #include <thread>
23
24 #include <C2Config.h>
25 #include <C2Debug.h>
26 #include <C2ParamInternal.h>
27 #include <C2PlatformSupport.h>
28
29 #include <android/IOMXBufferSource.h>
30 #include <android/hardware/media/c2/1.0/IInputSurface.h>
31 #include <android/hardware/media/omx/1.0/IGraphicBufferSource.h>
32 #include <android/hardware/media/omx/1.0/IOmx.h>
33 #include <android-base/stringprintf.h>
34 #include <cutils/properties.h>
35 #include <gui/IGraphicBufferProducer.h>
36 #include <gui/Surface.h>
37 #include <gui/bufferqueue/1.0/H2BGraphicBufferProducer.h>
38 #include <media/omx/1.0/WOmxNode.h>
39 #include <media/openmax/OMX_Core.h>
40 #include <media/openmax/OMX_IndexExt.h>
41 #include <media/stagefright/omx/1.0/WGraphicBufferSource.h>
42 #include <media/stagefright/omx/OmxGraphicBufferSource.h>
43 #include <media/stagefright/CCodec.h>
44 #include <media/stagefright/BufferProducerWrapper.h>
45 #include <media/stagefright/MediaCodecConstants.h>
46 #include <media/stagefright/PersistentSurface.h>
47
48 #include "C2OMXNode.h"
49 #include "CCodecBufferChannel.h"
50 #include "CCodecConfig.h"
51 #include "Codec2Mapper.h"
52 #include "InputSurfaceWrapper.h"
53
54 extern "C" android::PersistentSurface *CreateInputSurface();
55
56 namespace android {
57
58 using namespace std::chrono_literals;
59 using ::android::hardware::graphics::bufferqueue::V1_0::utils::H2BGraphicBufferProducer;
60 using android::base::StringPrintf;
61 using ::android::hardware::media::c2::V1_0::IInputSurface;
62
63 typedef hardware::media::omx::V1_0::IGraphicBufferSource HGraphicBufferSource;
64 typedef CCodecConfig Config;
65
66 namespace {
67
68 class CCodecWatchdog : public AHandler {
69 private:
70 enum {
71 kWhatWatch,
72 };
73 constexpr static int64_t kWatchIntervalUs = 3300000; // 3.3 secs
74
75 public:
getInstance()76 static sp<CCodecWatchdog> getInstance() {
77 static sp<CCodecWatchdog> instance(new CCodecWatchdog);
78 static std::once_flag flag;
79 // Call Init() only once.
80 std::call_once(flag, Init, instance);
81 return instance;
82 }
83
84 ~CCodecWatchdog() = default;
85
watch(sp<CCodec> codec)86 void watch(sp<CCodec> codec) {
87 bool shouldPost = false;
88 {
89 Mutexed<std::set<wp<CCodec>>>::Locked codecs(mCodecsToWatch);
90 // If a watch message is in flight, piggy-back this instance as well.
91 // Otherwise, post a new watch message.
92 shouldPost = codecs->empty();
93 codecs->emplace(codec);
94 }
95 if (shouldPost) {
96 ALOGV("posting watch message");
97 (new AMessage(kWhatWatch, this))->post(kWatchIntervalUs);
98 }
99 }
100
101 protected:
onMessageReceived(const sp<AMessage> & msg)102 void onMessageReceived(const sp<AMessage> &msg) {
103 switch (msg->what()) {
104 case kWhatWatch: {
105 Mutexed<std::set<wp<CCodec>>>::Locked codecs(mCodecsToWatch);
106 ALOGV("watch for %zu codecs", codecs->size());
107 for (auto it = codecs->begin(); it != codecs->end(); ++it) {
108 sp<CCodec> codec = it->promote();
109 if (codec == nullptr) {
110 continue;
111 }
112 codec->initiateReleaseIfStuck();
113 }
114 codecs->clear();
115 break;
116 }
117
118 default: {
119 TRESPASS("CCodecWatchdog: unrecognized message");
120 }
121 }
122 }
123
124 private:
CCodecWatchdog()125 CCodecWatchdog() : mLooper(new ALooper) {}
126
Init(const sp<CCodecWatchdog> & thiz)127 static void Init(const sp<CCodecWatchdog> &thiz) {
128 ALOGV("Init");
129 thiz->mLooper->setName("CCodecWatchdog");
130 thiz->mLooper->registerHandler(thiz);
131 thiz->mLooper->start();
132 }
133
134 sp<ALooper> mLooper;
135
136 Mutexed<std::set<wp<CCodec>>> mCodecsToWatch;
137 };
138
139 class C2InputSurfaceWrapper : public InputSurfaceWrapper {
140 public:
C2InputSurfaceWrapper(const std::shared_ptr<Codec2Client::InputSurface> & surface)141 explicit C2InputSurfaceWrapper(
142 const std::shared_ptr<Codec2Client::InputSurface> &surface) :
143 mSurface(surface) {
144 }
145
146 ~C2InputSurfaceWrapper() override = default;
147
connect(const std::shared_ptr<Codec2Client::Component> & comp)148 status_t connect(const std::shared_ptr<Codec2Client::Component> &comp) override {
149 if (mConnection != nullptr) {
150 return ALREADY_EXISTS;
151 }
152 return toStatusT(comp->connectToInputSurface(mSurface, &mConnection));
153 }
154
disconnect()155 void disconnect() override {
156 if (mConnection != nullptr) {
157 mConnection->disconnect();
158 mConnection = nullptr;
159 }
160 }
161
start()162 status_t start() override {
163 // InputSurface does not distinguish started state
164 return OK;
165 }
166
signalEndOfInputStream()167 status_t signalEndOfInputStream() override {
168 C2InputSurfaceEosTuning eos(true);
169 std::vector<std::unique_ptr<C2SettingResult>> failures;
170 c2_status_t err = mSurface->config({&eos}, C2_MAY_BLOCK, &failures);
171 if (err != C2_OK) {
172 return UNKNOWN_ERROR;
173 }
174 return OK;
175 }
176
configure(Config & config __unused)177 status_t configure(Config &config __unused) {
178 // TODO
179 return OK;
180 }
181
182 private:
183 std::shared_ptr<Codec2Client::InputSurface> mSurface;
184 std::shared_ptr<Codec2Client::InputSurfaceConnection> mConnection;
185 };
186
187 class GraphicBufferSourceWrapper : public InputSurfaceWrapper {
188 public:
189 typedef hardware::media::omx::V1_0::Status OmxStatus;
190
GraphicBufferSourceWrapper(const sp<HGraphicBufferSource> & source,uint32_t width,uint32_t height,uint64_t usage)191 GraphicBufferSourceWrapper(
192 const sp<HGraphicBufferSource> &source,
193 uint32_t width,
194 uint32_t height,
195 uint64_t usage)
196 : mSource(source), mWidth(width), mHeight(height) {
197 mDataSpace = HAL_DATASPACE_BT709;
198 mConfig.mUsage = usage;
199 }
200 ~GraphicBufferSourceWrapper() override = default;
201
connect(const std::shared_ptr<Codec2Client::Component> & comp)202 status_t connect(const std::shared_ptr<Codec2Client::Component> &comp) override {
203 mNode = new C2OMXNode(comp);
204 mOmxNode = new hardware::media::omx::V1_0::utils::TWOmxNode(mNode);
205 mNode->setFrameSize(mWidth, mHeight);
206
207 // Usage is queried during configure(), so setting it beforehand.
208 OMX_U32 usage = mConfig.mUsage & 0xFFFFFFFF;
209 (void)mNode->setParameter(
210 (OMX_INDEXTYPE)OMX_IndexParamConsumerUsageBits,
211 &usage, sizeof(usage));
212
213 // NOTE: we do not use/pass through color aspects from GraphicBufferSource as we
214 // communicate that directly to the component.
215 mSource->configure(
216 mOmxNode, static_cast<hardware::graphics::common::V1_0::Dataspace>(mDataSpace));
217 return OK;
218 }
219
disconnect()220 void disconnect() override {
221 if (mNode == nullptr) {
222 return;
223 }
224 sp<IOMXBufferSource> source = mNode->getSource();
225 if (source == nullptr) {
226 ALOGD("GBSWrapper::disconnect: node is not configured with OMXBufferSource.");
227 return;
228 }
229 source->onOmxIdle();
230 source->onOmxLoaded();
231 mNode.clear();
232 mOmxNode.clear();
233 }
234
GetStatus(hardware::Return<OmxStatus> && status)235 status_t GetStatus(hardware::Return<OmxStatus> &&status) {
236 if (status.isOk()) {
237 return static_cast<status_t>(status.withDefault(OmxStatus::UNKNOWN_ERROR));
238 } else if (status.isDeadObject()) {
239 return DEAD_OBJECT;
240 }
241 return UNKNOWN_ERROR;
242 }
243
start()244 status_t start() override {
245 sp<IOMXBufferSource> source = mNode->getSource();
246 if (source == nullptr) {
247 return NO_INIT;
248 }
249 constexpr size_t kNumSlots = 16;
250 for (size_t i = 0; i < kNumSlots; ++i) {
251 source->onInputBufferAdded(i);
252 }
253
254 source->onOmxExecuting();
255 return OK;
256 }
257
signalEndOfInputStream()258 status_t signalEndOfInputStream() override {
259 return GetStatus(mSource->signalEndOfInputStream());
260 }
261
configure(Config & config)262 status_t configure(Config &config) {
263 std::stringstream status;
264 status_t err = OK;
265
266 // handle each configuration granually, in case we need to handle part of the configuration
267 // elsewhere
268
269 // TRICKY: we do not unset frame delay repeating
270 if (config.mMinFps > 0 && config.mMinFps != mConfig.mMinFps) {
271 int64_t us = 1e6 / config.mMinFps + 0.5;
272 status_t res = GetStatus(mSource->setRepeatPreviousFrameDelayUs(us));
273 status << " minFps=" << config.mMinFps << " => repeatDelayUs=" << us;
274 if (res != OK) {
275 status << " (=> " << asString(res) << ")";
276 err = res;
277 }
278 mConfig.mMinFps = config.mMinFps;
279 }
280
281 // pts gap
282 if (config.mMinAdjustedFps > 0 || config.mFixedAdjustedFps > 0) {
283 if (mNode != nullptr) {
284 OMX_PARAM_U32TYPE ptrGapParam = {};
285 ptrGapParam.nSize = sizeof(OMX_PARAM_U32TYPE);
286 float gap = (config.mMinAdjustedFps > 0)
287 ? c2_min(INT32_MAX + 0., 1e6 / config.mMinAdjustedFps + 0.5)
288 : c2_max(0. - INT32_MAX, -1e6 / config.mFixedAdjustedFps - 0.5);
289 // float -> uint32_t is undefined if the value is negative.
290 // First convert to int32_t to ensure the expected behavior.
291 ptrGapParam.nU32 = int32_t(gap);
292 (void)mNode->setParameter(
293 (OMX_INDEXTYPE)OMX_IndexParamMaxFrameDurationForBitrateControl,
294 &ptrGapParam, sizeof(ptrGapParam));
295 }
296 }
297
298 // max fps
299 // TRICKY: we do not unset max fps to 0 unless using fixed fps
300 if ((config.mMaxFps > 0 || (config.mFixedAdjustedFps > 0 && config.mMaxFps == -1))
301 && config.mMaxFps != mConfig.mMaxFps) {
302 status_t res = GetStatus(mSource->setMaxFps(config.mMaxFps));
303 status << " maxFps=" << config.mMaxFps;
304 if (res != OK) {
305 status << " (=> " << asString(res) << ")";
306 err = res;
307 }
308 mConfig.mMaxFps = config.mMaxFps;
309 }
310
311 if (config.mTimeOffsetUs != mConfig.mTimeOffsetUs) {
312 status_t res = GetStatus(mSource->setTimeOffsetUs(config.mTimeOffsetUs));
313 status << " timeOffset " << config.mTimeOffsetUs << "us";
314 if (res != OK) {
315 status << " (=> " << asString(res) << ")";
316 err = res;
317 }
318 mConfig.mTimeOffsetUs = config.mTimeOffsetUs;
319 }
320
321 if (config.mCaptureFps != mConfig.mCaptureFps || config.mCodedFps != mConfig.mCodedFps) {
322 status_t res =
323 GetStatus(mSource->setTimeLapseConfig(config.mCodedFps, config.mCaptureFps));
324 status << " timeLapse " << config.mCaptureFps << "fps as " << config.mCodedFps << "fps";
325 if (res != OK) {
326 status << " (=> " << asString(res) << ")";
327 err = res;
328 }
329 mConfig.mCaptureFps = config.mCaptureFps;
330 mConfig.mCodedFps = config.mCodedFps;
331 }
332
333 if (config.mStartAtUs != mConfig.mStartAtUs
334 || (config.mStopped != mConfig.mStopped && !config.mStopped)) {
335 status_t res = GetStatus(mSource->setStartTimeUs(config.mStartAtUs));
336 status << " start at " << config.mStartAtUs << "us";
337 if (res != OK) {
338 status << " (=> " << asString(res) << ")";
339 err = res;
340 }
341 mConfig.mStartAtUs = config.mStartAtUs;
342 mConfig.mStopped = config.mStopped;
343 }
344
345 // suspend-resume
346 if (config.mSuspended != mConfig.mSuspended) {
347 status_t res = GetStatus(mSource->setSuspend(config.mSuspended, config.mSuspendAtUs));
348 status << " " << (config.mSuspended ? "suspend" : "resume")
349 << " at " << config.mSuspendAtUs << "us";
350 if (res != OK) {
351 status << " (=> " << asString(res) << ")";
352 err = res;
353 }
354 mConfig.mSuspended = config.mSuspended;
355 mConfig.mSuspendAtUs = config.mSuspendAtUs;
356 }
357
358 if (config.mStopped != mConfig.mStopped && config.mStopped) {
359 status_t res = GetStatus(mSource->setStopTimeUs(config.mStopAtUs));
360 status << " stop at " << config.mStopAtUs << "us";
361 if (res != OK) {
362 status << " (=> " << asString(res) << ")";
363 err = res;
364 } else {
365 status << " delayUs";
366 hardware::Return<void> trans = mSource->getStopTimeOffsetUs(
367 [&res, &delayUs = config.mInputDelayUs](
368 auto status, auto stopTimeOffsetUs) {
369 res = static_cast<status_t>(status);
370 delayUs = stopTimeOffsetUs;
371 });
372 if (!trans.isOk()) {
373 res = trans.isDeadObject() ? DEAD_OBJECT : UNKNOWN_ERROR;
374 }
375 if (res != OK) {
376 status << " (=> " << asString(res) << ")";
377 } else {
378 status << "=" << config.mInputDelayUs << "us";
379 }
380 mConfig.mInputDelayUs = config.mInputDelayUs;
381 }
382 mConfig.mStopAtUs = config.mStopAtUs;
383 mConfig.mStopped = config.mStopped;
384 }
385
386 // color aspects (android._color-aspects)
387
388 // consumer usage is queried earlier.
389
390 if (status.str().empty()) {
391 ALOGD("ISConfig not changed");
392 } else {
393 ALOGD("ISConfig%s", status.str().c_str());
394 }
395 return err;
396 }
397
onInputBufferDone(c2_cntr64_t index)398 void onInputBufferDone(c2_cntr64_t index) override {
399 mNode->onInputBufferDone(index);
400 }
401
402 private:
403 sp<HGraphicBufferSource> mSource;
404 sp<C2OMXNode> mNode;
405 sp<hardware::media::omx::V1_0::IOmxNode> mOmxNode;
406 uint32_t mWidth;
407 uint32_t mHeight;
408 Config mConfig;
409 };
410
411 class Codec2ClientInterfaceWrapper : public C2ComponentStore {
412 std::shared_ptr<Codec2Client> mClient;
413
414 public:
Codec2ClientInterfaceWrapper(std::shared_ptr<Codec2Client> client)415 Codec2ClientInterfaceWrapper(std::shared_ptr<Codec2Client> client)
416 : mClient(client) { }
417
418 virtual ~Codec2ClientInterfaceWrapper() = default;
419
config_sm(const std::vector<C2Param * > & params,std::vector<std::unique_ptr<C2SettingResult>> * const failures)420 virtual c2_status_t config_sm(
421 const std::vector<C2Param *> ¶ms,
422 std::vector<std::unique_ptr<C2SettingResult>> *const failures) {
423 return mClient->config(params, C2_MAY_BLOCK, failures);
424 };
425
copyBuffer(std::shared_ptr<C2GraphicBuffer>,std::shared_ptr<C2GraphicBuffer>)426 virtual c2_status_t copyBuffer(
427 std::shared_ptr<C2GraphicBuffer>,
428 std::shared_ptr<C2GraphicBuffer>) {
429 return C2_OMITTED;
430 }
431
createComponent(C2String,std::shared_ptr<C2Component> * const component)432 virtual c2_status_t createComponent(
433 C2String, std::shared_ptr<C2Component> *const component) {
434 component->reset();
435 return C2_OMITTED;
436 }
437
createInterface(C2String,std::shared_ptr<C2ComponentInterface> * const interface)438 virtual c2_status_t createInterface(
439 C2String, std::shared_ptr<C2ComponentInterface> *const interface) {
440 interface->reset();
441 return C2_OMITTED;
442 }
443
query_sm(const std::vector<C2Param * > & stackParams,const std::vector<C2Param::Index> & heapParamIndices,std::vector<std::unique_ptr<C2Param>> * const heapParams) const444 virtual c2_status_t query_sm(
445 const std::vector<C2Param *> &stackParams,
446 const std::vector<C2Param::Index> &heapParamIndices,
447 std::vector<std::unique_ptr<C2Param>> *const heapParams) const {
448 return mClient->query(stackParams, heapParamIndices, C2_MAY_BLOCK, heapParams);
449 }
450
querySupportedParams_nb(std::vector<std::shared_ptr<C2ParamDescriptor>> * const params) const451 virtual c2_status_t querySupportedParams_nb(
452 std::vector<std::shared_ptr<C2ParamDescriptor>> *const params) const {
453 return mClient->querySupportedParams(params);
454 }
455
querySupportedValues_sm(std::vector<C2FieldSupportedValuesQuery> & fields) const456 virtual c2_status_t querySupportedValues_sm(
457 std::vector<C2FieldSupportedValuesQuery> &fields) const {
458 return mClient->querySupportedValues(fields, C2_MAY_BLOCK);
459 }
460
getName() const461 virtual C2String getName() const {
462 return mClient->getName();
463 }
464
getParamReflector() const465 virtual std::shared_ptr<C2ParamReflector> getParamReflector() const {
466 return mClient->getParamReflector();
467 }
468
listComponents()469 virtual std::vector<std::shared_ptr<const C2Component::Traits>> listComponents() {
470 return std::vector<std::shared_ptr<const C2Component::Traits>>();
471 }
472 };
473
474 } // namespace
475
476 // CCodec::ClientListener
477
478 struct CCodec::ClientListener : public Codec2Client::Listener {
479
ClientListenerandroid::CCodec::ClientListener480 explicit ClientListener(const wp<CCodec> &codec) : mCodec(codec) {}
481
onWorkDoneandroid::CCodec::ClientListener482 virtual void onWorkDone(
483 const std::weak_ptr<Codec2Client::Component>& component,
484 std::list<std::unique_ptr<C2Work>>& workItems) override {
485 (void)component;
486 sp<CCodec> codec(mCodec.promote());
487 if (!codec) {
488 return;
489 }
490 codec->onWorkDone(workItems);
491 }
492
onTrippedandroid::CCodec::ClientListener493 virtual void onTripped(
494 const std::weak_ptr<Codec2Client::Component>& component,
495 const std::vector<std::shared_ptr<C2SettingResult>>& settingResult
496 ) override {
497 // TODO
498 (void)component;
499 (void)settingResult;
500 }
501
onErrorandroid::CCodec::ClientListener502 virtual void onError(
503 const std::weak_ptr<Codec2Client::Component>& component,
504 uint32_t errorCode) override {
505 // TODO
506 (void)component;
507 (void)errorCode;
508 }
509
onDeathandroid::CCodec::ClientListener510 virtual void onDeath(
511 const std::weak_ptr<Codec2Client::Component>& component) override {
512 { // Log the death of the component.
513 std::shared_ptr<Codec2Client::Component> comp = component.lock();
514 if (!comp) {
515 ALOGE("Codec2 component died.");
516 } else {
517 ALOGE("Codec2 component \"%s\" died.", comp->getName().c_str());
518 }
519 }
520
521 // Report to MediaCodec.
522 sp<CCodec> codec(mCodec.promote());
523 if (!codec || !codec->mCallback) {
524 return;
525 }
526 codec->mCallback->onError(DEAD_OBJECT, ACTION_CODE_FATAL);
527 }
528
onFrameRenderedandroid::CCodec::ClientListener529 virtual void onFrameRendered(uint64_t bufferQueueId,
530 int32_t slotId,
531 int64_t timestampNs) override {
532 // TODO: implement
533 (void)bufferQueueId;
534 (void)slotId;
535 (void)timestampNs;
536 }
537
onInputBufferDoneandroid::CCodec::ClientListener538 virtual void onInputBufferDone(
539 uint64_t frameIndex, size_t arrayIndex) override {
540 sp<CCodec> codec(mCodec.promote());
541 if (codec) {
542 codec->onInputBufferDone(frameIndex, arrayIndex);
543 }
544 }
545
546 private:
547 wp<CCodec> mCodec;
548 };
549
550 // CCodecCallbackImpl
551
552 class CCodecCallbackImpl : public CCodecCallback {
553 public:
CCodecCallbackImpl(CCodec * codec)554 explicit CCodecCallbackImpl(CCodec *codec) : mCodec(codec) {}
555 ~CCodecCallbackImpl() override = default;
556
onError(status_t err,enum ActionCode actionCode)557 void onError(status_t err, enum ActionCode actionCode) override {
558 mCodec->mCallback->onError(err, actionCode);
559 }
560
onOutputFramesRendered(int64_t mediaTimeUs,nsecs_t renderTimeNs)561 void onOutputFramesRendered(int64_t mediaTimeUs, nsecs_t renderTimeNs) override {
562 mCodec->mCallback->onOutputFramesRendered(
563 {RenderedFrameInfo(mediaTimeUs, renderTimeNs)});
564 }
565
onOutputBuffersChanged()566 void onOutputBuffersChanged() override {
567 mCodec->mCallback->onOutputBuffersChanged();
568 }
569
570 private:
571 CCodec *mCodec;
572 };
573
574 // CCodec
575
CCodec()576 CCodec::CCodec()
577 : mChannel(new CCodecBufferChannel(std::make_shared<CCodecCallbackImpl>(this))),
578 mConfig(new CCodecConfig) {
579 }
580
~CCodec()581 CCodec::~CCodec() {
582 }
583
getBufferChannel()584 std::shared_ptr<BufferChannelBase> CCodec::getBufferChannel() {
585 return mChannel;
586 }
587
tryAndReportOnError(std::function<status_t ()> job)588 status_t CCodec::tryAndReportOnError(std::function<status_t()> job) {
589 status_t err = job();
590 if (err != C2_OK) {
591 mCallback->onError(err, ACTION_CODE_FATAL);
592 }
593 return err;
594 }
595
initiateAllocateComponent(const sp<AMessage> & msg)596 void CCodec::initiateAllocateComponent(const sp<AMessage> &msg) {
597 auto setAllocating = [this] {
598 Mutexed<State>::Locked state(mState);
599 if (state->get() != RELEASED) {
600 return INVALID_OPERATION;
601 }
602 state->set(ALLOCATING);
603 return OK;
604 };
605 if (tryAndReportOnError(setAllocating) != OK) {
606 return;
607 }
608
609 sp<RefBase> codecInfo;
610 CHECK(msg->findObject("codecInfo", &codecInfo));
611 // For Codec 2.0 components, componentName == codecInfo->getCodecName().
612
613 sp<AMessage> allocMsg(new AMessage(kWhatAllocate, this));
614 allocMsg->setObject("codecInfo", codecInfo);
615 allocMsg->post();
616 }
617
allocate(const sp<MediaCodecInfo> & codecInfo)618 void CCodec::allocate(const sp<MediaCodecInfo> &codecInfo) {
619 if (codecInfo == nullptr) {
620 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
621 return;
622 }
623 ALOGD("allocate(%s)", codecInfo->getCodecName());
624 mClientListener.reset(new ClientListener(this));
625
626 AString componentName = codecInfo->getCodecName();
627 std::shared_ptr<Codec2Client> client;
628
629 // set up preferred component store to access vendor store parameters
630 client = Codec2Client::CreateFromService("default");
631 if (client) {
632 ALOGI("setting up '%s' as default (vendor) store", client->getServiceName().c_str());
633 SetPreferredCodec2ComponentStore(
634 std::make_shared<Codec2ClientInterfaceWrapper>(client));
635 }
636
637 std::shared_ptr<Codec2Client::Component> comp =
638 Codec2Client::CreateComponentByName(
639 componentName.c_str(),
640 mClientListener,
641 &client);
642 if (!comp) {
643 ALOGE("Failed Create component: %s", componentName.c_str());
644 Mutexed<State>::Locked state(mState);
645 state->set(RELEASED);
646 state.unlock();
647 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
648 state.lock();
649 return;
650 }
651 ALOGI("Created component [%s]", componentName.c_str());
652 mChannel->setComponent(comp);
653 auto setAllocated = [this, comp, client] {
654 Mutexed<State>::Locked state(mState);
655 if (state->get() != ALLOCATING) {
656 state->set(RELEASED);
657 return UNKNOWN_ERROR;
658 }
659 state->set(ALLOCATED);
660 state->comp = comp;
661 mClient = client;
662 return OK;
663 };
664 if (tryAndReportOnError(setAllocated) != OK) {
665 return;
666 }
667
668 // initialize config here in case setParameters is called prior to configure
669 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
670 const std::unique_ptr<Config> &config = *configLocked;
671 status_t err = config->initialize(mClient->getParamReflector(), comp);
672 if (err != OK) {
673 ALOGW("Failed to initialize configuration support");
674 // TODO: report error once we complete implementation.
675 }
676 config->queryConfiguration(comp);
677
678 mCallback->onComponentAllocated(componentName.c_str());
679 }
680
initiateConfigureComponent(const sp<AMessage> & format)681 void CCodec::initiateConfigureComponent(const sp<AMessage> &format) {
682 auto checkAllocated = [this] {
683 Mutexed<State>::Locked state(mState);
684 return (state->get() != ALLOCATED) ? UNKNOWN_ERROR : OK;
685 };
686 if (tryAndReportOnError(checkAllocated) != OK) {
687 return;
688 }
689
690 sp<AMessage> msg(new AMessage(kWhatConfigure, this));
691 msg->setMessage("format", format);
692 msg->post();
693 }
694
configure(const sp<AMessage> & msg)695 void CCodec::configure(const sp<AMessage> &msg) {
696 std::shared_ptr<Codec2Client::Component> comp;
697 auto checkAllocated = [this, &comp] {
698 Mutexed<State>::Locked state(mState);
699 if (state->get() != ALLOCATED) {
700 state->set(RELEASED);
701 return UNKNOWN_ERROR;
702 }
703 comp = state->comp;
704 return OK;
705 };
706 if (tryAndReportOnError(checkAllocated) != OK) {
707 return;
708 }
709
710 auto doConfig = [msg, comp, this]() -> status_t {
711 AString mime;
712 if (!msg->findString("mime", &mime)) {
713 return BAD_VALUE;
714 }
715
716 int32_t encoder;
717 if (!msg->findInt32("encoder", &encoder)) {
718 encoder = false;
719 }
720
721 int32_t flags;
722 if (!msg->findInt32("flags", &flags)) {
723 return BAD_VALUE;
724 }
725
726 // TODO: read from intf()
727 if ((!encoder) != (comp->getName().find("encoder") == std::string::npos)) {
728 return UNKNOWN_ERROR;
729 }
730
731 int32_t storeMeta;
732 if (encoder
733 && msg->findInt32("android._input-metadata-buffer-type", &storeMeta)
734 && storeMeta != kMetadataBufferTypeInvalid) {
735 if (storeMeta != kMetadataBufferTypeANWBuffer) {
736 ALOGD("Only ANW buffers are supported for legacy metadata mode");
737 return BAD_VALUE;
738 }
739 mChannel->setMetaMode(CCodecBufferChannel::MODE_ANW);
740 }
741
742 sp<RefBase> obj;
743 sp<Surface> surface;
744 if (msg->findObject("native-window", &obj)) {
745 surface = static_cast<Surface *>(obj.get());
746 setSurface(surface);
747 }
748
749 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
750 const std::unique_ptr<Config> &config = *configLocked;
751 config->mUsingSurface = surface != nullptr;
752 config->mBuffersBoundToCodec = ((flags & CONFIGURE_FLAG_USE_BLOCK_MODEL) == 0);
753 ALOGD("[%s] buffers are %sbound to CCodec for this session",
754 comp->getName().c_str(), config->mBuffersBoundToCodec ? "" : "not ");
755
756 // Enforce required parameters
757 int32_t i32;
758 float flt;
759 if (config->mDomain & Config::IS_AUDIO) {
760 if (!msg->findInt32(KEY_SAMPLE_RATE, &i32)) {
761 ALOGD("sample rate is missing, which is required for audio components.");
762 return BAD_VALUE;
763 }
764 if (!msg->findInt32(KEY_CHANNEL_COUNT, &i32)) {
765 ALOGD("channel count is missing, which is required for audio components.");
766 return BAD_VALUE;
767 }
768 if ((config->mDomain & Config::IS_ENCODER)
769 && !mime.equalsIgnoreCase(MEDIA_MIMETYPE_AUDIO_FLAC)
770 && !msg->findInt32(KEY_BIT_RATE, &i32)
771 && !msg->findFloat(KEY_BIT_RATE, &flt)) {
772 ALOGD("bitrate is missing, which is required for audio encoders.");
773 return BAD_VALUE;
774 }
775 }
776 if (config->mDomain & (Config::IS_IMAGE | Config::IS_VIDEO)) {
777 if (!msg->findInt32(KEY_WIDTH, &i32)) {
778 ALOGD("width is missing, which is required for image/video components.");
779 return BAD_VALUE;
780 }
781 if (!msg->findInt32(KEY_HEIGHT, &i32)) {
782 ALOGD("height is missing, which is required for image/video components.");
783 return BAD_VALUE;
784 }
785 if ((config->mDomain & Config::IS_ENCODER) && (config->mDomain & Config::IS_VIDEO)) {
786 int32_t mode = BITRATE_MODE_VBR;
787 if (msg->findInt32(KEY_BITRATE_MODE, &mode) && mode == BITRATE_MODE_CQ) {
788 if (!msg->findInt32(KEY_QUALITY, &i32)) {
789 ALOGD("quality is missing, which is required for video encoders in CQ.");
790 return BAD_VALUE;
791 }
792 } else {
793 if (!msg->findInt32(KEY_BIT_RATE, &i32)
794 && !msg->findFloat(KEY_BIT_RATE, &flt)) {
795 ALOGD("bitrate is missing, which is required for video encoders.");
796 return BAD_VALUE;
797 }
798 }
799 if (!msg->findInt32(KEY_I_FRAME_INTERVAL, &i32)
800 && !msg->findFloat(KEY_I_FRAME_INTERVAL, &flt)) {
801 ALOGD("I frame interval is missing, which is required for video encoders.");
802 return BAD_VALUE;
803 }
804 if (!msg->findInt32(KEY_FRAME_RATE, &i32)
805 && !msg->findFloat(KEY_FRAME_RATE, &flt)) {
806 ALOGD("frame rate is missing, which is required for video encoders.");
807 return BAD_VALUE;
808 }
809 }
810 }
811
812 /*
813 * Handle input surface configuration
814 */
815 if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))
816 && (config->mDomain & Config::IS_ENCODER)) {
817 config->mISConfig.reset(new InputSurfaceWrapper::Config{});
818 {
819 config->mISConfig->mMinFps = 0;
820 int64_t value;
821 if (msg->findInt64(KEY_REPEAT_PREVIOUS_FRAME_AFTER, &value) && value > 0) {
822 config->mISConfig->mMinFps = 1e6 / value;
823 }
824 if (!msg->findFloat(
825 KEY_MAX_FPS_TO_ENCODER, &config->mISConfig->mMaxFps)) {
826 config->mISConfig->mMaxFps = -1;
827 }
828 config->mISConfig->mMinAdjustedFps = 0;
829 config->mISConfig->mFixedAdjustedFps = 0;
830 if (msg->findInt64(KEY_MAX_PTS_GAP_TO_ENCODER, &value)) {
831 if (value < 0 && value >= INT32_MIN) {
832 config->mISConfig->mFixedAdjustedFps = -1e6 / value;
833 config->mISConfig->mMaxFps = -1;
834 } else if (value > 0 && value <= INT32_MAX) {
835 config->mISConfig->mMinAdjustedFps = 1e6 / value;
836 }
837 }
838 }
839
840 {
841 bool captureFpsFound = false;
842 double timeLapseFps;
843 float captureRate;
844 if (msg->findDouble("time-lapse-fps", &timeLapseFps)) {
845 config->mISConfig->mCaptureFps = timeLapseFps;
846 captureFpsFound = true;
847 } else if (msg->findAsFloat(KEY_CAPTURE_RATE, &captureRate)) {
848 config->mISConfig->mCaptureFps = captureRate;
849 captureFpsFound = true;
850 }
851 if (captureFpsFound) {
852 (void)msg->findAsFloat(KEY_FRAME_RATE, &config->mISConfig->mCodedFps);
853 }
854 }
855
856 {
857 config->mISConfig->mSuspended = false;
858 config->mISConfig->mSuspendAtUs = -1;
859 int32_t value;
860 if (msg->findInt32(KEY_CREATE_INPUT_SURFACE_SUSPENDED, &value) && value) {
861 config->mISConfig->mSuspended = true;
862 }
863 }
864 config->mISConfig->mUsage = 0;
865 }
866
867 /*
868 * Handle desired color format.
869 */
870 if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))) {
871 int32_t format = -1;
872 if (!msg->findInt32(KEY_COLOR_FORMAT, &format)) {
873 /*
874 * Also handle default color format (encoders require color format, so this is only
875 * needed for decoders.
876 */
877 if (!(config->mDomain & Config::IS_ENCODER)) {
878 format = (surface == nullptr) ? COLOR_FormatYUV420Planar : COLOR_FormatSurface;
879 }
880 }
881
882 if (format >= 0) {
883 msg->setInt32("android._color-format", format);
884 }
885 }
886
887 int32_t subscribeToAllVendorParams;
888 if (msg->findInt32("x-*", &subscribeToAllVendorParams) && subscribeToAllVendorParams) {
889 if (config->subscribeToAllVendorParams(comp, C2_MAY_BLOCK) != OK) {
890 ALOGD("[%s] Failed to subscribe to all vendor params", comp->getName().c_str());
891 }
892 }
893
894 std::vector<std::unique_ptr<C2Param>> configUpdate;
895 // NOTE: We used to ignore "video-bitrate" at configure; replicate
896 // the behavior here.
897 sp<AMessage> sdkParams = msg;
898 int32_t videoBitrate;
899 if (sdkParams->findInt32(PARAMETER_KEY_VIDEO_BITRATE, &videoBitrate)) {
900 sdkParams = msg->dup();
901 sdkParams->removeEntryAt(sdkParams->findEntryByName(PARAMETER_KEY_VIDEO_BITRATE));
902 }
903 status_t err = config->getConfigUpdateFromSdkParams(
904 comp, sdkParams, Config::IS_CONFIG, C2_DONT_BLOCK, &configUpdate);
905 if (err != OK) {
906 ALOGW("failed to convert configuration to c2 params");
907 }
908
909 int32_t maxBframes = 0;
910 if ((config->mDomain & Config::IS_ENCODER)
911 && (config->mDomain & Config::IS_VIDEO)
912 && sdkParams->findInt32(KEY_MAX_B_FRAMES, &maxBframes)
913 && maxBframes > 0) {
914 std::unique_ptr<C2StreamGopTuning::output> gop =
915 C2StreamGopTuning::output::AllocUnique(2 /* flexCount */, 0u /* stream */);
916 gop->m.values[0] = { P_FRAME, UINT32_MAX };
917 gop->m.values[1] = {
918 C2Config::picture_type_t(P_FRAME | B_FRAME),
919 uint32_t(maxBframes)
920 };
921 configUpdate.push_back(std::move(gop));
922 }
923
924 err = config->setParameters(comp, configUpdate, C2_DONT_BLOCK);
925 if (err != OK) {
926 ALOGW("failed to configure c2 params");
927 return err;
928 }
929
930 std::vector<std::unique_ptr<C2Param>> params;
931 C2StreamUsageTuning::input usage(0u, 0u);
932 C2StreamMaxBufferSizeInfo::input maxInputSize(0u, 0u);
933 C2PrependHeaderModeSetting prepend(PREPEND_HEADER_TO_NONE);
934
935 std::initializer_list<C2Param::Index> indices {
936 };
937 c2_status_t c2err = comp->query(
938 { &usage, &maxInputSize, &prepend },
939 indices,
940 C2_DONT_BLOCK,
941 ¶ms);
942 if (c2err != C2_OK && c2err != C2_BAD_INDEX) {
943 ALOGE("Failed to query component interface: %d", c2err);
944 return UNKNOWN_ERROR;
945 }
946 if (params.size() != indices.size()) {
947 ALOGE("Component returns wrong number of params: expected %zu actual %zu",
948 indices.size(), params.size());
949 return UNKNOWN_ERROR;
950 }
951 if (usage) {
952 if (usage.value & C2MemoryUsage::CPU_READ) {
953 config->mInputFormat->setInt32("using-sw-read-often", true);
954 }
955 if (config->mISConfig) {
956 C2AndroidMemoryUsage androidUsage(C2MemoryUsage(usage.value));
957 config->mISConfig->mUsage = androidUsage.asGrallocUsage();
958 }
959 }
960
961 // NOTE: we don't blindly use client specified input size if specified as clients
962 // at times specify too small size. Instead, mimic the behavior from OMX, where the
963 // client specified size is only used to ask for bigger buffers than component suggested
964 // size.
965 int32_t clientInputSize = 0;
966 bool clientSpecifiedInputSize =
967 msg->findInt32(KEY_MAX_INPUT_SIZE, &clientInputSize) && clientInputSize > 0;
968 // TEMP: enforce minimum buffer size of 1MB for video decoders
969 // and 16K / 4K for audio encoders/decoders
970 if (maxInputSize.value == 0) {
971 if (config->mDomain & Config::IS_AUDIO) {
972 maxInputSize.value = encoder ? 16384 : 4096;
973 } else if (!encoder) {
974 maxInputSize.value = 1048576u;
975 }
976 }
977
978 // verify that CSD fits into this size (if defined)
979 if ((config->mDomain & Config::IS_DECODER) && maxInputSize.value > 0) {
980 sp<ABuffer> csd;
981 for (size_t ix = 0; msg->findBuffer(StringPrintf("csd-%zu", ix).c_str(), &csd); ++ix) {
982 if (csd && csd->size() > maxInputSize.value) {
983 maxInputSize.value = csd->size();
984 }
985 }
986 }
987
988 // TODO: do this based on component requiring linear allocator for input
989 if ((config->mDomain & Config::IS_DECODER) || (config->mDomain & Config::IS_AUDIO)) {
990 if (clientSpecifiedInputSize) {
991 // Warn that we're overriding client's max input size if necessary.
992 if ((uint32_t)clientInputSize < maxInputSize.value) {
993 ALOGD("client requested max input size %d, which is smaller than "
994 "what component recommended (%u); overriding with component "
995 "recommendation.", clientInputSize, maxInputSize.value);
996 ALOGW("This behavior is subject to change. It is recommended that "
997 "app developers double check whether the requested "
998 "max input size is in reasonable range.");
999 } else {
1000 maxInputSize.value = clientInputSize;
1001 }
1002 }
1003 // Pass max input size on input format to the buffer channel (if supplied by the
1004 // component or by a default)
1005 if (maxInputSize.value) {
1006 config->mInputFormat->setInt32(
1007 KEY_MAX_INPUT_SIZE,
1008 (int32_t)(c2_min(maxInputSize.value, uint32_t(INT32_MAX))));
1009 }
1010 }
1011
1012 int32_t clientPrepend;
1013 if ((config->mDomain & Config::IS_VIDEO)
1014 && (config->mDomain & Config::IS_ENCODER)
1015 && msg->findInt32(KEY_PREPEND_HEADERS_TO_SYNC_FRAMES, &clientPrepend)
1016 && clientPrepend
1017 && (!prepend || prepend.value != PREPEND_HEADER_TO_ALL_SYNC)) {
1018 ALOGE("Failed to set KEY_PREPEND_HEADERS_TO_SYNC_FRAMES");
1019 return BAD_VALUE;
1020 }
1021
1022 if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))) {
1023 // propagate HDR static info to output format for both encoders and decoders
1024 // if component supports this info, we will update from component, but only the raw port,
1025 // so don't propagate if component already filled it in.
1026 sp<ABuffer> hdrInfo;
1027 if (msg->findBuffer(KEY_HDR_STATIC_INFO, &hdrInfo)
1028 && !config->mOutputFormat->findBuffer(KEY_HDR_STATIC_INFO, &hdrInfo)) {
1029 config->mOutputFormat->setBuffer(KEY_HDR_STATIC_INFO, hdrInfo);
1030 }
1031
1032 // Set desired color format from configuration parameter
1033 int32_t format;
1034 if (msg->findInt32("android._color-format", &format)) {
1035 if (config->mDomain & Config::IS_ENCODER) {
1036 config->mInputFormat->setInt32(KEY_COLOR_FORMAT, format);
1037 } else {
1038 config->mOutputFormat->setInt32(KEY_COLOR_FORMAT, format);
1039 }
1040 }
1041 }
1042
1043 // propagate encoder delay and padding to output format
1044 if ((config->mDomain & Config::IS_DECODER) && (config->mDomain & Config::IS_AUDIO)) {
1045 int delay = 0;
1046 if (msg->findInt32("encoder-delay", &delay)) {
1047 config->mOutputFormat->setInt32("encoder-delay", delay);
1048 }
1049 int padding = 0;
1050 if (msg->findInt32("encoder-padding", &padding)) {
1051 config->mOutputFormat->setInt32("encoder-padding", padding);
1052 }
1053 }
1054
1055 // set channel-mask
1056 if (config->mDomain & Config::IS_AUDIO) {
1057 int32_t mask;
1058 if (msg->findInt32(KEY_CHANNEL_MASK, &mask)) {
1059 if (config->mDomain & Config::IS_ENCODER) {
1060 config->mInputFormat->setInt32(KEY_CHANNEL_MASK, mask);
1061 } else {
1062 config->mOutputFormat->setInt32(KEY_CHANNEL_MASK, mask);
1063 }
1064 }
1065 }
1066
1067 ALOGD("setup formats input: %s and output: %s",
1068 config->mInputFormat->debugString().c_str(),
1069 config->mOutputFormat->debugString().c_str());
1070 return OK;
1071 };
1072 if (tryAndReportOnError(doConfig) != OK) {
1073 return;
1074 }
1075
1076 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1077 const std::unique_ptr<Config> &config = *configLocked;
1078
1079 mCallback->onComponentConfigured(config->mInputFormat, config->mOutputFormat);
1080 }
1081
initiateCreateInputSurface()1082 void CCodec::initiateCreateInputSurface() {
1083 status_t err = [this] {
1084 Mutexed<State>::Locked state(mState);
1085 if (state->get() != ALLOCATED) {
1086 return UNKNOWN_ERROR;
1087 }
1088 // TODO: read it from intf() properly.
1089 if (state->comp->getName().find("encoder") == std::string::npos) {
1090 return INVALID_OPERATION;
1091 }
1092 return OK;
1093 }();
1094 if (err != OK) {
1095 mCallback->onInputSurfaceCreationFailed(err);
1096 return;
1097 }
1098
1099 (new AMessage(kWhatCreateInputSurface, this))->post();
1100 }
1101
CreateOmxInputSurface()1102 sp<PersistentSurface> CCodec::CreateOmxInputSurface() {
1103 using namespace android::hardware::media::omx::V1_0;
1104 using namespace android::hardware::media::omx::V1_0::utils;
1105 using namespace android::hardware::graphics::bufferqueue::V1_0::utils;
1106 typedef android::hardware::media::omx::V1_0::Status OmxStatus;
1107 android::sp<IOmx> omx = IOmx::getService();
1108 typedef android::hardware::graphics::bufferqueue::V1_0::
1109 IGraphicBufferProducer HGraphicBufferProducer;
1110 typedef android::hardware::media::omx::V1_0::
1111 IGraphicBufferSource HGraphicBufferSource;
1112 OmxStatus s;
1113 android::sp<HGraphicBufferProducer> gbp;
1114 android::sp<HGraphicBufferSource> gbs;
1115
1116 using ::android::hardware::Return;
1117 Return<void> transStatus = omx->createInputSurface(
1118 [&s, &gbp, &gbs](
1119 OmxStatus status,
1120 const android::sp<HGraphicBufferProducer>& producer,
1121 const android::sp<HGraphicBufferSource>& source) {
1122 s = status;
1123 gbp = producer;
1124 gbs = source;
1125 });
1126 if (transStatus.isOk() && s == OmxStatus::OK) {
1127 return new PersistentSurface(new H2BGraphicBufferProducer(gbp), gbs);
1128 }
1129
1130 return nullptr;
1131 }
1132
CreateCompatibleInputSurface()1133 sp<PersistentSurface> CCodec::CreateCompatibleInputSurface() {
1134 sp<PersistentSurface> surface(CreateInputSurface());
1135
1136 if (surface == nullptr) {
1137 surface = CreateOmxInputSurface();
1138 }
1139
1140 return surface;
1141 }
1142
createInputSurface()1143 void CCodec::createInputSurface() {
1144 status_t err;
1145 sp<IGraphicBufferProducer> bufferProducer;
1146
1147 sp<AMessage> inputFormat;
1148 sp<AMessage> outputFormat;
1149 uint64_t usage = 0;
1150 {
1151 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1152 const std::unique_ptr<Config> &config = *configLocked;
1153 inputFormat = config->mInputFormat;
1154 outputFormat = config->mOutputFormat;
1155 usage = config->mISConfig ? config->mISConfig->mUsage : 0;
1156 }
1157
1158 sp<PersistentSurface> persistentSurface = CreateCompatibleInputSurface();
1159 sp<hidl::base::V1_0::IBase> hidlTarget = persistentSurface->getHidlTarget();
1160 sp<IInputSurface> hidlInputSurface = IInputSurface::castFrom(hidlTarget);
1161 sp<HGraphicBufferSource> gbs = HGraphicBufferSource::castFrom(hidlTarget);
1162
1163 if (hidlInputSurface) {
1164 std::shared_ptr<Codec2Client::InputSurface> inputSurface =
1165 std::make_shared<Codec2Client::InputSurface>(hidlInputSurface);
1166 err = setupInputSurface(std::make_shared<C2InputSurfaceWrapper>(
1167 inputSurface));
1168 bufferProducer = inputSurface->getGraphicBufferProducer();
1169 } else if (gbs) {
1170 int32_t width = 0;
1171 (void)outputFormat->findInt32("width", &width);
1172 int32_t height = 0;
1173 (void)outputFormat->findInt32("height", &height);
1174 err = setupInputSurface(std::make_shared<GraphicBufferSourceWrapper>(
1175 gbs, width, height, usage));
1176 bufferProducer = persistentSurface->getBufferProducer();
1177 } else {
1178 ALOGE("Corrupted input surface");
1179 mCallback->onInputSurfaceCreationFailed(UNKNOWN_ERROR);
1180 return;
1181 }
1182
1183 if (err != OK) {
1184 ALOGE("Failed to set up input surface: %d", err);
1185 mCallback->onInputSurfaceCreationFailed(err);
1186 return;
1187 }
1188
1189 mCallback->onInputSurfaceCreated(
1190 inputFormat,
1191 outputFormat,
1192 new BufferProducerWrapper(bufferProducer));
1193 }
1194
setupInputSurface(const std::shared_ptr<InputSurfaceWrapper> & surface)1195 status_t CCodec::setupInputSurface(const std::shared_ptr<InputSurfaceWrapper> &surface) {
1196 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1197 const std::unique_ptr<Config> &config = *configLocked;
1198 config->mUsingSurface = true;
1199
1200 // we are now using surface - apply default color aspects to input format - as well as
1201 // get dataspace
1202 bool inputFormatChanged = config->updateFormats(Config::IS_INPUT);
1203 ALOGD("input format %s to %s",
1204 inputFormatChanged ? "changed" : "unchanged",
1205 config->mInputFormat->debugString().c_str());
1206
1207 // configure dataspace
1208 static_assert(sizeof(int32_t) == sizeof(android_dataspace), "dataspace size mismatch");
1209 android_dataspace dataSpace = HAL_DATASPACE_UNKNOWN;
1210 (void)config->mInputFormat->findInt32("android._dataspace", (int32_t*)&dataSpace);
1211 surface->setDataSpace(dataSpace);
1212
1213 status_t err = mChannel->setInputSurface(surface);
1214 if (err != OK) {
1215 // undo input format update
1216 config->mUsingSurface = false;
1217 (void)config->updateFormats(Config::IS_INPUT);
1218 return err;
1219 }
1220 config->mInputSurface = surface;
1221
1222 if (config->mISConfig) {
1223 surface->configure(*config->mISConfig);
1224 } else {
1225 ALOGD("ISConfig: no configuration");
1226 }
1227
1228 return OK;
1229 }
1230
initiateSetInputSurface(const sp<PersistentSurface> & surface)1231 void CCodec::initiateSetInputSurface(const sp<PersistentSurface> &surface) {
1232 sp<AMessage> msg = new AMessage(kWhatSetInputSurface, this);
1233 msg->setObject("surface", surface);
1234 msg->post();
1235 }
1236
setInputSurface(const sp<PersistentSurface> & surface)1237 void CCodec::setInputSurface(const sp<PersistentSurface> &surface) {
1238 sp<AMessage> inputFormat;
1239 sp<AMessage> outputFormat;
1240 uint64_t usage = 0;
1241 {
1242 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1243 const std::unique_ptr<Config> &config = *configLocked;
1244 inputFormat = config->mInputFormat;
1245 outputFormat = config->mOutputFormat;
1246 usage = config->mISConfig ? config->mISConfig->mUsage : 0;
1247 }
1248 sp<hidl::base::V1_0::IBase> hidlTarget = surface->getHidlTarget();
1249 sp<IInputSurface> inputSurface = IInputSurface::castFrom(hidlTarget);
1250 sp<HGraphicBufferSource> gbs = HGraphicBufferSource::castFrom(hidlTarget);
1251 if (inputSurface) {
1252 status_t err = setupInputSurface(std::make_shared<C2InputSurfaceWrapper>(
1253 std::make_shared<Codec2Client::InputSurface>(inputSurface)));
1254 if (err != OK) {
1255 ALOGE("Failed to set up input surface: %d", err);
1256 mCallback->onInputSurfaceDeclined(err);
1257 return;
1258 }
1259 } else if (gbs) {
1260 int32_t width = 0;
1261 (void)outputFormat->findInt32("width", &width);
1262 int32_t height = 0;
1263 (void)outputFormat->findInt32("height", &height);
1264 status_t err = setupInputSurface(std::make_shared<GraphicBufferSourceWrapper>(
1265 gbs, width, height, usage));
1266 if (err != OK) {
1267 ALOGE("Failed to set up input surface: %d", err);
1268 mCallback->onInputSurfaceDeclined(err);
1269 return;
1270 }
1271 } else {
1272 ALOGE("Failed to set input surface: Corrupted surface.");
1273 mCallback->onInputSurfaceDeclined(UNKNOWN_ERROR);
1274 return;
1275 }
1276 mCallback->onInputSurfaceAccepted(inputFormat, outputFormat);
1277 }
1278
initiateStart()1279 void CCodec::initiateStart() {
1280 auto setStarting = [this] {
1281 Mutexed<State>::Locked state(mState);
1282 if (state->get() != ALLOCATED) {
1283 return UNKNOWN_ERROR;
1284 }
1285 state->set(STARTING);
1286 return OK;
1287 };
1288 if (tryAndReportOnError(setStarting) != OK) {
1289 return;
1290 }
1291
1292 (new AMessage(kWhatStart, this))->post();
1293 }
1294
start()1295 void CCodec::start() {
1296 std::shared_ptr<Codec2Client::Component> comp;
1297 auto checkStarting = [this, &comp] {
1298 Mutexed<State>::Locked state(mState);
1299 if (state->get() != STARTING) {
1300 return UNKNOWN_ERROR;
1301 }
1302 comp = state->comp;
1303 return OK;
1304 };
1305 if (tryAndReportOnError(checkStarting) != OK) {
1306 return;
1307 }
1308
1309 c2_status_t err = comp->start();
1310 if (err != C2_OK) {
1311 mCallback->onError(toStatusT(err, C2_OPERATION_Component_start),
1312 ACTION_CODE_FATAL);
1313 return;
1314 }
1315 sp<AMessage> inputFormat;
1316 sp<AMessage> outputFormat;
1317 status_t err2 = OK;
1318 bool buffersBoundToCodec = false;
1319 {
1320 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1321 const std::unique_ptr<Config> &config = *configLocked;
1322 inputFormat = config->mInputFormat;
1323 // start triggers format dup
1324 outputFormat = config->mOutputFormat = config->mOutputFormat->dup();
1325 if (config->mInputSurface) {
1326 err2 = config->mInputSurface->start();
1327 }
1328 buffersBoundToCodec = config->mBuffersBoundToCodec;
1329 }
1330 if (err2 != OK) {
1331 mCallback->onError(err2, ACTION_CODE_FATAL);
1332 return;
1333 }
1334 // We're not starting after flush.
1335 (void)mSentConfigAfterResume.test_and_set();
1336 err2 = mChannel->start(inputFormat, outputFormat, buffersBoundToCodec);
1337 if (err2 != OK) {
1338 mCallback->onError(err2, ACTION_CODE_FATAL);
1339 return;
1340 }
1341
1342 auto setRunning = [this] {
1343 Mutexed<State>::Locked state(mState);
1344 if (state->get() != STARTING) {
1345 return UNKNOWN_ERROR;
1346 }
1347 state->set(RUNNING);
1348 return OK;
1349 };
1350 if (tryAndReportOnError(setRunning) != OK) {
1351 return;
1352 }
1353 mCallback->onStartCompleted();
1354
1355 (void)mChannel->requestInitialInputBuffers();
1356 }
1357
initiateShutdown(bool keepComponentAllocated)1358 void CCodec::initiateShutdown(bool keepComponentAllocated) {
1359 if (keepComponentAllocated) {
1360 initiateStop();
1361 } else {
1362 initiateRelease();
1363 }
1364 }
1365
initiateStop()1366 void CCodec::initiateStop() {
1367 {
1368 Mutexed<State>::Locked state(mState);
1369 if (state->get() == ALLOCATED
1370 || state->get() == RELEASED
1371 || state->get() == STOPPING
1372 || state->get() == RELEASING) {
1373 // We're already stopped, released, or doing it right now.
1374 state.unlock();
1375 mCallback->onStopCompleted();
1376 state.lock();
1377 return;
1378 }
1379 state->set(STOPPING);
1380 }
1381
1382 mChannel->reset();
1383 (new AMessage(kWhatStop, this))->post();
1384 }
1385
stop()1386 void CCodec::stop() {
1387 std::shared_ptr<Codec2Client::Component> comp;
1388 {
1389 Mutexed<State>::Locked state(mState);
1390 if (state->get() == RELEASING) {
1391 state.unlock();
1392 // We're already stopped or release is in progress.
1393 mCallback->onStopCompleted();
1394 state.lock();
1395 return;
1396 } else if (state->get() != STOPPING) {
1397 state.unlock();
1398 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1399 state.lock();
1400 return;
1401 }
1402 comp = state->comp;
1403 }
1404 status_t err = comp->stop();
1405 if (err != C2_OK) {
1406 // TODO: convert err into status_t
1407 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1408 }
1409
1410 {
1411 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1412 const std::unique_ptr<Config> &config = *configLocked;
1413 if (config->mInputSurface) {
1414 config->mInputSurface->disconnect();
1415 config->mInputSurface = nullptr;
1416 }
1417 }
1418 {
1419 Mutexed<State>::Locked state(mState);
1420 if (state->get() == STOPPING) {
1421 state->set(ALLOCATED);
1422 }
1423 }
1424 mCallback->onStopCompleted();
1425 }
1426
initiateRelease(bool sendCallback)1427 void CCodec::initiateRelease(bool sendCallback /* = true */) {
1428 bool clearInputSurfaceIfNeeded = false;
1429 {
1430 Mutexed<State>::Locked state(mState);
1431 if (state->get() == RELEASED || state->get() == RELEASING) {
1432 // We're already released or doing it right now.
1433 if (sendCallback) {
1434 state.unlock();
1435 mCallback->onReleaseCompleted();
1436 state.lock();
1437 }
1438 return;
1439 }
1440 if (state->get() == ALLOCATING) {
1441 state->set(RELEASING);
1442 // With the altered state allocate() would fail and clean up.
1443 if (sendCallback) {
1444 state.unlock();
1445 mCallback->onReleaseCompleted();
1446 state.lock();
1447 }
1448 return;
1449 }
1450 if (state->get() == STARTING
1451 || state->get() == RUNNING
1452 || state->get() == STOPPING) {
1453 // Input surface may have been started, so clean up is needed.
1454 clearInputSurfaceIfNeeded = true;
1455 }
1456 state->set(RELEASING);
1457 }
1458
1459 if (clearInputSurfaceIfNeeded) {
1460 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1461 const std::unique_ptr<Config> &config = *configLocked;
1462 if (config->mInputSurface) {
1463 config->mInputSurface->disconnect();
1464 config->mInputSurface = nullptr;
1465 }
1466 }
1467
1468 mChannel->reset();
1469 // thiz holds strong ref to this while the thread is running.
1470 sp<CCodec> thiz(this);
1471 std::thread([thiz, sendCallback] { thiz->release(sendCallback); }).detach();
1472 }
1473
release(bool sendCallback)1474 void CCodec::release(bool sendCallback) {
1475 std::shared_ptr<Codec2Client::Component> comp;
1476 {
1477 Mutexed<State>::Locked state(mState);
1478 if (state->get() == RELEASED) {
1479 if (sendCallback) {
1480 state.unlock();
1481 mCallback->onReleaseCompleted();
1482 state.lock();
1483 }
1484 return;
1485 }
1486 comp = state->comp;
1487 }
1488 comp->release();
1489
1490 {
1491 Mutexed<State>::Locked state(mState);
1492 state->set(RELEASED);
1493 state->comp.reset();
1494 }
1495 (new AMessage(kWhatRelease, this))->post();
1496 if (sendCallback) {
1497 mCallback->onReleaseCompleted();
1498 }
1499 }
1500
setSurface(const sp<Surface> & surface)1501 status_t CCodec::setSurface(const sp<Surface> &surface) {
1502 return mChannel->setSurface(surface);
1503 }
1504
signalFlush()1505 void CCodec::signalFlush() {
1506 status_t err = [this] {
1507 Mutexed<State>::Locked state(mState);
1508 if (state->get() == FLUSHED) {
1509 return ALREADY_EXISTS;
1510 }
1511 if (state->get() != RUNNING) {
1512 return UNKNOWN_ERROR;
1513 }
1514 state->set(FLUSHING);
1515 return OK;
1516 }();
1517 switch (err) {
1518 case ALREADY_EXISTS:
1519 mCallback->onFlushCompleted();
1520 return;
1521 case OK:
1522 break;
1523 default:
1524 mCallback->onError(err, ACTION_CODE_FATAL);
1525 return;
1526 }
1527
1528 mChannel->stop();
1529 (new AMessage(kWhatFlush, this))->post();
1530 }
1531
flush()1532 void CCodec::flush() {
1533 std::shared_ptr<Codec2Client::Component> comp;
1534 auto checkFlushing = [this, &comp] {
1535 Mutexed<State>::Locked state(mState);
1536 if (state->get() != FLUSHING) {
1537 return UNKNOWN_ERROR;
1538 }
1539 comp = state->comp;
1540 return OK;
1541 };
1542 if (tryAndReportOnError(checkFlushing) != OK) {
1543 return;
1544 }
1545
1546 std::list<std::unique_ptr<C2Work>> flushedWork;
1547 c2_status_t err = comp->flush(C2Component::FLUSH_COMPONENT, &flushedWork);
1548 {
1549 Mutexed<std::list<std::unique_ptr<C2Work>>>::Locked queue(mWorkDoneQueue);
1550 flushedWork.splice(flushedWork.end(), *queue);
1551 }
1552 if (err != C2_OK) {
1553 // TODO: convert err into status_t
1554 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1555 }
1556
1557 mChannel->flush(flushedWork);
1558
1559 {
1560 Mutexed<State>::Locked state(mState);
1561 if (state->get() == FLUSHING) {
1562 state->set(FLUSHED);
1563 }
1564 }
1565 mCallback->onFlushCompleted();
1566 }
1567
signalResume()1568 void CCodec::signalResume() {
1569 std::shared_ptr<Codec2Client::Component> comp;
1570 auto setResuming = [this, &comp] {
1571 Mutexed<State>::Locked state(mState);
1572 if (state->get() != FLUSHED) {
1573 return UNKNOWN_ERROR;
1574 }
1575 state->set(RESUMING);
1576 comp = state->comp;
1577 return OK;
1578 };
1579 if (tryAndReportOnError(setResuming) != OK) {
1580 return;
1581 }
1582
1583 mSentConfigAfterResume.clear();
1584 {
1585 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1586 const std::unique_ptr<Config> &config = *configLocked;
1587 config->queryConfiguration(comp);
1588 }
1589
1590 (void)mChannel->start(nullptr, nullptr, [&]{
1591 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1592 const std::unique_ptr<Config> &config = *configLocked;
1593 return config->mBuffersBoundToCodec;
1594 }());
1595
1596 {
1597 Mutexed<State>::Locked state(mState);
1598 if (state->get() != RESUMING) {
1599 state.unlock();
1600 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1601 state.lock();
1602 return;
1603 }
1604 state->set(RUNNING);
1605 }
1606
1607 (void)mChannel->requestInitialInputBuffers();
1608 }
1609
signalSetParameters(const sp<AMessage> & msg)1610 void CCodec::signalSetParameters(const sp<AMessage> &msg) {
1611 std::shared_ptr<Codec2Client::Component> comp;
1612 auto checkState = [this, &comp] {
1613 Mutexed<State>::Locked state(mState);
1614 if (state->get() == RELEASED) {
1615 return INVALID_OPERATION;
1616 }
1617 comp = state->comp;
1618 return OK;
1619 };
1620 if (tryAndReportOnError(checkState) != OK) {
1621 return;
1622 }
1623
1624 // NOTE: We used to ignore "bitrate" at setParameters; replicate
1625 // the behavior here.
1626 sp<AMessage> params = msg;
1627 int32_t bitrate;
1628 if (params->findInt32(KEY_BIT_RATE, &bitrate)) {
1629 params = msg->dup();
1630 params->removeEntryAt(params->findEntryByName(KEY_BIT_RATE));
1631 }
1632
1633 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1634 const std::unique_ptr<Config> &config = *configLocked;
1635
1636 /**
1637 * Handle input surface parameters
1638 */
1639 if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))
1640 && (config->mDomain & Config::IS_ENCODER)
1641 && config->mInputSurface && config->mISConfig) {
1642 (void)params->findInt64(PARAMETER_KEY_OFFSET_TIME, &config->mISConfig->mTimeOffsetUs);
1643
1644 if (params->findInt64("skip-frames-before", &config->mISConfig->mStartAtUs)) {
1645 config->mISConfig->mStopped = false;
1646 } else if (params->findInt64("stop-time-us", &config->mISConfig->mStopAtUs)) {
1647 config->mISConfig->mStopped = true;
1648 }
1649
1650 int32_t value;
1651 if (params->findInt32(PARAMETER_KEY_SUSPEND, &value)) {
1652 config->mISConfig->mSuspended = value;
1653 config->mISConfig->mSuspendAtUs = -1;
1654 (void)params->findInt64(PARAMETER_KEY_SUSPEND_TIME, &config->mISConfig->mSuspendAtUs);
1655 }
1656
1657 (void)config->mInputSurface->configure(*config->mISConfig);
1658 if (config->mISConfig->mStopped) {
1659 config->mInputFormat->setInt64(
1660 "android._stop-time-offset-us", config->mISConfig->mInputDelayUs);
1661 }
1662 }
1663
1664 std::vector<std::unique_ptr<C2Param>> configUpdate;
1665 (void)config->getConfigUpdateFromSdkParams(
1666 comp, params, Config::IS_PARAM, C2_MAY_BLOCK, &configUpdate);
1667 // Prefer to pass parameters to the buffer channel, so they can be synchronized with the frames.
1668 // Parameter synchronization is not defined when using input surface. For now, route
1669 // these directly to the component.
1670 if (config->mInputSurface == nullptr
1671 && (property_get_bool("debug.stagefright.ccodec_delayed_params", false)
1672 || comp->getName().find("c2.android.") == 0)) {
1673 mChannel->setParameters(configUpdate);
1674 } else {
1675 (void)config->setParameters(comp, configUpdate, C2_MAY_BLOCK);
1676 }
1677 }
1678
signalEndOfInputStream()1679 void CCodec::signalEndOfInputStream() {
1680 mCallback->onSignaledInputEOS(mChannel->signalEndOfInputStream());
1681 }
1682
signalRequestIDRFrame()1683 void CCodec::signalRequestIDRFrame() {
1684 std::shared_ptr<Codec2Client::Component> comp;
1685 {
1686 Mutexed<State>::Locked state(mState);
1687 if (state->get() == RELEASED) {
1688 ALOGD("no IDR request sent since component is released");
1689 return;
1690 }
1691 comp = state->comp;
1692 }
1693 ALOGV("request IDR");
1694 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1695 const std::unique_ptr<Config> &config = *configLocked;
1696 std::vector<std::unique_ptr<C2Param>> params;
1697 params.push_back(
1698 std::make_unique<C2StreamRequestSyncFrameTuning::output>(0u, true));
1699 config->setParameters(comp, params, C2_MAY_BLOCK);
1700 }
1701
onWorkDone(std::list<std::unique_ptr<C2Work>> & workItems)1702 void CCodec::onWorkDone(std::list<std::unique_ptr<C2Work>> &workItems) {
1703 if (!workItems.empty()) {
1704 Mutexed<std::list<std::unique_ptr<C2Work>>>::Locked queue(mWorkDoneQueue);
1705 queue->splice(queue->end(), workItems);
1706 }
1707 (new AMessage(kWhatWorkDone, this))->post();
1708 }
1709
onInputBufferDone(uint64_t frameIndex,size_t arrayIndex)1710 void CCodec::onInputBufferDone(uint64_t frameIndex, size_t arrayIndex) {
1711 mChannel->onInputBufferDone(frameIndex, arrayIndex);
1712 if (arrayIndex == 0) {
1713 // We always put no more than one buffer per work, if we use an input surface.
1714 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1715 const std::unique_ptr<Config> &config = *configLocked;
1716 if (config->mInputSurface) {
1717 config->mInputSurface->onInputBufferDone(frameIndex);
1718 }
1719 }
1720 }
1721
onMessageReceived(const sp<AMessage> & msg)1722 void CCodec::onMessageReceived(const sp<AMessage> &msg) {
1723 TimePoint now = std::chrono::steady_clock::now();
1724 CCodecWatchdog::getInstance()->watch(this);
1725 switch (msg->what()) {
1726 case kWhatAllocate: {
1727 // C2ComponentStore::createComponent() should return within 100ms.
1728 setDeadline(now, 1500ms, "allocate");
1729 sp<RefBase> obj;
1730 CHECK(msg->findObject("codecInfo", &obj));
1731 allocate((MediaCodecInfo *)obj.get());
1732 break;
1733 }
1734 case kWhatConfigure: {
1735 // C2Component::commit_sm() should return within 5ms.
1736 setDeadline(now, 1500ms, "configure");
1737 sp<AMessage> format;
1738 CHECK(msg->findMessage("format", &format));
1739 configure(format);
1740 break;
1741 }
1742 case kWhatStart: {
1743 // C2Component::start() should return within 500ms.
1744 setDeadline(now, 1500ms, "start");
1745 start();
1746 break;
1747 }
1748 case kWhatStop: {
1749 // C2Component::stop() should return within 500ms.
1750 setDeadline(now, 1500ms, "stop");
1751 stop();
1752 break;
1753 }
1754 case kWhatFlush: {
1755 // C2Component::flush_sm() should return within 5ms.
1756 setDeadline(now, 1500ms, "flush");
1757 flush();
1758 break;
1759 }
1760 case kWhatRelease: {
1761 mChannel->release();
1762 mClient.reset();
1763 mClientListener.reset();
1764 break;
1765 }
1766 case kWhatCreateInputSurface: {
1767 // Surface operations may be briefly blocking.
1768 setDeadline(now, 1500ms, "createInputSurface");
1769 createInputSurface();
1770 break;
1771 }
1772 case kWhatSetInputSurface: {
1773 // Surface operations may be briefly blocking.
1774 setDeadline(now, 1500ms, "setInputSurface");
1775 sp<RefBase> obj;
1776 CHECK(msg->findObject("surface", &obj));
1777 sp<PersistentSurface> surface(static_cast<PersistentSurface *>(obj.get()));
1778 setInputSurface(surface);
1779 break;
1780 }
1781 case kWhatWorkDone: {
1782 std::unique_ptr<C2Work> work;
1783 bool shouldPost = false;
1784 {
1785 Mutexed<std::list<std::unique_ptr<C2Work>>>::Locked queue(mWorkDoneQueue);
1786 if (queue->empty()) {
1787 break;
1788 }
1789 work.swap(queue->front());
1790 queue->pop_front();
1791 shouldPost = !queue->empty();
1792 }
1793 if (shouldPost) {
1794 (new AMessage(kWhatWorkDone, this))->post();
1795 }
1796
1797 // handle configuration changes in work done
1798 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1799 const std::unique_ptr<Config> &config = *configLocked;
1800 bool changed = !mSentConfigAfterResume.test_and_set();
1801 Config::Watcher<C2StreamInitDataInfo::output> initData =
1802 config->watch<C2StreamInitDataInfo::output>();
1803 if (!work->worklets.empty()
1804 && (work->worklets.front()->output.flags
1805 & C2FrameData::FLAG_DISCARD_FRAME) == 0) {
1806
1807 // copy buffer info to config
1808 std::vector<std::unique_ptr<C2Param>> updates;
1809 for (const std::unique_ptr<C2Param> ¶m
1810 : work->worklets.front()->output.configUpdate) {
1811 updates.push_back(C2Param::Copy(*param));
1812 }
1813 unsigned stream = 0;
1814 for (const std::shared_ptr<C2Buffer> &buf : work->worklets.front()->output.buffers) {
1815 for (const std::shared_ptr<const C2Info> &info : buf->info()) {
1816 // move all info into output-stream #0 domain
1817 updates.emplace_back(C2Param::CopyAsStream(*info, true /* output */, stream));
1818 }
1819 for (const C2ConstGraphicBlock &block : buf->data().graphicBlocks()) {
1820 // ALOGV("got output buffer with crop %u,%u+%u,%u and size %u,%u",
1821 // block.crop().left, block.crop().top,
1822 // block.crop().width, block.crop().height,
1823 // block.width(), block.height());
1824 updates.emplace_back(new C2StreamCropRectInfo::output(stream, block.crop()));
1825 updates.emplace_back(new C2StreamPictureSizeInfo::output(
1826 stream, block.crop().width, block.crop().height));
1827 break; // for now only do the first block
1828 }
1829 ++stream;
1830 }
1831
1832 if (config->updateConfiguration(updates, config->mOutputDomain)) {
1833 changed = true;
1834 }
1835
1836 // copy standard infos to graphic buffers if not already present (otherwise, we
1837 // may overwrite the actual intermediate value with a final value)
1838 stream = 0;
1839 const static std::vector<C2Param::Index> stdGfxInfos = {
1840 C2StreamRotationInfo::output::PARAM_TYPE,
1841 C2StreamColorAspectsInfo::output::PARAM_TYPE,
1842 C2StreamDataSpaceInfo::output::PARAM_TYPE,
1843 C2StreamHdrStaticInfo::output::PARAM_TYPE,
1844 C2StreamHdr10PlusInfo::output::PARAM_TYPE,
1845 C2StreamPixelAspectRatioInfo::output::PARAM_TYPE,
1846 C2StreamSurfaceScalingInfo::output::PARAM_TYPE
1847 };
1848 for (const std::shared_ptr<C2Buffer> &buf : work->worklets.front()->output.buffers) {
1849 if (buf->data().graphicBlocks().size()) {
1850 for (C2Param::Index ix : stdGfxInfos) {
1851 if (!buf->hasInfo(ix)) {
1852 const C2Param *param =
1853 config->getConfigParameterValue(ix.withStream(stream));
1854 if (param) {
1855 std::shared_ptr<C2Param> info(C2Param::Copy(*param));
1856 buf->setInfo(std::static_pointer_cast<C2Info>(info));
1857 }
1858 }
1859 }
1860 }
1861 ++stream;
1862 }
1863 }
1864 if (config->mInputSurface) {
1865 config->mInputSurface->onInputBufferDone(work->input.ordinal.frameIndex);
1866 }
1867 mChannel->onWorkDone(
1868 std::move(work), changed ? config->mOutputFormat->dup() : nullptr,
1869 initData.hasChanged() ? initData.update().get() : nullptr);
1870 break;
1871 }
1872 case kWhatWatch: {
1873 // watch message already posted; no-op.
1874 break;
1875 }
1876 default: {
1877 ALOGE("unrecognized message");
1878 break;
1879 }
1880 }
1881 setDeadline(TimePoint::max(), 0ms, "none");
1882 }
1883
setDeadline(const TimePoint & now,const std::chrono::milliseconds & timeout,const char * name)1884 void CCodec::setDeadline(
1885 const TimePoint &now,
1886 const std::chrono::milliseconds &timeout,
1887 const char *name) {
1888 int32_t mult = std::max(1, property_get_int32("debug.stagefright.ccodec_timeout_mult", 1));
1889 Mutexed<NamedTimePoint>::Locked deadline(mDeadline);
1890 deadline->set(now + (timeout * mult), name);
1891 }
1892
initiateReleaseIfStuck()1893 void CCodec::initiateReleaseIfStuck() {
1894 std::string name;
1895 bool pendingDeadline = false;
1896 {
1897 Mutexed<NamedTimePoint>::Locked deadline(mDeadline);
1898 if (deadline->get() < std::chrono::steady_clock::now()) {
1899 name = deadline->getName();
1900 }
1901 if (deadline->get() != TimePoint::max()) {
1902 pendingDeadline = true;
1903 }
1904 }
1905 if (name.empty()) {
1906 constexpr std::chrono::steady_clock::duration kWorkDurationThreshold = 3s;
1907 std::chrono::steady_clock::duration elapsed = mChannel->elapsed();
1908 if (elapsed >= kWorkDurationThreshold) {
1909 name = "queue";
1910 }
1911 if (elapsed > 0s) {
1912 pendingDeadline = true;
1913 }
1914 }
1915 if (name.empty()) {
1916 // We're not stuck.
1917 if (pendingDeadline) {
1918 // If we are not stuck yet but still has deadline coming up,
1919 // post watch message to check back later.
1920 (new AMessage(kWhatWatch, this))->post();
1921 }
1922 return;
1923 }
1924
1925 ALOGW("previous call to %s exceeded timeout", name.c_str());
1926 initiateRelease(false);
1927 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1928 }
1929
1930 // static
CreateInputSurface()1931 PersistentSurface *CCodec::CreateInputSurface() {
1932 using namespace android;
1933 using ::android::hardware::media::omx::V1_0::implementation::TWGraphicBufferSource;
1934 // Attempt to create a Codec2's input surface.
1935 std::shared_ptr<Codec2Client::InputSurface> inputSurface =
1936 Codec2Client::CreateInputSurface();
1937 if (!inputSurface) {
1938 if (property_get_int32("debug.stagefright.c2inputsurface", 0) == -1) {
1939 sp<IGraphicBufferProducer> gbp;
1940 sp<OmxGraphicBufferSource> gbs = new OmxGraphicBufferSource();
1941 status_t err = gbs->initCheck();
1942 if (err != OK) {
1943 ALOGE("Failed to create persistent input surface: error %d", err);
1944 return nullptr;
1945 }
1946 return new PersistentSurface(
1947 gbs->getIGraphicBufferProducer(), new TWGraphicBufferSource(gbs));
1948 } else {
1949 return nullptr;
1950 }
1951 }
1952 return new PersistentSurface(
1953 inputSurface->getGraphicBufferProducer(),
1954 static_cast<sp<android::hidl::base::V1_0::IBase>>(
1955 inputSurface->getHalInterface()));
1956 }
1957
1958 class IntfCache {
1959 public:
1960 IntfCache() = default;
1961
init(const std::string & name)1962 status_t init(const std::string &name) {
1963 std::shared_ptr<Codec2Client::Interface> intf{
1964 Codec2Client::CreateInterfaceByName(name.c_str())};
1965 if (!intf) {
1966 ALOGW("IntfCache [%s]: Unrecognized interface name", name.c_str());
1967 mInitStatus = NO_INIT;
1968 return NO_INIT;
1969 }
1970 const static C2StreamUsageTuning::input sUsage{0u /* stream id */};
1971 mFields.push_back(C2FieldSupportedValuesQuery::Possible(
1972 C2ParamField{&sUsage, &sUsage.value}));
1973 c2_status_t err = intf->querySupportedValues(mFields, C2_MAY_BLOCK);
1974 if (err != C2_OK) {
1975 ALOGW("IntfCache [%s]: failed to query usage supported value (err=%d)",
1976 name.c_str(), err);
1977 mFields[0].status = err;
1978 }
1979 std::vector<std::unique_ptr<C2Param>> params;
1980 err = intf->query(
1981 {&mApiFeatures},
1982 {C2PortAllocatorsTuning::input::PARAM_TYPE},
1983 C2_MAY_BLOCK,
1984 ¶ms);
1985 if (err != C2_OK && err != C2_BAD_INDEX) {
1986 ALOGW("IntfCache [%s]: failed to query api features (err=%d)",
1987 name.c_str(), err);
1988 }
1989 while (!params.empty()) {
1990 C2Param *param = params.back().release();
1991 params.pop_back();
1992 if (!param) {
1993 continue;
1994 }
1995 if (param->type() == C2PortAllocatorsTuning::input::PARAM_TYPE) {
1996 mInputAllocators.reset(
1997 C2PortAllocatorsTuning::input::From(params[0].get()));
1998 }
1999 }
2000 mInitStatus = OK;
2001 return OK;
2002 }
2003
initCheck() const2004 status_t initCheck() const { return mInitStatus; }
2005
getUsageSupportedValues() const2006 const C2FieldSupportedValuesQuery &getUsageSupportedValues() const {
2007 CHECK_EQ(1u, mFields.size());
2008 return mFields[0];
2009 }
2010
getApiFeatures() const2011 const C2ApiFeaturesSetting &getApiFeatures() const {
2012 return mApiFeatures;
2013 }
2014
getInputAllocators() const2015 const C2PortAllocatorsTuning::input &getInputAllocators() const {
2016 static std::unique_ptr<C2PortAllocatorsTuning::input> sInvalidated = []{
2017 std::unique_ptr<C2PortAllocatorsTuning::input> param =
2018 C2PortAllocatorsTuning::input::AllocUnique(0);
2019 param->invalidate();
2020 return param;
2021 }();
2022 return mInputAllocators ? *mInputAllocators : *sInvalidated;
2023 }
2024
2025 private:
2026 status_t mInitStatus{NO_INIT};
2027
2028 std::vector<C2FieldSupportedValuesQuery> mFields;
2029 C2ApiFeaturesSetting mApiFeatures;
2030 std::unique_ptr<C2PortAllocatorsTuning::input> mInputAllocators;
2031 };
2032
GetIntfCache(const std::string & name)2033 static const IntfCache &GetIntfCache(const std::string &name) {
2034 static IntfCache sNullIntfCache;
2035 static std::mutex sMutex;
2036 static std::map<std::string, IntfCache> sCache;
2037 std::unique_lock<std::mutex> lock{sMutex};
2038 auto it = sCache.find(name);
2039 if (it == sCache.end()) {
2040 lock.unlock();
2041 IntfCache intfCache;
2042 status_t err = intfCache.init(name);
2043 if (err != OK) {
2044 return sNullIntfCache;
2045 }
2046 lock.lock();
2047 it = sCache.insert({name, std::move(intfCache)}).first;
2048 }
2049 return it->second;
2050 }
2051
GetCommonAllocatorIds(const std::vector<std::string> & names,C2Allocator::type_t type,std::set<C2Allocator::id_t> * ids)2052 static status_t GetCommonAllocatorIds(
2053 const std::vector<std::string> &names,
2054 C2Allocator::type_t type,
2055 std::set<C2Allocator::id_t> *ids) {
2056 int poolMask = GetCodec2PoolMask();
2057 C2PlatformAllocatorStore::id_t preferredLinearId = GetPreferredLinearAllocatorId(poolMask);
2058 C2Allocator::id_t defaultAllocatorId =
2059 (type == C2Allocator::LINEAR) ? preferredLinearId : C2PlatformAllocatorStore::GRALLOC;
2060
2061 ids->clear();
2062 if (names.empty()) {
2063 return OK;
2064 }
2065 bool firstIteration = true;
2066 for (const std::string &name : names) {
2067 const IntfCache &intfCache = GetIntfCache(name);
2068 if (intfCache.initCheck() != OK) {
2069 continue;
2070 }
2071 const C2PortAllocatorsTuning::input &allocators = intfCache.getInputAllocators();
2072 if (firstIteration) {
2073 firstIteration = false;
2074 if (allocators && allocators.flexCount() > 0) {
2075 ids->insert(allocators.m.values,
2076 allocators.m.values + allocators.flexCount());
2077 }
2078 if (ids->empty()) {
2079 // The component does not advertise allocators. Use default.
2080 ids->insert(defaultAllocatorId);
2081 }
2082 continue;
2083 }
2084 bool filtered = false;
2085 if (allocators && allocators.flexCount() > 0) {
2086 filtered = true;
2087 for (auto it = ids->begin(); it != ids->end(); ) {
2088 bool found = false;
2089 for (size_t j = 0; j < allocators.flexCount(); ++j) {
2090 if (allocators.m.values[j] == *it) {
2091 found = true;
2092 break;
2093 }
2094 }
2095 if (found) {
2096 ++it;
2097 } else {
2098 it = ids->erase(it);
2099 }
2100 }
2101 }
2102 if (!filtered) {
2103 // The component does not advertise supported allocators. Use default.
2104 bool containsDefault = (ids->count(defaultAllocatorId) > 0u);
2105 if (ids->size() != (containsDefault ? 1 : 0)) {
2106 ids->clear();
2107 if (containsDefault) {
2108 ids->insert(defaultAllocatorId);
2109 }
2110 }
2111 }
2112 }
2113 // Finally, filter with pool masks
2114 for (auto it = ids->begin(); it != ids->end(); ) {
2115 if ((poolMask >> *it) & 1) {
2116 ++it;
2117 } else {
2118 it = ids->erase(it);
2119 }
2120 }
2121 return OK;
2122 }
2123
CalculateMinMaxUsage(const std::vector<std::string> & names,uint64_t * minUsage,uint64_t * maxUsage)2124 static status_t CalculateMinMaxUsage(
2125 const std::vector<std::string> &names, uint64_t *minUsage, uint64_t *maxUsage) {
2126 static C2StreamUsageTuning::input sUsage{0u /* stream id */};
2127 *minUsage = 0;
2128 *maxUsage = ~0ull;
2129 for (const std::string &name : names) {
2130 const IntfCache &intfCache = GetIntfCache(name);
2131 if (intfCache.initCheck() != OK) {
2132 continue;
2133 }
2134 const C2FieldSupportedValuesQuery &usageSupportedValues =
2135 intfCache.getUsageSupportedValues();
2136 if (usageSupportedValues.status != C2_OK) {
2137 continue;
2138 }
2139 const C2FieldSupportedValues &supported = usageSupportedValues.values;
2140 if (supported.type != C2FieldSupportedValues::FLAGS) {
2141 continue;
2142 }
2143 if (supported.values.empty()) {
2144 *maxUsage = 0;
2145 continue;
2146 }
2147 *minUsage |= supported.values[0].u64;
2148 int64_t currentMaxUsage = 0;
2149 for (const C2Value::Primitive &flags : supported.values) {
2150 currentMaxUsage |= flags.u64;
2151 }
2152 *maxUsage &= currentMaxUsage;
2153 }
2154 return OK;
2155 }
2156
2157 // static
CanFetchLinearBlock(const std::vector<std::string> & names,const C2MemoryUsage & usage,bool * isCompatible)2158 status_t CCodec::CanFetchLinearBlock(
2159 const std::vector<std::string> &names, const C2MemoryUsage &usage, bool *isCompatible) {
2160 for (const std::string &name : names) {
2161 const IntfCache &intfCache = GetIntfCache(name);
2162 if (intfCache.initCheck() != OK) {
2163 continue;
2164 }
2165 const C2ApiFeaturesSetting &features = intfCache.getApiFeatures();
2166 if (features && !(features.value & API_SAME_INPUT_BUFFER)) {
2167 *isCompatible = false;
2168 return OK;
2169 }
2170 }
2171 uint64_t minUsage = usage.expected;
2172 uint64_t maxUsage = ~0ull;
2173 std::set<C2Allocator::id_t> allocators;
2174 GetCommonAllocatorIds(names, C2Allocator::LINEAR, &allocators);
2175 if (allocators.empty()) {
2176 *isCompatible = false;
2177 return OK;
2178 }
2179 CalculateMinMaxUsage(names, &minUsage, &maxUsage);
2180 *isCompatible = ((maxUsage & minUsage) == minUsage);
2181 return OK;
2182 }
2183
GetPool(C2Allocator::id_t allocId)2184 static std::shared_ptr<C2BlockPool> GetPool(C2Allocator::id_t allocId) {
2185 static std::mutex sMutex{};
2186 static std::map<C2Allocator::id_t, std::shared_ptr<C2BlockPool>> sPools;
2187 std::unique_lock<std::mutex> lock{sMutex};
2188 std::shared_ptr<C2BlockPool> pool;
2189 auto it = sPools.find(allocId);
2190 if (it == sPools.end()) {
2191 c2_status_t err = CreateCodec2BlockPool(allocId, nullptr, &pool);
2192 if (err == OK) {
2193 sPools.emplace(allocId, pool);
2194 } else {
2195 pool.reset();
2196 }
2197 } else {
2198 pool = it->second;
2199 }
2200 return pool;
2201 }
2202
2203 // static
FetchLinearBlock(size_t capacity,const C2MemoryUsage & usage,const std::vector<std::string> & names)2204 std::shared_ptr<C2LinearBlock> CCodec::FetchLinearBlock(
2205 size_t capacity, const C2MemoryUsage &usage, const std::vector<std::string> &names) {
2206 uint64_t minUsage = usage.expected;
2207 uint64_t maxUsage = ~0ull;
2208 std::set<C2Allocator::id_t> allocators;
2209 GetCommonAllocatorIds(names, C2Allocator::LINEAR, &allocators);
2210 if (allocators.empty()) {
2211 allocators.insert(C2PlatformAllocatorStore::DEFAULT_LINEAR);
2212 }
2213 CalculateMinMaxUsage(names, &minUsage, &maxUsage);
2214 if ((maxUsage & minUsage) != minUsage) {
2215 allocators.clear();
2216 allocators.insert(C2PlatformAllocatorStore::DEFAULT_LINEAR);
2217 }
2218 std::shared_ptr<C2LinearBlock> block;
2219 for (C2Allocator::id_t allocId : allocators) {
2220 std::shared_ptr<C2BlockPool> pool = GetPool(allocId);
2221 if (!pool) {
2222 continue;
2223 }
2224 c2_status_t err = pool->fetchLinearBlock(capacity, C2MemoryUsage{minUsage}, &block);
2225 if (err != C2_OK || !block) {
2226 block.reset();
2227 continue;
2228 }
2229 break;
2230 }
2231 return block;
2232 }
2233
2234 // static
CanFetchGraphicBlock(const std::vector<std::string> & names,bool * isCompatible)2235 status_t CCodec::CanFetchGraphicBlock(
2236 const std::vector<std::string> &names, bool *isCompatible) {
2237 uint64_t minUsage = 0;
2238 uint64_t maxUsage = ~0ull;
2239 std::set<C2Allocator::id_t> allocators;
2240 GetCommonAllocatorIds(names, C2Allocator::GRAPHIC, &allocators);
2241 if (allocators.empty()) {
2242 *isCompatible = false;
2243 return OK;
2244 }
2245 CalculateMinMaxUsage(names, &minUsage, &maxUsage);
2246 *isCompatible = ((maxUsage & minUsage) == minUsage);
2247 return OK;
2248 }
2249
2250 // static
FetchGraphicBlock(int32_t width,int32_t height,int32_t format,uint64_t usage,const std::vector<std::string> & names)2251 std::shared_ptr<C2GraphicBlock> CCodec::FetchGraphicBlock(
2252 int32_t width,
2253 int32_t height,
2254 int32_t format,
2255 uint64_t usage,
2256 const std::vector<std::string> &names) {
2257 uint32_t halPixelFormat = HAL_PIXEL_FORMAT_YCBCR_420_888;
2258 if (!C2Mapper::mapPixelFormatFrameworkToCodec(format, &halPixelFormat)) {
2259 ALOGD("Unrecognized pixel format: %d", format);
2260 return nullptr;
2261 }
2262 uint64_t minUsage = 0;
2263 uint64_t maxUsage = ~0ull;
2264 std::set<C2Allocator::id_t> allocators;
2265 GetCommonAllocatorIds(names, C2Allocator::GRAPHIC, &allocators);
2266 if (allocators.empty()) {
2267 allocators.insert(C2PlatformAllocatorStore::DEFAULT_GRAPHIC);
2268 }
2269 CalculateMinMaxUsage(names, &minUsage, &maxUsage);
2270 minUsage |= usage;
2271 if ((maxUsage & minUsage) != minUsage) {
2272 allocators.clear();
2273 allocators.insert(C2PlatformAllocatorStore::DEFAULT_GRAPHIC);
2274 }
2275 std::shared_ptr<C2GraphicBlock> block;
2276 for (C2Allocator::id_t allocId : allocators) {
2277 std::shared_ptr<C2BlockPool> pool;
2278 c2_status_t err = CreateCodec2BlockPool(allocId, nullptr, &pool);
2279 if (err != C2_OK || !pool) {
2280 continue;
2281 }
2282 err = pool->fetchGraphicBlock(
2283 width, height, halPixelFormat, C2MemoryUsage{minUsage}, &block);
2284 if (err != C2_OK || !block) {
2285 block.reset();
2286 continue;
2287 }
2288 break;
2289 }
2290 return block;
2291 }
2292
2293 } // namespace android
2294
2295