1 /*
2 * Copyright 2024, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "C2NodeImpl"
19 #include <log/log.h>
20
21 #include <C2AllocatorGralloc.h>
22 #include <C2BlockInternal.h>
23 #include <C2Component.h>
24 #include <C2Config.h>
25 #include <C2Debug.h>
26 #include <C2PlatformSupport.h>
27
28 #include <android_media_codec.h>
29 #include <android/fdsan.h>
30 #include <media/stagefright/foundation/ColorUtils.h>
31 #include <ui/Fence.h>
32 #include <ui/GraphicBuffer.h>
33 #include <utils/Errors.h>
34 #include <utils/Thread.h>
35
36 #include "utils/Codec2Mapper.h"
37 #include "C2NodeImpl.h"
38 #include "Codec2Buffer.h"
39
40 namespace android {
41
42 using ::aidl::android::media::IAidlBufferSource;
43 using ::aidl::android::media::IAidlNode;
44
45 using ::android::media::BUFFERFLAG_EOS;
46
47 namespace {
48
49 class Buffer2D : public C2Buffer {
50 public:
Buffer2D(C2ConstGraphicBlock block)51 explicit Buffer2D(C2ConstGraphicBlock block) : C2Buffer({ block }) {}
52 };
53
54 } // namespace
55
56 class C2NodeImpl::QueueThread : public Thread {
57 public:
QueueThread()58 QueueThread() : Thread(false) {}
59 ~QueueThread() override = default;
queue(const std::shared_ptr<Codec2Client::Component> & comp,int fenceFd,std::unique_ptr<C2Work> && work,android::base::unique_fd && fd0,android::base::unique_fd && fd1)60 void queue(
61 const std::shared_ptr<Codec2Client::Component> &comp,
62 int fenceFd,
63 std::unique_ptr<C2Work> &&work,
64 android::base::unique_fd &&fd0,
65 android::base::unique_fd &&fd1) {
66 Mutexed<Jobs>::Locked jobs(mJobs);
67 auto it = jobs->queues.try_emplace(comp, comp).first;
68 it->second.workList.emplace_back(
69 std::move(work), fenceFd, std::move(fd0), std::move(fd1));
70 jobs->cond.broadcast();
71 }
72
setDataspace(android_dataspace dataspace)73 void setDataspace(android_dataspace dataspace) {
74 Mutexed<Jobs>::Locked jobs(mJobs);
75 ColorUtils::convertDataSpaceToV0(dataspace);
76 jobs->configUpdate.emplace_back(new C2StreamDataSpaceInfo::input(0u, dataspace));
77 int32_t standard;
78 int32_t transfer;
79 int32_t range;
80 ColorUtils::getColorConfigFromDataSpace(dataspace, &range, &standard, &transfer);
81 std::unique_ptr<C2StreamColorAspectsInfo::input> colorAspects =
82 std::make_unique<C2StreamColorAspectsInfo::input>(0u);
83 if (C2Mapper::map(standard, &colorAspects->primaries, &colorAspects->matrix)
84 && C2Mapper::map(transfer, &colorAspects->transfer)
85 && C2Mapper::map(range, &colorAspects->range)) {
86 jobs->configUpdate.push_back(std::move(colorAspects));
87 }
88 }
89
setPriority(int priority)90 void setPriority(int priority) {
91 androidSetThreadPriority(getTid(), priority);
92 }
93
94 protected:
threadLoop()95 bool threadLoop() override {
96 constexpr nsecs_t kIntervalNs = nsecs_t(10) * 1000 * 1000; // 10ms
97 constexpr nsecs_t kWaitNs = kIntervalNs * 2;
98 for (int i = 0; i < 2; ++i) {
99 Mutexed<Jobs>::Locked jobs(mJobs);
100 nsecs_t nowNs = systemTime();
101 bool queued = false;
102 for (auto it = jobs->queues.begin(); it != jobs->queues.end(); ) {
103 Queue &queue = it->second;
104 if (queue.workList.empty()
105 || (queue.lastQueuedTimestampNs != 0 &&
106 nowNs - queue.lastQueuedTimestampNs < kIntervalNs)) {
107 ++it;
108 continue;
109 }
110 std::shared_ptr<Codec2Client::Component> comp = queue.component.lock();
111 if (!comp) {
112 it = jobs->queues.erase(it);
113 continue;
114 }
115 std::list<std::unique_ptr<C2Work>> items;
116 std::vector<int> fenceFds;
117 std::vector<android::base::unique_fd> uniqueFds;
118 while (!queue.workList.empty()) {
119 items.push_back(std::move(queue.workList.front().work));
120 fenceFds.push_back(queue.workList.front().fenceFd);
121 uniqueFds.push_back(std::move(queue.workList.front().fd0));
122 uniqueFds.push_back(std::move(queue.workList.front().fd1));
123 queue.workList.pop_front();
124 }
125 for (const std::unique_ptr<C2Param> ¶m : jobs->configUpdate) {
126 items.front()->input.configUpdate.emplace_back(C2Param::Copy(*param));
127 }
128
129 jobs.unlock();
130 for (int fenceFd : fenceFds) {
131 sp<Fence> fence(new Fence(fenceFd));
132 fence->waitForever(LOG_TAG);
133 }
134 queue.lastQueuedTimestampNs = nowNs;
135 comp->queue(&items);
136 for (android::base::unique_fd &ufd : uniqueFds) {
137 (void)ufd.release();
138 }
139 jobs.lock();
140
141 it = jobs->queues.upper_bound(comp);
142 queued = true;
143 }
144 if (queued) {
145 jobs->configUpdate.clear();
146 return true;
147 }
148 if (i == 0) {
149 jobs.waitForConditionRelative(jobs->cond, kWaitNs);
150 }
151 }
152 return true;
153 }
154
155 private:
156 struct WorkFence {
WorkFenceandroid::C2NodeImpl::QueueThread::WorkFence157 WorkFence(std::unique_ptr<C2Work> &&w, int fd) : work(std::move(w)), fenceFd(fd) {}
158
WorkFenceandroid::C2NodeImpl::QueueThread::WorkFence159 WorkFence(
160 std::unique_ptr<C2Work> &&w,
161 int fd,
162 android::base::unique_fd &&uniqueFd0,
163 android::base::unique_fd &&uniqueFd1)
164 : work(std::move(w)),
165 fenceFd(fd),
166 fd0(std::move(uniqueFd0)),
167 fd1(std::move(uniqueFd1)) {}
168
169 std::unique_ptr<C2Work> work;
170 int fenceFd;
171 android::base::unique_fd fd0;
172 android::base::unique_fd fd1;
173 };
174 struct Queue {
Queueandroid::C2NodeImpl::QueueThread::Queue175 Queue(const std::shared_ptr<Codec2Client::Component> &comp)
176 : component(comp), lastQueuedTimestampNs(0) {}
177 Queue(const Queue &) = delete;
178 Queue &operator =(const Queue &) = delete;
179
180 std::weak_ptr<Codec2Client::Component> component;
181 std::list<WorkFence> workList;
182 nsecs_t lastQueuedTimestampNs;
183 };
184 struct Jobs {
185 std::map<std::weak_ptr<Codec2Client::Component>,
186 Queue,
187 std::owner_less<std::weak_ptr<Codec2Client::Component>>> queues;
188 std::vector<std::unique_ptr<C2Param>> configUpdate;
189 Condition cond;
190 };
191 Mutexed<Jobs> mJobs;
192 };
193
C2NodeImpl(const std::shared_ptr<Codec2Client::Component> & comp,bool aidl)194 C2NodeImpl::C2NodeImpl(const std::shared_ptr<Codec2Client::Component> &comp, bool aidl)
195 : mComp(comp), mFrameIndex(0), mWidth(0), mHeight(0), mUsage(0),
196 mAdjustTimestampGapUs(0), mFirstInputFrame(true),
197 mQueueThread(new QueueThread), mAidlHal(aidl) {
198 android_fdsan_set_error_level(ANDROID_FDSAN_ERROR_LEVEL_WARN_ALWAYS);
199 mQueueThread->run("C2NodeImpl", PRIORITY_AUDIO);
200
201 android_dataspace ds = HAL_DATASPACE_UNKNOWN;
202 mDataspace.lock().set(ds);
203 uint32_t pf = PIXEL_FORMAT_UNKNOWN;
204 mPixelFormat.lock().set(pf);
205 }
206
~C2NodeImpl()207 C2NodeImpl::~C2NodeImpl() {
208 }
209
freeNode()210 status_t C2NodeImpl::freeNode() {
211 mComp.reset();
212 android_fdsan_set_error_level(ANDROID_FDSAN_ERROR_LEVEL_WARN_ONCE);
213 return mQueueThread->requestExitAndWait();
214 }
215
onFirstInputFrame()216 void C2NodeImpl::onFirstInputFrame() {
217 mFirstInputFrame = true;
218 }
219
getConsumerUsageBits(uint64_t * usage)220 void C2NodeImpl::getConsumerUsageBits(uint64_t *usage) {
221 *usage = mUsage;
222 }
223
getInputBufferParams(IAidlNode::InputBufferParams * params)224 void C2NodeImpl::getInputBufferParams(IAidlNode::InputBufferParams *params) {
225 params->bufferCountActual = 16;
226
227 // WORKAROUND: having more slots improve performance while consuming
228 // more memory. This is a temporary workaround to reduce memory for
229 // larger-than-4K scenario.
230 if (mWidth * mHeight > 4096 * 2340) {
231 std::shared_ptr<Codec2Client::Component> comp = mComp.lock();
232 C2PortActualDelayTuning::input inputDelay(0);
233 C2ActualPipelineDelayTuning pipelineDelay(0);
234 c2_status_t c2err = C2_NOT_FOUND;
235 if (comp) {
236 c2err = comp->query(
237 {&inputDelay, &pipelineDelay}, {}, C2_DONT_BLOCK, nullptr);
238 }
239 if (c2err == C2_OK || c2err == C2_BAD_INDEX) {
240 params->bufferCountActual = 4;
241 params->bufferCountActual += (inputDelay ? inputDelay.value : 0u);
242 params->bufferCountActual += (pipelineDelay ? pipelineDelay.value : 0u);
243 }
244 }
245
246 params->frameWidth = mWidth;
247 params->frameHeight = mHeight;
248 }
249
setConsumerUsageBits(uint64_t usage)250 void C2NodeImpl::setConsumerUsageBits(uint64_t usage) {
251 mUsage = usage;
252 }
253
setAdjustTimestampGapUs(int32_t gapUs)254 void C2NodeImpl::setAdjustTimestampGapUs(int32_t gapUs) {
255 mAdjustTimestampGapUs = gapUs;
256 }
257
setInputSurface(const sp<IOMXBufferSource> & bufferSource)258 status_t C2NodeImpl::setInputSurface(const sp<IOMXBufferSource> &bufferSource) {
259 c2_status_t err = GetCodec2PlatformAllocatorStore()->fetchAllocator(
260 C2PlatformAllocatorStore::GRALLOC,
261 &mAllocator);
262 if (err != OK) {
263 return UNKNOWN_ERROR;
264 }
265 CHECK(!mAidlHal);
266 mBufferSource = bufferSource;
267 return OK;
268 }
269
setAidlInputSurface(const std::shared_ptr<IAidlBufferSource> & aidlBufferSource)270 status_t C2NodeImpl::setAidlInputSurface(
271 const std::shared_ptr<IAidlBufferSource> &aidlBufferSource) {
272 c2_status_t err = GetCodec2PlatformAllocatorStore()->fetchAllocator(
273 C2PlatformAllocatorStore::GRALLOC,
274 &mAllocator);
275 if (err != OK) {
276 return UNKNOWN_ERROR;
277 }
278 CHECK(mAidlHal);
279 mAidlBufferSource = aidlBufferSource;
280 return OK;
281 }
282
submitBuffer(uint32_t buffer,const sp<GraphicBuffer> & graphicBuffer,uint32_t flags,int64_t timestamp,int fenceFd)283 status_t C2NodeImpl::submitBuffer(
284 uint32_t buffer, const sp<GraphicBuffer> &graphicBuffer,
285 uint32_t flags, int64_t timestamp, int fenceFd) {
286 std::shared_ptr<Codec2Client::Component> comp = mComp.lock();
287 if (!comp) {
288 return NO_INIT;
289 }
290
291 uint32_t c2Flags = (flags & BUFFERFLAG_EOS)
292 ? C2FrameData::FLAG_END_OF_STREAM : 0;
293 std::shared_ptr<C2GraphicBlock> block;
294
295 android::base::unique_fd fd0, fd1;
296 C2Handle *handle = nullptr;
297 if (graphicBuffer) {
298 std::shared_ptr<C2GraphicAllocation> alloc;
299 handle = WrapNativeCodec2GrallocHandle(
300 graphicBuffer->handle,
301 graphicBuffer->width,
302 graphicBuffer->height,
303 graphicBuffer->format,
304 graphicBuffer->usage,
305 graphicBuffer->stride);
306 if (handle != nullptr) {
307 // unique_fd takes ownership of the fds, we'll get warning if these
308 // fds get closed by somebody else. Onwership will be released before
309 // we return, so that the fds get closed as usually when this function
310 // goes out of scope (when both items and block are gone).
311 native_handle_t *nativeHandle = reinterpret_cast<native_handle_t*>(handle);
312 fd0.reset(nativeHandle->numFds > 0 ? nativeHandle->data[0] : -1);
313 fd1.reset(nativeHandle->numFds > 1 ? nativeHandle->data[1] : -1);
314 }
315 c2_status_t err = mAllocator->priorGraphicAllocation(handle, &alloc);
316 if (err != OK) {
317 (void)fd0.release();
318 (void)fd1.release();
319 native_handle_close(handle);
320 native_handle_delete(handle);
321 return UNKNOWN_ERROR;
322 }
323 block = _C2BlockFactory::CreateGraphicBlock(alloc);
324 } else if (!(flags & BUFFERFLAG_EOS)) {
325 return BAD_VALUE;
326 }
327
328 std::unique_ptr<C2Work> work(new C2Work);
329 work->input.flags = (C2FrameData::flags_t)c2Flags;
330 work->input.ordinal.timestamp = timestamp;
331
332 // WORKAROUND: adjust timestamp based on gapUs
333 {
334 work->input.ordinal.customOrdinal = timestamp; // save input timestamp
335 if (mFirstInputFrame) {
336 // grab timestamps on first frame
337 mPrevInputTimestamp = timestamp;
338 mPrevCodecTimestamp = timestamp;
339 mFirstInputFrame = false;
340 } else if (mAdjustTimestampGapUs > 0) {
341 work->input.ordinal.timestamp =
342 mPrevCodecTimestamp
343 + c2_min((timestamp - mPrevInputTimestamp).peek(), mAdjustTimestampGapUs);
344 } else if (mAdjustTimestampGapUs < 0) {
345 work->input.ordinal.timestamp = mPrevCodecTimestamp - mAdjustTimestampGapUs;
346 }
347 mPrevInputTimestamp = work->input.ordinal.customOrdinal;
348 mPrevCodecTimestamp = work->input.ordinal.timestamp;
349 ALOGV("adjusting %lld to %lld (gap=%lld)",
350 work->input.ordinal.customOrdinal.peekll(),
351 work->input.ordinal.timestamp.peekll(),
352 (long long)mAdjustTimestampGapUs);
353 }
354
355 work->input.ordinal.frameIndex = mFrameIndex++;
356 work->input.buffers.clear();
357 if (block) {
358 std::shared_ptr<C2Buffer> c2Buffer(
359 new Buffer2D(block->share(
360 C2Rect(block->width(), block->height()), ::C2Fence())));
361 work->input.buffers.push_back(c2Buffer);
362 std::shared_ptr<C2StreamHdrStaticInfo::input> staticInfo;
363 std::shared_ptr<C2StreamHdrDynamicMetadataInfo::input> dynamicInfo;
364 GetHdrMetadataFromGralloc4Handle(
365 block->handle(),
366 &staticInfo,
367 &dynamicInfo);
368 if (staticInfo && *staticInfo) {
369 c2Buffer->setInfo(staticInfo);
370 }
371 if (dynamicInfo && *dynamicInfo) {
372 c2Buffer->setInfo(dynamicInfo);
373 }
374 }
375 work->worklets.clear();
376 work->worklets.emplace_back(new C2Worklet);
377 {
378 Mutexed<BuffersTracker>::Locked buffers(mBuffersTracker);
379 buffers->mIdsInUse.emplace(work->input.ordinal.frameIndex.peeku(), buffer);
380 }
381 mQueueThread->queue(comp, fenceFd, std::move(work), std::move(fd0), std::move(fd1));
382
383 return OK;
384 }
385
onDataspaceChanged(uint32_t dataSpace,uint32_t pixelFormat)386 status_t C2NodeImpl::onDataspaceChanged(uint32_t dataSpace, uint32_t pixelFormat) {
387 ALOGD("dataspace changed to %#x pixel format: %#x", dataSpace, pixelFormat);
388 android_dataspace d = (android_dataspace)dataSpace;
389 mQueueThread->setDataspace(d);
390
391 mDataspace.lock().set(d);
392 mPixelFormat.lock().set(pixelFormat);
393 return OK;
394 }
395
getSource()396 sp<IOMXBufferSource> C2NodeImpl::getSource() {
397 CHECK(!mAidlHal);
398 return mBufferSource;
399 }
400
getAidlSource()401 std::shared_ptr<IAidlBufferSource> C2NodeImpl::getAidlSource() {
402 CHECK(mAidlHal);
403 return mAidlBufferSource;
404 }
405
setFrameSize(uint32_t width,uint32_t height)406 void C2NodeImpl::setFrameSize(uint32_t width, uint32_t height) {
407 mWidth = width;
408 mHeight = height;
409 }
410
onInputBufferDone(c2_cntr64_t index)411 void C2NodeImpl::onInputBufferDone(c2_cntr64_t index) {
412 if (android::media::codec::provider_->input_surface_throttle()) {
413 Mutexed<BuffersTracker>::Locked buffers(mBuffersTracker);
414 auto it = buffers->mIdsInUse.find(index.peeku());
415 if (it == buffers->mIdsInUse.end()) {
416 ALOGV("Untracked input index %llu (maybe already removed)", index.peekull());
417 return;
418 }
419 int32_t bufferId = it->second;
420 (void)buffers->mIdsInUse.erase(it);
421 buffers->mAvailableIds.push_back(bufferId);
422 } else {
423 if (!hasBufferSource()) {
424 return;
425 }
426 int32_t bufferId = 0;
427 {
428 Mutexed<BuffersTracker>::Locked buffers(mBuffersTracker);
429 auto it = buffers->mIdsInUse.find(index.peeku());
430 if (it == buffers->mIdsInUse.end()) {
431 ALOGV("Untracked input index %llu (maybe already removed)", index.peekull());
432 return;
433 }
434 bufferId = it->second;
435 (void)buffers->mIdsInUse.erase(it);
436 }
437 notifyInputBufferEmptied(bufferId);
438 }
439 }
440
onInputBufferEmptied()441 void C2NodeImpl::onInputBufferEmptied() {
442 if (!android::media::codec::provider_->input_surface_throttle()) {
443 ALOGE("onInputBufferEmptied should not be called "
444 "when input_surface_throttle is false");
445 return;
446 }
447 if (!hasBufferSource()) {
448 return;
449 }
450 int32_t bufferId = 0;
451 {
452 Mutexed<BuffersTracker>::Locked buffers(mBuffersTracker);
453 if (buffers->mAvailableIds.empty()) {
454 ALOGV("The codec is ready to take more input buffers "
455 "but no input buffers are ready yet.");
456 return;
457 }
458 bufferId = buffers->mAvailableIds.front();
459 buffers->mAvailableIds.pop_front();
460 }
461 notifyInputBufferEmptied(bufferId);
462 }
463
hasBufferSource()464 bool C2NodeImpl::hasBufferSource() {
465 if (mAidlHal) {
466 if (!mAidlBufferSource) {
467 ALOGD("Buffer source not set");
468 return false;
469 }
470 } else {
471 if (!mBufferSource) {
472 ALOGD("Buffer source not set");
473 return false;
474 }
475 }
476 return true;
477 }
478
notifyInputBufferEmptied(int32_t bufferId)479 void C2NodeImpl::notifyInputBufferEmptied(int32_t bufferId) {
480 if (mAidlHal) {
481 ::ndk::ScopedFileDescriptor nullFence;
482 (void)mAidlBufferSource->onInputBufferEmptied(bufferId, nullFence);
483 } else {
484 (void)mBufferSource->onInputBufferEmptied(bufferId, -1);
485 }
486 }
487
getDataspace()488 android_dataspace C2NodeImpl::getDataspace() {
489 return *mDataspace.lock();
490 }
491
getPixelFormat()492 uint32_t C2NodeImpl::getPixelFormat() {
493 return *mPixelFormat.lock();
494 }
495
setPriority(int priority)496 void C2NodeImpl::setPriority(int priority) {
497 mQueueThread->setPriority(priority);
498 }
499
500 } // namespace android
501