1 /*
2  * Copyright 2024, The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "C2NodeImpl"
19 #include <log/log.h>
20 
21 #include <C2AllocatorGralloc.h>
22 #include <C2BlockInternal.h>
23 #include <C2Component.h>
24 #include <C2Config.h>
25 #include <C2Debug.h>
26 #include <C2PlatformSupport.h>
27 
28 #include <android_media_codec.h>
29 #include <android/fdsan.h>
30 #include <media/stagefright/foundation/ColorUtils.h>
31 #include <ui/Fence.h>
32 #include <ui/GraphicBuffer.h>
33 #include <utils/Errors.h>
34 #include <utils/Thread.h>
35 
36 #include "utils/Codec2Mapper.h"
37 #include "C2NodeImpl.h"
38 #include "Codec2Buffer.h"
39 
40 namespace android {
41 
42 using ::aidl::android::media::IAidlBufferSource;
43 using ::aidl::android::media::IAidlNode;
44 
45 using ::android::media::BUFFERFLAG_EOS;
46 
47 namespace {
48 
49 class Buffer2D : public C2Buffer {
50 public:
Buffer2D(C2ConstGraphicBlock block)51     explicit Buffer2D(C2ConstGraphicBlock block) : C2Buffer({ block }) {}
52 };
53 
54 }  // namespace
55 
56 class C2NodeImpl::QueueThread : public Thread {
57 public:
QueueThread()58     QueueThread() : Thread(false) {}
59     ~QueueThread() override = default;
queue(const std::shared_ptr<Codec2Client::Component> & comp,int fenceFd,std::unique_ptr<C2Work> && work,android::base::unique_fd && fd0,android::base::unique_fd && fd1)60     void queue(
61             const std::shared_ptr<Codec2Client::Component> &comp,
62             int fenceFd,
63             std::unique_ptr<C2Work> &&work,
64             android::base::unique_fd &&fd0,
65             android::base::unique_fd &&fd1) {
66         Mutexed<Jobs>::Locked jobs(mJobs);
67         auto it = jobs->queues.try_emplace(comp, comp).first;
68         it->second.workList.emplace_back(
69                 std::move(work), fenceFd, std::move(fd0), std::move(fd1));
70         jobs->cond.broadcast();
71     }
72 
setDataspace(android_dataspace dataspace)73     void setDataspace(android_dataspace dataspace) {
74         Mutexed<Jobs>::Locked jobs(mJobs);
75         ColorUtils::convertDataSpaceToV0(dataspace);
76         jobs->configUpdate.emplace_back(new C2StreamDataSpaceInfo::input(0u, dataspace));
77         int32_t standard;
78         int32_t transfer;
79         int32_t range;
80         ColorUtils::getColorConfigFromDataSpace(dataspace, &range, &standard, &transfer);
81         std::unique_ptr<C2StreamColorAspectsInfo::input> colorAspects =
82             std::make_unique<C2StreamColorAspectsInfo::input>(0u);
83         if (C2Mapper::map(standard, &colorAspects->primaries, &colorAspects->matrix)
84                 && C2Mapper::map(transfer, &colorAspects->transfer)
85                 && C2Mapper::map(range, &colorAspects->range)) {
86             jobs->configUpdate.push_back(std::move(colorAspects));
87         }
88     }
89 
setPriority(int priority)90     void setPriority(int priority) {
91         androidSetThreadPriority(getTid(), priority);
92     }
93 
94 protected:
threadLoop()95     bool threadLoop() override {
96         constexpr nsecs_t kIntervalNs = nsecs_t(10) * 1000 * 1000;  // 10ms
97         constexpr nsecs_t kWaitNs = kIntervalNs * 2;
98         for (int i = 0; i < 2; ++i) {
99             Mutexed<Jobs>::Locked jobs(mJobs);
100             nsecs_t nowNs = systemTime();
101             bool queued = false;
102             for (auto it = jobs->queues.begin(); it != jobs->queues.end(); ) {
103                 Queue &queue = it->second;
104                 if (queue.workList.empty()
105                         || (queue.lastQueuedTimestampNs != 0 &&
106                             nowNs - queue.lastQueuedTimestampNs < kIntervalNs)) {
107                     ++it;
108                     continue;
109                 }
110                 std::shared_ptr<Codec2Client::Component> comp = queue.component.lock();
111                 if (!comp) {
112                     it = jobs->queues.erase(it);
113                     continue;
114                 }
115                 std::list<std::unique_ptr<C2Work>> items;
116                 std::vector<int> fenceFds;
117                 std::vector<android::base::unique_fd> uniqueFds;
118                 while (!queue.workList.empty()) {
119                     items.push_back(std::move(queue.workList.front().work));
120                     fenceFds.push_back(queue.workList.front().fenceFd);
121                     uniqueFds.push_back(std::move(queue.workList.front().fd0));
122                     uniqueFds.push_back(std::move(queue.workList.front().fd1));
123                     queue.workList.pop_front();
124                 }
125                 for (const std::unique_ptr<C2Param> &param : jobs->configUpdate) {
126                     items.front()->input.configUpdate.emplace_back(C2Param::Copy(*param));
127                 }
128 
129                 jobs.unlock();
130                 for (int fenceFd : fenceFds) {
131                     sp<Fence> fence(new Fence(fenceFd));
132                     fence->waitForever(LOG_TAG);
133                 }
134                 queue.lastQueuedTimestampNs = nowNs;
135                 comp->queue(&items);
136                 for (android::base::unique_fd &ufd : uniqueFds) {
137                     (void)ufd.release();
138                 }
139                 jobs.lock();
140 
141                 it = jobs->queues.upper_bound(comp);
142                 queued = true;
143             }
144             if (queued) {
145                 jobs->configUpdate.clear();
146                 return true;
147             }
148             if (i == 0) {
149                 jobs.waitForConditionRelative(jobs->cond, kWaitNs);
150             }
151         }
152         return true;
153     }
154 
155 private:
156     struct WorkFence {
WorkFenceandroid::C2NodeImpl::QueueThread::WorkFence157         WorkFence(std::unique_ptr<C2Work> &&w, int fd) : work(std::move(w)), fenceFd(fd) {}
158 
WorkFenceandroid::C2NodeImpl::QueueThread::WorkFence159         WorkFence(
160                 std::unique_ptr<C2Work> &&w,
161                 int fd,
162                 android::base::unique_fd &&uniqueFd0,
163                 android::base::unique_fd &&uniqueFd1)
164             : work(std::move(w)),
165               fenceFd(fd),
166               fd0(std::move(uniqueFd0)),
167               fd1(std::move(uniqueFd1)) {}
168 
169         std::unique_ptr<C2Work> work;
170         int fenceFd;
171         android::base::unique_fd fd0;
172         android::base::unique_fd fd1;
173     };
174     struct Queue {
Queueandroid::C2NodeImpl::QueueThread::Queue175         Queue(const std::shared_ptr<Codec2Client::Component> &comp)
176             : component(comp), lastQueuedTimestampNs(0) {}
177         Queue(const Queue &) = delete;
178         Queue &operator =(const Queue &) = delete;
179 
180         std::weak_ptr<Codec2Client::Component> component;
181         std::list<WorkFence> workList;
182         nsecs_t lastQueuedTimestampNs;
183     };
184     struct Jobs {
185         std::map<std::weak_ptr<Codec2Client::Component>,
186                  Queue,
187                  std::owner_less<std::weak_ptr<Codec2Client::Component>>> queues;
188         std::vector<std::unique_ptr<C2Param>> configUpdate;
189         Condition cond;
190     };
191     Mutexed<Jobs> mJobs;
192 };
193 
C2NodeImpl(const std::shared_ptr<Codec2Client::Component> & comp,bool aidl)194 C2NodeImpl::C2NodeImpl(const std::shared_ptr<Codec2Client::Component> &comp, bool aidl)
195     : mComp(comp), mFrameIndex(0), mWidth(0), mHeight(0), mUsage(0),
196       mAdjustTimestampGapUs(0), mFirstInputFrame(true),
197       mQueueThread(new QueueThread), mAidlHal(aidl) {
198     android_fdsan_set_error_level(ANDROID_FDSAN_ERROR_LEVEL_WARN_ALWAYS);
199     mQueueThread->run("C2NodeImpl", PRIORITY_AUDIO);
200 
201     android_dataspace ds = HAL_DATASPACE_UNKNOWN;
202     mDataspace.lock().set(ds);
203     uint32_t pf = PIXEL_FORMAT_UNKNOWN;
204     mPixelFormat.lock().set(pf);
205 }
206 
~C2NodeImpl()207 C2NodeImpl::~C2NodeImpl() {
208 }
209 
freeNode()210 status_t C2NodeImpl::freeNode() {
211     mComp.reset();
212     android_fdsan_set_error_level(ANDROID_FDSAN_ERROR_LEVEL_WARN_ONCE);
213     return mQueueThread->requestExitAndWait();
214 }
215 
onFirstInputFrame()216 void C2NodeImpl::onFirstInputFrame() {
217     mFirstInputFrame = true;
218 }
219 
getConsumerUsageBits(uint64_t * usage)220 void C2NodeImpl::getConsumerUsageBits(uint64_t *usage) {
221     *usage = mUsage;
222 }
223 
getInputBufferParams(IAidlNode::InputBufferParams * params)224 void C2NodeImpl::getInputBufferParams(IAidlNode::InputBufferParams *params) {
225     params->bufferCountActual = 16;
226 
227     // WORKAROUND: having more slots improve performance while consuming
228     // more memory. This is a temporary workaround to reduce memory for
229     // larger-than-4K scenario.
230     if (mWidth * mHeight > 4096 * 2340) {
231         std::shared_ptr<Codec2Client::Component> comp = mComp.lock();
232         C2PortActualDelayTuning::input inputDelay(0);
233         C2ActualPipelineDelayTuning pipelineDelay(0);
234         c2_status_t c2err = C2_NOT_FOUND;
235         if (comp) {
236             c2err = comp->query(
237                     {&inputDelay, &pipelineDelay}, {}, C2_DONT_BLOCK, nullptr);
238         }
239         if (c2err == C2_OK || c2err == C2_BAD_INDEX) {
240             params->bufferCountActual = 4;
241             params->bufferCountActual += (inputDelay ? inputDelay.value : 0u);
242             params->bufferCountActual += (pipelineDelay ? pipelineDelay.value : 0u);
243         }
244     }
245 
246     params->frameWidth = mWidth;
247     params->frameHeight = mHeight;
248 }
249 
setConsumerUsageBits(uint64_t usage)250 void C2NodeImpl::setConsumerUsageBits(uint64_t usage) {
251     mUsage = usage;
252 }
253 
setAdjustTimestampGapUs(int32_t gapUs)254 void C2NodeImpl::setAdjustTimestampGapUs(int32_t gapUs) {
255     mAdjustTimestampGapUs = gapUs;
256 }
257 
setInputSurface(const sp<IOMXBufferSource> & bufferSource)258 status_t C2NodeImpl::setInputSurface(const sp<IOMXBufferSource> &bufferSource) {
259     c2_status_t err = GetCodec2PlatformAllocatorStore()->fetchAllocator(
260             C2PlatformAllocatorStore::GRALLOC,
261             &mAllocator);
262     if (err != OK) {
263         return UNKNOWN_ERROR;
264     }
265     CHECK(!mAidlHal);
266     mBufferSource = bufferSource;
267     return OK;
268 }
269 
setAidlInputSurface(const std::shared_ptr<IAidlBufferSource> & aidlBufferSource)270 status_t C2NodeImpl::setAidlInputSurface(
271         const std::shared_ptr<IAidlBufferSource> &aidlBufferSource) {
272     c2_status_t err = GetCodec2PlatformAllocatorStore()->fetchAllocator(
273             C2PlatformAllocatorStore::GRALLOC,
274             &mAllocator);
275     if (err != OK) {
276         return UNKNOWN_ERROR;
277     }
278     CHECK(mAidlHal);
279     mAidlBufferSource = aidlBufferSource;
280     return OK;
281 }
282 
submitBuffer(uint32_t buffer,const sp<GraphicBuffer> & graphicBuffer,uint32_t flags,int64_t timestamp,int fenceFd)283 status_t C2NodeImpl::submitBuffer(
284         uint32_t buffer, const sp<GraphicBuffer> &graphicBuffer,
285         uint32_t flags, int64_t timestamp, int fenceFd) {
286     std::shared_ptr<Codec2Client::Component> comp = mComp.lock();
287     if (!comp) {
288         return NO_INIT;
289     }
290 
291     uint32_t c2Flags = (flags & BUFFERFLAG_EOS)
292             ? C2FrameData::FLAG_END_OF_STREAM : 0;
293     std::shared_ptr<C2GraphicBlock> block;
294 
295     android::base::unique_fd fd0, fd1;
296     C2Handle *handle = nullptr;
297     if (graphicBuffer) {
298         std::shared_ptr<C2GraphicAllocation> alloc;
299         handle = WrapNativeCodec2GrallocHandle(
300                 graphicBuffer->handle,
301                 graphicBuffer->width,
302                 graphicBuffer->height,
303                 graphicBuffer->format,
304                 graphicBuffer->usage,
305                 graphicBuffer->stride);
306         if (handle != nullptr) {
307             // unique_fd takes ownership of the fds, we'll get warning if these
308             // fds get closed by somebody else. Onwership will be released before
309             // we return, so that the fds get closed as usually when this function
310             // goes out of scope (when both items and block are gone).
311             native_handle_t *nativeHandle = reinterpret_cast<native_handle_t*>(handle);
312             fd0.reset(nativeHandle->numFds > 0 ? nativeHandle->data[0] : -1);
313             fd1.reset(nativeHandle->numFds > 1 ? nativeHandle->data[1] : -1);
314         }
315         c2_status_t err = mAllocator->priorGraphicAllocation(handle, &alloc);
316         if (err != OK) {
317             (void)fd0.release();
318             (void)fd1.release();
319             native_handle_close(handle);
320             native_handle_delete(handle);
321             return UNKNOWN_ERROR;
322         }
323         block = _C2BlockFactory::CreateGraphicBlock(alloc);
324     } else if (!(flags & BUFFERFLAG_EOS)) {
325         return BAD_VALUE;
326     }
327 
328     std::unique_ptr<C2Work> work(new C2Work);
329     work->input.flags = (C2FrameData::flags_t)c2Flags;
330     work->input.ordinal.timestamp = timestamp;
331 
332     // WORKAROUND: adjust timestamp based on gapUs
333     {
334         work->input.ordinal.customOrdinal = timestamp; // save input timestamp
335         if (mFirstInputFrame) {
336             // grab timestamps on first frame
337             mPrevInputTimestamp = timestamp;
338             mPrevCodecTimestamp = timestamp;
339             mFirstInputFrame = false;
340         } else if (mAdjustTimestampGapUs > 0) {
341             work->input.ordinal.timestamp =
342                 mPrevCodecTimestamp
343                         + c2_min((timestamp - mPrevInputTimestamp).peek(), mAdjustTimestampGapUs);
344         } else if (mAdjustTimestampGapUs < 0) {
345             work->input.ordinal.timestamp = mPrevCodecTimestamp - mAdjustTimestampGapUs;
346         }
347         mPrevInputTimestamp = work->input.ordinal.customOrdinal;
348         mPrevCodecTimestamp = work->input.ordinal.timestamp;
349         ALOGV("adjusting %lld to %lld (gap=%lld)",
350               work->input.ordinal.customOrdinal.peekll(),
351               work->input.ordinal.timestamp.peekll(),
352               (long long)mAdjustTimestampGapUs);
353     }
354 
355     work->input.ordinal.frameIndex = mFrameIndex++;
356     work->input.buffers.clear();
357     if (block) {
358         std::shared_ptr<C2Buffer> c2Buffer(
359                 new Buffer2D(block->share(
360                         C2Rect(block->width(), block->height()), ::C2Fence())));
361         work->input.buffers.push_back(c2Buffer);
362         std::shared_ptr<C2StreamHdrStaticInfo::input> staticInfo;
363         std::shared_ptr<C2StreamHdrDynamicMetadataInfo::input> dynamicInfo;
364         GetHdrMetadataFromGralloc4Handle(
365                 block->handle(),
366                 &staticInfo,
367                 &dynamicInfo);
368         if (staticInfo && *staticInfo) {
369             c2Buffer->setInfo(staticInfo);
370         }
371         if (dynamicInfo && *dynamicInfo) {
372             c2Buffer->setInfo(dynamicInfo);
373         }
374     }
375     work->worklets.clear();
376     work->worklets.emplace_back(new C2Worklet);
377     {
378         Mutexed<BuffersTracker>::Locked buffers(mBuffersTracker);
379         buffers->mIdsInUse.emplace(work->input.ordinal.frameIndex.peeku(), buffer);
380     }
381     mQueueThread->queue(comp, fenceFd, std::move(work), std::move(fd0), std::move(fd1));
382 
383     return OK;
384 }
385 
onDataspaceChanged(uint32_t dataSpace,uint32_t pixelFormat)386 status_t C2NodeImpl::onDataspaceChanged(uint32_t dataSpace, uint32_t pixelFormat) {
387     ALOGD("dataspace changed to %#x pixel format: %#x", dataSpace, pixelFormat);
388     android_dataspace d = (android_dataspace)dataSpace;
389     mQueueThread->setDataspace(d);
390 
391     mDataspace.lock().set(d);
392     mPixelFormat.lock().set(pixelFormat);
393     return OK;
394 }
395 
getSource()396 sp<IOMXBufferSource> C2NodeImpl::getSource() {
397     CHECK(!mAidlHal);
398     return mBufferSource;
399 }
400 
getAidlSource()401 std::shared_ptr<IAidlBufferSource> C2NodeImpl::getAidlSource() {
402     CHECK(mAidlHal);
403     return mAidlBufferSource;
404 }
405 
setFrameSize(uint32_t width,uint32_t height)406 void C2NodeImpl::setFrameSize(uint32_t width, uint32_t height) {
407     mWidth = width;
408     mHeight = height;
409 }
410 
onInputBufferDone(c2_cntr64_t index)411 void C2NodeImpl::onInputBufferDone(c2_cntr64_t index) {
412     if (android::media::codec::provider_->input_surface_throttle()) {
413         Mutexed<BuffersTracker>::Locked buffers(mBuffersTracker);
414         auto it = buffers->mIdsInUse.find(index.peeku());
415         if (it == buffers->mIdsInUse.end()) {
416             ALOGV("Untracked input index %llu (maybe already removed)", index.peekull());
417             return;
418         }
419         int32_t bufferId = it->second;
420         (void)buffers->mIdsInUse.erase(it);
421         buffers->mAvailableIds.push_back(bufferId);
422     } else {
423         if (!hasBufferSource()) {
424             return;
425         }
426         int32_t bufferId = 0;
427         {
428             Mutexed<BuffersTracker>::Locked buffers(mBuffersTracker);
429             auto it = buffers->mIdsInUse.find(index.peeku());
430             if (it == buffers->mIdsInUse.end()) {
431                 ALOGV("Untracked input index %llu (maybe already removed)", index.peekull());
432                 return;
433             }
434             bufferId = it->second;
435             (void)buffers->mIdsInUse.erase(it);
436         }
437         notifyInputBufferEmptied(bufferId);
438     }
439 }
440 
onInputBufferEmptied()441 void C2NodeImpl::onInputBufferEmptied() {
442     if (!android::media::codec::provider_->input_surface_throttle()) {
443         ALOGE("onInputBufferEmptied should not be called "
444               "when input_surface_throttle is false");
445         return;
446     }
447     if (!hasBufferSource()) {
448         return;
449     }
450     int32_t bufferId = 0;
451     {
452         Mutexed<BuffersTracker>::Locked buffers(mBuffersTracker);
453         if (buffers->mAvailableIds.empty()) {
454             ALOGV("The codec is ready to take more input buffers "
455                     "but no input buffers are ready yet.");
456             return;
457         }
458         bufferId = buffers->mAvailableIds.front();
459         buffers->mAvailableIds.pop_front();
460     }
461     notifyInputBufferEmptied(bufferId);
462 }
463 
hasBufferSource()464 bool C2NodeImpl::hasBufferSource() {
465     if (mAidlHal) {
466         if (!mAidlBufferSource) {
467             ALOGD("Buffer source not set");
468             return false;
469         }
470     } else {
471         if (!mBufferSource) {
472             ALOGD("Buffer source not set");
473             return false;
474         }
475     }
476     return true;
477 }
478 
notifyInputBufferEmptied(int32_t bufferId)479 void C2NodeImpl::notifyInputBufferEmptied(int32_t bufferId) {
480     if (mAidlHal) {
481         ::ndk::ScopedFileDescriptor nullFence;
482         (void)mAidlBufferSource->onInputBufferEmptied(bufferId, nullFence);
483     } else {
484         (void)mBufferSource->onInputBufferEmptied(bufferId, -1);
485     }
486 }
487 
getDataspace()488 android_dataspace C2NodeImpl::getDataspace() {
489     return *mDataspace.lock();
490 }
491 
getPixelFormat()492 uint32_t C2NodeImpl::getPixelFormat() {
493     return *mPixelFormat.lock();
494 }
495 
setPriority(int priority)496 void C2NodeImpl::setPriority(int priority) {
497     mQueueThread->setPriority(priority);
498 }
499 
500 }  // namespace android
501