1 /*
2  * Copyright 2023 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "Codec2-MultiAccessUnitHelper"
19 #include <android-base/logging.h>
20 
21 #include <com_android_media_codec_flags.h>
22 
23 #include <codec2/common/MultiAccessUnitHelper.h>
24 #include <android-base/properties.h>
25 
26 #include <C2BufferPriv.h>
27 #include <C2Debug.h>
28 #include <C2PlatformSupport.h>
29 
30 static inline constexpr  uint32_t MAX_SUPPORTED_SIZE = ( 10 * 512000 * 8 * 2u);
31 namespace android {
32 
MultiAccessUnitParamsSetter(bool mayBlock,C2InterfaceHelper::C2P<C2LargeFrame::output> & me)33 static C2R MultiAccessUnitParamsSetter(
34         bool mayBlock, C2InterfaceHelper::C2P<C2LargeFrame::output> &me) {
35     (void)mayBlock;
36     C2R res = C2R::Ok();
37     if (!me.F(me.v.maxSize).supportsAtAll(me.v.maxSize)) {
38         res = res.plus(C2SettingResultBuilder::BadValue(me.F(me.v.maxSize)));
39     } else if (!me.F(me.v.thresholdSize).supportsAtAll(me.v.thresholdSize)) {
40         res = res.plus(C2SettingResultBuilder::BadValue(me.F(me.v.thresholdSize)));
41     } else if (me.v.maxSize < me.v.thresholdSize) {
42         me.set().maxSize = me.v.thresholdSize;
43     }
44     std::vector<std::unique_ptr<C2SettingResult>> failures;
45     res.retrieveFailures(&failures);
46     if (!failures.empty()) {
47         me.set().maxSize = 0;
48         me.set().thresholdSize = 0;
49     }
50     return res;
51 }
52 
MultiAccessUnitInterface(const std::shared_ptr<C2ComponentInterface> & interface,std::shared_ptr<C2ReflectorHelper> helper)53 MultiAccessUnitInterface::MultiAccessUnitInterface(
54         const std::shared_ptr<C2ComponentInterface>& interface,
55         std::shared_ptr<C2ReflectorHelper> helper)
56         : C2InterfaceHelper(helper), mC2ComponentIntf(interface) {
57     setDerivedInstance(this);
58     addParameter(
59             DefineParam(mLargeFrameParams, C2_PARAMKEY_OUTPUT_LARGE_FRAME)
60             .withDefault(new C2LargeFrame::output(0u, 0, 0))
61             .withFields({
62                 C2F(mLargeFrameParams, maxSize).inRange(
63                         0, c2_min(UINT_MAX, MAX_SUPPORTED_SIZE)),
64                 C2F(mLargeFrameParams, thresholdSize).inRange(
65                         0, c2_min(UINT_MAX, MAX_SUPPORTED_SIZE))
66             })
67             .withSetter(MultiAccessUnitParamsSetter)
68             .build());
69     std::vector<std::shared_ptr<C2ParamDescriptor>> supportedParams;
70     querySupportedParams(&supportedParams);
71     // Adding to set to do intf seperation in query/config
72     for (std::shared_ptr<C2ParamDescriptor> &desc : supportedParams) {
73         mSupportedParamIndexSet.insert(desc->index());
74     }
75     mParamFields.emplace_back(mLargeFrameParams.get(), &(mLargeFrameParams.get()->maxSize));
76     mParamFields.emplace_back(mLargeFrameParams.get(), &(mLargeFrameParams.get()->thresholdSize));
77 
78     if (mC2ComponentIntf) {
79         c2_status_t err = mC2ComponentIntf->query_vb({&mKind}, {}, C2_MAY_BLOCK, nullptr);
80     }
81 }
82 
isValidField(const C2ParamField & field) const83 bool MultiAccessUnitInterface::isValidField(const C2ParamField &field) const {
84     return (std::find(mParamFields.begin(), mParamFields.end(), field) != mParamFields.end());
85 }
isParamSupported(C2Param::Index index)86 bool MultiAccessUnitInterface::isParamSupported(C2Param::Index index) {
87     return (mSupportedParamIndexSet.count(index) != 0);
88 }
89 
getLargeFrameParam() const90 C2LargeFrame::output MultiAccessUnitInterface::getLargeFrameParam() const {
91     return *mLargeFrameParams;
92 }
93 
kind() const94 C2Component::kind_t MultiAccessUnitInterface::kind() const {
95     return (C2Component::kind_t)(mKind.value);
96 }
97 
getDecoderSampleRateAndChannelCount(uint32_t * const sampleRate_,uint32_t * const channelCount_) const98 bool MultiAccessUnitInterface::getDecoderSampleRateAndChannelCount(
99         uint32_t * const sampleRate_, uint32_t * const channelCount_) const {
100     if (sampleRate_ == nullptr || channelCount_ == nullptr) {
101         return false;
102     }
103     if (mC2ComponentIntf) {
104         C2StreamSampleRateInfo::output sampleRate;
105         C2StreamChannelCountInfo::output channelCount;
106         c2_status_t res = mC2ComponentIntf->query_vb(
107                 {&sampleRate, &channelCount}, {}, C2_MAY_BLOCK, nullptr);
108         if (res == C2_OK && sampleRate.value > 0 && channelCount.value > 0) {
109             *sampleRate_ = sampleRate.value;
110             *channelCount_ = channelCount.value;
111             return true;
112         }
113     }
114     return false;
115 }
116 
getMaxInputSize(C2StreamMaxBufferSizeInfo::input * const maxInputSize) const117 bool MultiAccessUnitInterface::getMaxInputSize(
118         C2StreamMaxBufferSizeInfo::input* const maxInputSize) const {
119     if (maxInputSize == nullptr || mC2ComponentIntf == nullptr) {
120         return false;
121     }
122     c2_status_t err = mC2ComponentIntf->query_vb({maxInputSize}, {}, C2_MAY_BLOCK, nullptr);
123     if (err != OK) {
124         return false;
125     }
126     return true;
127 }
128 
129 //C2MultiAccessUnitBuffer
130 class C2MultiAccessUnitBuffer : public C2Buffer {
131     public:
C2MultiAccessUnitBuffer(const std::vector<C2ConstLinearBlock> & blocks)132         explicit C2MultiAccessUnitBuffer(
133                 const std::vector<C2ConstLinearBlock> &blocks):
134                 C2Buffer(blocks) {
135         }
136 };
137 
138 //MultiAccessUnitHelper
MultiAccessUnitHelper(const std::shared_ptr<MultiAccessUnitInterface> & intf,std::shared_ptr<C2BlockPool> & linearPool)139 MultiAccessUnitHelper::MultiAccessUnitHelper(
140         const std::shared_ptr<MultiAccessUnitInterface>& intf,
141         std::shared_ptr<C2BlockPool>& linearPool):
142         mMultiAccessOnOffAllowed(true),
143         mInit(false),
144         mInterface(intf),
145         mLinearPool(linearPool) {
146     if (mLinearPool) {
147         mInit = true;
148     }
149 }
150 
~MultiAccessUnitHelper()151 MultiAccessUnitHelper::~MultiAccessUnitHelper() {
152     std::unique_lock<std::mutex> l(mLock);
153     mFrameHolder.clear();
154 }
155 
isEnabledOnPlatform()156 bool MultiAccessUnitHelper::isEnabledOnPlatform() {
157     bool result = com::android::media::codec::flags::provider_->large_audio_frame();
158     if (!result) {
159         return false;
160     }
161     //TODO: remove this before launch
162     result = ::android::base::GetBoolProperty("debug.media.c2.large.audio.frame", true);
163     LOG(DEBUG) << "MultiAccessUnitHelper " << (result ? "enabled" : "disabled");
164     return result;
165 }
166 
tryReconfigure(const std::unique_ptr<C2Param> & param)167 bool MultiAccessUnitHelper::tryReconfigure(const std::unique_ptr<C2Param> &param) {
168     C2LargeFrame::output *lfp = C2LargeFrame::output::From(param.get());
169     if (lfp == nullptr) {
170         return false;
171     }
172     bool isDecoder = (mInterface->kind() == C2Component::KIND_DECODER) ? true : false;
173     if (!isDecoder) {
174         C2StreamMaxBufferSizeInfo::input maxInputSize(0);
175         if (!mInterface->getMaxInputSize(&maxInputSize)) {
176             LOG(ERROR) << "Error in reconfigure: "
177                     << "Encoder failed to respond with a valid max input size";
178             return false;
179         }
180         // This is assuming a worst case compression ratio of 1:1
181         // In no case the encoder should give an output more than
182         // what is being provided to the encoder in a single call.
183         if (lfp->maxSize < maxInputSize.value) {
184             lfp->maxSize = maxInputSize.value;
185         }
186     }
187     lfp->maxSize =
188             (lfp->maxSize > MAX_SUPPORTED_SIZE) ? MAX_SUPPORTED_SIZE :
189                     (lfp->maxSize < 0) ? 0 : lfp->maxSize;
190     lfp->thresholdSize =
191             (lfp->thresholdSize > MAX_SUPPORTED_SIZE) ? MAX_SUPPORTED_SIZE :
192                     (lfp->thresholdSize < 0) ? 0 : lfp->thresholdSize;
193     C2LargeFrame::output currentConfig = mInterface->getLargeFrameParam();
194     if ((currentConfig.maxSize == lfp->maxSize)
195             && (currentConfig.thresholdSize == lfp->thresholdSize)) {
196         // no need to update
197         return false;
198     }
199     if (isDecoder) {
200         bool isOnOffTransition =
201                 (currentConfig.maxSize == 0 && lfp->maxSize != 0)
202                 || (currentConfig.maxSize != 0 && lfp->maxSize == 0);
203             if (isOnOffTransition && !mMultiAccessOnOffAllowed) {
204                 LOG(ERROR) << "Setting new configs not allowed"
205                         << " MaxSize: " << lfp->maxSize
206                         << " ThresholdSize: " << lfp->thresholdSize;
207                 return false;
208             }
209     }
210     std::vector<C2Param*> config{lfp};
211     std::vector<std::unique_ptr<C2SettingResult>> failures;
212     if (C2_OK != mInterface->config(config, C2_MAY_BLOCK, &failures)) {
213         LOG(ERROR) << "Dynamic config not applied for"
214                 << " MaxSize: " << lfp->maxSize
215                 << " ThresholdSize: " << lfp->thresholdSize;
216         return false;
217     }
218     LOG(DEBUG) << "Updated from param maxSize "
219             << lfp->maxSize
220             << " ThresholdSize " << lfp->thresholdSize;
221     return true;
222 }
223 
getInterface()224 std::shared_ptr<MultiAccessUnitInterface> MultiAccessUnitHelper::getInterface() {
225     return mInterface;
226 }
227 
getStatus()228 bool MultiAccessUnitHelper::getStatus() {
229     return mInit;
230 }
231 
reset()232 void MultiAccessUnitHelper::reset() {
233     std::lock_guard<std::mutex> l(mLock);
234     mFrameHolder.clear();
235     mMultiAccessOnOffAllowed = true;
236 }
237 
error(std::list<std::unique_ptr<C2Work>> * const worklist)238 c2_status_t MultiAccessUnitHelper::error(
239         std::list<std::unique_ptr<C2Work>> * const worklist) {
240     if (worklist == nullptr) {
241         LOG(ERROR) << "Provided null worklist for error()";
242         mFrameHolder.clear();
243         return C2_OK;
244     }
245     std::unique_lock<std::mutex> l(mLock);
246     for (auto frame = mFrameHolder.begin(); frame != mFrameHolder.end(); frame++) {
247         if (frame->mLargeWork) {
248             finalizeWork(*frame, 0, true);
249             worklist->push_back(std::move(frame->mLargeWork));
250             frame->reset();
251         }
252     }
253     mFrameHolder.clear();
254     mMultiAccessOnOffAllowed = true;
255     return C2_OK;
256 }
257 
flush(std::list<std::unique_ptr<C2Work>> * const c2flushedWorks)258 c2_status_t MultiAccessUnitHelper::flush(
259         std::list<std::unique_ptr<C2Work>>* const c2flushedWorks) {
260     c2_status_t c2res = C2_OK;
261     std::lock_guard<std::mutex> l(mLock);
262     for (auto iterWork = c2flushedWorks->begin() ; iterWork != c2flushedWorks->end(); ) {
263         bool foundFlushedFrame = false;
264         std::list<MultiAccessUnitInfo>::iterator frame =
265                 mFrameHolder.begin();
266         while (frame != mFrameHolder.end() && !foundFlushedFrame) {
267             auto it = frame->mComponentFrameIds.find(
268                     (*iterWork)->input.ordinal.frameIndex.peekull());
269             if (it != frame->mComponentFrameIds.end()) {
270                 LOG(DEBUG) << "Multi access-unit flush "
271                         << (*iterWork)->input.ordinal.frameIndex.peekull()
272                         << " with " << frame->inOrdinal.frameIndex.peekull();
273                 (*iterWork)->input.ordinal.frameIndex = frame->inOrdinal.frameIndex;
274                 frame = mFrameHolder.erase(frame);
275                 foundFlushedFrame = true;
276             } else {
277                 ++frame;
278             }
279         }
280         if (!foundFlushedFrame) {
281             iterWork = c2flushedWorks->erase(iterWork);
282         } else {
283             ++iterWork;
284         }
285     }
286     return c2res;
287 }
288 
scatter(std::list<std::unique_ptr<C2Work>> & largeWork,std::list<std::list<std::unique_ptr<C2Work>>> * const processedWork)289 c2_status_t MultiAccessUnitHelper::scatter(
290         std::list<std::unique_ptr<C2Work>> &largeWork,
291         std::list<std::list<std::unique_ptr<C2Work>>>* const processedWork) {
292     LOG(DEBUG) << "Multiple access-unit: scatter";
293     if (processedWork == nullptr) {
294         LOG(ERROR) << "MultiAccessUnitHelper provided with no work list";
295         return C2_CORRUPTED;
296     }
297     for (std::unique_ptr<C2Work>& w : largeWork) {
298         std::list<std::unique_ptr<C2Work>> sliceWork;
299         C2WorkOrdinalStruct inputOrdinal = w->input.ordinal;
300         // To hold correspondence and processing bits b/w input and output
301         MultiAccessUnitInfo frameInfo(inputOrdinal);
302         std::set<uint64_t>& frameSet = frameInfo.mComponentFrameIds;
303         uint64_t newFrameIdx = mFrameIndex++;
304         // TODO: Do not split buffers if component inherantly supports MultipleFrames.
305         // if thats case, only replace frameindex.
306         auto cloneInputWork = [&frameInfo, &newFrameIdx, this]
307                 (std::unique_ptr<C2Work>& inWork, uint32_t flags) -> std::unique_ptr<C2Work> {
308             std::unique_ptr<C2Work> newWork(new C2Work);
309             newWork->input.flags = (C2FrameData::flags_t)flags;
310             newWork->input.ordinal = inWork->input.ordinal;
311             newWork->input.ordinal.frameIndex = newFrameIdx;
312             if (!inWork->input.configUpdate.empty()) {
313                 for (std::unique_ptr<C2Param>& param : inWork->input.configUpdate) {
314                     if (param->index() == C2LargeFrame::output::PARAM_TYPE) {
315                         if (tryReconfigure(param)) {
316                             frameInfo.mConfigUpdate.push_back(std::move(param));
317                         }
318                     } else {
319                         newWork->input.configUpdate.push_back(std::move(param));
320                     }
321                 }
322                 inWork->input.configUpdate.clear();
323             }
324             newWork->input.infoBuffers = (inWork->input.infoBuffers);
325             if (!inWork->worklets.empty() && inWork->worklets.front() != nullptr) {
326                 newWork->worklets.emplace_back(new C2Worklet);
327                 newWork->worklets.front()->component = inWork->worklets.front()->component;
328                 std::vector<std::unique_ptr<C2Tuning>> tunings;
329                 for (std::unique_ptr<C2Tuning>& tuning : inWork->worklets.front()->tunings) {
330                     tunings.push_back(std::move(
331                             std::unique_ptr<C2Tuning>(
332                                     static_cast<C2Tuning*>(
333                                             C2Param::Copy(*(tuning.get())).release()))));
334                 }
335                 newWork->worklets.front()->tunings = std::move(tunings);
336             }
337             return newWork;
338         };
339         if (w->input.buffers.empty()
340                 || (w->input.buffers.front() == nullptr)
341                 || (!w->input.buffers.front()->hasInfo(
342                         C2AccessUnitInfos::input::PARAM_TYPE))) {
343             LOG(DEBUG) << "Empty or MultiAU info buffer scatter frames with frameIndex "
344                     << inputOrdinal.frameIndex.peekull()
345                     << ") -> newFrameIndex " << newFrameIdx
346                     <<" : input ts " << inputOrdinal.timestamp.peekull();
347             sliceWork.push_back(std::move(cloneInputWork(w, w->input.flags)));
348             if (!w->input.buffers.empty() && w->input.buffers.front() != nullptr) {
349                 sliceWork.back()->input.buffers = std::move(w->input.buffers);
350             }
351             frameSet.insert(newFrameIdx);
352             processedWork->push_back(std::move(sliceWork));
353         }  else {
354             const std::vector<std::shared_ptr<C2Buffer>>& inBuffers = w->input.buffers;
355             if (inBuffers.front()->data().linearBlocks().size() == 0) {
356                 LOG(ERROR) << "ERROR: Work has Large frame info but has no linear blocks.";
357                 return C2_CORRUPTED;
358             }
359             frameInfo.mInputC2Ref = inBuffers;
360             const std::vector<C2ConstLinearBlock>& multiAU =
361                     inBuffers.front()->data().linearBlocks();
362             std::shared_ptr<const C2AccessUnitInfos::input> auInfo =
363                     std::static_pointer_cast<const C2AccessUnitInfos::input>(
364                     w->input.buffers.front()->getInfo(C2AccessUnitInfos::input::PARAM_TYPE));
365             uint32_t offset = 0; uint32_t multiAUSize = multiAU.front().size();
366             bool sendEos = false;
367             for (int idx = 0; idx < auInfo->flexCount(); ++idx) {
368                 std::vector<C2ConstLinearBlock> au;
369                 const C2AccessUnitInfosStruct &info = auInfo->m.values[idx];
370                 sendEos |= (info.flags & C2FrameData::FLAG_END_OF_STREAM);
371                 std::unique_ptr<C2Work> newWork = cloneInputWork(w, info.flags);
372                 frameSet.insert(newFrameIdx);
373                 newFrameIdx = mFrameIndex++;
374                 newWork->input.ordinal.timestamp = info.timestamp;
375                 au.push_back(multiAU.front().subBlock(offset, info.size));
376                 if ((offset + info.size) > multiAUSize) {
377                     LOG(ERROR) << "ERROR: access-unit offset > buffer size"
378                             << " current offset " << (offset + info.size)
379                             << " buffer size " << multiAUSize;
380                     return C2_CORRUPTED;
381                 }
382                 newWork->input.buffers.push_back(
383                         std::shared_ptr<C2Buffer>(new C2MultiAccessUnitBuffer(au)));
384                 LOG(DEBUG) << "Frame scatter queuing frames WITH info in ordinal "
385                         << inputOrdinal.frameIndex.peekull()
386                         << " info.size " << info.size
387                         << " : TS " << newWork->input.ordinal.timestamp.peekull()
388                         << " with index " << newFrameIdx - 1;
389                 // add to worklist
390                 sliceWork.push_back(std::move(newWork));
391                 processedWork->push_back(std::move(sliceWork));
392                 offset += info.size;
393             }
394             mFrameIndex--;
395             if (!sendEos && (w->input.flags & C2FrameData::FLAG_END_OF_STREAM)) {
396                 if (!processedWork->empty()) {
397                     std::list<std::unique_ptr<C2Work>> &sliceWork = processedWork->back();
398                     if (!sliceWork.empty()) {
399                         std::unique_ptr<C2Work> &work = sliceWork.back();
400                         if (work) {
401                             work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
402                         }
403                     }
404                 }
405             }
406         }
407         if (!processedWork->empty()) {
408             C2LargeFrame::output multiAccessParams = mInterface->getLargeFrameParam();
409             frameInfo.mLargeFrameTuning = multiAccessParams;
410             std::lock_guard<std::mutex> l(mLock);
411             mFrameHolder.push_back(std::move(frameInfo));
412             mMultiAccessOnOffAllowed = false;
413         }
414     }
415     return C2_OK;
416 }
417 
gather(std::list<std::unique_ptr<C2Work>> & c2workItems,std::list<std::unique_ptr<C2Work>> * const processedWork)418 c2_status_t MultiAccessUnitHelper::gather(
419         std::list<std::unique_ptr<C2Work>> &c2workItems,
420         std::list<std::unique_ptr<C2Work>>* const processedWork) {
421     LOG(DEBUG) << "Multi access-unit gather process";
422     if (processedWork == nullptr) {
423         LOG(ERROR) << "Nothing provided for processed work";
424         return C2_CORRUPTED;
425     }
426     auto addOutWork = [&processedWork](std::unique_ptr<C2Work>& work) {
427         processedWork->push_back(std::move(work));
428     };
429     {
430         std::lock_guard<std::mutex> l(mLock);
431         for (auto& work : c2workItems) {
432             LOG(DEBUG) << "FrameHolder Size: " << mFrameHolder.size();
433             uint64_t thisFrameIndex = work->input.ordinal.frameIndex.peekull();
434             bool removeEntry = work->worklets.empty()
435                     || !work->worklets.front()
436                     || (work->worklets.front()->output.flags
437                         & C2FrameData::FLAG_INCOMPLETE) == 0;
438             bool foundFrame = false;
439             std::list<MultiAccessUnitInfo>::iterator frame =
440                     mFrameHolder.begin();
441             while (!foundFrame && frame != mFrameHolder.end()) {
442                 c2_status_t res = C2_OK;
443                 auto it = frame->mComponentFrameIds.find(thisFrameIndex);
444                 if (it != frame->mComponentFrameIds.end()) {
445                     foundFrame = true;
446                     LOG(DEBUG) << "onWorkDone (frameIndex " << thisFrameIndex
447                             << " worklstsSze " << work->worklets.size()
448                             << ") -> frameIndex " << frame->inOrdinal.frameIndex.peekull();
449                     if (work->result != C2_OK
450                             || work->worklets.empty()
451                             || !work->worklets.front()
452                             || frame->mLargeFrameTuning.maxSize == 0) {
453                         if (removeEntry) {
454                             frame->mComponentFrameIds.erase(it);
455                             removeEntry = false;
456                         }
457                         if (frame->mLargeWork) {
458                             finalizeWork(*frame);
459                             addOutWork(frame->mLargeWork);
460                             frame->reset();
461                         }
462                         c2_status_t workResult = work->result;
463                         frame->mLargeWork = std::move(work);
464                         frame->mLargeWork->input.ordinal.frameIndex =
465                                 frame->inOrdinal.frameIndex;
466                         finalizeWork(*frame);
467                         addOutWork(frame->mLargeWork);
468                         frame->reset();
469                         if (workResult != C2_OK) {
470                             frame->mComponentFrameIds.clear();
471                             removeEntry = false;
472                         }
473                     } else if (C2_OK != (res = processWorklets(*frame, work, addOutWork))) {
474                         // Upon error in processing worklets, we return the work with
475                         // result set to the error. This should indicate the error to the
476                         // framework and thus doing what is necessary to handle the
477                         // error.
478                         LOG(DEBUG) << "Error while processing worklets";
479                         if (frame->mLargeWork == nullptr) {
480                             frame->mLargeWork.reset(new C2Work);
481                             frame->mLargeWork->input.ordinal = frame->inOrdinal;
482                             frame->mLargeWork->input.ordinal.frameIndex =
483                                     frame->inOrdinal.frameIndex;
484                         }
485                         frame->mLargeWork->result = res;
486                         finalizeWork(*frame);
487                         addOutWork(frame->mLargeWork);
488                         frame->reset();
489                         frame->mComponentFrameIds.clear();
490                         removeEntry = false;
491                     }
492                     if (removeEntry) {
493                         LOG(DEBUG) << "Removing entry: " << thisFrameIndex
494                                 << " -> " << frame->inOrdinal.frameIndex.peekull();
495                         frame->mComponentFrameIds.erase(it);
496                     }
497                     // This is to take care of the last bytes and to decide to send with
498                     // FLAG_INCOMPLETE or not.
499                     if ((frame->mWview
500                             && (frame->mWview->offset() >= frame->mLargeFrameTuning.thresholdSize))
501                             || frame->mComponentFrameIds.empty()) {
502                         if (frame->mLargeWork) {
503                             frame->mLargeWork->result = C2_OK;
504                             finalizeWork(*frame);
505                             addOutWork(frame->mLargeWork);
506                             frame->reset();
507                         }
508                     }
509                     if (frame->mComponentFrameIds.empty()) {
510                         LOG(DEBUG) << "This frame is finished ID " << thisFrameIndex;
511                         frame = mFrameHolder.erase(frame);
512                         continue;
513                     }
514                 } else {
515                     LOG(DEBUG) << "Received an out-of-order output " << thisFrameIndex
516                             << " expected: " <<mFrameHolder.front().inOrdinal.frameIndex.peekull();
517                 }
518                 frame++;
519             }
520             if (!foundFrame) {
521                 LOG(ERROR) <<" Error: Frame Holder reports no frame " << thisFrameIndex;
522             }
523         }
524     }
525     return C2_OK;
526 }
527 
createLinearBlock(MultiAccessUnitInfo & frame)528 c2_status_t MultiAccessUnitHelper::createLinearBlock(MultiAccessUnitInfo &frame) {
529     if (!mInit) {
530         LOG(ERROR) << "Large buffer allocator failed";
531         return C2_NO_MEMORY;
532     }
533     C2MemoryUsage usage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE };
534     uint32_t maxOutSize = frame.mLargeFrameTuning.maxSize;
535     c2_status_t err = mLinearPool->fetchLinearBlock(maxOutSize, usage, &frame.mBlock);
536     LOG(DEBUG) << "Allocated block with offset : " << frame.mBlock->offset()
537             << " size " << frame.mBlock->size() << " Capacity " << frame.mBlock->capacity();
538     if (err != C2_OK) {
539         LOG(ERROR) << "Error allocating Multi access-unit Buffer";
540         return err;
541     }
542     frame.mWview = std::make_shared<C2WriteView>(frame.mBlock->map().get());
543     LOG(DEBUG) << "Allocated buffer : requested size : " <<
544             frame.mLargeFrameTuning.maxSize
545             << " alloc size " << frame.mWview->size();
546     return C2_OK;
547 }
548 
549 /*
550  * For every work from the component, we try to do aggregation of work here.
551 */
processWorklets(MultiAccessUnitInfo & frame,std::unique_ptr<C2Work> & work,const std::function<void (std::unique_ptr<C2Work> &)> & addWork)552 c2_status_t MultiAccessUnitHelper::processWorklets(MultiAccessUnitInfo &frame,
553         std::unique_ptr<C2Work>& work,
554         const std::function <void(std::unique_ptr<C2Work>&)>& addWork) {
555     // This will allocate work, worklet, c2Block
556     auto allocateWork = [&](MultiAccessUnitInfo &frame,
557             bool allocateWorket = false,
558             bool allocateBuffer = false) {
559         c2_status_t ret = C2_OK;
560         if (frame.mLargeWork == nullptr) {
561             frame.mLargeWork.reset(new C2Work);
562             frame.mLargeWork->result = C2_OK;
563             frame.mLargeWork->input.flags = (C2FrameData::flags_t)0;
564             frame.mLargeWork->input.ordinal = frame.inOrdinal;
565             frame.mLargeWork->input.ordinal.frameIndex = frame.inOrdinal.frameIndex;
566         }
567         if (allocateWorket) {
568             if (frame.mLargeWork->worklets.size() == 0) {
569                 frame.mLargeWork->worklets.emplace_back(new C2Worklet);
570                 frame.mLargeWork->worklets.back()->output.flags = (C2FrameData::flags_t)0;
571             }
572         }
573         if (allocateBuffer) {
574             if (frame.mWview == nullptr) {
575                 ret = createLinearBlock(frame);
576             }
577         }
578         return ret;
579     };
580     // we will only have one worklet.
581     bool foundEndOfStream = false;
582     for (auto worklet = work->worklets.begin();
583              worklet != work->worklets.end() && (*worklet) != nullptr; ++worklet) {
584         uint32_t flagsForNoCopy = C2FrameData::FLAG_DROP_FRAME
585                 | C2FrameData::FLAG_DISCARD_FRAME
586                 | C2FrameData::FLAG_CORRUPT;
587         if ((*worklet)->output.flags & flagsForNoCopy) {
588             if (frame.mLargeWork) {
589                 finalizeWork(frame);
590                 addWork(frame.mLargeWork);
591                 frame.reset();
592             }
593             frame.mLargeWork = std::move(work);
594             frame.mLargeWork->input.ordinal.frameIndex = frame.inOrdinal.frameIndex;
595             finalizeWork(frame, (*worklet)->output.flags, true);
596             addWork(frame.mLargeWork);
597             frame.reset();
598             return C2_OK;
599         }
600         int64_t sampleTimeUs = 0;
601         uint32_t frameSize = 0;
602         uint32_t sampleRate = 0;
603         uint32_t channelCount = 0;
604         if (mInterface->getDecoderSampleRateAndChannelCount(&sampleRate, &channelCount)) {
605             sampleTimeUs = (1000000u) / (sampleRate * channelCount * 2);
606             frameSize = channelCount * 2;
607             if (mInterface->kind() == C2Component::KIND_DECODER) {
608                 frame.mLargeFrameTuning.maxSize =
609                         (frame.mLargeFrameTuning.maxSize / frameSize) * frameSize;
610                 frame.mLargeFrameTuning.thresholdSize =
611                         (frame.mLargeFrameTuning.thresholdSize / frameSize) * frameSize;
612             }
613         }
614         c2_status_t c2ret = allocateWork(frame, true);
615         if (c2ret != C2_OK) {
616             return c2ret;
617         }
618         uint32_t flags = work->input.flags;
619         flags |= frame.mLargeWork->input.flags;
620         frame.mLargeWork->input.flags = (C2FrameData::flags_t)flags;
621         C2FrameData& outputFramedata = frame.mLargeWork->worklets.front()->output;
622         if (!(*worklet)->output.configUpdate.empty()) {
623             for (auto& configUpdate : (*worklet)->output.configUpdate) {
624                 outputFramedata.configUpdate.push_back(std::move(configUpdate));
625             }
626             (*worklet)->output.configUpdate.clear();
627         }
628         outputFramedata.infoBuffers.insert(outputFramedata.infoBuffers.begin(),
629                 (*worklet)->output.infoBuffers.begin(),
630                 (*worklet)->output.infoBuffers.end());
631 
632         LOG(DEBUG) << "maxOutSize " << frame.mLargeFrameTuning.maxSize
633                 << " threshold " << frame.mLargeFrameTuning.thresholdSize;
634         LOG(DEBUG) << "This worklet has " << (*worklet)->output.buffers.size() << " buffers"
635                 << " ts: " << (*worklet)->output.ordinal.timestamp.peekull();
636         int64_t workletTimestamp = (*worklet)->output.ordinal.timestamp.peekull();
637         int64_t timestamp = workletTimestamp;
638         uint32_t flagsForCopy =  ((*worklet)->output.flags) & C2FrameData::FLAG_CODEC_CONFIG;
639         for (int bufIdx = 0; bufIdx < (*worklet)->output.buffers.size(); ++bufIdx) {
640             std::shared_ptr<C2Buffer>& buffer = (*worklet)->output.buffers[bufIdx];
641             if (!buffer || buffer->data().linearBlocks().empty()) {
642                 continue;
643             }
644             const std::vector<C2ConstLinearBlock>& blocks = buffer->data().linearBlocks();
645             if (blocks.size() > 0) {
646                 uint32_t inputOffset = 0;
647                 uint32_t inputSize = blocks.front().size();
648                 frame.mInfos.insert(frame.mInfos.end(),
649                         buffer->info().begin(), buffer->info().end());
650                 if (frameSize != 0 && (mInterface->kind() == C2Component::KIND_DECODER)) {
651                     // For decoders we only split multiples of 16bChannelCount*2
652                     inputSize -= (inputSize % frameSize);
653                 }
654                 while (inputOffset < inputSize) {
655                     if ((frame.mWview != nullptr)
656                             && (frame.mWview->offset() >= frame.mLargeFrameTuning.thresholdSize)) {
657                         frame.mLargeWork->result = C2_OK;
658                         finalizeWork(frame, flagsForCopy);
659                         addWork(frame.mLargeWork);
660                         frame.reset();
661                     }
662                     if (mInterface->kind() == C2Component::KIND_ENCODER) {
663                         if (inputSize > frame.mLargeFrameTuning.maxSize) {
664                             LOG(WARNING) << "WARNING Encoder:"
665                                     << " Output buffer too small for configuration"
666                                     << " configured max size " << frame.mLargeFrameTuning.maxSize
667                                     << " access unit size " << inputSize;
668                             if (frame.mLargeWork && (frame.mWview && frame.mWview->offset() > 0)) {
669                                 frame.mLargeWork->result = C2_OK;
670                                 finalizeWork(frame, flagsForCopy);
671                                 addWork(frame.mLargeWork);
672                                 frame.reset();
673                             }
674                             frame.mLargeFrameTuning.maxSize = inputSize;
675                         } else if ((frame.mWview != nullptr)
676                                 && (inputSize > frame.mWview->size())) {
677                             LOG(DEBUG) << "Enc: Large frame hitting bufer limit, current size "
678                                 << frame.mWview->offset();
679                             if (frame.mWview->offset() > 0) {
680                                 frame.mLargeWork->result = C2_OK;
681                                 finalizeWork(frame, flagsForCopy);
682                                 addWork(frame.mLargeWork);
683                                 frame.reset();
684                             }
685                         }
686                     }
687                     allocateWork(frame, true, true);
688                     uint32_t flags = work->input.flags;
689                     flags |= frame.mLargeWork->input.flags;
690                     frame.mLargeWork->input.flags = (C2FrameData::flags_t)flags;
691                     C2ReadView rView = blocks.front().map().get();
692                     if (rView.error()) {
693                         LOG(ERROR) << "Buffer read view error";
694                         frame.mLargeWork->result = rView.error();
695                         frame.mLargeWork->worklets.clear();
696                         finalizeWork(frame, 0, true);
697                         addWork(frame.mLargeWork);
698                         frame.reset();
699                         return C2_NO_MEMORY;
700                     }
701                     uint32_t toCopy = 0;
702                     if (mInterface->kind() == C2Component::KIND_ENCODER) {
703                         toCopy = inputSize;
704                     } else {
705                         toCopy = c2_min(frame.mWview->size(), (inputSize - inputOffset));
706                         timestamp = workletTimestamp + inputOffset * sampleTimeUs;
707                         LOG(DEBUG) << "ts " << timestamp
708                                 << " copiedOutput " << inputOffset
709                                 << " sampleTimeUs " << sampleTimeUs;
710                     }
711                     LOG(DEBUG) << " Copy size " << toCopy
712                             << " ts " << timestamp;
713                     memcpy(frame.mWview->data(), rView.data() + inputOffset, toCopy);
714                     frame.mWview->setOffset(frame.mWview->offset() + toCopy);
715                     inputOffset += toCopy;
716                     mergeAccessUnitInfo(frame, flagsForCopy, toCopy, timestamp);
717                 }
718             } else {
719                 frame.mLargeWork->worklets.front()->output.buffers.push_back(std::move(buffer));
720                 LOG(DEBUG) << "Copying worklets without linear buffer";
721             }
722         }
723         uint32_t flagsForCsdOrEnd = (*worklet)->output.flags
724                 & (C2FrameData::FLAG_END_OF_STREAM | C2FrameData::FLAG_CODEC_CONFIG);
725         if (flagsForCsdOrEnd) {
726             LOG(DEBUG) << "Output worklet has CSD/EOS data";
727             frame.mLargeWork->result = C2_OK;
728             // we can assign timestamp as this will be evaluated in finalizeWork
729             frame.mLargeWork->worklets.front()->output.ordinal.timestamp = timestamp;
730             finalizeWork(frame, flagsForCsdOrEnd, true);
731             addWork(frame.mLargeWork);
732             frame.reset();
733         }
734     }
735     return C2_OK;
736 }
737 
finalizeWork(MultiAccessUnitInfo & frame,uint32_t inFlags,bool forceComplete)738 c2_status_t MultiAccessUnitHelper::finalizeWork(
739         MultiAccessUnitInfo& frame, uint32_t inFlags, bool forceComplete) {
740     if (frame.mLargeWork == nullptr) {
741         return C2_OK;
742     }
743     //prepare input ordinal
744     frame.mLargeWork->input.ordinal = frame.inOrdinal;
745     // remove this
746     int64_t timeStampUs = frame.inOrdinal.timestamp.peekull();
747     if (!frame.mAccessUnitInfos.empty()) {
748         timeStampUs = frame.mAccessUnitInfos.front().timestamp;
749     } else if (!frame.mLargeWork->worklets.empty()) {
750         std::unique_ptr<C2Worklet> &worklet = frame.mLargeWork->worklets.front();
751         if (worklet) {
752             timeStampUs = worklet->output.ordinal.timestamp.peekull();
753         }
754     }
755     LOG(DEBUG) << "Finalizing work with input Idx "
756             << frame.mLargeWork->input.ordinal.frameIndex.peekull()
757             << " timestamp " << timeStampUs
758             << " inFlags " << inFlags;
759     uint32_t finalFlags = 0;
760     if ((!forceComplete)
761             && (frame.mLargeWork->result == C2_OK)
762             && (!frame.mComponentFrameIds.empty())) {
763         finalFlags |= C2FrameData::FLAG_INCOMPLETE;
764     }
765     if (frame.mLargeWork->result == C2_OK) {
766         finalFlags |= inFlags;
767     }
768     // update worklet if present
769     if (!frame.mLargeWork->worklets.empty() &&
770             frame.mLargeWork->worklets.front() != nullptr) {
771         frame.mLargeWork->workletsProcessed = 1;
772         C2FrameData& outFrameData = frame.mLargeWork->worklets.front()->output;
773         outFrameData.ordinal.frameIndex = frame.inOrdinal.frameIndex.peekull();
774         outFrameData.ordinal.timestamp = timeStampUs;
775         finalFlags |= frame.mLargeWork->worklets.front()->output.flags;
776         outFrameData.flags = (C2FrameData::flags_t)finalFlags;
777         // update buffers
778         if (frame.mBlock && (frame.mWview->offset() > 0)) {
779             size_t size = frame.mWview->offset();
780             LOG(DEBUG) << "Finalize : Block: Large frame size set as " << size
781                     << " timestamp as " << timeStampUs
782                     << "frameIndex " << outFrameData.ordinal.frameIndex.peekull();
783             frame.mWview->setOffset(0);
784             std::shared_ptr<C2Buffer> c2Buffer = C2Buffer::CreateLinearBuffer(
785                     frame.mBlock->share(0, size, ::C2Fence()));
786             frame.mLargeWork->worklets.front()->output.buffers.push_back(std::move(c2Buffer));
787         }
788         if (frame.mLargeWork->worklets.front()->output.buffers.size() > 0) {
789             std::shared_ptr<C2Buffer>& c2Buffer =
790                 frame.mLargeWork->worklets.front()->output.buffers.front();
791             if (c2Buffer != nullptr) {
792                 if (frame.mAccessUnitInfos.size() > 0) {
793                     if (finalFlags & C2FrameData::FLAG_END_OF_STREAM) {
794                         frame.mAccessUnitInfos.back().flags |= C2FrameData::FLAG_END_OF_STREAM;
795                     }
796                     std::shared_ptr<C2AccessUnitInfos::output> largeFrame =
797                             C2AccessUnitInfos::output::AllocShared(
798                                     frame.mAccessUnitInfos.size(), 0u, frame.mAccessUnitInfos);
799                     frame.mInfos.push_back(largeFrame);
800                     frame.mAccessUnitInfos.clear();
801                 }
802                 for (auto &info : frame.mInfos) {
803                     c2Buffer->setInfo(std::const_pointer_cast<C2Info>(info));
804                 }
805             }
806         }
807         if (frame.mConfigUpdate.size() > 0) {
808             outFrameData.configUpdate.insert(
809                     outFrameData.configUpdate.end(),
810                     make_move_iterator(frame.mConfigUpdate.begin()),
811                     make_move_iterator(frame.mConfigUpdate.end()));
812         }
813     }
814     frame.mConfigUpdate.clear();
815     frame.mInfos.clear();
816     frame.mBlock.reset();
817     frame.mWview.reset();
818 
819     LOG(DEBUG) << "Multi access-unitflag setting as " << finalFlags;
820     return C2_OK;
821 }
822 
mergeAccessUnitInfo(MultiAccessUnitInfo & frame,uint32_t flags_,uint32_t size,int64_t timestamp)823 void MultiAccessUnitHelper::mergeAccessUnitInfo(
824         MultiAccessUnitInfo &frame,
825         uint32_t flags_,
826         uint32_t size,
827         int64_t timestamp) {
828     // Remove flags that are not part of Access unit info
829     uint32_t flags = flags_ & ~(C2FrameData::FLAG_INCOMPLETE
830             | C2FrameData::FLAG_DISCARD_FRAME
831             | C2FrameData::FLAG_CORRUPT
832             | C2FrameData::FLAG_CORRECTED);
833     if (frame.mAccessUnitInfos.empty()) {
834         frame.mAccessUnitInfos.emplace_back(flags, size, timestamp);
835         return;
836     }
837     if ((mInterface->kind() == C2Component::KIND_DECODER) &&
838             (frame.mAccessUnitInfos.back().flags == flags)) {
839         // merge access units here
840         C2AccessUnitInfosStruct &s = frame.mAccessUnitInfos.back();
841         s.size += size; // don't have to update timestamp
842     } else {
843         frame.mAccessUnitInfos.emplace_back(flags, size, timestamp);
844     }
845 }
846 
reset()847 void MultiAccessUnitHelper::MultiAccessUnitInfo::reset() {
848     mBlock.reset();
849     mWview.reset();
850     mInfos.clear();
851     mConfigUpdate.clear();
852     mAccessUnitInfos.clear();
853     mLargeWork.reset();
854 }
855 
856 }  // namespace android