1 /*
2  * Copyright 2017, The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef CCODEC_BUFFER_CHANNEL_H_
18 
19 #define CCODEC_BUFFER_CHANNEL_H_
20 
21 #include <deque>
22 #include <map>
23 #include <memory>
24 #include <vector>
25 
26 #include <C2Buffer.h>
27 #include <C2Component.h>
28 #include <Codec2Mapper.h>
29 
30 #include <codec2/hidl/client.h>
31 #include <media/stagefright/foundation/Mutexed.h>
32 #include <media/stagefright/CodecBase.h>
33 
34 #include "CCodecBuffers.h"
35 #include "FrameReassembler.h"
36 #include "InputSurfaceWrapper.h"
37 #include "PipelineWatcher.h"
38 
39 namespace android {
40 
41 class MemoryDealer;
42 
43 class CCodecCallback {
44 public:
45     virtual ~CCodecCallback() = default;
46     virtual void onError(status_t err, enum ActionCode actionCode) = 0;
47     virtual void onOutputFramesRendered(int64_t mediaTimeUs, nsecs_t renderTimeNs) = 0;
48     virtual void onOutputBuffersChanged() = 0;
49     virtual void onFirstTunnelFrameReady() = 0;
50 };
51 
52 /**
53  * BufferChannelBase implementation for CCodec.
54  */
55 class CCodecBufferChannel
56     : public BufferChannelBase, public std::enable_shared_from_this<CCodecBufferChannel> {
57 public:
58     explicit CCodecBufferChannel(const std::shared_ptr<CCodecCallback> &callback);
59     virtual ~CCodecBufferChannel();
60 
61     // BufferChannelBase interface
62     void setCrypto(const sp<ICrypto> &crypto) override;
63     void setDescrambler(const sp<IDescrambler> &descrambler) override;
64 
65     status_t queueInputBuffer(const sp<MediaCodecBuffer> &buffer) override;
66     status_t queueSecureInputBuffer(
67             const sp<MediaCodecBuffer> &buffer,
68             bool secure,
69             const uint8_t *key,
70             const uint8_t *iv,
71             CryptoPlugin::Mode mode,
72             CryptoPlugin::Pattern pattern,
73             const CryptoPlugin::SubSample *subSamples,
74             size_t numSubSamples,
75             AString *errorDetailMsg) override;
76     status_t queueSecureInputBuffers(
77             const sp<MediaCodecBuffer> &buffer,
78             bool secure,
79             AString *errorDetailMsg) override;
80     status_t attachBuffer(
81             const std::shared_ptr<C2Buffer> &c2Buffer,
82             const sp<MediaCodecBuffer> &buffer) override;
83     status_t attachEncryptedBuffer(
84             const sp<hardware::HidlMemory> &memory,
85             bool secure,
86             const uint8_t *key,
87             const uint8_t *iv,
88             CryptoPlugin::Mode mode,
89             CryptoPlugin::Pattern pattern,
90             size_t offset,
91             const CryptoPlugin::SubSample *subSamples,
92             size_t numSubSamples,
93             const sp<MediaCodecBuffer> &buffer,
94             AString* errorDetailMsg) override;
95     status_t attachEncryptedBuffers(
96             const sp<hardware::HidlMemory> &memory,
97             size_t offset,
98             const sp<MediaCodecBuffer> &buffer,
99             bool secure,
100             AString* errorDetailMsg) override;
101     status_t renderOutputBuffer(
102             const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) override;
103     void pollForRenderedBuffers() override;
104     void onBufferReleasedFromOutputSurface(uint32_t generation) override;
105     status_t discardBuffer(const sp<MediaCodecBuffer> &buffer) override;
106     void getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) override;
107     void getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) override;
108 
109     // Methods below are interface for CCodec to use.
110 
111     /**
112      * Set the component object for buffer processing.
113      */
114     void setComponent(const std::shared_ptr<Codec2Client::Component> &component);
115 
116     /**
117      * Set output graphic surface for rendering.
118      */
119     status_t setSurface(const sp<Surface> &surface, uint32_t generation, bool pushBlankBuffer);
120 
121     /**
122      * Set GraphicBufferSource object from which the component extracts input
123      * buffers.
124      */
125     status_t setInputSurface(const std::shared_ptr<InputSurfaceWrapper> &surface);
126 
127     /**
128      * Signal EOS to input surface.
129      */
130     status_t signalEndOfInputStream();
131 
132     /**
133      * Set parameters.
134      */
135     status_t setParameters(std::vector<std::unique_ptr<C2Param>> &params);
136 
137     /**
138      * Start queueing buffers to the component. This object should never queue
139      * buffers before this call has completed.
140      */
141     status_t start(
142             const sp<AMessage> &inputFormat,
143             const sp<AMessage> &outputFormat,
144             bool buffersBoundToCodec);
145 
146     /**
147      * Prepare initial input buffers to be filled by client.
148      *
149      * \param clientInputBuffers[out]   pointer to slot index -> buffer map.
150      *                                  On success, it contains prepared
151      *                                  initial input buffers.
152      */
153     status_t prepareInitialInputBuffers(
154             std::map<size_t, sp<MediaCodecBuffer>> *clientInputBuffers,
155             bool retry = false);
156 
157     /**
158      * Request initial input buffers as prepared in clientInputBuffers.
159      *
160      * \param clientInputBuffers[in]    slot index -> buffer map with prepared
161      *                                  initial input buffers.
162      */
163     status_t requestInitialInputBuffers(
164             std::map<size_t, sp<MediaCodecBuffer>> &&clientInputBuffers);
165 
166     /**
167      * Stop using buffers of the current output surface for other Codec
168      * instances to use the surface safely.
169      *
170      * \param pushBlankBuffer[in]       push a blank buffer at the end if true
171      */
172     void stopUseOutputSurface(bool pushBlankBuffer);
173 
174     /**
175      * Stop queueing buffers to the component. This object should never queue
176      * buffers after this call, until start() is called.
177      */
178     void stop();
179 
180     /**
181      * Stop queueing buffers to the component and release all buffers.
182      */
183     void reset();
184 
185     /**
186      * Release all resources.
187      */
188     void release();
189 
190     void flush(const std::list<std::unique_ptr<C2Work>> &flushedWork);
191 
192     /**
193      * Notify input client about work done.
194      *
195      * @param workItems   finished work item.
196      * @param outputFormat new output format if it has changed, otherwise nullptr
197      * @param initData    new init data (CSD) if it has changed, otherwise nullptr
198      */
199     void onWorkDone(
200             std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat,
201             const C2StreamInitDataInfo::output *initData);
202 
203     /**
204      * Make an input buffer available for the client as it is no longer needed
205      * by the codec.
206      *
207      * @param frameIndex The index of input work
208      * @param arrayIndex The index of buffer in the input work buffers.
209      */
210     void onInputBufferDone(uint64_t frameIndex, size_t arrayIndex);
211 
212     PipelineWatcher::Clock::duration elapsed();
213 
214     enum MetaMode {
215         MODE_NONE,
216         MODE_ANW,
217     };
218 
219     void setMetaMode(MetaMode mode);
220 
221     /**
222      * get pixel format from output buffers.
223      *
224      * @return 0 if no valid pixel format found.
225      */
226     uint32_t getBuffersPixelFormat(bool isEncoder);
227 
228     void resetBuffersPixelFormat(bool isEncoder);
229 
230     /**
231      * Queue a C2 info buffer that will be sent to codec in the subsequent
232      * queueInputBuffer
233      *
234      * @param buffer C2 info buffer
235      */
236     void setInfoBuffer(const std::shared_ptr<C2InfoBuffer> &buffer);
237 
238 private:
239     uint32_t getInputBuffersPixelFormat();
240 
241     uint32_t getOutputBuffersPixelFormat();
242 
243     class QueueGuard;
244 
245     /**
246      * Special mutex-like object with the following properties:
247      *
248      * - At STOPPED state (initial, or after stop())
249      *   - QueueGuard object gets created at STOPPED state, and the client is
250      *     supposed to return immediately.
251      * - At RUNNING state (after start())
252      *   - Each QueueGuard object
253      */
254     class QueueSync {
255     public:
256         /**
257          * At construction the sync object is in STOPPED state.
258          */
QueueSync()259         inline QueueSync() {}
260         ~QueueSync() = default;
261 
262         /**
263          * Transition to RUNNING state when stopped. No-op if already in RUNNING
264          * state.
265          */
266         void start();
267 
268         /**
269          * At RUNNING state, wait until all QueueGuard object created during
270          * RUNNING state are destroyed, and then transition to STOPPED state.
271          * No-op if already in STOPPED state.
272          */
273         void stop();
274 
275     private:
276         Mutex mGuardLock;
277 
278         struct Counter {
CounterCounter279             inline Counter() : value(-1) {}
280             int32_t value;
281             Condition cond;
282         };
283         Mutexed<Counter> mCount;
284 
285         friend class CCodecBufferChannel::QueueGuard;
286     };
287 
288     class QueueGuard {
289     public:
290         QueueGuard(QueueSync &sync);
291         ~QueueGuard();
isRunning()292         inline bool isRunning() { return mRunning; }
293 
294     private:
295         QueueSync &mSync;
296         bool mRunning;
297     };
298 
299     struct TrackedFrame {
300         uint64_t number;
301         int64_t mediaTimeUs;
302         int64_t desiredRenderTimeNs;
303         nsecs_t latchTime;
304         sp<Fence> presentFence;
305     };
306 
307     void feedInputBufferIfAvailable();
308     void feedInputBufferIfAvailableInternal();
309     status_t queueInputBufferInternal(sp<MediaCodecBuffer> buffer,
310                                       std::shared_ptr<C2LinearBlock> encryptedBlock = nullptr,
311                                       size_t blockSize = 0);
312     bool handleWork(
313             std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat,
314             const C2StreamInitDataInfo::output *initData);
315     void sendOutputBuffers();
316     void ensureDecryptDestination(size_t size);
317     int32_t getHeapSeqNum(const sp<hardware::HidlMemory> &memory);
318 
319     void initializeFrameTrackingFor(ANativeWindow * window);
320     void trackReleasedFrame(const IGraphicBufferProducer::QueueBufferOutput& qbo,
321                             int64_t mediaTimeUs, int64_t desiredRenderTimeNs);
322     void processRenderedFrames(const FrameEventHistoryDelta& delta);
323     int64_t getRenderTimeNs(const TrackedFrame& frame);
324 
325     QueueSync mSync;
326     sp<MemoryDealer> mDealer;
327     sp<IMemory> mDecryptDestination;
328     int32_t mHeapSeqNum;
329     std::map<wp<hardware::HidlMemory>, int32_t> mHeapSeqNumMap;
330 
331     std::shared_ptr<Codec2Client::Component> mComponent;
332     std::string mComponentName; ///< component name for debugging
333     const char *mName; ///< C-string version of component name
334     std::shared_ptr<CCodecCallback> mCCodecCallback;
335     std::shared_ptr<C2BlockPool> mInputAllocator;
336     QueueSync mQueueSync;
337     std::vector<std::unique_ptr<C2Param>> mParamsToBeSet;
338     sp<AMessage> mOutputFormat;
339 
340     struct Input {
341         Input();
342 
343         std::unique_ptr<InputBuffers> buffers;
344         size_t numSlots;
345         FlexBuffersImpl extraBuffers;
346         size_t numExtraSlots;
347         uint32_t inputDelay;
348         uint32_t pipelineDelay;
349         c2_cntr64_t lastFlushIndex;
350 
351         FrameReassembler frameReassembler;
352     };
353     Mutexed<Input> mInput;
354     struct Output {
355         std::unique_ptr<OutputBuffers> buffers;
356         size_t numSlots;
357         uint32_t outputDelay;
358         // true iff the underlying block pool is bounded --- for example,
359         // a BufferQueue-based block pool would be bounded by the BufferQueue.
360         bool bounded;
361     };
362     Mutexed<Output> mOutput;
363     Mutexed<std::list<std::unique_ptr<C2Work>>> mFlushedConfigs;
364 
365     std::atomic_uint64_t mFrameIndex;
366     std::atomic_uint64_t mFirstValidFrameIndex;
367 
368     sp<MemoryDealer> makeMemoryDealer(size_t heapSize);
369 
370     std::deque<TrackedFrame> mTrackedFrames;
371     bool mAreRenderMetricsEnabled;
372     bool mIsSurfaceToDisplay;
373     bool mHasPresentFenceTimes;
374 
375     struct OutputSurface {
376         sp<Surface> surface;
377         uint32_t generation;
378         int maxDequeueBuffers;
379         std::map<uint64_t, int> rotation;
380     };
381     Mutexed<OutputSurface> mOutputSurface;
382     int mRenderingDepth;
383 
384     struct BlockPools {
385         C2Allocator::id_t inputAllocatorId;
386         std::shared_ptr<C2BlockPool> inputPool;
387         C2Allocator::id_t outputAllocatorId;
388         C2BlockPool::local_id_t outputPoolId;
389         std::shared_ptr<Codec2Client::Configurable> outputPoolIntf;
390     };
391     Mutexed<BlockPools> mBlockPools;
392 
393     std::shared_ptr<InputSurfaceWrapper> mInputSurface;
394 
395     MetaMode mMetaMode;
396 
397     Mutexed<PipelineWatcher> mPipelineWatcher;
398 
399     std::atomic_bool mInputMetEos;
400     std::once_flag mRenderWarningFlag;
401 
402     sp<ICrypto> mCrypto;
403     sp<IDescrambler> mDescrambler;
404 
hasCryptoOrDescrambler()405     inline bool hasCryptoOrDescrambler() {
406         return mCrypto != nullptr || mDescrambler != nullptr;
407     }
408     std::atomic_bool mSendEncryptedInfoBuffer;
409 
410     std::atomic_bool mTunneled;
411 
412     std::vector<std::shared_ptr<C2InfoBuffer>> mInfoBuffers;
413 };
414 
415 // Conversion of a c2_status_t value to a status_t value may depend on the
416 // operation that returns the c2_status_t value.
417 enum c2_operation_t {
418     C2_OPERATION_NONE,
419     C2_OPERATION_Component_connectToOmxInputSurface,
420     C2_OPERATION_Component_createBlockPool,
421     C2_OPERATION_Component_destroyBlockPool,
422     C2_OPERATION_Component_disconnectFromInputSurface,
423     C2_OPERATION_Component_drain,
424     C2_OPERATION_Component_flush,
425     C2_OPERATION_Component_queue,
426     C2_OPERATION_Component_release,
427     C2_OPERATION_Component_reset,
428     C2_OPERATION_Component_setOutputSurface,
429     C2_OPERATION_Component_start,
430     C2_OPERATION_Component_stop,
431     C2_OPERATION_ComponentStore_copyBuffer,
432     C2_OPERATION_ComponentStore_createComponent,
433     C2_OPERATION_ComponentStore_createInputSurface,
434     C2_OPERATION_ComponentStore_createInterface,
435     C2_OPERATION_Configurable_config,
436     C2_OPERATION_Configurable_query,
437     C2_OPERATION_Configurable_querySupportedParams,
438     C2_OPERATION_Configurable_querySupportedValues,
439     C2_OPERATION_InputSurface_connectToComponent,
440     C2_OPERATION_InputSurfaceConnection_disconnect,
441 };
442 
443 status_t toStatusT(c2_status_t c2s, c2_operation_t c2op = C2_OPERATION_NONE);
444 
445 }  // namespace android
446 
447 #endif  // CCODEC_BUFFER_CHANNEL_H_
448