1 /*
2  * Copyright 2012, The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef MEDIA_CODEC_H_
18 
19 #define MEDIA_CODEC_H_
20 
21 #include <memory>
22 #include <vector>
23 
24 #include <gui/IGraphicBufferProducer.h>
25 #include <media/hardware/CryptoAPI.h>
26 #include <media/MediaCodecInfo.h>
27 #include <media/MediaMetrics.h>
28 #include <media/stagefright/foundation/AHandler.h>
29 #include <media/stagefright/FrameRenderTracker.h>
30 #include <utils/Vector.h>
31 
32 class C2Buffer;
33 class C2GraphicBlock;
34 class C2LinearBlock;
35 
36 namespace aidl {
37 namespace android {
38 namespace media {
39 class MediaResourceParcel;
40 } // media
41 } // android
42 } // aidl
43 
44 namespace android {
45 
46 struct ABuffer;
47 struct AMessage;
48 struct AReplyToken;
49 struct AString;
50 struct BatteryChecker;
51 class BufferChannelBase;
52 struct CodecBase;
53 class IBatteryStats;
54 struct ICrypto;
55 class MediaCodecBuffer;
56 class IMemory;
57 struct PersistentSurface;
58 class SoftwareRenderer;
59 class Surface;
60 namespace hardware {
61 namespace cas {
62 namespace native {
63 namespace V1_0 {
64 struct IDescrambler;
65 }}}}
66 
67 using hardware::cas::native::V1_0::IDescrambler;
68 using aidl::android::media::MediaResourceParcel;
69 
70 struct MediaCodec : public AHandler {
71     enum ConfigureFlags {
72         CONFIGURE_FLAG_ENCODE           = 1,
73         CONFIGURE_FLAG_USE_BLOCK_MODEL  = 2,
74     };
75 
76     enum BufferFlags {
77         BUFFER_FLAG_SYNCFRAME     = 1,
78         BUFFER_FLAG_CODECCONFIG   = 2,
79         BUFFER_FLAG_EOS           = 4,
80         BUFFER_FLAG_PARTIAL_FRAME = 8,
81         BUFFER_FLAG_MUXER_DATA    = 16,
82     };
83 
84     enum {
85         CB_INPUT_AVAILABLE = 1,
86         CB_OUTPUT_AVAILABLE = 2,
87         CB_ERROR = 3,
88         CB_OUTPUT_FORMAT_CHANGED = 4,
89         CB_RESOURCE_RECLAIMED = 5,
90     };
91 
92     static const pid_t kNoPid = -1;
93     static const uid_t kNoUid = -1;
94 
95     static sp<MediaCodec> CreateByType(
96             const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err = NULL,
97             pid_t pid = kNoPid, uid_t uid = kNoUid);
98 
99     static sp<MediaCodec> CreateByComponentName(
100             const sp<ALooper> &looper, const AString &name, status_t *err = NULL,
101             pid_t pid = kNoPid, uid_t uid = kNoUid);
102 
103     static sp<PersistentSurface> CreatePersistentInputSurface();
104 
105     status_t configure(
106             const sp<AMessage> &format,
107             const sp<Surface> &nativeWindow,
108             const sp<ICrypto> &crypto,
109             uint32_t flags);
110 
111     status_t configure(
112             const sp<AMessage> &format,
113             const sp<Surface> &nativeWindow,
114             const sp<ICrypto> &crypto,
115             const sp<IDescrambler> &descrambler,
116             uint32_t flags);
117 
118     status_t releaseCrypto();
119 
120     status_t setCallback(const sp<AMessage> &callback);
121 
122     status_t setOnFrameRenderedNotification(const sp<AMessage> &notify);
123 
124     status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
125 
126     status_t setInputSurface(const sp<PersistentSurface> &surface);
127 
128     status_t start();
129 
130     // Returns to a state in which the component remains allocated but
131     // unconfigured.
132     status_t stop();
133 
134     // Resets the codec to the INITIALIZED state.  Can be called after an error
135     // has occured to make the codec usable.
136     status_t reset();
137 
138     // Client MUST call release before releasing final reference to this
139     // object.
140     status_t release();
141 
142     status_t releaseAsync(const sp<AMessage> &notify);
143 
144     status_t flush();
145 
146     status_t queueInputBuffer(
147             size_t index,
148             size_t offset,
149             size_t size,
150             int64_t presentationTimeUs,
151             uint32_t flags,
152             AString *errorDetailMsg = NULL);
153 
154     status_t queueSecureInputBuffer(
155             size_t index,
156             size_t offset,
157             const CryptoPlugin::SubSample *subSamples,
158             size_t numSubSamples,
159             const uint8_t key[16],
160             const uint8_t iv[16],
161             CryptoPlugin::Mode mode,
162             const CryptoPlugin::Pattern &pattern,
163             int64_t presentationTimeUs,
164             uint32_t flags,
165             AString *errorDetailMsg = NULL);
166 
167     status_t queueBuffer(
168             size_t index,
169             const std::shared_ptr<C2Buffer> &buffer,
170             int64_t presentationTimeUs,
171             uint32_t flags,
172             const sp<AMessage> &tunings,
173             AString *errorDetailMsg = NULL);
174 
175     status_t queueEncryptedBuffer(
176             size_t index,
177             const sp<hardware::HidlMemory> &memory,
178             size_t offset,
179             const CryptoPlugin::SubSample *subSamples,
180             size_t numSubSamples,
181             const uint8_t key[16],
182             const uint8_t iv[16],
183             CryptoPlugin::Mode mode,
184             const CryptoPlugin::Pattern &pattern,
185             int64_t presentationTimeUs,
186             uint32_t flags,
187             const sp<AMessage> &tunings,
188             AString *errorDetailMsg = NULL);
189 
190     std::shared_ptr<C2Buffer> decrypt(
191             const std::shared_ptr<C2Buffer> &buffer,
192             const CryptoPlugin::SubSample *subSamples,
193             size_t numSubSamples,
194             const uint8_t key[16],
195             const uint8_t iv[16],
196             CryptoPlugin::Mode mode,
197             const CryptoPlugin::Pattern &pattern);
198 
199     status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll);
200 
201     status_t dequeueOutputBuffer(
202             size_t *index,
203             size_t *offset,
204             size_t *size,
205             int64_t *presentationTimeUs,
206             uint32_t *flags,
207             int64_t timeoutUs = 0ll);
208 
209     status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs);
210     status_t renderOutputBufferAndRelease(size_t index);
211     status_t releaseOutputBuffer(size_t index);
212 
213     status_t signalEndOfInputStream();
214 
215     status_t getOutputFormat(sp<AMessage> *format) const;
216     status_t getInputFormat(sp<AMessage> *format) const;
217 
218     status_t getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
219     status_t getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
220 
221     status_t getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
222     status_t getOutputFormat(size_t index, sp<AMessage> *format);
223     status_t getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
224 
225     status_t setSurface(const sp<Surface> &nativeWindow);
226 
227     status_t requestIDRFrame();
228 
229     // Notification will be posted once there "is something to do", i.e.
230     // an input/output buffer has become available, a format change is
231     // pending, an error is pending.
232     void requestActivityNotification(const sp<AMessage> &notify);
233 
234     status_t getName(AString *componentName) const;
235 
236     status_t getCodecInfo(sp<MediaCodecInfo> *codecInfo) const;
237 
238     status_t getMetrics(mediametrics_handle_t &reply);
239 
240     status_t setParameters(const sp<AMessage> &params);
241 
242     // Create a MediaCodec notification message from a list of rendered or dropped render infos
243     // by adding rendered frame information to a base notification message. Returns the number
244     // of frames that were rendered.
245     static size_t CreateFramesRenderedMessage(
246             const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg);
247 
248     static status_t CanFetchLinearBlock(
249             const std::vector<std::string> &names, bool *isCompatible);
250 
251     static std::shared_ptr<C2LinearBlock> FetchLinearBlock(
252             size_t capacity, const std::vector<std::string> &names);
253 
254     static status_t CanFetchGraphicBlock(
255             const std::vector<std::string> &names, bool *isCompatible);
256 
257     static std::shared_ptr<C2GraphicBlock> FetchGraphicBlock(
258             int32_t width,
259             int32_t height,
260             int32_t format,
261             uint64_t usage,
262             const std::vector<std::string> &names);
263 
264     template <typename T>
265     struct WrapperObject : public RefBase {
WrapperObjectMediaCodec::WrapperObject266         WrapperObject(const T& v) : value(v) {}
WrapperObjectMediaCodec::WrapperObject267         WrapperObject(T&& v) : value(std::move(v)) {}
268         T value;
269     };
270 
271 protected:
272     virtual ~MediaCodec();
273     virtual void onMessageReceived(const sp<AMessage> &msg);
274 
275 private:
276     // used by ResourceManagerClient
277     status_t reclaim(bool force = false);
278     friend struct ResourceManagerClient;
279 
280 private:
281     enum State {
282         UNINITIALIZED,
283         INITIALIZING,
284         INITIALIZED,
285         CONFIGURING,
286         CONFIGURED,
287         STARTING,
288         STARTED,
289         FLUSHING,
290         FLUSHED,
291         STOPPING,
292         RELEASING,
293     };
294     std::string stateString(State state);
295 
296     enum {
297         kPortIndexInput         = 0,
298         kPortIndexOutput        = 1,
299     };
300 
301     enum {
302         kWhatInit                           = 'init',
303         kWhatConfigure                      = 'conf',
304         kWhatSetSurface                     = 'sSur',
305         kWhatCreateInputSurface             = 'cisf',
306         kWhatSetInputSurface                = 'sisf',
307         kWhatStart                          = 'strt',
308         kWhatStop                           = 'stop',
309         kWhatRelease                        = 'rele',
310         kWhatDequeueInputBuffer             = 'deqI',
311         kWhatQueueInputBuffer               = 'queI',
312         kWhatDequeueOutputBuffer            = 'deqO',
313         kWhatReleaseOutputBuffer            = 'relO',
314         kWhatSignalEndOfInputStream         = 'eois',
315         kWhatGetBuffers                     = 'getB',
316         kWhatFlush                          = 'flus',
317         kWhatGetOutputFormat                = 'getO',
318         kWhatGetInputFormat                 = 'getI',
319         kWhatDequeueInputTimedOut           = 'dITO',
320         kWhatDequeueOutputTimedOut          = 'dOTO',
321         kWhatCodecNotify                    = 'codc',
322         kWhatRequestIDRFrame                = 'ridr',
323         kWhatRequestActivityNotification    = 'racN',
324         kWhatGetName                        = 'getN',
325         kWhatGetCodecInfo                   = 'gCoI',
326         kWhatSetParameters                  = 'setP',
327         kWhatSetCallback                    = 'setC',
328         kWhatSetNotification                = 'setN',
329         kWhatDrmReleaseCrypto               = 'rDrm',
330         kWhatCheckBatteryStats              = 'chkB',
331     };
332 
333     enum {
334         kFlagUsesSoftwareRenderer       = 1,
335         kFlagOutputFormatChanged        = 2,
336         kFlagOutputBuffersChanged       = 4,
337         kFlagStickyError                = 8,
338         kFlagDequeueInputPending        = 16,
339         kFlagDequeueOutputPending       = 32,
340         kFlagIsSecure                   = 64,
341         kFlagSawMediaServerDie          = 128,
342         kFlagIsEncoder                  = 256,
343         // 512 skipped
344         kFlagIsAsync                    = 1024,
345         kFlagIsComponentAllocated       = 2048,
346         kFlagPushBlankBuffersOnShutdown = 4096,
347         kFlagUseBlockModel              = 8192,
348     };
349 
350     struct BufferInfo {
351         BufferInfo();
352 
353         sp<MediaCodecBuffer> mData;
354         bool mOwnedByClient;
355     };
356 
357     struct ResourceManagerServiceProxy;
358 
359     State mState;
360     uid_t mUid;
361     bool mReleasedByResourceManager;
362     sp<ALooper> mLooper;
363     sp<ALooper> mCodecLooper;
364     sp<CodecBase> mCodec;
365     AString mComponentName;
366     AString mOwnerName;
367     sp<MediaCodecInfo> mCodecInfo;
368     sp<AReplyToken> mReplyID;
369     uint32_t mFlags;
370     status_t mStickyError;
371     sp<Surface> mSurface;
372     SoftwareRenderer *mSoftRenderer;
373 
374     mediametrics_handle_t mMetricsHandle = 0;
375     nsecs_t mLifetimeStartNs = 0;
376     void initMediametrics();
377     void updateMediametrics();
378     void flushMediametrics();
379     void updateEphemeralMediametrics(mediametrics_handle_t item);
380     void updateLowLatency(const sp<AMessage> &msg);
381 
382     sp<AMessage> mOutputFormat;
383     sp<AMessage> mInputFormat;
384     sp<AMessage> mCallback;
385     sp<AMessage> mOnFrameRenderedNotification;
386     sp<AMessage> mAsyncReleaseCompleteNotification;
387 
388     sp<ResourceManagerServiceProxy> mResourceManagerProxy;
389 
390     bool mIsVideo;
391     int32_t mVideoWidth;
392     int32_t mVideoHeight;
393     int32_t mRotationDegrees;
394     int32_t mAllowFrameDroppingBySurface;
395 
396     // initial create parameters
397     AString mInitName;
398 
399     // configure parameter
400     sp<AMessage> mConfigureMsg;
401 
402     // Used only to synchronize asynchronous getBufferAndFormat
403     // across all the other (synchronous) buffer state change
404     // operations, such as de/queueIn/OutputBuffer, start and
405     // stop/flush/reset/release.
406     Mutex mBufferLock;
407 
408     List<size_t> mAvailPortBuffers[2];
409     std::vector<BufferInfo> mPortBuffers[2];
410 
411     int32_t mDequeueInputTimeoutGeneration;
412     sp<AReplyToken> mDequeueInputReplyID;
413 
414     int32_t mDequeueOutputTimeoutGeneration;
415     sp<AReplyToken> mDequeueOutputReplyID;
416 
417     sp<ICrypto> mCrypto;
418 
419     sp<IDescrambler> mDescrambler;
420 
421     List<sp<ABuffer> > mCSD;
422 
423     sp<AMessage> mActivityNotify;
424 
425     bool mHaveInputSurface;
426     bool mHavePendingInputBuffers;
427     bool mCpuBoostRequested;
428 
429     std::shared_ptr<BufferChannelBase> mBufferChannel;
430 
431     MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid);
432 
433     static sp<CodecBase> GetCodecBase(const AString &name, const char *owner = nullptr);
434 
435     static status_t PostAndAwaitResponse(
436             const sp<AMessage> &msg, sp<AMessage> *response);
437 
438     void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err);
439 
440     status_t init(const AString &name);
441 
442     void setState(State newState);
443     void returnBuffersToCodec(bool isReclaim = false);
444     void returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim = false);
445     size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg);
446     status_t onQueueInputBuffer(const sp<AMessage> &msg);
447     status_t onReleaseOutputBuffer(const sp<AMessage> &msg);
448     ssize_t dequeuePortBuffer(int32_t portIndex);
449 
450     status_t getBufferAndFormat(
451             size_t portIndex, size_t index,
452             sp<MediaCodecBuffer> *buffer, sp<AMessage> *format);
453 
454     bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
455     bool handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
456     void cancelPendingDequeueOperations();
457 
458     void extractCSD(const sp<AMessage> &format);
459     status_t queueCSDInputBuffer(size_t bufferIndex);
460 
461     status_t handleSetSurface(const sp<Surface> &surface);
462     status_t connectToSurface(const sp<Surface> &surface);
463     status_t disconnectFromSurface();
464 
hasCryptoOrDescramblerMediaCodec465     bool hasCryptoOrDescrambler() {
466         return mCrypto != NULL || mDescrambler != NULL;
467     }
468 
469     void postActivityNotificationIfPossible();
470 
471     void onInputBufferAvailable();
472     void onOutputBufferAvailable();
473     void onError(status_t err, int32_t actionCode, const char *detail = NULL);
474     void onOutputFormatChanged();
475 
476     status_t onSetParameters(const sp<AMessage> &params);
477 
478     status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer);
479     bool isExecuting() const;
480 
481     uint64_t getGraphicBufferSize();
482     void requestCpuBoostIfNeeded();
483 
484     bool hasPendingBuffer(int portIndex);
485     bool hasPendingBuffer();
486 
487     /* called to get the last codec error when the sticky flag is set.
488      * if no such codec error is found, returns UNKNOWN_ERROR.
489      */
getStickyErrorMediaCodec490     inline status_t getStickyError() const {
491         return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR;
492     }
493 
setStickyErrorMediaCodec494     inline void setStickyError(status_t err) {
495         mFlags |= kFlagStickyError;
496         mStickyError = err;
497     }
498 
499     void onReleaseCrypto(const sp<AMessage>& msg);
500 
501     // managing time-of-flight aka latency
502     typedef struct {
503             int64_t presentationUs;
504             int64_t startedNs;
505     } BufferFlightTiming_t;
506     std::deque<BufferFlightTiming_t> mBuffersInFlight;
507     Mutex mLatencyLock;
508     int64_t mLatencyUnknown;    // buffers for which we couldn't calculate latency
509     int64_t mNumLowLatencyEnables;  // how many times low latency mode is enabled
510     int64_t mNumLowLatencyDisables;  // how many times low latency mode is disabled
511     bool mIsLowLatencyModeOn;  // is low latency mode on currently
512     int64_t mIndexOfFirstFrameWhenLowLatencyOn;  // index of the first frame queued
513                                                  // when low latency is on
514     int64_t mInputBufferCounter;  // number of input buffers queued since last reset/flush
515 
516     class ReleaseSurface;
517     std::unique_ptr<ReleaseSurface> mReleaseSurface;
518 
519     std::list<sp<AMessage>> mLeftover;
520     status_t handleLeftover(size_t index);
521 
522     sp<BatteryChecker> mBatteryChecker;
523 
524     void statsBufferSent(int64_t presentationUs);
525     void statsBufferReceived(int64_t presentationUs);
526 
527     enum {
528         // the default shape of our latency histogram buckets
529         // XXX: should these be configurable in some way?
530         kLatencyHistBuckets = 20,
531         kLatencyHistWidth = 2000,
532         kLatencyHistFloor = 2000,
533 
534         // how many samples are in the 'recent latency' histogram
535         // 300 frames = 5 sec @ 60fps or ~12 sec @ 24fps
536         kRecentLatencyFrames = 300,
537 
538         // how we initialize mRecentSamples
539         kRecentSampleInvalid = -1,
540     };
541 
542     int64_t mRecentSamples[kRecentLatencyFrames];
543     int mRecentHead;
544     Mutex mRecentLock;
545 
546     class Histogram {
547       public:
HistogramMediaCodec548         Histogram() : mFloor(0), mWidth(0), mBelow(0), mAbove(0),
549                       mMin(INT64_MAX), mMax(INT64_MIN), mSum(0), mCount(0),
550                       mBucketCount(0), mBuckets(NULL) {};
~HistogramMediaCodec551         ~Histogram() { clear(); };
clearMediaCodec552         void clear() { if (mBuckets != NULL) free(mBuckets); mBuckets = NULL; };
553         bool setup(int nbuckets, int64_t width, int64_t floor = 0);
554         void insert(int64_t sample);
getMinMediaCodec555         int64_t getMin() const { return mMin; }
getMaxMediaCodec556         int64_t getMax() const { return mMax; }
getCountMediaCodec557         int64_t getCount() const { return mCount; }
getSumMediaCodec558         int64_t getSum() const { return mSum; }
getAvgMediaCodec559         int64_t getAvg() const { return mSum / (mCount == 0 ? 1 : mCount); }
560         std::string emit();
561       private:
562         int64_t mFloor, mCeiling, mWidth;
563         int64_t mBelow, mAbove;
564         int64_t mMin, mMax, mSum, mCount;
565 
566         int mBucketCount;
567         int64_t *mBuckets;
568     };
569 
570     Histogram mLatencyHist;
571 
572     DISALLOW_EVIL_CONSTRUCTORS(MediaCodec);
573 };
574 
575 }  // namespace android
576 
577 #endif  // MEDIA_CODEC_H_
578