1 /*
2  * Copyright 2012, The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef MEDIA_CODEC_H_
18 
19 #define MEDIA_CODEC_H_
20 
21 #include <memory>
22 #include <vector>
23 
24 #include <gui/IGraphicBufferProducer.h>
25 #include <media/hardware/CryptoAPI.h>
26 #include <media/MediaCodecInfo.h>
27 #include <media/MediaResource.h>
28 #include <media/MediaAnalyticsItem.h>
29 #include <media/stagefright/foundation/AHandler.h>
30 #include <media/stagefright/FrameRenderTracker.h>
31 #include <utils/Vector.h>
32 
33 namespace android {
34 
35 struct ABuffer;
36 struct AMessage;
37 struct AReplyToken;
38 struct AString;
39 class BufferChannelBase;
40 struct CodecBase;
41 class IBatteryStats;
42 struct ICrypto;
43 class MediaCodecBuffer;
44 class IMemory;
45 class IResourceManagerClient;
46 class IResourceManagerService;
47 struct PersistentSurface;
48 class SoftwareRenderer;
49 class Surface;
50 namespace hardware {
51 namespace cas {
52 namespace native {
53 namespace V1_0 {
54 struct IDescrambler;
55 }}}}
56 using hardware::cas::native::V1_0::IDescrambler;
57 
58 struct MediaCodec : public AHandler {
59     enum ConfigureFlags {
60         CONFIGURE_FLAG_ENCODE   = 1,
61     };
62 
63     enum BufferFlags {
64         BUFFER_FLAG_SYNCFRAME     = 1,
65         BUFFER_FLAG_CODECCONFIG   = 2,
66         BUFFER_FLAG_EOS           = 4,
67         BUFFER_FLAG_PARTIAL_FRAME = 8,
68         BUFFER_FLAG_MUXER_DATA    = 16,
69     };
70 
71     enum {
72         CB_INPUT_AVAILABLE = 1,
73         CB_OUTPUT_AVAILABLE = 2,
74         CB_ERROR = 3,
75         CB_OUTPUT_FORMAT_CHANGED = 4,
76         CB_RESOURCE_RECLAIMED = 5,
77     };
78 
79     static const pid_t kNoPid = -1;
80     static const uid_t kNoUid = -1;
81 
82     static sp<MediaCodec> CreateByType(
83             const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err = NULL,
84             pid_t pid = kNoPid, uid_t uid = kNoUid);
85 
86     static sp<MediaCodec> CreateByComponentName(
87             const sp<ALooper> &looper, const AString &name, status_t *err = NULL,
88             pid_t pid = kNoPid, uid_t uid = kNoUid);
89 
90     static sp<PersistentSurface> CreatePersistentInputSurface();
91 
92     status_t configure(
93             const sp<AMessage> &format,
94             const sp<Surface> &nativeWindow,
95             const sp<ICrypto> &crypto,
96             uint32_t flags);
97 
98     status_t configure(
99             const sp<AMessage> &format,
100             const sp<Surface> &nativeWindow,
101             const sp<ICrypto> &crypto,
102             const sp<IDescrambler> &descrambler,
103             uint32_t flags);
104 
105     status_t releaseCrypto();
106 
107     status_t setCallback(const sp<AMessage> &callback);
108 
109     status_t setOnFrameRenderedNotification(const sp<AMessage> &notify);
110 
111     status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
112 
113     status_t setInputSurface(const sp<PersistentSurface> &surface);
114 
115     status_t start();
116 
117     // Returns to a state in which the component remains allocated but
118     // unconfigured.
119     status_t stop();
120 
121     // Resets the codec to the INITIALIZED state.  Can be called after an error
122     // has occured to make the codec usable.
123     status_t reset();
124 
125     // Client MUST call release before releasing final reference to this
126     // object.
127     status_t release();
128 
129     status_t flush();
130 
131     status_t queueInputBuffer(
132             size_t index,
133             size_t offset,
134             size_t size,
135             int64_t presentationTimeUs,
136             uint32_t flags,
137             AString *errorDetailMsg = NULL);
138 
139     status_t queueSecureInputBuffer(
140             size_t index,
141             size_t offset,
142             const CryptoPlugin::SubSample *subSamples,
143             size_t numSubSamples,
144             const uint8_t key[16],
145             const uint8_t iv[16],
146             CryptoPlugin::Mode mode,
147             const CryptoPlugin::Pattern &pattern,
148             int64_t presentationTimeUs,
149             uint32_t flags,
150             AString *errorDetailMsg = NULL);
151 
152     status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll);
153 
154     status_t dequeueOutputBuffer(
155             size_t *index,
156             size_t *offset,
157             size_t *size,
158             int64_t *presentationTimeUs,
159             uint32_t *flags,
160             int64_t timeoutUs = 0ll);
161 
162     status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs);
163     status_t renderOutputBufferAndRelease(size_t index);
164     status_t releaseOutputBuffer(size_t index);
165 
166     status_t signalEndOfInputStream();
167 
168     status_t getOutputFormat(sp<AMessage> *format) const;
169     status_t getInputFormat(sp<AMessage> *format) const;
170 
171     status_t getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
172     status_t getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
173 
174     status_t getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
175     status_t getOutputFormat(size_t index, sp<AMessage> *format);
176     status_t getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
177 
178     status_t setSurface(const sp<Surface> &nativeWindow);
179 
180     status_t requestIDRFrame();
181 
182     // Notification will be posted once there "is something to do", i.e.
183     // an input/output buffer has become available, a format change is
184     // pending, an error is pending.
185     void requestActivityNotification(const sp<AMessage> &notify);
186 
187     status_t getName(AString *componentName) const;
188 
189     status_t getCodecInfo(sp<MediaCodecInfo> *codecInfo) const;
190 
191     status_t getMetrics(MediaAnalyticsItem * &reply);
192 
193     status_t setParameters(const sp<AMessage> &params);
194 
195     // Create a MediaCodec notification message from a list of rendered or dropped render infos
196     // by adding rendered frame information to a base notification message. Returns the number
197     // of frames that were rendered.
198     static size_t CreateFramesRenderedMessage(
199             const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg);
200 
201 protected:
202     virtual ~MediaCodec();
203     virtual void onMessageReceived(const sp<AMessage> &msg);
204 
205 private:
206     // used by ResourceManagerClient
207     status_t reclaim(bool force = false);
208     friend struct ResourceManagerClient;
209 
210 private:
211     enum State {
212         UNINITIALIZED,
213         INITIALIZING,
214         INITIALIZED,
215         CONFIGURING,
216         CONFIGURED,
217         STARTING,
218         STARTED,
219         FLUSHING,
220         FLUSHED,
221         STOPPING,
222         RELEASING,
223     };
224     std::string stateString(State state);
225 
226     enum {
227         kPortIndexInput         = 0,
228         kPortIndexOutput        = 1,
229     };
230 
231     enum {
232         kWhatInit                           = 'init',
233         kWhatConfigure                      = 'conf',
234         kWhatSetSurface                     = 'sSur',
235         kWhatCreateInputSurface             = 'cisf',
236         kWhatSetInputSurface                = 'sisf',
237         kWhatStart                          = 'strt',
238         kWhatStop                           = 'stop',
239         kWhatRelease                        = 'rele',
240         kWhatDequeueInputBuffer             = 'deqI',
241         kWhatQueueInputBuffer               = 'queI',
242         kWhatDequeueOutputBuffer            = 'deqO',
243         kWhatReleaseOutputBuffer            = 'relO',
244         kWhatSignalEndOfInputStream         = 'eois',
245         kWhatGetBuffers                     = 'getB',
246         kWhatFlush                          = 'flus',
247         kWhatGetOutputFormat                = 'getO',
248         kWhatGetInputFormat                 = 'getI',
249         kWhatDequeueInputTimedOut           = 'dITO',
250         kWhatDequeueOutputTimedOut          = 'dOTO',
251         kWhatCodecNotify                    = 'codc',
252         kWhatRequestIDRFrame                = 'ridr',
253         kWhatRequestActivityNotification    = 'racN',
254         kWhatGetName                        = 'getN',
255         kWhatGetCodecInfo                   = 'gCoI',
256         kWhatSetParameters                  = 'setP',
257         kWhatSetCallback                    = 'setC',
258         kWhatSetNotification                = 'setN',
259         kWhatDrmReleaseCrypto               = 'rDrm',
260     };
261 
262     enum {
263         kFlagUsesSoftwareRenderer       = 1,
264         kFlagOutputFormatChanged        = 2,
265         kFlagOutputBuffersChanged       = 4,
266         kFlagStickyError                = 8,
267         kFlagDequeueInputPending        = 16,
268         kFlagDequeueOutputPending       = 32,
269         kFlagIsSecure                   = 64,
270         kFlagSawMediaServerDie          = 128,
271         kFlagIsEncoder                  = 256,
272         // 512 skipped
273         kFlagIsAsync                    = 1024,
274         kFlagIsComponentAllocated       = 2048,
275         kFlagPushBlankBuffersOnShutdown = 4096,
276     };
277 
278     struct BufferInfo {
279         BufferInfo();
280 
281         sp<MediaCodecBuffer> mData;
282         bool mOwnedByClient;
283     };
284 
285     struct ResourceManagerServiceProxy : public IBinder::DeathRecipient {
286         ResourceManagerServiceProxy(pid_t pid);
287         ~ResourceManagerServiceProxy();
288 
289         void init();
290 
291         // implements DeathRecipient
292         virtual void binderDied(const wp<IBinder>& /*who*/);
293 
294         void addResource(
295                 int64_t clientId,
296                 const sp<IResourceManagerClient> &client,
297                 const Vector<MediaResource> &resources);
298 
299         void removeResource(int64_t clientId);
300 
301         bool reclaimResource(const Vector<MediaResource> &resources);
302 
303     private:
304         Mutex mLock;
305         sp<IResourceManagerService> mService;
306         pid_t mPid;
307     };
308 
309     State mState;
310     uid_t mUid;
311     bool mReleasedByResourceManager;
312     sp<ALooper> mLooper;
313     sp<ALooper> mCodecLooper;
314     sp<CodecBase> mCodec;
315     AString mComponentName;
316     sp<MediaCodecInfo> mCodecInfo;
317     sp<AReplyToken> mReplyID;
318     uint32_t mFlags;
319     status_t mStickyError;
320     sp<Surface> mSurface;
321     SoftwareRenderer *mSoftRenderer;
322 
323     MediaAnalyticsItem *mAnalyticsItem;
324     void initAnalyticsItem();
325     void updateAnalyticsItem();
326     void flushAnalyticsItem();
327     void updateEphemeralAnalytics(MediaAnalyticsItem *item);
328 
329     sp<AMessage> mOutputFormat;
330     sp<AMessage> mInputFormat;
331     sp<AMessage> mCallback;
332     sp<AMessage> mOnFrameRenderedNotification;
333 
334     sp<IResourceManagerClient> mResourceManagerClient;
335     sp<ResourceManagerServiceProxy> mResourceManagerService;
336 
337     bool mBatteryStatNotified;
338     bool mIsVideo;
339     int32_t mVideoWidth;
340     int32_t mVideoHeight;
341     int32_t mRotationDegrees;
342 
343     // initial create parameters
344     AString mInitName;
345 
346     // configure parameter
347     sp<AMessage> mConfigureMsg;
348 
349     // Used only to synchronize asynchronous getBufferAndFormat
350     // across all the other (synchronous) buffer state change
351     // operations, such as de/queueIn/OutputBuffer, start and
352     // stop/flush/reset/release.
353     Mutex mBufferLock;
354 
355     List<size_t> mAvailPortBuffers[2];
356     std::vector<BufferInfo> mPortBuffers[2];
357 
358     int32_t mDequeueInputTimeoutGeneration;
359     sp<AReplyToken> mDequeueInputReplyID;
360 
361     int32_t mDequeueOutputTimeoutGeneration;
362     sp<AReplyToken> mDequeueOutputReplyID;
363 
364     sp<ICrypto> mCrypto;
365 
366     sp<IDescrambler> mDescrambler;
367 
368     List<sp<ABuffer> > mCSD;
369 
370     sp<AMessage> mActivityNotify;
371 
372     bool mHaveInputSurface;
373     bool mHavePendingInputBuffers;
374     bool mCpuBoostRequested;
375 
376     std::shared_ptr<BufferChannelBase> mBufferChannel;
377 
378     MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid);
379 
380     static sp<CodecBase> GetCodecBase(const AString &name);
381 
382     static status_t PostAndAwaitResponse(
383             const sp<AMessage> &msg, sp<AMessage> *response);
384 
385     void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err);
386 
387     status_t init(const AString &name);
388 
389     void setState(State newState);
390     void returnBuffersToCodec(bool isReclaim = false);
391     void returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim = false);
392     size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg);
393     status_t onQueueInputBuffer(const sp<AMessage> &msg);
394     status_t onReleaseOutputBuffer(const sp<AMessage> &msg);
395     ssize_t dequeuePortBuffer(int32_t portIndex);
396 
397     status_t getBufferAndFormat(
398             size_t portIndex, size_t index,
399             sp<MediaCodecBuffer> *buffer, sp<AMessage> *format);
400 
401     bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
402     bool handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
403     void cancelPendingDequeueOperations();
404 
405     void extractCSD(const sp<AMessage> &format);
406     status_t queueCSDInputBuffer(size_t bufferIndex);
407 
408     status_t handleSetSurface(const sp<Surface> &surface);
409     status_t connectToSurface(const sp<Surface> &surface);
410     status_t disconnectFromSurface();
411 
hasCryptoOrDescramblerMediaCodec412     bool hasCryptoOrDescrambler() {
413         return mCrypto != NULL || mDescrambler != NULL;
414     }
415 
416     void postActivityNotificationIfPossible();
417 
418     void onInputBufferAvailable();
419     void onOutputBufferAvailable();
420     void onError(status_t err, int32_t actionCode, const char *detail = NULL);
421     void onOutputFormatChanged();
422 
423     status_t onSetParameters(const sp<AMessage> &params);
424 
425     status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer);
426     void updateBatteryStat();
427     bool isExecuting() const;
428 
429     uint64_t getGraphicBufferSize();
430     void addResource(MediaResource::Type type, MediaResource::SubType subtype, uint64_t value);
431     void requestCpuBoostIfNeeded();
432 
433     bool hasPendingBuffer(int portIndex);
434     bool hasPendingBuffer();
435 
436     /* called to get the last codec error when the sticky flag is set.
437      * if no such codec error is found, returns UNKNOWN_ERROR.
438      */
getStickyErrorMediaCodec439     inline status_t getStickyError() const {
440         return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR;
441     }
442 
setStickyErrorMediaCodec443     inline void setStickyError(status_t err) {
444         mFlags |= kFlagStickyError;
445         mStickyError = err;
446     }
447 
448     void onReleaseCrypto(const sp<AMessage>& msg);
449 
450     // managing time-of-flight aka latency
451     typedef struct {
452             int64_t presentationUs;
453             int64_t startedNs;
454     } BufferFlightTiming_t;
455     std::deque<BufferFlightTiming_t> mBuffersInFlight;
456     Mutex mLatencyLock;
457     int64_t mLatencyUnknown;    // buffers for which we couldn't calculate latency
458 
459     void statsBufferSent(int64_t presentationUs);
460     void statsBufferReceived(int64_t presentationUs);
461 
462     enum {
463         // the default shape of our latency histogram buckets
464         // XXX: should these be configurable in some way?
465         kLatencyHistBuckets = 20,
466         kLatencyHistWidth = 2000,
467         kLatencyHistFloor = 2000,
468 
469         // how many samples are in the 'recent latency' histogram
470         // 300 frames = 5 sec @ 60fps or ~12 sec @ 24fps
471         kRecentLatencyFrames = 300,
472 
473         // how we initialize mRecentSamples
474         kRecentSampleInvalid = -1,
475     };
476 
477     int64_t mRecentSamples[kRecentLatencyFrames];
478     int mRecentHead;
479     Mutex mRecentLock;
480 
481     class Histogram {
482       public:
HistogramMediaCodec483         Histogram() : mFloor(0), mWidth(0), mBelow(0), mAbove(0),
484                       mMin(INT64_MAX), mMax(INT64_MIN), mSum(0), mCount(0),
485                       mBucketCount(0), mBuckets(NULL) {};
~HistogramMediaCodec486         ~Histogram() { clear(); };
clearMediaCodec487         void clear() { if (mBuckets != NULL) free(mBuckets); mBuckets = NULL; };
488         bool setup(int nbuckets, int64_t width, int64_t floor = 0);
489         void insert(int64_t sample);
getMinMediaCodec490         int64_t getMin() const { return mMin; }
getMaxMediaCodec491         int64_t getMax() const { return mMax; }
getCountMediaCodec492         int64_t getCount() const { return mCount; }
getSumMediaCodec493         int64_t getSum() const { return mSum; }
getAvgMediaCodec494         int64_t getAvg() const { return mSum / (mCount == 0 ? 1 : mCount); }
495         std::string emit();
496       private:
497         int64_t mFloor, mCeiling, mWidth;
498         int64_t mBelow, mAbove;
499         int64_t mMin, mMax, mSum, mCount;
500 
501         int mBucketCount;
502         int64_t *mBuckets;
503     };
504 
505     Histogram mLatencyHist;
506 
507     DISALLOW_EVIL_CONSTRUCTORS(MediaCodec);
508 };
509 
510 }  // namespace android
511 
512 #endif  // MEDIA_CODEC_H_
513