1 /*
2  * Copyright 2012, The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef MEDIA_CODEC_H_
18 
19 #define MEDIA_CODEC_H_
20 
21 #include <memory>
22 #include <vector>
23 
24 #include <gui/IGraphicBufferProducer.h>
25 #include <media/hardware/CryptoAPI.h>
26 #include <media/MediaCodecInfo.h>
27 #include <media/MediaResource.h>
28 #include <media/MediaAnalyticsItem.h>
29 #include <media/stagefright/foundation/AHandler.h>
30 #include <media/stagefright/FrameRenderTracker.h>
31 #include <utils/Vector.h>
32 
33 namespace android {
34 
35 struct ABuffer;
36 struct AMessage;
37 struct AReplyToken;
38 struct AString;
39 class BufferChannelBase;
40 struct CodecBase;
41 class IBatteryStats;
42 struct ICrypto;
43 class MediaCodecBuffer;
44 class IMemory;
45 class IResourceManagerClient;
46 class IResourceManagerService;
47 struct PersistentSurface;
48 class SoftwareRenderer;
49 class Surface;
50 namespace media {
51 class IDescrambler;
52 };
53 using namespace media;
54 
55 struct MediaCodec : public AHandler {
56     enum ConfigureFlags {
57         CONFIGURE_FLAG_ENCODE   = 1,
58     };
59 
60     enum BufferFlags {
61         BUFFER_FLAG_SYNCFRAME   = 1,
62         BUFFER_FLAG_CODECCONFIG = 2,
63         BUFFER_FLAG_EOS         = 4,
64     };
65 
66     enum {
67         CB_INPUT_AVAILABLE = 1,
68         CB_OUTPUT_AVAILABLE = 2,
69         CB_ERROR = 3,
70         CB_OUTPUT_FORMAT_CHANGED = 4,
71         CB_RESOURCE_RECLAIMED = 5,
72     };
73 
74     static const pid_t kNoPid = -1;
75     static const uid_t kNoUid = -1;
76 
77     static sp<MediaCodec> CreateByType(
78             const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err = NULL,
79             pid_t pid = kNoPid, uid_t uid = kNoUid);
80 
81     static sp<MediaCodec> CreateByComponentName(
82             const sp<ALooper> &looper, const AString &name, status_t *err = NULL,
83             pid_t pid = kNoPid, uid_t uid = kNoUid);
84 
85     static sp<PersistentSurface> CreatePersistentInputSurface();
86 
87     // utility method to query capabilities
88     static status_t QueryCapabilities(
89             const AString &name, const AString &mime, bool isEncoder,
90             sp<MediaCodecInfo::Capabilities> *caps /* nonnull */);
91 
92     status_t configure(
93             const sp<AMessage> &format,
94             const sp<Surface> &nativeWindow,
95             const sp<ICrypto> &crypto,
96             uint32_t flags);
97 
98     status_t configure(
99             const sp<AMessage> &format,
100             const sp<Surface> &nativeWindow,
101             const sp<ICrypto> &crypto,
102             const sp<IDescrambler> &descrambler,
103             uint32_t flags);
104 
105     status_t releaseCrypto();
106 
107     status_t setCallback(const sp<AMessage> &callback);
108 
109     status_t setOnFrameRenderedNotification(const sp<AMessage> &notify);
110 
111     status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
112 
113     status_t setInputSurface(const sp<PersistentSurface> &surface);
114 
115     status_t start();
116 
117     // Returns to a state in which the component remains allocated but
118     // unconfigured.
119     status_t stop();
120 
121     // Resets the codec to the INITIALIZED state.  Can be called after an error
122     // has occured to make the codec usable.
123     status_t reset();
124 
125     // Client MUST call release before releasing final reference to this
126     // object.
127     status_t release();
128 
129     status_t flush();
130 
131     status_t queueInputBuffer(
132             size_t index,
133             size_t offset,
134             size_t size,
135             int64_t presentationTimeUs,
136             uint32_t flags,
137             AString *errorDetailMsg = NULL);
138 
139     status_t queueSecureInputBuffer(
140             size_t index,
141             size_t offset,
142             const CryptoPlugin::SubSample *subSamples,
143             size_t numSubSamples,
144             const uint8_t key[16],
145             const uint8_t iv[16],
146             CryptoPlugin::Mode mode,
147             const CryptoPlugin::Pattern &pattern,
148             int64_t presentationTimeUs,
149             uint32_t flags,
150             AString *errorDetailMsg = NULL);
151 
152     status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll);
153 
154     status_t dequeueOutputBuffer(
155             size_t *index,
156             size_t *offset,
157             size_t *size,
158             int64_t *presentationTimeUs,
159             uint32_t *flags,
160             int64_t timeoutUs = 0ll);
161 
162     status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs);
163     status_t renderOutputBufferAndRelease(size_t index);
164     status_t releaseOutputBuffer(size_t index);
165 
166     status_t signalEndOfInputStream();
167 
168     status_t getOutputFormat(sp<AMessage> *format) const;
169     status_t getInputFormat(sp<AMessage> *format) const;
170 
171     status_t getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
172     status_t getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
173 
174     status_t getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
175     status_t getOutputFormat(size_t index, sp<AMessage> *format);
176     status_t getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
177 
178     status_t setSurface(const sp<Surface> &nativeWindow);
179 
180     status_t requestIDRFrame();
181 
182     // Notification will be posted once there "is something to do", i.e.
183     // an input/output buffer has become available, a format change is
184     // pending, an error is pending.
185     void requestActivityNotification(const sp<AMessage> &notify);
186 
187     status_t getName(AString *componentName) const;
188 
189     status_t getMetrics(MediaAnalyticsItem * &reply);
190 
191     status_t setParameters(const sp<AMessage> &params);
192 
193     // Create a MediaCodec notification message from a list of rendered or dropped render infos
194     // by adding rendered frame information to a base notification message. Returns the number
195     // of frames that were rendered.
196     static size_t CreateFramesRenderedMessage(
197             const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg);
198 
199 protected:
200     virtual ~MediaCodec();
201     virtual void onMessageReceived(const sp<AMessage> &msg);
202 
203 private:
204     // used by ResourceManagerClient
205     status_t reclaim(bool force = false);
206     friend struct ResourceManagerClient;
207 
208 private:
209     enum State {
210         UNINITIALIZED,
211         INITIALIZING,
212         INITIALIZED,
213         CONFIGURING,
214         CONFIGURED,
215         STARTING,
216         STARTED,
217         FLUSHING,
218         FLUSHED,
219         STOPPING,
220         RELEASING,
221     };
222 
223     enum {
224         kPortIndexInput         = 0,
225         kPortIndexOutput        = 1,
226     };
227 
228     enum {
229         kWhatInit                           = 'init',
230         kWhatConfigure                      = 'conf',
231         kWhatSetSurface                     = 'sSur',
232         kWhatCreateInputSurface             = 'cisf',
233         kWhatSetInputSurface                = 'sisf',
234         kWhatStart                          = 'strt',
235         kWhatStop                           = 'stop',
236         kWhatRelease                        = 'rele',
237         kWhatDequeueInputBuffer             = 'deqI',
238         kWhatQueueInputBuffer               = 'queI',
239         kWhatDequeueOutputBuffer            = 'deqO',
240         kWhatReleaseOutputBuffer            = 'relO',
241         kWhatSignalEndOfInputStream         = 'eois',
242         kWhatGetBuffers                     = 'getB',
243         kWhatFlush                          = 'flus',
244         kWhatGetOutputFormat                = 'getO',
245         kWhatGetInputFormat                 = 'getI',
246         kWhatDequeueInputTimedOut           = 'dITO',
247         kWhatDequeueOutputTimedOut          = 'dOTO',
248         kWhatCodecNotify                    = 'codc',
249         kWhatRequestIDRFrame                = 'ridr',
250         kWhatRequestActivityNotification    = 'racN',
251         kWhatGetName                        = 'getN',
252         kWhatSetParameters                  = 'setP',
253         kWhatSetCallback                    = 'setC',
254         kWhatSetNotification                = 'setN',
255         kWhatDrmReleaseCrypto               = 'rDrm',
256     };
257 
258     enum {
259         kFlagUsesSoftwareRenderer       = 1,
260         kFlagOutputFormatChanged        = 2,
261         kFlagOutputBuffersChanged       = 4,
262         kFlagStickyError                = 8,
263         kFlagDequeueInputPending        = 16,
264         kFlagDequeueOutputPending       = 32,
265         kFlagIsSecure                   = 64,
266         kFlagSawMediaServerDie          = 128,
267         kFlagIsEncoder                  = 256,
268         // 512 skipped
269         kFlagIsAsync                    = 1024,
270         kFlagIsComponentAllocated       = 2048,
271         kFlagPushBlankBuffersOnShutdown = 4096,
272     };
273 
274     struct BufferInfo {
275         BufferInfo();
276 
277         sp<MediaCodecBuffer> mData;
278         bool mOwnedByClient;
279     };
280 
281     struct ResourceManagerServiceProxy : public IBinder::DeathRecipient {
282         ResourceManagerServiceProxy(pid_t pid);
283         ~ResourceManagerServiceProxy();
284 
285         void init();
286 
287         // implements DeathRecipient
288         virtual void binderDied(const wp<IBinder>& /*who*/);
289 
290         void addResource(
291                 int64_t clientId,
292                 const sp<IResourceManagerClient> &client,
293                 const Vector<MediaResource> &resources);
294 
295         void removeResource(int64_t clientId);
296 
297         bool reclaimResource(const Vector<MediaResource> &resources);
298 
299     private:
300         Mutex mLock;
301         sp<IResourceManagerService> mService;
302         pid_t mPid;
303     };
304 
305     State mState;
306     uid_t mUid;
307     bool mReleasedByResourceManager;
308     sp<ALooper> mLooper;
309     sp<ALooper> mCodecLooper;
310     sp<CodecBase> mCodec;
311     AString mComponentName;
312     sp<AReplyToken> mReplyID;
313     uint32_t mFlags;
314     status_t mStickyError;
315     sp<Surface> mSurface;
316     SoftwareRenderer *mSoftRenderer;
317 
318     MediaAnalyticsItem *mAnalyticsItem;
319 
320     sp<AMessage> mOutputFormat;
321     sp<AMessage> mInputFormat;
322     sp<AMessage> mCallback;
323     sp<AMessage> mOnFrameRenderedNotification;
324 
325     sp<IResourceManagerClient> mResourceManagerClient;
326     sp<ResourceManagerServiceProxy> mResourceManagerService;
327 
328     bool mBatteryStatNotified;
329     bool mIsVideo;
330     int32_t mVideoWidth;
331     int32_t mVideoHeight;
332     int32_t mRotationDegrees;
333 
334     // initial create parameters
335     AString mInitName;
336     bool mInitNameIsType;
337     bool mInitIsEncoder;
338 
339     // configure parameter
340     sp<AMessage> mConfigureMsg;
341 
342     // Used only to synchronize asynchronous getBufferAndFormat
343     // across all the other (synchronous) buffer state change
344     // operations, such as de/queueIn/OutputBuffer, start and
345     // stop/flush/reset/release.
346     Mutex mBufferLock;
347 
348     List<size_t> mAvailPortBuffers[2];
349     std::vector<BufferInfo> mPortBuffers[2];
350 
351     int32_t mDequeueInputTimeoutGeneration;
352     sp<AReplyToken> mDequeueInputReplyID;
353 
354     int32_t mDequeueOutputTimeoutGeneration;
355     sp<AReplyToken> mDequeueOutputReplyID;
356 
357     sp<ICrypto> mCrypto;
358 
359     sp<IDescrambler> mDescrambler;
360 
361     List<sp<ABuffer> > mCSD;
362 
363     sp<AMessage> mActivityNotify;
364 
365     bool mHaveInputSurface;
366     bool mHavePendingInputBuffers;
367 
368     std::shared_ptr<BufferChannelBase> mBufferChannel;
369 
370     MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid);
371 
372     static sp<CodecBase> GetCodecBase(const AString &name, bool nameIsType = false);
373 
374     static status_t PostAndAwaitResponse(
375             const sp<AMessage> &msg, sp<AMessage> *response);
376 
377     void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err);
378 
379     status_t init(const AString &name, bool nameIsType, bool encoder);
380 
381     void setState(State newState);
382     void returnBuffersToCodec(bool isReclaim = false);
383     void returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim = false);
384     size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg);
385     status_t onQueueInputBuffer(const sp<AMessage> &msg);
386     status_t onReleaseOutputBuffer(const sp<AMessage> &msg);
387     ssize_t dequeuePortBuffer(int32_t portIndex);
388 
389     status_t getBufferAndFormat(
390             size_t portIndex, size_t index,
391             sp<MediaCodecBuffer> *buffer, sp<AMessage> *format);
392 
393     bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
394     bool handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
395     void cancelPendingDequeueOperations();
396 
397     void extractCSD(const sp<AMessage> &format);
398     status_t queueCSDInputBuffer(size_t bufferIndex);
399 
400     status_t handleSetSurface(const sp<Surface> &surface);
401     status_t connectToSurface(const sp<Surface> &surface);
402     status_t disconnectFromSurface();
403 
hasCryptoOrDescramblerMediaCodec404     bool hasCryptoOrDescrambler() {
405         return mCrypto != NULL || mDescrambler != NULL;
406     }
407 
408     void postActivityNotificationIfPossible();
409 
410     void onInputBufferAvailable();
411     void onOutputBufferAvailable();
412     void onError(status_t err, int32_t actionCode, const char *detail = NULL);
413     void onOutputFormatChanged();
414 
415     status_t onSetParameters(const sp<AMessage> &params);
416 
417     status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer);
418     void updateBatteryStat();
419     bool isExecuting() const;
420 
421     uint64_t getGraphicBufferSize();
422     void addResource(MediaResource::Type type, MediaResource::SubType subtype, uint64_t value);
423 
424     bool hasPendingBuffer(int portIndex);
425     bool hasPendingBuffer();
426 
427     /* called to get the last codec error when the sticky flag is set.
428      * if no such codec error is found, returns UNKNOWN_ERROR.
429      */
getStickyErrorMediaCodec430     inline status_t getStickyError() const {
431         return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR;
432     }
433 
setStickyErrorMediaCodec434     inline void setStickyError(status_t err) {
435         mFlags |= kFlagStickyError;
436         mStickyError = err;
437     }
438 
439     void onReleaseCrypto(const sp<AMessage>& msg);
440 
441     DISALLOW_EVIL_CONSTRUCTORS(MediaCodec);
442 };
443 
444 }  // namespace android
445 
446 #endif  // MEDIA_CODEC_H_
447