1 /*
2  * Copyright 2012, The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef MEDIA_CODEC_H_
18 
19 #define MEDIA_CODEC_H_
20 
21 #include <gui/IGraphicBufferProducer.h>
22 #include <media/hardware/CryptoAPI.h>
23 #include <media/MediaCodecInfo.h>
24 #include <media/MediaResource.h>
25 #include <media/stagefright/foundation/AHandler.h>
26 #include <media/stagefright/FrameRenderTracker.h>
27 #include <utils/Vector.h>
28 
29 namespace android {
30 
31 struct ABuffer;
32 struct AMessage;
33 struct AReplyToken;
34 struct AString;
35 struct CodecBase;
36 struct IBatteryStats;
37 struct ICrypto;
38 class IMemory;
39 struct MemoryDealer;
40 class IResourceManagerClient;
41 class IResourceManagerService;
42 struct PersistentSurface;
43 struct SoftwareRenderer;
44 struct Surface;
45 
46 struct MediaCodec : public AHandler {
47     enum ConfigureFlags {
48         CONFIGURE_FLAG_ENCODE   = 1,
49     };
50 
51     enum BufferFlags {
52         BUFFER_FLAG_SYNCFRAME   = 1,
53         BUFFER_FLAG_CODECCONFIG = 2,
54         BUFFER_FLAG_EOS         = 4,
55     };
56 
57     enum {
58         CB_INPUT_AVAILABLE = 1,
59         CB_OUTPUT_AVAILABLE = 2,
60         CB_ERROR = 3,
61         CB_OUTPUT_FORMAT_CHANGED = 4,
62         CB_RESOURCE_RECLAIMED = 5,
63     };
64 
65     static const pid_t kNoPid = -1;
66 
67     static sp<MediaCodec> CreateByType(
68             const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err = NULL,
69             pid_t pid = kNoPid);
70 
71     static sp<MediaCodec> CreateByComponentName(
72             const sp<ALooper> &looper, const AString &name, status_t *err = NULL,
73             pid_t pid = kNoPid);
74 
75     static sp<PersistentSurface> CreatePersistentInputSurface();
76 
77     // utility method to query capabilities
78     static status_t QueryCapabilities(
79             const AString &name, const AString &mime, bool isEncoder,
80             sp<MediaCodecInfo::Capabilities> *caps /* nonnull */);
81 
82     status_t configure(
83             const sp<AMessage> &format,
84             const sp<Surface> &nativeWindow,
85             const sp<ICrypto> &crypto,
86             uint32_t flags);
87 
88     status_t setCallback(const sp<AMessage> &callback);
89 
90     status_t setOnFrameRenderedNotification(const sp<AMessage> &notify);
91 
92     status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
93 
94     status_t setInputSurface(const sp<PersistentSurface> &surface);
95 
96     status_t start();
97 
98     // Returns to a state in which the component remains allocated but
99     // unconfigured.
100     status_t stop();
101 
102     // Resets the codec to the INITIALIZED state.  Can be called after an error
103     // has occured to make the codec usable.
104     status_t reset();
105 
106     // Client MUST call release before releasing final reference to this
107     // object.
108     status_t release();
109 
110     status_t flush();
111 
112     status_t queueInputBuffer(
113             size_t index,
114             size_t offset,
115             size_t size,
116             int64_t presentationTimeUs,
117             uint32_t flags,
118             AString *errorDetailMsg = NULL);
119 
120     status_t queueSecureInputBuffer(
121             size_t index,
122             size_t offset,
123             const CryptoPlugin::SubSample *subSamples,
124             size_t numSubSamples,
125             const uint8_t key[16],
126             const uint8_t iv[16],
127             CryptoPlugin::Mode mode,
128             const CryptoPlugin::Pattern &pattern,
129             int64_t presentationTimeUs,
130             uint32_t flags,
131             AString *errorDetailMsg = NULL);
132 
133     status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll);
134 
135     status_t dequeueOutputBuffer(
136             size_t *index,
137             size_t *offset,
138             size_t *size,
139             int64_t *presentationTimeUs,
140             uint32_t *flags,
141             int64_t timeoutUs = 0ll);
142 
143     status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs);
144     status_t renderOutputBufferAndRelease(size_t index);
145     status_t releaseOutputBuffer(size_t index);
146 
147     status_t signalEndOfInputStream();
148 
149     status_t getOutputFormat(sp<AMessage> *format) const;
150     status_t getInputFormat(sp<AMessage> *format) const;
151 
152     status_t getWidevineLegacyBuffers(Vector<sp<ABuffer> > *buffers) const;
153 
154     status_t getInputBuffers(Vector<sp<ABuffer> > *buffers) const;
155     status_t getOutputBuffers(Vector<sp<ABuffer> > *buffers) const;
156 
157     status_t getOutputBuffer(size_t index, sp<ABuffer> *buffer);
158     status_t getOutputFormat(size_t index, sp<AMessage> *format);
159     status_t getInputBuffer(size_t index, sp<ABuffer> *buffer);
160 
161     status_t setSurface(const sp<Surface> &nativeWindow);
162 
163     status_t requestIDRFrame();
164 
165     // Notification will be posted once there "is something to do", i.e.
166     // an input/output buffer has become available, a format change is
167     // pending, an error is pending.
168     void requestActivityNotification(const sp<AMessage> &notify);
169 
170     status_t getName(AString *componentName) const;
171 
172     status_t setParameters(const sp<AMessage> &params);
173 
174     // Create a MediaCodec notification message from a list of rendered or dropped render infos
175     // by adding rendered frame information to a base notification message. Returns the number
176     // of frames that were rendered.
177     static size_t CreateFramesRenderedMessage(
178             std::list<FrameRenderTracker::Info> done, sp<AMessage> &msg);
179 
180 protected:
181     virtual ~MediaCodec();
182     virtual void onMessageReceived(const sp<AMessage> &msg);
183 
184 private:
185     // used by ResourceManagerClient
186     status_t reclaim(bool force = false);
187     friend struct ResourceManagerClient;
188 
189 private:
190     enum State {
191         UNINITIALIZED,
192         INITIALIZING,
193         INITIALIZED,
194         CONFIGURING,
195         CONFIGURED,
196         STARTING,
197         STARTED,
198         FLUSHING,
199         FLUSHED,
200         STOPPING,
201         RELEASING,
202     };
203 
204     enum {
205         kPortIndexInput         = 0,
206         kPortIndexOutput        = 1,
207     };
208 
209     enum {
210         kWhatInit                           = 'init',
211         kWhatConfigure                      = 'conf',
212         kWhatSetSurface                     = 'sSur',
213         kWhatCreateInputSurface             = 'cisf',
214         kWhatSetInputSurface                = 'sisf',
215         kWhatStart                          = 'strt',
216         kWhatStop                           = 'stop',
217         kWhatRelease                        = 'rele',
218         kWhatDequeueInputBuffer             = 'deqI',
219         kWhatQueueInputBuffer               = 'queI',
220         kWhatDequeueOutputBuffer            = 'deqO',
221         kWhatReleaseOutputBuffer            = 'relO',
222         kWhatSignalEndOfInputStream         = 'eois',
223         kWhatGetBuffers                     = 'getB',
224         kWhatFlush                          = 'flus',
225         kWhatGetOutputFormat                = 'getO',
226         kWhatGetInputFormat                 = 'getI',
227         kWhatDequeueInputTimedOut           = 'dITO',
228         kWhatDequeueOutputTimedOut          = 'dOTO',
229         kWhatCodecNotify                    = 'codc',
230         kWhatRequestIDRFrame                = 'ridr',
231         kWhatRequestActivityNotification    = 'racN',
232         kWhatGetName                        = 'getN',
233         kWhatSetParameters                  = 'setP',
234         kWhatSetCallback                    = 'setC',
235         kWhatSetNotification                = 'setN',
236     };
237 
238     enum {
239         kFlagUsesSoftwareRenderer       = 1,
240         kFlagOutputFormatChanged        = 2,
241         kFlagOutputBuffersChanged       = 4,
242         kFlagStickyError                = 8,
243         kFlagDequeueInputPending        = 16,
244         kFlagDequeueOutputPending       = 32,
245         kFlagIsSecure                   = 64,
246         kFlagSawMediaServerDie          = 128,
247         kFlagIsEncoder                  = 256,
248         kFlagGatherCodecSpecificData    = 512,
249         kFlagIsAsync                    = 1024,
250         kFlagIsComponentAllocated       = 2048,
251         kFlagPushBlankBuffersOnShutdown = 4096,
252     };
253 
254     struct BufferInfo {
255         uint32_t mBufferID;
256         sp<ABuffer> mData;
257         sp<NativeHandle> mNativeHandle;
258         sp<RefBase> mMemRef;
259         sp<ABuffer> mEncryptedData;
260         sp<IMemory> mSharedEncryptedBuffer;
261         sp<AMessage> mNotify;
262         sp<AMessage> mFormat;
263         bool mOwnedByClient;
264     };
265 
266     struct ResourceManagerServiceProxy : public IBinder::DeathRecipient {
267         ResourceManagerServiceProxy(pid_t pid);
268         ~ResourceManagerServiceProxy();
269 
270         void init();
271 
272         // implements DeathRecipient
273         virtual void binderDied(const wp<IBinder>& /*who*/);
274 
275         void addResource(
276                 int64_t clientId,
277                 const sp<IResourceManagerClient> client,
278                 const Vector<MediaResource> &resources);
279 
280         void removeResource(int64_t clientId);
281 
282         bool reclaimResource(const Vector<MediaResource> &resources);
283 
284     private:
285         Mutex mLock;
286         sp<IResourceManagerService> mService;
287         pid_t mPid;
288     };
289 
290     State mState;
291     bool mReleasedByResourceManager;
292     sp<ALooper> mLooper;
293     sp<ALooper> mCodecLooper;
294     sp<CodecBase> mCodec;
295     AString mComponentName;
296     sp<AReplyToken> mReplyID;
297     uint32_t mFlags;
298     status_t mStickyError;
299     sp<Surface> mSurface;
300     SoftwareRenderer *mSoftRenderer;
301 
302     sp<AMessage> mOutputFormat;
303     sp<AMessage> mInputFormat;
304     sp<AMessage> mCallback;
305     sp<AMessage> mOnFrameRenderedNotification;
306     sp<MemoryDealer> mDealer;
307 
308     sp<IResourceManagerClient> mResourceManagerClient;
309     sp<ResourceManagerServiceProxy> mResourceManagerService;
310 
311     bool mBatteryStatNotified;
312     bool mIsVideo;
313     int32_t mVideoWidth;
314     int32_t mVideoHeight;
315     int32_t mRotationDegrees;
316 
317     // initial create parameters
318     AString mInitName;
319     bool mInitNameIsType;
320     bool mInitIsEncoder;
321 
322     // configure parameter
323     sp<AMessage> mConfigureMsg;
324 
325     // Used only to synchronize asynchronous getBufferAndFormat
326     // across all the other (synchronous) buffer state change
327     // operations, such as de/queueIn/OutputBuffer, start and
328     // stop/flush/reset/release.
329     Mutex mBufferLock;
330 
331     List<size_t> mAvailPortBuffers[2];
332     Vector<BufferInfo> mPortBuffers[2];
333 
334     int32_t mDequeueInputTimeoutGeneration;
335     sp<AReplyToken> mDequeueInputReplyID;
336 
337     int32_t mDequeueOutputTimeoutGeneration;
338     sp<AReplyToken> mDequeueOutputReplyID;
339 
340     sp<ICrypto> mCrypto;
341 
342     List<sp<ABuffer> > mCSD;
343 
344     sp<AMessage> mActivityNotify;
345 
346     bool mHaveInputSurface;
347     bool mHavePendingInputBuffers;
348 
349     MediaCodec(const sp<ALooper> &looper, pid_t pid);
350 
351     static sp<CodecBase> GetCodecBase(const AString &name, bool nameIsType = false);
352 
353     static status_t PostAndAwaitResponse(
354             const sp<AMessage> &msg, sp<AMessage> *response);
355 
356     void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err);
357 
358     status_t init(const AString &name, bool nameIsType, bool encoder);
359 
360     void setState(State newState);
361     void returnBuffersToCodec(bool isReclaim = false);
362     void returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim = false);
363     size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg);
364     status_t onQueueInputBuffer(const sp<AMessage> &msg);
365     status_t onReleaseOutputBuffer(const sp<AMessage> &msg);
366     ssize_t dequeuePortBuffer(int32_t portIndex);
367 
368     status_t getBufferAndFormat(
369             size_t portIndex, size_t index,
370             sp<ABuffer> *buffer, sp<AMessage> *format);
371 
372     bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
373     bool handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
374     void cancelPendingDequeueOperations();
375 
376     void extractCSD(const sp<AMessage> &format);
377     status_t queueCSDInputBuffer(size_t bufferIndex);
378 
379     status_t handleSetSurface(const sp<Surface> &surface);
380     status_t connectToSurface(const sp<Surface> &surface);
381     status_t disconnectFromSurface();
382 
383     void postActivityNotificationIfPossible();
384 
385     void onInputBufferAvailable();
386     void onOutputBufferAvailable();
387     void onError(status_t err, int32_t actionCode, const char *detail = NULL);
388     void onOutputFormatChanged();
389 
390     status_t onSetParameters(const sp<AMessage> &params);
391 
392     status_t amendOutputFormatWithCodecSpecificData(const sp<ABuffer> &buffer);
393     void updateBatteryStat();
394     bool isExecuting() const;
395 
396     uint64_t getGraphicBufferSize();
397     void addResource(MediaResource::Type type, MediaResource::SubType subtype, uint64_t value);
398 
399     bool hasPendingBuffer(int portIndex);
400     bool hasPendingBuffer();
401 
402     /* called to get the last codec error when the sticky flag is set.
403      * if no such codec error is found, returns UNKNOWN_ERROR.
404      */
getStickyErrorMediaCodec405     inline status_t getStickyError() const {
406         return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR;
407     }
408 
setStickyErrorMediaCodec409     inline void setStickyError(status_t err) {
410         mFlags |= kFlagStickyError;
411         mStickyError = err;
412     }
413 
414     DISALLOW_EVIL_CONSTRUCTORS(MediaCodec);
415 };
416 
417 }  // namespace android
418 
419 #endif  // MEDIA_CODEC_H_
420