1 /*
2  * Copyright 2012, The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef MEDIA_CODEC_H_
18 
19 #define MEDIA_CODEC_H_
20 
21 #include <gui/IGraphicBufferProducer.h>
22 #include <media/hardware/CryptoAPI.h>
23 #include <media/MediaResource.h>
24 #include <media/stagefright/foundation/AHandler.h>
25 #include <media/stagefright/FrameRenderTracker.h>
26 #include <utils/Vector.h>
27 
28 namespace android {
29 
30 struct ABuffer;
31 struct AMessage;
32 struct AReplyToken;
33 struct AString;
34 struct CodecBase;
35 struct IBatteryStats;
36 struct ICrypto;
37 class IMemory;
38 struct MemoryDealer;
39 class IResourceManagerClient;
40 class IResourceManagerService;
41 struct PersistentSurface;
42 struct SoftwareRenderer;
43 struct Surface;
44 
45 struct MediaCodec : public AHandler {
46     enum ConfigureFlags {
47         CONFIGURE_FLAG_ENCODE   = 1,
48     };
49 
50     enum BufferFlags {
51         BUFFER_FLAG_SYNCFRAME   = 1,
52         BUFFER_FLAG_CODECCONFIG = 2,
53         BUFFER_FLAG_EOS         = 4,
54     };
55 
56     enum {
57         CB_INPUT_AVAILABLE = 1,
58         CB_OUTPUT_AVAILABLE = 2,
59         CB_ERROR = 3,
60         CB_OUTPUT_FORMAT_CHANGED = 4,
61         CB_RESOURCE_RECLAIMED = 5,
62     };
63 
64     static const pid_t kNoPid = -1;
65 
66     static sp<MediaCodec> CreateByType(
67             const sp<ALooper> &looper, const char *mime, bool encoder, status_t *err = NULL,
68             pid_t pid = kNoPid);
69 
70     static sp<MediaCodec> CreateByComponentName(
71             const sp<ALooper> &looper, const char *name, status_t *err = NULL,
72             pid_t pid = kNoPid);
73 
74     static sp<PersistentSurface> CreatePersistentInputSurface();
75 
76     status_t configure(
77             const sp<AMessage> &format,
78             const sp<Surface> &nativeWindow,
79             const sp<ICrypto> &crypto,
80             uint32_t flags);
81 
82     status_t setCallback(const sp<AMessage> &callback);
83 
84     status_t setOnFrameRenderedNotification(const sp<AMessage> &notify);
85 
86     status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
87 
88     status_t setInputSurface(const sp<PersistentSurface> &surface);
89 
90     status_t start();
91 
92     // Returns to a state in which the component remains allocated but
93     // unconfigured.
94     status_t stop();
95 
96     // Resets the codec to the INITIALIZED state.  Can be called after an error
97     // has occured to make the codec usable.
98     status_t reset();
99 
100     // Client MUST call release before releasing final reference to this
101     // object.
102     status_t release();
103 
104     status_t flush();
105 
106     status_t queueInputBuffer(
107             size_t index,
108             size_t offset,
109             size_t size,
110             int64_t presentationTimeUs,
111             uint32_t flags,
112             AString *errorDetailMsg = NULL);
113 
114     status_t queueSecureInputBuffer(
115             size_t index,
116             size_t offset,
117             const CryptoPlugin::SubSample *subSamples,
118             size_t numSubSamples,
119             const uint8_t key[16],
120             const uint8_t iv[16],
121             CryptoPlugin::Mode mode,
122             int64_t presentationTimeUs,
123             uint32_t flags,
124             AString *errorDetailMsg = NULL);
125 
126     status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll);
127 
128     status_t dequeueOutputBuffer(
129             size_t *index,
130             size_t *offset,
131             size_t *size,
132             int64_t *presentationTimeUs,
133             uint32_t *flags,
134             int64_t timeoutUs = 0ll);
135 
136     status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs);
137     status_t renderOutputBufferAndRelease(size_t index);
138     status_t releaseOutputBuffer(size_t index);
139 
140     status_t signalEndOfInputStream();
141 
142     status_t getOutputFormat(sp<AMessage> *format) const;
143     status_t getInputFormat(sp<AMessage> *format) const;
144 
145     status_t getWidevineLegacyBuffers(Vector<sp<ABuffer> > *buffers) const;
146 
147     status_t getInputBuffers(Vector<sp<ABuffer> > *buffers) const;
148     status_t getOutputBuffers(Vector<sp<ABuffer> > *buffers) const;
149 
150     status_t getOutputBuffer(size_t index, sp<ABuffer> *buffer);
151     status_t getOutputFormat(size_t index, sp<AMessage> *format);
152     status_t getInputBuffer(size_t index, sp<ABuffer> *buffer);
153 
154     status_t setSurface(const sp<Surface> &nativeWindow);
155 
156     status_t requestIDRFrame();
157 
158     // Notification will be posted once there "is something to do", i.e.
159     // an input/output buffer has become available, a format change is
160     // pending, an error is pending.
161     void requestActivityNotification(const sp<AMessage> &notify);
162 
163     status_t getName(AString *componentName) const;
164 
165     status_t setParameters(const sp<AMessage> &params);
166 
167     // Create a MediaCodec notification message from a list of rendered or dropped render infos
168     // by adding rendered frame information to a base notification message. Returns the number
169     // of frames that were rendered.
170     static size_t CreateFramesRenderedMessage(
171             std::list<FrameRenderTracker::Info> done, sp<AMessage> &msg);
172 
173 protected:
174     virtual ~MediaCodec();
175     virtual void onMessageReceived(const sp<AMessage> &msg);
176 
177 private:
178     // used by ResourceManagerClient
179     status_t reclaim();
180     friend struct ResourceManagerClient;
181 
182 private:
183     enum State {
184         UNINITIALIZED,
185         INITIALIZING,
186         INITIALIZED,
187         CONFIGURING,
188         CONFIGURED,
189         STARTING,
190         STARTED,
191         FLUSHING,
192         FLUSHED,
193         STOPPING,
194         RELEASING,
195     };
196 
197     enum {
198         kPortIndexInput         = 0,
199         kPortIndexOutput        = 1,
200     };
201 
202     enum {
203         kWhatInit                           = 'init',
204         kWhatConfigure                      = 'conf',
205         kWhatSetSurface                     = 'sSur',
206         kWhatCreateInputSurface             = 'cisf',
207         kWhatSetInputSurface                = 'sisf',
208         kWhatStart                          = 'strt',
209         kWhatStop                           = 'stop',
210         kWhatRelease                        = 'rele',
211         kWhatDequeueInputBuffer             = 'deqI',
212         kWhatQueueInputBuffer               = 'queI',
213         kWhatDequeueOutputBuffer            = 'deqO',
214         kWhatReleaseOutputBuffer            = 'relO',
215         kWhatSignalEndOfInputStream         = 'eois',
216         kWhatGetBuffers                     = 'getB',
217         kWhatFlush                          = 'flus',
218         kWhatGetOutputFormat                = 'getO',
219         kWhatGetInputFormat                 = 'getI',
220         kWhatDequeueInputTimedOut           = 'dITO',
221         kWhatDequeueOutputTimedOut          = 'dOTO',
222         kWhatCodecNotify                    = 'codc',
223         kWhatRequestIDRFrame                = 'ridr',
224         kWhatRequestActivityNotification    = 'racN',
225         kWhatGetName                        = 'getN',
226         kWhatSetParameters                  = 'setP',
227         kWhatSetCallback                    = 'setC',
228         kWhatSetNotification                = 'setN',
229     };
230 
231     enum {
232         kFlagUsesSoftwareRenderer       = 1,
233         kFlagOutputFormatChanged        = 2,
234         kFlagOutputBuffersChanged       = 4,
235         kFlagStickyError                = 8,
236         kFlagDequeueInputPending        = 16,
237         kFlagDequeueOutputPending       = 32,
238         kFlagIsSecure                   = 64,
239         kFlagSawMediaServerDie          = 128,
240         kFlagIsEncoder                  = 256,
241         kFlagGatherCodecSpecificData    = 512,
242         kFlagIsAsync                    = 1024,
243         kFlagIsComponentAllocated       = 2048,
244         kFlagPushBlankBuffersOnShutdown = 4096,
245     };
246 
247     struct BufferInfo {
248         uint32_t mBufferID;
249         sp<ABuffer> mData;
250         sp<ABuffer> mEncryptedData;
251         sp<IMemory> mSharedEncryptedBuffer;
252         sp<AMessage> mNotify;
253         sp<AMessage> mFormat;
254         bool mOwnedByClient;
255     };
256 
257     struct ResourceManagerServiceProxy : public IBinder::DeathRecipient {
258         ResourceManagerServiceProxy(pid_t pid);
259         ~ResourceManagerServiceProxy();
260 
261         void init();
262 
263         // implements DeathRecipient
264         virtual void binderDied(const wp<IBinder>& /*who*/);
265 
266         void addResource(
267                 int64_t clientId,
268                 const sp<IResourceManagerClient> client,
269                 const Vector<MediaResource> &resources);
270 
271         void removeResource(int64_t clientId);
272 
273         bool reclaimResource(const Vector<MediaResource> &resources);
274 
275     private:
276         Mutex mLock;
277         sp<IResourceManagerService> mService;
278         pid_t mPid;
279     };
280 
281     State mState;
282     bool mReleasedByResourceManager;
283     sp<ALooper> mLooper;
284     sp<ALooper> mCodecLooper;
285     sp<CodecBase> mCodec;
286     AString mComponentName;
287     sp<AReplyToken> mReplyID;
288     uint32_t mFlags;
289     status_t mStickyError;
290     sp<Surface> mSurface;
291     SoftwareRenderer *mSoftRenderer;
292 
293     sp<AMessage> mOutputFormat;
294     sp<AMessage> mInputFormat;
295     sp<AMessage> mCallback;
296     sp<AMessage> mOnFrameRenderedNotification;
297     sp<MemoryDealer> mDealer;
298 
299     sp<IResourceManagerClient> mResourceManagerClient;
300     sp<ResourceManagerServiceProxy> mResourceManagerService;
301 
302     bool mBatteryStatNotified;
303     bool mIsVideo;
304     int32_t mVideoWidth;
305     int32_t mVideoHeight;
306     int32_t mRotationDegrees;
307 
308     // initial create parameters
309     AString mInitName;
310     bool mInitNameIsType;
311     bool mInitIsEncoder;
312 
313     // configure parameter
314     sp<AMessage> mConfigureMsg;
315 
316     // Used only to synchronize asynchronous getBufferAndFormat
317     // across all the other (synchronous) buffer state change
318     // operations, such as de/queueIn/OutputBuffer, start and
319     // stop/flush/reset/release.
320     Mutex mBufferLock;
321 
322     List<size_t> mAvailPortBuffers[2];
323     Vector<BufferInfo> mPortBuffers[2];
324 
325     int32_t mDequeueInputTimeoutGeneration;
326     sp<AReplyToken> mDequeueInputReplyID;
327 
328     int32_t mDequeueOutputTimeoutGeneration;
329     sp<AReplyToken> mDequeueOutputReplyID;
330 
331     sp<ICrypto> mCrypto;
332 
333     List<sp<ABuffer> > mCSD;
334 
335     sp<AMessage> mActivityNotify;
336 
337     bool mHaveInputSurface;
338     bool mHavePendingInputBuffers;
339 
340     MediaCodec(const sp<ALooper> &looper, pid_t pid);
341 
342     static status_t PostAndAwaitResponse(
343             const sp<AMessage> &msg, sp<AMessage> *response);
344 
345     void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err);
346 
347     status_t init(const AString &name, bool nameIsType, bool encoder);
348 
349     void setState(State newState);
350     void returnBuffersToCodec();
351     void returnBuffersToCodecOnPort(int32_t portIndex);
352     size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg);
353     status_t onQueueInputBuffer(const sp<AMessage> &msg);
354     status_t onReleaseOutputBuffer(const sp<AMessage> &msg);
355     ssize_t dequeuePortBuffer(int32_t portIndex);
356 
357     status_t getBufferAndFormat(
358             size_t portIndex, size_t index,
359             sp<ABuffer> *buffer, sp<AMessage> *format);
360 
361     bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
362     bool handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
363     void cancelPendingDequeueOperations();
364 
365     void extractCSD(const sp<AMessage> &format);
366     status_t queueCSDInputBuffer(size_t bufferIndex);
367 
368     status_t handleSetSurface(const sp<Surface> &surface);
369     status_t connectToSurface(const sp<Surface> &surface);
370     status_t disconnectFromSurface();
371 
372     void postActivityNotificationIfPossible();
373 
374     void onInputBufferAvailable();
375     void onOutputBufferAvailable();
376     void onError(status_t err, int32_t actionCode, const char *detail = NULL);
377     void onOutputFormatChanged();
378 
379     status_t onSetParameters(const sp<AMessage> &params);
380 
381     status_t amendOutputFormatWithCodecSpecificData(const sp<ABuffer> &buffer);
382     void updateBatteryStat();
383     bool isExecuting() const;
384 
385     uint64_t getGraphicBufferSize();
386     void addResource(const String8 &type, const String8 &subtype, uint64_t value);
387 
388     /* called to get the last codec error when the sticky flag is set.
389      * if no such codec error is found, returns UNKNOWN_ERROR.
390      */
getStickyErrorMediaCodec391     inline status_t getStickyError() const {
392         return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR;
393     }
394 
setStickyErrorMediaCodec395     inline void setStickyError(status_t err) {
396         mFlags |= kFlagStickyError;
397         mStickyError = err;
398     }
399 
400     DISALLOW_EVIL_CONSTRUCTORS(MediaCodec);
401 };
402 
403 }  // namespace android
404 
405 #endif  // MEDIA_CODEC_H_
406