1 /*
2  * Copyright 2012, The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef MEDIA_CODEC_H_
18 
19 #define MEDIA_CODEC_H_
20 
21 #include <list>
22 #include <memory>
23 #include <vector>
24 
25 #include <gui/IGraphicBufferProducer.h>
26 #include <media/hardware/CryptoAPI.h>
27 #include <media/MediaCodecInfo.h>
28 #include <media/MediaMetrics.h>
29 #include <media/MediaProfiles.h>
30 #include <media/stagefright/foundation/AHandler.h>
31 #include <media/stagefright/foundation/AMessage.h>
32 #include <media/stagefright/CodecErrorLog.h>
33 #include <media/stagefright/FrameRenderTracker.h>
34 #include <media/stagefright/MediaHistogram.h>
35 #include <media/stagefright/PlaybackDurationAccumulator.h>
36 #include <media/stagefright/VideoRenderQualityTracker.h>
37 #include <utils/Vector.h>
38 
39 class C2Buffer;
40 class C2GraphicBlock;
41 class C2LinearBlock;
42 
43 namespace aidl {
44 namespace android {
45 namespace media {
46 class MediaResourceParcel;
47 class ClientConfigParcel;
48 } // media
49 } // android
50 } // aidl
51 
52 namespace android {
53 
54 struct ABuffer;
55 struct AMessage;
56 struct AReplyToken;
57 struct AString;
58 struct BatteryChecker;
59 class BufferChannelBase;
60 struct AccessUnitInfo;
61 struct CodecBase;
62 struct CodecCryptoInfo;
63 struct CodecParameterDescriptor;
64 class IBatteryStats;
65 struct ICrypto;
66 class CryptoAsync;
67 class MediaCodecBuffer;
68 class IMemory;
69 struct PersistentSurface;
70 class RenderedFrameInfo;
71 class SoftwareRenderer;
72 class Surface;
73 namespace hardware {
74 namespace cas {
75 namespace native {
76 namespace V1_0 {
77 struct IDescrambler;
78 }}}}
79 
80 using hardware::cas::native::V1_0::IDescrambler;
81 using aidl::android::media::MediaResourceParcel;
82 using aidl::android::media::ClientConfigParcel;
83 
84 typedef WrapperObject<std::vector<AccessUnitInfo>> BufferInfosWrapper;
85 typedef WrapperObject<std::vector<std::unique_ptr<CodecCryptoInfo>>> CryptoInfosWrapper;
86 
87 struct MediaCodec : public AHandler {
88     enum Domain {
89         DOMAIN_UNKNOWN = 0,
90         DOMAIN_VIDEO = 1,
91         DOMAIN_AUDIO = 2,
92         DOMAIN_IMAGE = 3
93     };
94 
95     enum ConfigureFlags {
96         CONFIGURE_FLAG_ENCODE           = 1,
97         CONFIGURE_FLAG_USE_BLOCK_MODEL  = 2,
98         CONFIGURE_FLAG_USE_CRYPTO_ASYNC = 4,
99         CONFIGURE_FLAG_DETACHED_SURFACE = 8,
100     };
101 
102     enum BufferFlags {
103         BUFFER_FLAG_SYNCFRAME     = 1,
104         BUFFER_FLAG_CODECCONFIG   = 2,
105         BUFFER_FLAG_EOS           = 4,
106         BUFFER_FLAG_PARTIAL_FRAME = 8,
107         BUFFER_FLAG_MUXER_DATA    = 16,
108         BUFFER_FLAG_DECODE_ONLY   = 32,
109     };
110 
111     enum CVODegree {
112         CVO_DEGREE_0   = 0,
113         CVO_DEGREE_90  = 90,
114         CVO_DEGREE_180 = 180,
115         CVO_DEGREE_270 = 270,
116     };
117 
118     enum {
119         CB_INPUT_AVAILABLE = 1,
120         CB_OUTPUT_AVAILABLE = 2,
121         CB_ERROR = 3,
122         CB_OUTPUT_FORMAT_CHANGED = 4,
123         CB_RESOURCE_RECLAIMED = 5,
124         CB_CRYPTO_ERROR = 6,
125         CB_LARGE_FRAME_OUTPUT_AVAILABLE = 7,
126     };
127 
128     static const pid_t kNoPid = -1;
129     static const uid_t kNoUid = -1;
130 
131     static sp<MediaCodec> CreateByType(
132             const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err = NULL,
133             pid_t pid = kNoPid, uid_t uid = kNoUid);
134 
135     static sp<MediaCodec> CreateByType(
136             const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err,
137             pid_t pid, uid_t uid, sp<AMessage> format);
138 
139     static sp<MediaCodec> CreateByComponentName(
140             const sp<ALooper> &looper, const AString &name, status_t *err = NULL,
141             pid_t pid = kNoPid, uid_t uid = kNoUid);
142 
143     static sp<PersistentSurface> CreatePersistentInputSurface();
144 
145     status_t configure(
146             const sp<AMessage> &format,
147             const sp<Surface> &nativeWindow,
148             const sp<ICrypto> &crypto,
149             uint32_t flags);
150 
151     status_t configure(
152             const sp<AMessage> &format,
153             const sp<Surface> &nativeWindow,
154             const sp<ICrypto> &crypto,
155             const sp<IDescrambler> &descrambler,
156             uint32_t flags);
157 
158     status_t releaseCrypto();
159 
160     status_t setCallback(const sp<AMessage> &callback);
161 
162     status_t setOnFrameRenderedNotification(const sp<AMessage> &notify);
163 
164     status_t setOnFirstTunnelFrameReadyNotification(const sp<AMessage> &notify);
165 
166     status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
167 
168     status_t setInputSurface(const sp<PersistentSurface> &surface);
169 
170     status_t start();
171 
172     // Returns to a state in which the component remains allocated but
173     // unconfigured.
174     status_t stop();
175 
176     // Resets the codec to the INITIALIZED state.  Can be called after an error
177     // has occured to make the codec usable.
178     status_t reset();
179 
180     // Client MUST call release before releasing final reference to this
181     // object.
182     status_t release();
183 
184     status_t releaseAsync(const sp<AMessage> &notify);
185 
186     status_t flush();
187 
188     status_t queueInputBuffer(
189             size_t index,
190             size_t offset,
191             size_t size,
192             int64_t presentationTimeUs,
193             uint32_t flags,
194             AString *errorDetailMsg = NULL);
195 
196     status_t queueInputBuffers(
197             size_t index,
198             size_t offset,
199             size_t size,
200             const sp<BufferInfosWrapper> &accessUnitInfo,
201             AString *errorDetailMsg = NULL);
202 
203     status_t queueSecureInputBuffer(
204             size_t index,
205             size_t offset,
206             const CryptoPlugin::SubSample *subSamples,
207             size_t numSubSamples,
208             const uint8_t key[16],
209             const uint8_t iv[16],
210             CryptoPlugin::Mode mode,
211             const CryptoPlugin::Pattern &pattern,
212             int64_t presentationTimeUs,
213             uint32_t flags,
214             AString *errorDetailMsg = NULL);
215 
216     status_t queueSecureInputBuffers(
217             size_t index,
218             size_t offset,
219             size_t size,
220             const sp<BufferInfosWrapper> &accessUnitInfo,
221             const sp<CryptoInfosWrapper> &cryptoInfos,
222             AString *errorDetailMsg = NULL);
223 
224     status_t queueBuffer(
225             size_t index,
226             const std::shared_ptr<C2Buffer> &buffer,
227             const sp<BufferInfosWrapper> &bufferInfos,
228             const sp<AMessage> &tunings,
229             AString *errorDetailMsg = NULL);
230 
231     status_t queueEncryptedBuffer(
232             size_t index,
233             const sp<hardware::HidlMemory> &memory,
234             size_t offset,
235             size_t size,
236             const sp<BufferInfosWrapper> &bufferInfos,
237             const sp<CryptoInfosWrapper> &cryptoInfos,
238             const sp<AMessage> &tunings,
239             AString *errorDetailMsg = NULL);
240 
241     std::shared_ptr<C2Buffer> decrypt(
242             const std::shared_ptr<C2Buffer> &buffer,
243             const CryptoPlugin::SubSample *subSamples,
244             size_t numSubSamples,
245             const uint8_t key[16],
246             const uint8_t iv[16],
247             CryptoPlugin::Mode mode,
248             const CryptoPlugin::Pattern &pattern);
249 
250     status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll);
251 
252     status_t dequeueOutputBuffer(
253             size_t *index,
254             size_t *offset,
255             size_t *size,
256             int64_t *presentationTimeUs,
257             uint32_t *flags,
258             int64_t timeoutUs = 0ll);
259 
260     status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs);
261     status_t renderOutputBufferAndRelease(size_t index);
262     status_t releaseOutputBuffer(size_t index);
263 
264     status_t signalEndOfInputStream();
265 
266     status_t getOutputFormat(sp<AMessage> *format) const;
267     status_t getInputFormat(sp<AMessage> *format) const;
268 
269     status_t getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
270     status_t getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
271 
272     status_t getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
273     status_t getOutputFormat(size_t index, sp<AMessage> *format);
274     status_t getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
275 
276     status_t setSurface(const sp<Surface> &nativeWindow);
277 
278     status_t detachOutputSurface();
279 
280     status_t requestIDRFrame();
281 
282     // Notification will be posted once there "is something to do", i.e.
283     // an input/output buffer has become available, a format change is
284     // pending, an error is pending.
285     void requestActivityNotification(const sp<AMessage> &notify);
286 
287     status_t getName(AString *componentName) const;
288 
289     status_t getCodecInfo(sp<MediaCodecInfo> *codecInfo) const;
290 
291     status_t getMetrics(mediametrics_handle_t &reply);
292 
293     status_t setParameters(const sp<AMessage> &params);
294 
295     status_t querySupportedVendorParameters(std::vector<std::string> *names);
296     status_t describeParameter(const std::string &name, CodecParameterDescriptor *desc);
297     status_t subscribeToVendorParameters(const std::vector<std::string> &names);
298     status_t unsubscribeFromVendorParameters(const std::vector<std::string> &names);
299 
300     // Create a MediaCodec notification message from a list of rendered or dropped render infos
301     // by adding rendered frame information to a base notification message. Returns the number
302     // of frames that were rendered.
303     static size_t CreateFramesRenderedMessage(
304             const std::list<RenderedFrameInfo> &done, sp<AMessage> &msg);
305     static size_t CreateFramesRenderedMessage(
306             const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg);
307 
308     static status_t CanFetchLinearBlock(
309             const std::vector<std::string> &names, bool *isCompatible);
310 
311     static std::shared_ptr<C2LinearBlock> FetchLinearBlock(
312             size_t capacity, const std::vector<std::string> &names);
313 
314     static status_t CanFetchGraphicBlock(
315             const std::vector<std::string> &names, bool *isCompatible);
316 
317     static std::shared_ptr<C2GraphicBlock> FetchGraphicBlock(
318             int32_t width,
319             int32_t height,
320             int32_t format,
321             uint64_t usage,
322             const std::vector<std::string> &names);
323 
324     template <typename T>
325     struct WrapperObject : public RefBase {
WrapperObjectMediaCodec::WrapperObject326         WrapperObject(const T& v) : value(v) {}
WrapperObjectMediaCodec::WrapperObject327         WrapperObject(T&& v) : value(std::move(v)) {}
328         T value;
329     };
330 
getErrorLogMediaCodec331     inline CodecErrorLog &getErrorLog() { return mErrorLog; }
332 
333 protected:
334     virtual ~MediaCodec();
335     virtual void onMessageReceived(const sp<AMessage> &msg);
336 
337 private:
338     // used by ResourceManagerClient
339     status_t reclaim(bool force = false);
340     friend struct ResourceManagerClient;
341 
342     // to create the metrics associated with this codec.
343     // Any error in this function will be captured by the output argument err.
344     mediametrics_handle_t createMediaMetrics(const sp<AMessage>& format,
345                                              uint32_t flags,
346                                              status_t* err);
347 
348 private:
349     enum State {
350         UNINITIALIZED,
351         INITIALIZING,
352         INITIALIZED,
353         CONFIGURING,
354         CONFIGURED,
355         STARTING,
356         STARTED,
357         FLUSHING,
358         FLUSHED,
359         STOPPING,
360         RELEASING,
361     };
362     std::string stateString(State state);
363     std::string apiStateString();
364 
365     enum {
366         kPortIndexInput         = 0,
367         kPortIndexOutput        = 1,
368     };
369 
370     enum {
371         kWhatInit                           = 'init',
372         kWhatConfigure                      = 'conf',
373         kWhatSetSurface                     = 'sSur',
374         kWhatDetachSurface                  = 'dSur',
375         kWhatCreateInputSurface             = 'cisf',
376         kWhatSetInputSurface                = 'sisf',
377         kWhatStart                          = 'strt',
378         kWhatStop                           = 'stop',
379         kWhatRelease                        = 'rele',
380         kWhatDequeueInputBuffer             = 'deqI',
381         kWhatQueueInputBuffer               = 'queI',
382         kWhatDequeueOutputBuffer            = 'deqO',
383         kWhatReleaseOutputBuffer            = 'relO',
384         kWhatSignalEndOfInputStream         = 'eois',
385         kWhatGetBuffers                     = 'getB',
386         kWhatFlush                          = 'flus',
387         kWhatGetOutputFormat                = 'getO',
388         kWhatGetInputFormat                 = 'getI',
389         kWhatDequeueInputTimedOut           = 'dITO',
390         kWhatDequeueOutputTimedOut          = 'dOTO',
391         kWhatCodecNotify                    = 'codc',
392         kWhatRequestIDRFrame                = 'ridr',
393         kWhatRequestActivityNotification    = 'racN',
394         kWhatGetName                        = 'getN',
395         kWhatGetCodecInfo                   = 'gCoI',
396         kWhatSetParameters                  = 'setP',
397         kWhatSetCallback                    = 'setC',
398         kWhatSetNotification                = 'setN',
399         kWhatDrmReleaseCrypto               = 'rDrm',
400         kWhatCheckBatteryStats              = 'chkB',
401         kWhatGetMetrics                     = 'getM',
402     };
403 
404     enum {
405         kFlagUsesSoftwareRenderer       = 1,
406         kFlagOutputFormatChanged        = 2,
407         kFlagOutputBuffersChanged       = 4,
408         kFlagStickyError                = 8,
409         kFlagDequeueInputPending        = 16,
410         kFlagDequeueOutputPending       = 32,
411         kFlagIsSecure                   = 64,
412         kFlagSawMediaServerDie          = 128,
413         kFlagIsEncoder                  = 256,
414         // 512 skipped
415         kFlagIsAsync                    = 1024,
416         kFlagIsComponentAllocated       = 2048,
417         kFlagPushBlankBuffersOnShutdown = 4096,
418         kFlagUseBlockModel              = 8192,
419         kFlagUseCryptoAsync             = 16384,
420     };
421 
422     struct BufferInfo {
423         BufferInfo();
424 
425         sp<MediaCodecBuffer> mData;
426         bool mOwnedByClient;
427     };
428 
429     // This type is used to track the tunnel mode video peek state machine:
430     //
431     // DisabledNoBuffer -> EnabledNoBuffer  when tunnel-peek = true
432     // DisabledQueued   -> EnabledQueued    when tunnel-peek = true
433     // DisabledNoBuffer -> DisabledQueued   when first frame queued
434     // EnabledNoBuffer  -> DisabledNoBuffer when tunnel-peek = false
435     // EnabledQueued    -> DisabledQueued   when tunnel-peek = false
436     // EnabledNoBuffer  -> EnabledQueued    when first frame queued
437     // DisabledNoBuffer -> BufferDecoded    when kWhatFirstTunnelFrameReady
438     // DisabledQueued   -> BufferDecoded    when kWhatFirstTunnelFrameReady
439     // EnabledNoBuffer  -> BufferDecoded    when kWhatFirstTunnelFrameReady
440     // EnabledQueued    -> BufferDecoded    when kWhatFirstTunnelFrameReady
441     // BufferDecoded    -> BufferRendered   when kWhatFrameRendered
442     // <all states>     -> EnabledNoBuffer  when flush
443     // <all states>     -> EnabledNoBuffer  when stop then configure then start
444     enum struct TunnelPeekState {
445         kLegacyMode,
446         kDisabledNoBuffer,
447         kEnabledNoBuffer,
448         kDisabledQueued,
449         kEnabledQueued,
450         kBufferDecoded,
451         kBufferRendered,
452     };
453 
454     enum class DequeueOutputResult {
455         kNoBuffer,
456         kDiscardedBuffer,
457         kRepliedWithError,
458         kSuccess,
459     };
460 
461     struct ResourceManagerServiceProxy;
462 
463     State mState;
464     bool mReleasedByResourceManager;
465     sp<ALooper> mLooper;
466     sp<ALooper> mCodecLooper;
467     sp<CodecBase> mCodec;
468     AString mComponentName;
469     AString mOwnerName;
470     sp<MediaCodecInfo> mCodecInfo;
471     sp<AReplyToken> mReplyID;
472     std::string mLastReplyOrigin;
473     std::vector<sp<AMessage>> mDeferredMessages;
474     uint32_t mFlags;
475     int64_t mPresentationTimeUs = 0;
476     status_t mStickyError;
477     sp<Surface> mSurface;
478     uint32_t mSurfaceGeneration = 0;
479     SoftwareRenderer *mSoftRenderer;
480 
481     // Get the detached BufferQueue surface for a video decoder, and create it
482     // if it did not yet exist.
483     sp<Surface> getOrCreateDetachedSurface();
484 
485     Mutex mMetricsLock;
486     mediametrics_handle_t mMetricsHandle = 0;
487     bool mMetricsToUpload = false;
488     nsecs_t mLifetimeStartNs = 0;
489     void initMediametrics();
490     void updateMediametrics();
491     void flushMediametrics();
492     void resetMetricsFields();
493     void updateEphemeralMediametrics(mediametrics_handle_t item);
494     void updateLowLatency(const sp<AMessage> &msg);
495     void updateCodecImportance(const sp<AMessage>& msg);
496     void onGetMetrics(const sp<AMessage>& msg);
497     constexpr const char *asString(TunnelPeekState state, const char *default_string="?");
498     void updateTunnelPeek(const sp<AMessage> &msg);
499     void processRenderedFrames(const sp<AMessage> &msg);
500 
501     inline void initClientConfigParcel(ClientConfigParcel& clientConfig);
502 
503     sp<AMessage> mOutputFormat;
504     sp<AMessage> mInputFormat;
505     sp<AMessage> mCallback;
506     sp<AMessage> mOnFrameRenderedNotification;
507     sp<AMessage> mAsyncReleaseCompleteNotification;
508     sp<AMessage> mOnFirstTunnelFrameReadyNotification;
509 
510     std::shared_ptr<ResourceManagerServiceProxy> mResourceManagerProxy;
511 
512     Domain mDomain;
513     AString mLogSessionId;
514     int32_t mWidth;
515     int32_t mHeight;
516     int32_t mRotationDegrees;
517     int32_t mAllowFrameDroppingBySurface;
518 
519     enum {
520         kFlagHasHdrStaticInfo   = 1,
521         kFlagHasHdr10PlusInfo   = 2,
522     };
523     uint32_t mHdrInfoFlags;
524     void updateHdrMetrics(bool isConfig);
525     hdr_format getHdrFormat(const AString &mime, const int32_t profile,
526             const int32_t colorTransfer);
527     hdr_format getHdrFormatForEncoder(const AString &mime, const int32_t profile,
528             const int32_t colorTransfer);
529     hdr_format getHdrFormatForDecoder(const AString &mime, const int32_t profile,
530             const int32_t colorTransfer);
531     bool profileSupport10Bits(const AString &mime, const int32_t profile);
532 
533     struct ApiUsageMetrics {
534         bool isArrayMode;
535         enum OperationMode {
536             kUnknownMode = 0,
537             kSynchronousMode = 1,
538             kAsynchronousMode = 2,
539             kBlockMode = 3,
540         };
541         OperationMode operationMode;
542         bool isUsingOutputSurface;
543         struct InputBufferSize {
544             int32_t appMax;  // max size configured by the app
545             int32_t usedMax;  // max size actually used
546             int32_t codecMax;  // max size suggested by the codec
547         } inputBufferSize;
548     } mApiUsageMetrics;
549     struct ReliabilityContextMetrics {
550         int32_t flushCount;
551         int32_t setOutputSurfaceCount;
552         int32_t resolutionChangeCount;
553     } mReliabilityContextMetrics;
554 
555     // initial create parameters
556     AString mInitName;
557 
558     // configure parameter
559     sp<AMessage> mConfigureMsg;
560 
561     // rewrites the format description during configure() for encoding.
562     // format and flags as they exist within configure()
563     // the (possibly) updated format is returned in place.
564     status_t shapeMediaFormat(
565             const sp<AMessage> &format,
566             uint32_t flags,
567             mediametrics_handle_t handle);
568 
569     // populate the format shaper library with information for this codec encoding
570     // for the indicated media type
571     status_t setupFormatShaper(AString mediaType);
572 
573     // Used only to synchronize asynchronous getBufferAndFormat
574     // across all the other (synchronous) buffer state change
575     // operations, such as de/queueIn/OutputBuffer, start and
576     // stop/flush/reset/release.
577     Mutex mBufferLock;
578 
579     std::list<size_t> mAvailPortBuffers[2];
580     std::vector<BufferInfo> mPortBuffers[2];
581 
582     int32_t mDequeueInputTimeoutGeneration;
583     sp<AReplyToken> mDequeueInputReplyID;
584 
585     int32_t mDequeueOutputTimeoutGeneration;
586     sp<AReplyToken> mDequeueOutputReplyID;
587 
588     sp<ICrypto> mCrypto;
589 
590     int32_t mTunneledInputWidth;
591     int32_t mTunneledInputHeight;
592     bool mTunneled;
593     TunnelPeekState mTunnelPeekState;
594     bool mTunnelPeekEnabled;
595 
596     sp<IDescrambler> mDescrambler;
597 
598     std::list<sp<ABuffer> > mCSD;
599 
600     sp<AMessage> mActivityNotify;
601 
602     bool mHaveInputSurface;
603     bool mHavePendingInputBuffers;
604     bool mCpuBoostRequested;
605 
606     std::shared_ptr<BufferChannelBase> mBufferChannel;
607     sp<CryptoAsync> mCryptoAsync;
608     sp<ALooper> mCryptoLooper;
609 
610     bool mIsSurfaceToDisplay;
611     bool mAreRenderMetricsEnabled;
612     PlaybackDurationAccumulator mPlaybackDurationAccumulator;
613     VideoRenderQualityTracker mVideoRenderQualityTracker;
614 
615     MediaCodec(
616             const sp<ALooper> &looper, pid_t pid, uid_t uid,
617             std::function<sp<CodecBase>(const AString &, const char *)> getCodecBase = nullptr,
618             std::function<status_t(const AString &, sp<MediaCodecInfo> *)> getCodecInfo = nullptr);
619 
620     static sp<CodecBase> GetCodecBase(const AString &name, const char *owner = nullptr);
621 
622     static status_t PostAndAwaitResponse(
623             const sp<AMessage> &msg, sp<AMessage> *response);
624 
625     void PostReplyWithError(const sp<AMessage> &msg, int32_t err);
626     void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err);
627 
628     status_t init(const AString &name);
629 
630     void setState(State newState);
631     void returnBuffersToCodec(bool isReclaim = false);
632     void returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim = false);
633     size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg);
634     status_t onQueueInputBuffer(const sp<AMessage> &msg);
635     status_t onReleaseOutputBuffer(const sp<AMessage> &msg);
636     BufferInfo *peekNextPortBuffer(int32_t portIndex);
637     ssize_t dequeuePortBuffer(int32_t portIndex);
638 
639     status_t getBufferAndFormat(
640             size_t portIndex, size_t index,
641             sp<MediaCodecBuffer> *buffer, sp<AMessage> *format);
642 
643     bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
644     DequeueOutputResult handleDequeueOutputBuffer(
645             const sp<AReplyToken> &replyID,
646             bool newRequest = false);
647     void cancelPendingDequeueOperations();
648 
649     void extractCSD(const sp<AMessage> &format);
650     status_t queueCSDInputBuffer(size_t bufferIndex);
651 
652     status_t handleSetSurface(const sp<Surface> &surface);
653 
654     // Common reimplementation of changing the output surface.
655     // Handles setting null surface, which is used during configure and init.
656     // Set |callCodec| to true if the codec needs to be notified (e.g. during executing state).
657     // Setting |onShutdown| to true will avoid extra work, if this is used for detaching on
658     // delayed release.
659     status_t handleSetSurface(const sp<Surface> &surface, bool callCodec, bool onShutdown = false);
660     status_t connectToSurface(const sp<Surface> &surface, uint32_t *generation);
661     status_t disconnectFromSurface();
662 
hasCryptoOrDescramblerMediaCodec663     bool hasCryptoOrDescrambler() {
664         return mCrypto != NULL || mDescrambler != NULL;
665     }
666 
667     void postActivityNotificationIfPossible();
668 
669     void onInputBufferAvailable();
670     void onOutputBufferAvailable();
671     void onCryptoError(const sp<AMessage> &msg);
672     void onError(status_t err, int32_t actionCode, const char *detail = NULL);
673     void onOutputFormatChanged();
674 
675     status_t onSetParameters(const sp<AMessage> &params);
676 
677     status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer);
678     void handleOutputFormatChangeIfNeeded(const sp<MediaCodecBuffer> &buffer);
679     bool isExecuting() const;
680 
681     uint64_t getGraphicBufferSize();
682     void requestCpuBoostIfNeeded();
683 
684     bool hasPendingBuffer(int portIndex);
685     bool hasPendingBuffer();
686 
687     void postPendingRepliesAndDeferredMessages(std::string origin, status_t err = OK);
688     void postPendingRepliesAndDeferredMessages(std::string origin, const sp<AMessage> &response);
689 
690     /* called to get the last codec error when the sticky flag is set.
691      * if no such codec error is found, returns UNKNOWN_ERROR.
692      */
getStickyErrorMediaCodec693     inline status_t getStickyError() const {
694         return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR;
695     }
696 
setStickyErrorMediaCodec697     inline void setStickyError(status_t err) {
698         mFlags |= kFlagStickyError;
699         mStickyError = err;
700     }
701 
702     void onReleaseCrypto(const sp<AMessage>& msg);
703 
704     // managing time-of-flight aka latency
705     typedef struct {
706             int64_t presentationUs;
707             int64_t startedNs;
708     } BufferFlightTiming_t;
709     std::deque<BufferFlightTiming_t> mBuffersInFlight;
710     Mutex mLatencyLock;
711     int64_t mLatencyUnknown;    // buffers for which we couldn't calculate latency
712 
713     Mutex mOutputStatsLock;
714     int64_t mBytesEncoded = 0;
715     int64_t mEarliestEncodedPtsUs = INT64_MAX;
716     int64_t mLatestEncodedPtsUs = INT64_MIN;
717     int64_t mFramesEncoded = 0;
718     int64_t mBytesInput = 0;
719     int64_t mFramesInput = 0;
720 
721     int64_t mNumLowLatencyEnables;  // how many times low latency mode is enabled
722     int64_t mNumLowLatencyDisables;  // how many times low latency mode is disabled
723     bool mIsLowLatencyModeOn;  // is low latency mode on currently
724     int64_t mIndexOfFirstFrameWhenLowLatencyOn;  // index of the first frame queued
725                                                  // when low latency is on
726     int64_t mInputBufferCounter;  // number of input buffers queued since last reset/flush
727 
728     // A rescheduleable message that periodically polls for rendered buffers
729     sp<AMessage> mMsgPollForRenderedBuffers;
730 
731     class ReleaseSurface;
732     std::unique_ptr<ReleaseSurface> mDetachedSurface;
733 
734     std::list<sp<AMessage>> mLeftover;
735     status_t handleLeftover(size_t index);
736 
737     sp<BatteryChecker> mBatteryChecker;
738 
739     void statsBufferSent(int64_t presentationUs, const sp<MediaCodecBuffer> &buffer);
740     void statsBufferReceived(int64_t presentationUs, const sp<MediaCodecBuffer> &buffer);
741     bool discardDecodeOnlyOutputBuffer(size_t index);
742 
743     enum {
744         // the default shape of our latency histogram buckets
745         // XXX: should these be configurable in some way?
746         kLatencyHistBuckets = 20,
747         kLatencyHistWidth = 2000,
748         kLatencyHistFloor = 2000,
749 
750         // how many samples are in the 'recent latency' histogram
751         // 300 frames = 5 sec @ 60fps or ~12 sec @ 24fps
752         kRecentLatencyFrames = 300,
753 
754         // how we initialize mRecentSamples
755         kRecentSampleInvalid = -1,
756     };
757 
758     int64_t mRecentSamples[kRecentLatencyFrames];
759     int mRecentHead;
760     Mutex mRecentLock;
761 
762     MediaHistogram<int64_t> mLatencyHist;
763 
764     // An unique ID for the codec - Used by the metrics.
765     uint64_t mCodecId = 0;
766     bool     mIsHardware = false;
767 
768     std::function<sp<CodecBase>(const AString &, const char *)> mGetCodecBase;
769     std::function<status_t(const AString &, sp<MediaCodecInfo> *)> mGetCodecInfo;
770     friend class MediaTestHelper;
771 
772     CodecErrorLog mErrorLog;
773 
774     DISALLOW_EVIL_CONSTRUCTORS(MediaCodec);
775 };
776 
777 }  // namespace android
778 
779 #endif  // MEDIA_CODEC_H_
780