1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef GRAPHIC_BUFFER_SOURCE_H_ 18 19 #define GRAPHIC_BUFFER_SOURCE_H_ 20 21 #include <gui/IGraphicBufferProducer.h> 22 #include <gui/BufferQueue.h> 23 #include <utils/RefBase.h> 24 25 #include <OMX_Core.h> 26 #include "../include/OMXNodeInstance.h" 27 #include <media/stagefright/foundation/ABase.h> 28 #include <media/stagefright/foundation/AHandlerReflector.h> 29 #include <media/stagefright/foundation/ALooper.h> 30 31 namespace android { 32 33 /* 34 * This class is used to feed OMX codecs from a Surface via BufferQueue. 35 * 36 * Instances of the class don't run on a dedicated thread. Instead, 37 * various events trigger data movement: 38 * 39 * - Availability of a new frame of data from the BufferQueue (notified 40 * via the onFrameAvailable callback). 41 * - The return of a codec buffer (via OnEmptyBufferDone). 42 * - Application signaling end-of-stream. 43 * - Transition to or from "executing" state. 44 * 45 * Frames of data (and, perhaps, the end-of-stream indication) can arrive 46 * before the codec is in the "executing" state, so we need to queue 47 * things up until we're ready to go. 48 */ 49 class GraphicBufferSource : public BufferQueue::ConsumerListener { 50 public: 51 GraphicBufferSource(OMXNodeInstance* nodeInstance, 52 uint32_t bufferWidth, uint32_t bufferHeight, uint32_t bufferCount, 53 bool useGraphicBufferInMeta = false); 54 virtual ~GraphicBufferSource(); 55 56 // We can't throw an exception if the constructor fails, so we just set 57 // this and require that the caller test the value. initCheck()58 status_t initCheck() const { 59 return mInitCheck; 60 } 61 62 // Returns the handle to the producer side of the BufferQueue. Buffers 63 // queued on this will be received by GraphicBufferSource. getIGraphicBufferProducer()64 sp<IGraphicBufferProducer> getIGraphicBufferProducer() const { 65 return mProducer; 66 } 67 68 // This is called when OMX transitions to OMX_StateExecuting, which means 69 // we can start handing it buffers. If we already have buffers of data 70 // sitting in the BufferQueue, this will send them to the codec. 71 void omxExecuting(); 72 73 // This is called when OMX transitions to OMX_StateIdle, indicating that 74 // the codec is meant to return all buffers back to the client for them 75 // to be freed. Do NOT submit any more buffers to the component. 76 void omxIdle(); 77 78 // This is called when OMX transitions to OMX_StateLoaded, indicating that 79 // we are shutting down. 80 void omxLoaded(); 81 82 // A "codec buffer", i.e. a buffer that can be used to pass data into 83 // the encoder, has been allocated. (This call does not call back into 84 // OMXNodeInstance.) 85 void addCodecBuffer(OMX_BUFFERHEADERTYPE* header); 86 87 // Called from OnEmptyBufferDone. If we have a BQ buffer available, 88 // fill it with a new frame of data; otherwise, just mark it as available. 89 void codecBufferEmptied(OMX_BUFFERHEADERTYPE* header); 90 91 // Called when omx_message::FILL_BUFFER_DONE is received. (Currently the 92 // buffer source will fix timestamp in the header if needed.) 93 void codecBufferFilled(OMX_BUFFERHEADERTYPE* header); 94 95 // This is called after the last input frame has been submitted. We 96 // need to submit an empty buffer with the EOS flag set. If we don't 97 // have a codec buffer ready, we just set the mEndOfStream flag. 98 status_t signalEndOfInputStream(); 99 100 // If suspend is true, all incoming buffers (including those currently 101 // in the BufferQueue) will be discarded until the suspension is lifted. 102 void suspend(bool suspend); 103 104 // Specifies the interval after which we requeue the buffer previously 105 // queued to the encoder. This is useful in the case of surface flinger 106 // providing the input surface if the resulting encoded stream is to 107 // be displayed "live". If we were not to push through the extra frame 108 // the decoder on the remote end would be unable to decode the latest frame. 109 // This API must be called before transitioning the encoder to "executing" 110 // state and once this behaviour is specified it cannot be reset. 111 status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs); 112 113 // When set, the timestamp fed to the encoder will be modified such that 114 // the gap between two adjacent frames is capped at maxGapUs. Timestamp 115 // will be restored to the original when the encoded frame is returned to 116 // the client. 117 // This is to solve a problem in certain real-time streaming case, where 118 // encoder's rate control logic produces huge frames after a long period 119 // of suspension on input. 120 status_t setMaxTimestampGapUs(int64_t maxGapUs); 121 122 // Sets the time lapse (or slow motion) parameters. 123 // data[0] is the time (us) between two frames for playback 124 // data[1] is the time (us) between two frames for capture 125 // When set, the sample's timestamp will be modified to playback framerate, 126 // and capture timestamp will be modified to capture rate. 127 status_t setTimeLapseUs(int64_t* data); 128 129 // Sets the start time us (in system time), samples before which should 130 // be dropped and not submitted to encoder 131 void setSkipFramesBeforeUs(int64_t startTimeUs); 132 133 protected: 134 // BufferQueue::ConsumerListener interface, called when a new frame of 135 // data is available. If we're executing and a codec buffer is 136 // available, we acquire the buffer, copy the GraphicBuffer reference 137 // into the codec buffer, and call Empty[This]Buffer. If we're not yet 138 // executing or there's no codec buffer available, we just increment 139 // mNumFramesAvailable and return. 140 virtual void onFrameAvailable(const BufferItem& item); 141 142 // BufferQueue::ConsumerListener interface, called when the client has 143 // released one or more GraphicBuffers. We clear out the appropriate 144 // set of mBufferSlot entries. 145 virtual void onBuffersReleased(); 146 147 // BufferQueue::ConsumerListener interface, called when the client has 148 // changed the sideband stream. GraphicBufferSource doesn't handle sideband 149 // streams so this is a no-op (and should never be called). 150 virtual void onSidebandStreamChanged(); 151 152 private: 153 // Keep track of codec input buffers. They may either be available 154 // (mGraphicBuffer == NULL) or in use by the codec. 155 struct CodecBuffer { 156 OMX_BUFFERHEADERTYPE* mHeader; 157 158 // buffer producer's frame-number for buffer 159 uint64_t mFrameNumber; 160 161 // buffer producer's buffer slot for buffer 162 int mBuf; 163 164 sp<GraphicBuffer> mGraphicBuffer; 165 }; 166 167 // Returns the index of an available codec buffer. If none are 168 // available, returns -1. Mutex must be held by caller. 169 int findAvailableCodecBuffer_l(); 170 171 // Returns true if a codec buffer is available. isCodecBufferAvailable_l()172 bool isCodecBufferAvailable_l() { 173 return findAvailableCodecBuffer_l() >= 0; 174 } 175 176 // Finds the mCodecBuffers entry that matches. Returns -1 if not found. 177 int findMatchingCodecBuffer_l(const OMX_BUFFERHEADERTYPE* header); 178 179 // Fills a codec buffer with a frame from the BufferQueue. This must 180 // only be called when we know that a frame of data is ready (i.e. we're 181 // in the onFrameAvailable callback, or if we're in codecBufferEmptied 182 // and mNumFramesAvailable is nonzero). Returns without doing anything if 183 // we don't have a codec buffer available. 184 // 185 // Returns true if we successfully filled a codec buffer with a BQ buffer. 186 bool fillCodecBuffer_l(); 187 188 // Marks the mCodecBuffers entry as in-use, copies the GraphicBuffer 189 // reference into the codec buffer, and submits the data to the codec. 190 status_t submitBuffer_l(const BufferQueue::BufferItem &item, int cbi); 191 192 // Submits an empty buffer, with the EOS flag set. Returns without 193 // doing anything if we don't have a codec buffer available. 194 void submitEndOfInputStream_l(); 195 196 void setLatestSubmittedBuffer_l(const BufferQueue::BufferItem &item); 197 bool repeatLatestSubmittedBuffer_l(); 198 int64_t getTimestamp(const BufferQueue::BufferItem &item); 199 200 // Lock, covers all member variables. 201 mutable Mutex mMutex; 202 203 // Used to report constructor failure. 204 status_t mInitCheck; 205 206 // Pointer back to the object that contains us. We send buffers here. 207 OMXNodeInstance* mNodeInstance; 208 209 // Set by omxExecuting() / omxIdling(). 210 bool mExecuting; 211 212 bool mSuspended; 213 214 // Our BufferQueue interfaces. mProducer is passed to the producer through 215 // getIGraphicBufferProducer, and mConsumer is used internally to retrieve 216 // the buffers queued by the producer. 217 sp<IGraphicBufferProducer> mProducer; 218 sp<IGraphicBufferConsumer> mConsumer; 219 220 // Number of frames pending in BufferQueue that haven't yet been 221 // forwarded to the codec. 222 size_t mNumFramesAvailable; 223 224 // Set to true if we want to send end-of-stream after we run out of 225 // frames in BufferQueue. 226 bool mEndOfStream; 227 bool mEndOfStreamSent; 228 229 // Cache of GraphicBuffers from the buffer queue. When the codec 230 // is done processing a GraphicBuffer, we can use this to map back 231 // to a slot number. 232 sp<GraphicBuffer> mBufferSlot[BufferQueue::NUM_BUFFER_SLOTS]; 233 234 // Tracks codec buffers. 235 Vector<CodecBuffer> mCodecBuffers; 236 237 //// 238 friend class AHandlerReflector<GraphicBufferSource>; 239 240 enum { 241 kWhatRepeatLastFrame, 242 }; 243 enum { 244 kRepeatLastFrameCount = 10, 245 }; 246 247 KeyedVector<int64_t, int64_t> mOriginalTimeUs; 248 int64_t mMaxTimestampGapUs; 249 int64_t mPrevOriginalTimeUs; 250 int64_t mPrevModifiedTimeUs; 251 int64_t mSkipFramesBeforeNs; 252 253 sp<ALooper> mLooper; 254 sp<AHandlerReflector<GraphicBufferSource> > mReflector; 255 256 int64_t mRepeatAfterUs; 257 int32_t mRepeatLastFrameGeneration; 258 int64_t mRepeatLastFrameTimestamp; 259 int32_t mRepeatLastFrameCount; 260 261 int mLatestSubmittedBufferId; 262 uint64_t mLatestSubmittedBufferFrameNum; 263 int32_t mLatestSubmittedBufferUseCount; 264 265 // The previously submitted buffer should've been repeated but 266 // no codec buffer was available at the time. 267 bool mRepeatBufferDeferred; 268 269 // Time lapse / slow motion configuration 270 int64_t mTimePerCaptureUs; 271 int64_t mTimePerFrameUs; 272 int64_t mPrevCaptureUs; 273 int64_t mPrevFrameUs; 274 275 bool mUseGraphicBufferInMeta; 276 277 void onMessageReceived(const sp<AMessage> &msg); 278 279 DISALLOW_EVIL_CONSTRUCTORS(GraphicBufferSource); 280 }; 281 282 } // namespace android 283 284 #endif // GRAPHIC_BUFFER_SOURCE_H_ 285