• Home
  • History
  • Annotate
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2013 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef GRAPHIC_BUFFER_SOURCE_H_
18 
19 #define GRAPHIC_BUFFER_SOURCE_H_
20 
21 #include <gui/IGraphicBufferProducer.h>
22 #include <gui/BufferQueue.h>
23 #include <utils/RefBase.h>
24 
25 #include <OMX_Core.h>
26 #include "../include/OMXNodeInstance.h"
27 #include <media/stagefright/foundation/ABase.h>
28 #include <media/stagefright/foundation/AHandlerReflector.h>
29 #include <media/stagefright/foundation/ALooper.h>
30 
31 namespace android {
32 
33 /*
34  * This class is used to feed OMX codecs from a Surface via BufferQueue.
35  *
36  * Instances of the class don't run on a dedicated thread.  Instead,
37  * various events trigger data movement:
38  *
39  *  - Availability of a new frame of data from the BufferQueue (notified
40  *    via the onFrameAvailable callback).
41  *  - The return of a codec buffer (via OnEmptyBufferDone).
42  *  - Application signaling end-of-stream.
43  *  - Transition to or from "executing" state.
44  *
45  * Frames of data (and, perhaps, the end-of-stream indication) can arrive
46  * before the codec is in the "executing" state, so we need to queue
47  * things up until we're ready to go.
48  */
49 class GraphicBufferSource : public BufferQueue::ConsumerListener {
50 public:
51     GraphicBufferSource(OMXNodeInstance* nodeInstance,
52             uint32_t bufferWidth, uint32_t bufferHeight, uint32_t bufferCount,
53             bool useGraphicBufferInMeta = false);
54     virtual ~GraphicBufferSource();
55 
56     // We can't throw an exception if the constructor fails, so we just set
57     // this and require that the caller test the value.
initCheck()58     status_t initCheck() const {
59         return mInitCheck;
60     }
61 
62     // Returns the handle to the producer side of the BufferQueue.  Buffers
63     // queued on this will be received by GraphicBufferSource.
getIGraphicBufferProducer()64     sp<IGraphicBufferProducer> getIGraphicBufferProducer() const {
65         return mProducer;
66     }
67 
68     // This is called when OMX transitions to OMX_StateExecuting, which means
69     // we can start handing it buffers.  If we already have buffers of data
70     // sitting in the BufferQueue, this will send them to the codec.
71     void omxExecuting();
72 
73     // This is called when OMX transitions to OMX_StateIdle, indicating that
74     // the codec is meant to return all buffers back to the client for them
75     // to be freed. Do NOT submit any more buffers to the component.
76     void omxIdle();
77 
78     // This is called when OMX transitions to OMX_StateLoaded, indicating that
79     // we are shutting down.
80     void omxLoaded();
81 
82     // A "codec buffer", i.e. a buffer that can be used to pass data into
83     // the encoder, has been allocated.  (This call does not call back into
84     // OMXNodeInstance.)
85     void addCodecBuffer(OMX_BUFFERHEADERTYPE* header);
86 
87     // Called from OnEmptyBufferDone.  If we have a BQ buffer available,
88     // fill it with a new frame of data; otherwise, just mark it as available.
89     void codecBufferEmptied(OMX_BUFFERHEADERTYPE* header);
90 
91     // Called when omx_message::FILL_BUFFER_DONE is received. (Currently the
92     // buffer source will fix timestamp in the header if needed.)
93     void codecBufferFilled(OMX_BUFFERHEADERTYPE* header);
94 
95     // This is called after the last input frame has been submitted.  We
96     // need to submit an empty buffer with the EOS flag set.  If we don't
97     // have a codec buffer ready, we just set the mEndOfStream flag.
98     status_t signalEndOfInputStream();
99 
100     // If suspend is true, all incoming buffers (including those currently
101     // in the BufferQueue) will be discarded until the suspension is lifted.
102     void suspend(bool suspend);
103 
104     // Specifies the interval after which we requeue the buffer previously
105     // queued to the encoder. This is useful in the case of surface flinger
106     // providing the input surface if the resulting encoded stream is to
107     // be displayed "live". If we were not to push through the extra frame
108     // the decoder on the remote end would be unable to decode the latest frame.
109     // This API must be called before transitioning the encoder to "executing"
110     // state and once this behaviour is specified it cannot be reset.
111     status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs);
112 
113     // When set, the timestamp fed to the encoder will be modified such that
114     // the gap between two adjacent frames is capped at maxGapUs. Timestamp
115     // will be restored to the original when the encoded frame is returned to
116     // the client.
117     // This is to solve a problem in certain real-time streaming case, where
118     // encoder's rate control logic produces huge frames after a long period
119     // of suspension on input.
120     status_t setMaxTimestampGapUs(int64_t maxGapUs);
121 
122     // Sets the time lapse (or slow motion) parameters.
123     // data[0] is the time (us) between two frames for playback
124     // data[1] is the time (us) between two frames for capture
125     // When set, the sample's timestamp will be modified to playback framerate,
126     // and capture timestamp will be modified to capture rate.
127     status_t setTimeLapseUs(int64_t* data);
128 
129     // Sets the start time us (in system time), samples before which should
130     // be dropped and not submitted to encoder
131     void setSkipFramesBeforeUs(int64_t startTimeUs);
132 
133 protected:
134     // BufferQueue::ConsumerListener interface, called when a new frame of
135     // data is available.  If we're executing and a codec buffer is
136     // available, we acquire the buffer, copy the GraphicBuffer reference
137     // into the codec buffer, and call Empty[This]Buffer.  If we're not yet
138     // executing or there's no codec buffer available, we just increment
139     // mNumFramesAvailable and return.
140     virtual void onFrameAvailable(const BufferItem& item);
141 
142     // BufferQueue::ConsumerListener interface, called when the client has
143     // released one or more GraphicBuffers.  We clear out the appropriate
144     // set of mBufferSlot entries.
145     virtual void onBuffersReleased();
146 
147     // BufferQueue::ConsumerListener interface, called when the client has
148     // changed the sideband stream. GraphicBufferSource doesn't handle sideband
149     // streams so this is a no-op (and should never be called).
150     virtual void onSidebandStreamChanged();
151 
152 private:
153     // Keep track of codec input buffers.  They may either be available
154     // (mGraphicBuffer == NULL) or in use by the codec.
155     struct CodecBuffer {
156         OMX_BUFFERHEADERTYPE* mHeader;
157 
158         // buffer producer's frame-number for buffer
159         uint64_t mFrameNumber;
160 
161         // buffer producer's buffer slot for buffer
162         int mBuf;
163 
164         sp<GraphicBuffer> mGraphicBuffer;
165     };
166 
167     // Returns the index of an available codec buffer.  If none are
168     // available, returns -1.  Mutex must be held by caller.
169     int findAvailableCodecBuffer_l();
170 
171     // Returns true if a codec buffer is available.
isCodecBufferAvailable_l()172     bool isCodecBufferAvailable_l() {
173         return findAvailableCodecBuffer_l() >= 0;
174     }
175 
176     // Finds the mCodecBuffers entry that matches.  Returns -1 if not found.
177     int findMatchingCodecBuffer_l(const OMX_BUFFERHEADERTYPE* header);
178 
179     // Fills a codec buffer with a frame from the BufferQueue.  This must
180     // only be called when we know that a frame of data is ready (i.e. we're
181     // in the onFrameAvailable callback, or if we're in codecBufferEmptied
182     // and mNumFramesAvailable is nonzero).  Returns without doing anything if
183     // we don't have a codec buffer available.
184     //
185     // Returns true if we successfully filled a codec buffer with a BQ buffer.
186     bool fillCodecBuffer_l();
187 
188     // Marks the mCodecBuffers entry as in-use, copies the GraphicBuffer
189     // reference into the codec buffer, and submits the data to the codec.
190     status_t submitBuffer_l(const BufferQueue::BufferItem &item, int cbi);
191 
192     // Submits an empty buffer, with the EOS flag set.   Returns without
193     // doing anything if we don't have a codec buffer available.
194     void submitEndOfInputStream_l();
195 
196     void setLatestSubmittedBuffer_l(const BufferQueue::BufferItem &item);
197     bool repeatLatestSubmittedBuffer_l();
198     int64_t getTimestamp(const BufferQueue::BufferItem &item);
199 
200     // Lock, covers all member variables.
201     mutable Mutex mMutex;
202 
203     // Used to report constructor failure.
204     status_t mInitCheck;
205 
206     // Pointer back to the object that contains us.  We send buffers here.
207     OMXNodeInstance* mNodeInstance;
208 
209     // Set by omxExecuting() / omxIdling().
210     bool mExecuting;
211 
212     bool mSuspended;
213 
214     // Our BufferQueue interfaces. mProducer is passed to the producer through
215     // getIGraphicBufferProducer, and mConsumer is used internally to retrieve
216     // the buffers queued by the producer.
217     sp<IGraphicBufferProducer> mProducer;
218     sp<IGraphicBufferConsumer> mConsumer;
219 
220     // Number of frames pending in BufferQueue that haven't yet been
221     // forwarded to the codec.
222     size_t mNumFramesAvailable;
223 
224     // Set to true if we want to send end-of-stream after we run out of
225     // frames in BufferQueue.
226     bool mEndOfStream;
227     bool mEndOfStreamSent;
228 
229     // Cache of GraphicBuffers from the buffer queue.  When the codec
230     // is done processing a GraphicBuffer, we can use this to map back
231     // to a slot number.
232     sp<GraphicBuffer> mBufferSlot[BufferQueue::NUM_BUFFER_SLOTS];
233 
234     // Tracks codec buffers.
235     Vector<CodecBuffer> mCodecBuffers;
236 
237     ////
238     friend class AHandlerReflector<GraphicBufferSource>;
239 
240     enum {
241         kWhatRepeatLastFrame,
242     };
243     enum {
244         kRepeatLastFrameCount = 10,
245     };
246 
247     KeyedVector<int64_t, int64_t> mOriginalTimeUs;
248     int64_t mMaxTimestampGapUs;
249     int64_t mPrevOriginalTimeUs;
250     int64_t mPrevModifiedTimeUs;
251     int64_t mSkipFramesBeforeNs;
252 
253     sp<ALooper> mLooper;
254     sp<AHandlerReflector<GraphicBufferSource> > mReflector;
255 
256     int64_t mRepeatAfterUs;
257     int32_t mRepeatLastFrameGeneration;
258     int64_t mRepeatLastFrameTimestamp;
259     int32_t mRepeatLastFrameCount;
260 
261     int mLatestSubmittedBufferId;
262     uint64_t mLatestSubmittedBufferFrameNum;
263     int32_t mLatestSubmittedBufferUseCount;
264 
265     // The previously submitted buffer should've been repeated but
266     // no codec buffer was available at the time.
267     bool mRepeatBufferDeferred;
268 
269     // Time lapse / slow motion configuration
270     int64_t mTimePerCaptureUs;
271     int64_t mTimePerFrameUs;
272     int64_t mPrevCaptureUs;
273     int64_t mPrevFrameUs;
274 
275     bool mUseGraphicBufferInMeta;
276 
277     void onMessageReceived(const sp<AMessage> &msg);
278 
279     DISALLOW_EVIL_CONSTRUCTORS(GraphicBufferSource);
280 };
281 
282 }  // namespace android
283 
284 #endif  // GRAPHIC_BUFFER_SOURCE_H_
285