1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef NUPLAYER_RENDERER_H_
18 
19 #define NUPLAYER_RENDERER_H_
20 
21 #include <media/AudioResamplerPublic.h>
22 #include <media/AVSyncSettings.h>
23 
24 #include "NuPlayer.h"
25 
26 namespace android {
27 
28 struct ABuffer;
29 class  AWakeLock;
30 struct MediaClock;
31 struct VideoFrameScheduler;
32 
33 struct NuPlayer::Renderer : public AHandler {
34     enum Flags {
35         FLAG_REAL_TIME = 1,
36         FLAG_OFFLOAD_AUDIO = 2,
37     };
38     Renderer(const sp<MediaPlayerBase::AudioSink> &sink,
39              const sp<AMessage> &notify,
40              uint32_t flags = 0);
41 
42     static size_t AudioSinkCallback(
43             MediaPlayerBase::AudioSink *audioSink,
44             void *data, size_t size, void *me,
45             MediaPlayerBase::AudioSink::cb_event_t event);
46 
47     void queueBuffer(
48             bool audio,
49             const sp<ABuffer> &buffer,
50             const sp<AMessage> &notifyConsumed);
51 
52     void queueEOS(bool audio, status_t finalResult);
53 
54     status_t setPlaybackSettings(const AudioPlaybackRate &rate /* sanitized */);
55     status_t getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */);
56     status_t setSyncSettings(const AVSyncSettings &sync, float videoFpsHint);
57     status_t getSyncSettings(AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */);
58 
59     void flush(bool audio, bool notifyComplete);
60 
61     void signalTimeDiscontinuity();
62 
63     void signalAudioSinkChanged();
64 
65     void signalDisableOffloadAudio();
66     void signalEnableOffloadAudio();
67 
68     void pause();
69     void resume();
70 
71     void setVideoFrameRate(float fps);
72 
73     status_t getCurrentPosition(int64_t *mediaUs);
74     int64_t getVideoLateByUs();
75 
76     status_t openAudioSink(
77             const sp<AMessage> &format,
78             bool offloadOnly,
79             bool hasVideo,
80             uint32_t flags,
81             bool *isOffloaded);
82     void closeAudioSink();
83 
84     enum {
85         kWhatEOS                      = 'eos ',
86         kWhatFlushComplete            = 'fluC',
87         kWhatPosition                 = 'posi',
88         kWhatVideoRenderingStart      = 'vdrd',
89         kWhatMediaRenderingStart      = 'mdrd',
90         kWhatAudioTearDown            = 'adTD',
91         kWhatAudioOffloadPauseTimeout = 'aOPT',
92     };
93 
94     enum AudioTearDownReason {
95         kDueToError = 0,   // Could restart with either offload or non-offload.
96         kDueToTimeout,
97         kForceNonOffload,  // Restart only with non-offload.
98     };
99 
100 protected:
101     virtual ~Renderer();
102 
103     virtual void onMessageReceived(const sp<AMessage> &msg);
104 
105 private:
106     enum {
107         kWhatDrainAudioQueue     = 'draA',
108         kWhatDrainVideoQueue     = 'draV',
109         kWhatPostDrainVideoQueue = 'pDVQ',
110         kWhatQueueBuffer         = 'queB',
111         kWhatQueueEOS            = 'qEOS',
112         kWhatConfigPlayback      = 'cfPB',
113         kWhatConfigSync          = 'cfSy',
114         kWhatGetPlaybackSettings = 'gPbS',
115         kWhatGetSyncSettings     = 'gSyS',
116         kWhatFlush               = 'flus',
117         kWhatPause               = 'paus',
118         kWhatResume              = 'resm',
119         kWhatOpenAudioSink       = 'opnA',
120         kWhatCloseAudioSink      = 'clsA',
121         kWhatStopAudioSink       = 'stpA',
122         kWhatDisableOffloadAudio = 'noOA',
123         kWhatEnableOffloadAudio  = 'enOA',
124         kWhatSetVideoFrameRate   = 'sVFR',
125     };
126 
127     struct QueueEntry {
128         sp<ABuffer> mBuffer;
129         sp<AMessage> mNotifyConsumed;
130         size_t mOffset;
131         status_t mFinalResult;
132         int32_t mBufferOrdinal;
133     };
134 
135     static const int64_t kMinPositionUpdateDelayUs;
136 
137     sp<MediaPlayerBase::AudioSink> mAudioSink;
138     bool mUseVirtualAudioSink;
139     sp<AMessage> mNotify;
140     Mutex mLock;
141     uint32_t mFlags;
142     List<QueueEntry> mAudioQueue;
143     List<QueueEntry> mVideoQueue;
144     uint32_t mNumFramesWritten;
145     sp<VideoFrameScheduler> mVideoScheduler;
146 
147     bool mDrainAudioQueuePending;
148     bool mDrainVideoQueuePending;
149     int32_t mAudioQueueGeneration;
150     int32_t mVideoQueueGeneration;
151     int32_t mAudioDrainGeneration;
152     int32_t mVideoDrainGeneration;
153     int32_t mAudioEOSGeneration;
154 
155     sp<MediaClock> mMediaClock;
156     float mPlaybackRate; // audio track rate
157 
158     AudioPlaybackRate mPlaybackSettings;
159     AVSyncSettings mSyncSettings;
160     float mVideoFpsHint;
161 
162     int64_t mAudioFirstAnchorTimeMediaUs;
163     int64_t mAnchorTimeMediaUs;
164     int64_t mAnchorNumFramesWritten;
165     int64_t mVideoLateByUs;
166     bool mHasAudio;
167     bool mHasVideo;
168 
169     bool mNotifyCompleteAudio;
170     bool mNotifyCompleteVideo;
171 
172     bool mSyncQueues;
173 
174     // modified on only renderer's thread.
175     bool mPaused;
176     int64_t mPauseDrainAudioAllowedUs; // time when we can drain/deliver audio in pause mode.
177 
178     bool mVideoSampleReceived;
179     bool mVideoRenderingStarted;
180     int32_t mVideoRenderingStartGeneration;
181     int32_t mAudioRenderingStartGeneration;
182     bool mRenderingDataDelivered;
183 
184     int64_t mNextAudioClockUpdateTimeUs;
185     // the media timestamp of last audio sample right before EOS.
186     int64_t mLastAudioMediaTimeUs;
187 
188     int32_t mAudioOffloadPauseTimeoutGeneration;
189     bool mAudioTornDown;
190     audio_offload_info_t mCurrentOffloadInfo;
191 
192     struct PcmInfo {
193         audio_channel_mask_t mChannelMask;
194         audio_output_flags_t mFlags;
195         audio_format_t mFormat;
196         int32_t mNumChannels;
197         int32_t mSampleRate;
198     };
199     PcmInfo mCurrentPcmInfo;
200     static const PcmInfo AUDIO_PCMINFO_INITIALIZER;
201 
202     int32_t mTotalBuffersQueued;
203     int32_t mLastAudioBufferDrained;
204     bool mUseAudioCallback;
205 
206     sp<AWakeLock> mWakeLock;
207 
208     status_t getCurrentPositionOnLooper(int64_t *mediaUs);
209     status_t getCurrentPositionOnLooper(
210             int64_t *mediaUs, int64_t nowUs, bool allowPastQueuedVideo = false);
211     bool getCurrentPositionIfPaused_l(int64_t *mediaUs);
212     status_t getCurrentPositionFromAnchor(
213             int64_t *mediaUs, int64_t nowUs, bool allowPastQueuedVideo = false);
214 
215     size_t fillAudioBuffer(void *buffer, size_t size);
216 
217     bool onDrainAudioQueue();
218     void drainAudioQueueUntilLastEOS();
219     int64_t getPendingAudioPlayoutDurationUs(int64_t nowUs);
220     void postDrainAudioQueue_l(int64_t delayUs = 0);
221 
222     void clearAnchorTime_l();
223     void clearAudioFirstAnchorTime_l();
224     void setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs);
225     void setVideoLateByUs(int64_t lateUs);
226 
227     void onNewAudioMediaTime(int64_t mediaTimeUs);
228     int64_t getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs);
229 
230     void onDrainVideoQueue();
231     void postDrainVideoQueue();
232 
233     void prepareForMediaRenderingStart_l();
234     void notifyIfMediaRenderingStarted_l();
235 
236     void onQueueBuffer(const sp<AMessage> &msg);
237     void onQueueEOS(const sp<AMessage> &msg);
238     void onFlush(const sp<AMessage> &msg);
239     void onAudioSinkChanged();
240     void onDisableOffloadAudio();
241     void onEnableOffloadAudio();
242     status_t onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */);
243     status_t onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */);
244     status_t onConfigSync(const AVSyncSettings &sync, float videoFpsHint);
245     status_t onGetSyncSettings(AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */);
246 
247     void onPause();
248     void onResume();
249     void onSetVideoFrameRate(float fps);
250     int32_t getQueueGeneration(bool audio);
251     int32_t getDrainGeneration(bool audio);
252     bool getSyncQueues();
253     void onAudioTearDown(AudioTearDownReason reason);
254     status_t onOpenAudioSink(
255             const sp<AMessage> &format,
256             bool offloadOnly,
257             bool hasVideo,
258             uint32_t flags);
259     void onCloseAudioSink();
260 
261     void notifyEOS(bool audio, status_t finalResult, int64_t delayUs = 0);
262     void notifyFlushComplete(bool audio);
263     void notifyPosition();
264     void notifyVideoLateBy(int64_t lateByUs);
265     void notifyVideoRenderingStart();
266     void notifyAudioTearDown(AudioTearDownReason reason);
267 
268     void flushQueue(List<QueueEntry> *queue);
269     bool dropBufferIfStale(bool audio, const sp<AMessage> &msg);
270     void syncQueuesDone_l();
271 
offloadingAudioRenderer272     bool offloadingAudio() const { return (mFlags & FLAG_OFFLOAD_AUDIO) != 0; }
273 
274     void startAudioOffloadPauseTimeout();
275     void cancelAudioOffloadPauseTimeout();
276 
277     int64_t getDurationUsIfPlayedAtSampleRate(uint32_t numFrames);
278 
279     DISALLOW_EVIL_CONSTRUCTORS(Renderer);
280 };
281 
282 } // namespace android
283 
284 #endif  // NUPLAYER_RENDERER_H_
285