1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef NUPLAYER_RENDERER_H_ 18 19 #define NUPLAYER_RENDERER_H_ 20 21 #include <atomic> 22 23 #include <media/AudioResamplerPublic.h> 24 #include <media/AVSyncSettings.h> 25 26 #include "NuPlayer.h" 27 28 namespace android { 29 30 class AWakeLock; 31 struct MediaClock; 32 class MediaCodecBuffer; 33 struct VideoFrameSchedulerBase; 34 35 struct NuPlayer::Renderer : public AHandler { 36 enum Flags { 37 FLAG_REAL_TIME = 1, 38 FLAG_OFFLOAD_AUDIO = 2, 39 }; 40 Renderer(const sp<MediaPlayerBase::AudioSink> &sink, 41 const sp<MediaClock> &mediaClock, 42 const sp<AMessage> ¬ify, 43 uint32_t flags = 0); 44 45 static size_t AudioSinkCallback( 46 MediaPlayerBase::AudioSink *audioSink, 47 void *data, size_t size, void *me, 48 MediaPlayerBase::AudioSink::cb_event_t event); 49 50 void queueBuffer( 51 bool audio, 52 const sp<MediaCodecBuffer> &buffer, 53 const sp<AMessage> ¬ifyConsumed); 54 55 void queueEOS(bool audio, status_t finalResult); 56 57 status_t setPlaybackSettings(const AudioPlaybackRate &rate /* sanitized */); 58 status_t getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */); 59 status_t setSyncSettings(const AVSyncSettings &sync, float videoFpsHint); 60 status_t getSyncSettings(AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */); 61 62 void flush(bool audio, bool notifyComplete); 63 64 void signalTimeDiscontinuity(); 65 66 void signalDisableOffloadAudio(); 67 void signalEnableOffloadAudio(); 68 69 void pause(); 70 void resume(); 71 72 void setVideoFrameRate(float fps); 73 74 status_t getCurrentPosition(int64_t *mediaUs); 75 int64_t getVideoLateByUs(); 76 77 status_t openAudioSink( 78 const sp<AMessage> &format, 79 bool offloadOnly, 80 bool hasVideo, 81 uint32_t flags, 82 bool *isOffloaded, 83 bool isStreaming); 84 void closeAudioSink(); 85 86 void dump(AString& logString); 87 88 // re-open audio sink after all pending audio buffers played. 89 void changeAudioFormat( 90 const sp<AMessage> &format, 91 bool offloadOnly, 92 bool hasVideo, 93 uint32_t flags, 94 bool isStreaming, 95 const sp<AMessage> ¬ify); 96 97 enum { 98 kWhatEOS = 'eos ', 99 kWhatFlushComplete = 'fluC', 100 kWhatPosition = 'posi', 101 kWhatVideoRenderingStart = 'vdrd', 102 kWhatMediaRenderingStart = 'mdrd', 103 kWhatAudioTearDown = 'adTD', 104 kWhatAudioOffloadPauseTimeout = 'aOPT', 105 kWhatReleaseWakeLock = 'adRL', 106 }; 107 108 enum AudioTearDownReason { 109 kDueToError = 0, // Could restart with either offload or non-offload. 110 kDueToTimeout, 111 kForceNonOffload, // Restart only with non-offload. 112 }; 113 114 protected: 115 virtual ~Renderer(); 116 117 virtual void onMessageReceived(const sp<AMessage> &msg); 118 119 private: 120 enum { 121 kWhatDrainAudioQueue = 'draA', 122 kWhatDrainVideoQueue = 'draV', 123 kWhatPostDrainVideoQueue = 'pDVQ', 124 kWhatQueueBuffer = 'queB', 125 kWhatQueueEOS = 'qEOS', 126 kWhatConfigPlayback = 'cfPB', 127 kWhatConfigSync = 'cfSy', 128 kWhatGetPlaybackSettings = 'gPbS', 129 kWhatGetSyncSettings = 'gSyS', 130 kWhatFlush = 'flus', 131 kWhatPause = 'paus', 132 kWhatResume = 'resm', 133 kWhatOpenAudioSink = 'opnA', 134 kWhatCloseAudioSink = 'clsA', 135 kWhatChangeAudioFormat = 'chgA', 136 kWhatStopAudioSink = 'stpA', 137 kWhatDisableOffloadAudio = 'noOA', 138 kWhatEnableOffloadAudio = 'enOA', 139 kWhatSetVideoFrameRate = 'sVFR', 140 }; 141 142 // if mBuffer != nullptr, it's a buffer containing real data. 143 // else if mNotifyConsumed == nullptr, it's EOS. 144 // else it's a tag for re-opening audio sink in different format. 145 struct QueueEntry { 146 sp<MediaCodecBuffer> mBuffer; 147 sp<AMessage> mMeta; 148 sp<AMessage> mNotifyConsumed; 149 size_t mOffset; 150 status_t mFinalResult; 151 int32_t mBufferOrdinal; 152 }; 153 154 static const int64_t kMinPositionUpdateDelayUs; 155 156 sp<MediaPlayerBase::AudioSink> mAudioSink; 157 bool mUseVirtualAudioSink; 158 sp<AMessage> mNotify; 159 Mutex mLock; 160 uint32_t mFlags; 161 List<QueueEntry> mAudioQueue; 162 List<QueueEntry> mVideoQueue; 163 uint32_t mNumFramesWritten; 164 sp<VideoFrameSchedulerBase> mVideoScheduler; 165 166 bool mDrainAudioQueuePending; 167 bool mDrainVideoQueuePending; 168 int32_t mAudioQueueGeneration; 169 int32_t mVideoQueueGeneration; 170 int32_t mAudioDrainGeneration; 171 int32_t mVideoDrainGeneration; 172 int32_t mAudioEOSGeneration; 173 174 const sp<MediaClock> mMediaClock; 175 float mPlaybackRate; // audio track rate 176 177 AudioPlaybackRate mPlaybackSettings; 178 AVSyncSettings mSyncSettings; 179 float mVideoFpsHint; 180 181 int64_t mAudioFirstAnchorTimeMediaUs; 182 // previous audio anchor timestamp, in media time base. 183 int64_t mAudioAnchorTimeMediaUs; 184 // previous anchor timestamp (audio or video), in media time base. 185 int64_t mAnchorTimeMediaUs; 186 int64_t mAnchorNumFramesWritten; 187 int64_t mVideoLateByUs; 188 int64_t mNextVideoTimeMediaUs; 189 bool mHasAudio; 190 bool mHasVideo; 191 192 bool mNotifyCompleteAudio; 193 bool mNotifyCompleteVideo; 194 195 bool mSyncQueues; 196 197 // modified on only renderer's thread. 198 bool mPaused; 199 int64_t mPauseDrainAudioAllowedUs; // time when we can drain/deliver audio in pause mode. 200 201 bool mVideoSampleReceived; 202 bool mVideoRenderingStarted; 203 int32_t mVideoRenderingStartGeneration; 204 int32_t mAudioRenderingStartGeneration; 205 bool mRenderingDataDelivered; 206 207 int64_t mNextAudioClockUpdateTimeUs; 208 // the media timestamp of last audio sample right before EOS. 209 int64_t mLastAudioMediaTimeUs; 210 211 int32_t mAudioOffloadPauseTimeoutGeneration; 212 bool mAudioTornDown; 213 audio_offload_info_t mCurrentOffloadInfo; 214 215 struct PcmInfo { 216 audio_channel_mask_t mChannelMask; 217 audio_output_flags_t mFlags; 218 audio_format_t mFormat; 219 int32_t mNumChannels; 220 int32_t mSampleRate; 221 }; 222 PcmInfo mCurrentPcmInfo; 223 static const PcmInfo AUDIO_PCMINFO_INITIALIZER; 224 225 int32_t mTotalBuffersQueued; 226 int32_t mLastAudioBufferDrained; 227 bool mUseAudioCallback; 228 229 sp<AWakeLock> mWakeLock; 230 231 std::atomic_flag mSyncFlag = ATOMIC_FLAG_INIT; 232 Mutex mSyncLock; 233 Condition mSyncCondition; 234 int64_t mSyncCount{0}; 235 236 status_t getCurrentPositionOnLooper(int64_t *mediaUs); 237 status_t getCurrentPositionOnLooper( 238 int64_t *mediaUs, int64_t nowUs, bool allowPastQueuedVideo = false); 239 bool getCurrentPositionIfPaused_l(int64_t *mediaUs); 240 status_t getCurrentPositionFromAnchor( 241 int64_t *mediaUs, int64_t nowUs, bool allowPastQueuedVideo = false); 242 243 struct WakeLockEvent{ 244 int64_t mTimeMs; 245 int32_t mEventTimeoutGeneration; 246 int32_t mRendererTimeoutGeneration; 247 WakeLockEventRenderer::WakeLockEvent248 WakeLockEvent(): 249 mTimeMs(0), 250 mEventTimeoutGeneration(0), 251 mRendererTimeoutGeneration(0) {} 252 updateValuesRenderer::WakeLockEvent253 void updateValues(int64_t timeMs, 254 int32_t eventGeneration, 255 int32_t rendererGeneration) { 256 mTimeMs = timeMs; 257 mEventTimeoutGeneration = eventGeneration; 258 mRendererTimeoutGeneration = rendererGeneration; 259 } 260 261 void dump(AString& logString); 262 }; 263 264 WakeLockEvent mWakelockAcquireEvent; 265 WakeLockEvent mWakelockTimeoutEvent; 266 WakeLockEvent mWakelockReleaseEvent; 267 WakeLockEvent mWakelockCancelEvent; 268 269 void notifyEOSCallback(); 270 size_t fillAudioBuffer(void *buffer, size_t size); 271 272 bool onDrainAudioQueue(); 273 void drainAudioQueueUntilLastEOS(); 274 int64_t getPendingAudioPlayoutDurationUs(int64_t nowUs); 275 void postDrainAudioQueue_l(int64_t delayUs = 0); 276 277 void clearAnchorTime(); 278 void clearAudioFirstAnchorTime_l(); 279 void setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs); 280 void setVideoLateByUs(int64_t lateUs); 281 282 void onNewAudioMediaTime(int64_t mediaTimeUs); 283 int64_t getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs); 284 285 void onDrainVideoQueue(); 286 void postDrainVideoQueue(); 287 288 void prepareForMediaRenderingStart_l(); 289 void notifyIfMediaRenderingStarted_l(); 290 291 void onQueueBuffer(const sp<AMessage> &msg); 292 void onQueueEOS(const sp<AMessage> &msg); 293 void onFlush(const sp<AMessage> &msg); 294 void onAudioSinkChanged(); 295 void onDisableOffloadAudio(); 296 void onEnableOffloadAudio(); 297 status_t onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */); 298 status_t onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */); 299 status_t onConfigSync(const AVSyncSettings &sync, float videoFpsHint); 300 status_t onGetSyncSettings(AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */); 301 302 void onPause(); 303 void onResume(); 304 void onSetVideoFrameRate(float fps); 305 int32_t getQueueGeneration(bool audio); 306 int32_t getDrainGeneration(bool audio); 307 bool getSyncQueues(); 308 void onAudioTearDown(AudioTearDownReason reason); 309 status_t onOpenAudioSink( 310 const sp<AMessage> &format, 311 bool offloadOnly, 312 bool hasVideo, 313 uint32_t flags, 314 bool isStreaming); 315 void onCloseAudioSink(); 316 void onChangeAudioFormat(const sp<AMessage> &meta, const sp<AMessage> ¬ify); 317 318 void notifyEOS(bool audio, status_t finalResult, int64_t delayUs = 0); 319 void notifyEOS_l(bool audio, status_t finalResult, int64_t delayUs = 0); 320 void notifyFlushComplete(bool audio); 321 void notifyPosition(); 322 void notifyVideoLateBy(int64_t lateByUs); 323 void notifyVideoRenderingStart(); 324 void notifyAudioTearDown(AudioTearDownReason reason); 325 326 void flushQueue(List<QueueEntry> *queue); 327 bool dropBufferIfStale(bool audio, const sp<AMessage> &msg); 328 void syncQueuesDone_l(); 329 offloadingAudioRenderer330 bool offloadingAudio() const { return (mFlags & FLAG_OFFLOAD_AUDIO) != 0; } 331 332 void startAudioOffloadPauseTimeout(); 333 void cancelAudioOffloadPauseTimeout(); 334 335 int64_t getDurationUsIfPlayedAtSampleRate(uint32_t numFrames); 336 337 DISALLOW_EVIL_CONSTRUCTORS(Renderer); 338 339 private: 340 bool mNeedVideoClearAnchor; 341 }; 342 343 } // namespace android 344 345 #endif // NUPLAYER_RENDERER_H_ 346