1 /*
2  *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef VIDEO_VIDEO_STREAM_DECODER_IMPL_H_
12 #define VIDEO_VIDEO_STREAM_DECODER_IMPL_H_
13 
14 #include <map>
15 #include <memory>
16 #include <utility>
17 
18 #include "absl/types/optional.h"
19 #include "api/video/video_stream_decoder.h"
20 #include "modules/video_coding/frame_buffer2.h"
21 #include "modules/video_coding/timing.h"
22 #include "rtc_base/platform_thread.h"
23 #include "rtc_base/synchronization/mutex.h"
24 #include "rtc_base/task_queue.h"
25 #include "rtc_base/thread_checker.h"
26 #include "system_wrappers/include/clock.h"
27 
28 namespace webrtc {
29 
30 class VideoStreamDecoderImpl : public VideoStreamDecoderInterface {
31  public:
32   VideoStreamDecoderImpl(
33       VideoStreamDecoderInterface::Callbacks* callbacks,
34       VideoDecoderFactory* decoder_factory,
35       TaskQueueFactory* task_queue_factory,
36       std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings);
37 
38   ~VideoStreamDecoderImpl() override;
39 
40   void OnFrame(std::unique_ptr<video_coding::EncodedFrame> frame) override;
41 
42   void SetMinPlayoutDelay(TimeDelta min_delay) override;
43   void SetMaxPlayoutDelay(TimeDelta max_delay) override;
44 
45  private:
46   class DecodeCallbacks : public DecodedImageCallback {
47    public:
48     explicit DecodeCallbacks(VideoStreamDecoderImpl* video_stream_decoder_impl);
49     int32_t Decoded(VideoFrame& decodedImage) override;
50     int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override;
51     void Decoded(VideoFrame& decodedImage,
52                  absl::optional<int32_t> decode_time_ms,
53                  absl::optional<uint8_t> qp) override;
54 
55    private:
56     VideoStreamDecoderImpl* const video_stream_decoder_impl_;
57   };
58 
59   enum DecodeResult {
60     kOk,
61     kOkRequestKeyframe,
62     kDecodeFailure,
63   };
64 
65   struct FrameTimestamps {
66     int64_t timestamp;
67     int64_t decode_start_time_ms;
68     int64_t render_time_us;
69   };
70 
71   void SaveFrameTimestamps(const video_coding::EncodedFrame& frame)
72       RTC_RUN_ON(bookkeeping_queue_);
73   FrameTimestamps* GetFrameTimestamps(int64_t timestamp)
74       RTC_RUN_ON(bookkeeping_queue_);
75   void StartNextDecode() RTC_RUN_ON(bookkeeping_queue_);
76   void OnNextFrameCallback(std::unique_ptr<video_coding::EncodedFrame> frame,
77                            video_coding::FrameBuffer::ReturnReason res)
78       RTC_RUN_ON(bookkeeping_queue_);
79   void OnDecodedFrameCallback(VideoFrame& decodedImage,  // NOLINT
80                               absl::optional<int32_t> decode_time_ms,
81                               absl::optional<uint8_t> qp);
82 
83   VideoDecoder* GetDecoder(int payload_type) RTC_RUN_ON(decode_queue_);
84   VideoStreamDecoderImpl::DecodeResult DecodeFrame(
85       std::unique_ptr<video_coding::EncodedFrame> frame)
86       RTC_RUN_ON(decode_queue_);
87 
88   VCMTiming timing_;
89   DecodeCallbacks decode_callbacks_;
90 
91   // Some decoders are pipelined so it is not sufficient to save frame info
92   // for the last frame only.
93   static constexpr int kFrameTimestampsMemory = 8;
94   std::array<FrameTimestamps, kFrameTimestampsMemory> frame_timestamps_
95       RTC_GUARDED_BY(bookkeeping_queue_);
96   int next_frame_timestamps_index_ RTC_GUARDED_BY(bookkeeping_queue_);
97   VideoStreamDecoderInterface::Callbacks* const callbacks_
98       RTC_PT_GUARDED_BY(bookkeeping_queue_);
99   video_coding::VideoLayerFrameId last_continuous_id_
100       RTC_GUARDED_BY(bookkeeping_queue_);
101   bool keyframe_required_ RTC_GUARDED_BY(bookkeeping_queue_);
102 
103   absl::optional<int> current_payload_type_ RTC_GUARDED_BY(decode_queue_);
104   VideoDecoderFactory* const decoder_factory_ RTC_PT_GUARDED_BY(decode_queue_);
105   std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings_
106       RTC_GUARDED_BY(decode_queue_);
107 
108   // The |bookkeeping_queue_| use the |frame_buffer_| and also posts tasks to
109   // the |decode_queue_|. The |decode_queue_| in turn use the |decoder_| to
110   // decode frames. When the |decoder_| is done it will post back to the
111   // |bookkeeping_queue_| with the decoded frame. During shutdown we start by
112   // isolating the |bookkeeping_queue_| from the |decode_queue_|, so now it's
113   // safe for the |decode_queue_| to be destructed. After that the |decoder_|
114   // can be destructed, and then the |bookkeeping_queue_|. Finally the
115   // |frame_buffer_| can be destructed.
116   Mutex shut_down_mutex_;
117   bool shut_down_ RTC_GUARDED_BY(shut_down_mutex_);
118   video_coding::FrameBuffer frame_buffer_ RTC_GUARDED_BY(bookkeeping_queue_);
119   rtc::TaskQueue bookkeeping_queue_;
120   std::unique_ptr<VideoDecoder> decoder_ RTC_GUARDED_BY(decode_queue_);
121   rtc::TaskQueue decode_queue_;
122 };
123 
124 }  // namespace webrtc
125 
126 #endif  // VIDEO_VIDEO_STREAM_DECODER_IMPL_H_
127