1 /*
2  *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 #ifndef TEST_SCENARIO_VIDEO_FRAME_MATCHER_H_
11 #define TEST_SCENARIO_VIDEO_FRAME_MATCHER_H_
12 
13 #include <deque>
14 #include <map>
15 #include <memory>
16 #include <set>
17 #include <string>
18 #include <vector>
19 
20 #include "api/units/timestamp.h"
21 #include "api/video/video_frame.h"
22 #include "api/video/video_sink_interface.h"
23 #include "api/video/video_source_interface.h"
24 #include "rtc_base/ref_counted_object.h"
25 #include "rtc_base/task_queue_for_test.h"
26 #include "system_wrappers/include/clock.h"
27 #include "test/scenario/performance_stats.h"
28 
29 namespace webrtc {
30 namespace test {
31 
32 class VideoFrameMatcher {
33  public:
34   explicit VideoFrameMatcher(
35       std::vector<std::function<void(const VideoFramePair&)>>
36           frame_pair_handlers);
37   ~VideoFrameMatcher();
38   void RegisterLayer(int layer_id);
39   void OnCapturedFrame(const VideoFrame& frame, Timestamp at_time);
40   void OnDecodedFrame(const VideoFrame& frame,
41                       int layer_id,
42                       Timestamp render_time,
43                       Timestamp at_time);
44   bool Active() const;
45 
46  private:
47   struct DecodedFrameBase {
48     int id;
49     Timestamp decoded_time = Timestamp::PlusInfinity();
50     Timestamp render_time = Timestamp::PlusInfinity();
51     rtc::scoped_refptr<VideoFrameBuffer> frame;
52     rtc::scoped_refptr<VideoFrameBuffer> thumb;
53     int repeat_count = 0;
54   };
55   using DecodedFrame = rtc::RefCountedObject<DecodedFrameBase>;
56   struct CapturedFrame {
57     int id;
58     Timestamp capture_time = Timestamp::PlusInfinity();
59     rtc::scoped_refptr<VideoFrameBuffer> frame;
60     rtc::scoped_refptr<VideoFrameBuffer> thumb;
61     double best_score = INFINITY;
62     rtc::scoped_refptr<DecodedFrame> best_decode;
63     bool matched = false;
64   };
65   struct VideoLayer {
66     int layer_id;
67     std::deque<CapturedFrame> captured_frames;
68     rtc::scoped_refptr<DecodedFrame> last_decode;
69     int next_decoded_id = 1;
70   };
71   void HandleMatch(CapturedFrame captured, int layer_id);
72   void Finalize();
73   int next_capture_id_ = 1;
74   std::vector<std::function<void(const VideoFramePair&)>> frame_pair_handlers_;
75   std::map<int, VideoLayer> layers_;
76   TaskQueueForTest task_queue_;
77 };
78 
79 class CapturedFrameTap : public rtc::VideoSinkInterface<VideoFrame> {
80  public:
81   CapturedFrameTap(Clock* clock, VideoFrameMatcher* matcher);
82   CapturedFrameTap(CapturedFrameTap&) = delete;
83   CapturedFrameTap& operator=(CapturedFrameTap&) = delete;
84 
85   void OnFrame(const VideoFrame& frame) override;
86   void OnDiscardedFrame() override;
87 
88  private:
89   Clock* const clock_;
90   VideoFrameMatcher* const matcher_;
91   int discarded_count_ = 0;
92 };
93 
94 class ForwardingCapturedFrameTap
95     : public rtc::VideoSinkInterface<VideoFrame>,
96       public rtc::VideoSourceInterface<VideoFrame> {
97  public:
98   ForwardingCapturedFrameTap(Clock* clock,
99                              VideoFrameMatcher* matcher,
100                              rtc::VideoSourceInterface<VideoFrame>* source);
101   ForwardingCapturedFrameTap(ForwardingCapturedFrameTap&) = delete;
102   ForwardingCapturedFrameTap& operator=(ForwardingCapturedFrameTap&) = delete;
103 
104   // VideoSinkInterface interface
105   void OnFrame(const VideoFrame& frame) override;
106   void OnDiscardedFrame() override;
107 
108   // VideoSourceInterface interface
109   void AddOrUpdateSink(VideoSinkInterface<VideoFrame>* sink,
110                        const rtc::VideoSinkWants& wants) override;
111   void RemoveSink(VideoSinkInterface<VideoFrame>* sink) override;
112 
113  private:
114   Clock* const clock_;
115   VideoFrameMatcher* const matcher_;
116   rtc::VideoSourceInterface<VideoFrame>* const source_;
117   VideoSinkInterface<VideoFrame>* sink_ = nullptr;
118   int discarded_count_ = 0;
119 };
120 
121 class DecodedFrameTap : public rtc::VideoSinkInterface<VideoFrame> {
122  public:
123   DecodedFrameTap(Clock* clock, VideoFrameMatcher* matcher, int layer_id);
124   // VideoSinkInterface interface
125   void OnFrame(const VideoFrame& frame) override;
126 
127  private:
128   Clock* const clock_;
129   VideoFrameMatcher* const matcher_;
130   int layer_id_;
131 };
132 }  // namespace test
133 }  // namespace webrtc
134 #endif  // TEST_SCENARIO_VIDEO_FRAME_MATCHER_H_
135