1 /*
2  * cl_video_stabilizer.h -  Digital Video Stabilization using IMU (Gyroscope, Accelerometer)
3  *
4  *  Copyright (c) 2017 Intel Corporation
5  *
6  * Licensed under the Apache License, Version 2.0 (the "License");
7  * you may not use this file except in compliance with the License.
8  * You may obtain a copy of the License at
9  *
10  *      http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  *
18  * Author: Zong Wei <wei.zong@intel.com>
19  */
20 
21 #ifndef XCAM_CL_VIDEO_STABILIZER_H
22 #define XCAM_CL_VIDEO_STABILIZER_H
23 
24 #include <xcam_std.h>
25 #include <meta_data.h>
26 #include <vec_mat.h>
27 #include <image_projector.h>
28 #include <ocl/cl_image_warp_handler.h>
29 
30 namespace XCam {
31 
32 class MotionFilter;
33 class ImageProjector;
34 class CLVideoStabilizer;
35 class CLImageWarpKernel;
36 class CLImageWarpHandler;
37 
38 class CLVideoStabilizerKernel
39     : public CLImageWarpKernel
40 {
41 public:
42     explicit CLVideoStabilizerKernel (
43         const SmartPtr<CLContext> &context,
44         const char *name,
45         uint32_t channel,
46         SmartPtr<CLImageHandler> &handler);
47 
48 private:
49     XCAM_DEAD_COPY (CLVideoStabilizerKernel);
50 
51     SmartPtr<CLVideoStabilizer> _handler;
52 };
53 
54 class CLVideoStabilizer
55     : public CLImageWarpHandler
56 {
57     typedef std::list<SmartPtr<VideoBuffer>> CLImageBufferList;
58 
59 public:
60     explicit CLVideoStabilizer (
61         const SmartPtr<CLContext> &context,
62         const char *name = "CLVideoStabilizer");
63 
~CLVideoStabilizer()64     virtual ~CLVideoStabilizer () {
65         _input_buf_list.clear ();
66     }
67 
68     virtual SmartPtr<VideoBuffer> get_warp_input_buf ();
69 
70     virtual bool is_ready ();
71 
72     void reset_counter ();
73 
74     XCamReturn set_sensor_calibration (CalibrationParams &params);
75     XCamReturn set_camera_intrinsics (
76         double focal_x,
77         double focal_y,
78         double offset_x,
79         double offset_y,
80         double skew);
81 
82     XCamReturn align_coordinate_system (
83         CoordinateSystemConv& world_to_device,
84         CoordinateSystemConv& device_to_image);
85 
86     XCamReturn set_motion_filter (uint32_t radius, float stdev);
filter_radius()87     uint32_t filter_radius () const {
88         return _filter_radius;
89     };
90 
91     Mat3d analyze_motion (
92         int64_t frame0_ts,
93         DevicePoseList pose0_list,
94         int64_t frame1_ts,
95         DevicePoseList pose1_list);
96 
97     Mat3d stabilize_motion (int32_t stab_frame_id, std::list<Mat3d> &motions);
98 
99 protected:
100     virtual XCamReturn prepare_parameters (SmartPtr<VideoBuffer> &input, SmartPtr<VideoBuffer> &output);
101     virtual XCamReturn execute_done (SmartPtr<VideoBuffer> &output);
102 
103 private:
104     XCAM_DEAD_COPY (CLVideoStabilizer);
105 
106 private:
107     Mat3d                    _intrinsics;
108     CalibrationParams        _calib_params;
109     SmartPtr<ImageProjector> _projector;
110     SmartPtr<MotionFilter>   _motion_filter;
111     CoordinateSystemConv     _world_to_device;
112     CoordinateSystemConv     _device_to_image;
113     int64_t                  _input_frame_id;
114     int64_t                  _frame_ts[2];
115     int64_t                  _stabilized_frame_id;
116     DevicePoseList           _device_pose[2];
117     std::list<Mat3d>         _motions; //motions[i] calculated from frame i to i+1
118     uint32_t                 _filter_radius;
119     CLImageBufferList        _input_buf_list;
120 };
121 
122 SmartPtr<CLImageHandler>
123 create_cl_video_stab_handler (const SmartPtr<CLContext> &context);
124 
125 
126 class MotionFilter
127 {
128 public:
129     MotionFilter (uint32_t radius = 15, float stdev = 10);
130     virtual ~MotionFilter ();
131 
132     void set_filters (uint32_t radius, float stdev);
133 
radius()134     uint32_t radius () const {
135         return _radius;
136     };
stdev()137     float stdev () const {
138         return _stdev;
139     };
140 
141     Mat3d stabilize (int32_t index,
142                      std::list<Mat3d> &motions,
143                      int32_t max);
144 
145 protected:
146     Mat3d cumulate_motion (uint32_t index, uint32_t from, std::list<Mat3d> &motions);
147 
148 private:
149     XCAM_DEAD_COPY (MotionFilter);
150 
151 private:
152     int32_t            _radius;
153     float              _stdev;
154     std::vector<float> _weight;
155 };
156 
157 }
158 #endif
159