1 /*
2  * cv_image_deblurring.cpp - iterative blind deblurring
3  *
4  *  Copyright (c) 2016-2017 Intel Corporation
5  *
6  * Licensed under the Apache License, Version 2.0 (the "License");
7  * you may not use this file except in compliance with the License.
8  * You may obtain a copy of the License at
9  *
10  *   http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  *
18  * Author: Andrey Parfenov <a1994ndrey@gmail.com>
19  * Author: Wind Yuan <feng.yuan@intel.com>
20  */
21 
22 #include "cv_image_deblurring.h"
23 
24 namespace XCam {
25 
26 
CVImageDeblurring()27 CVImageDeblurring::CVImageDeblurring ()
28     : CVBaseClass ()
29 {
30     _helper = new CVImageProcessHelper ();
31     _sharp = new CVImageSharp ();
32     _edgetaper = new CVEdgetaper ();
33     _wiener = new CVWienerFilter ();
34 }
35 
36 void
set_config(CVIDConfig config)37 CVImageDeblurring::set_config (CVIDConfig config)
38 {
39     _config = config;
40 }
41 
42 CVIDConfig
get_config()43 CVImageDeblurring::get_config ()
44 {
45     return _config;
46 }
47 
48 void
crop_border(cv::Mat & thresholded)49 CVImageDeblurring::crop_border (cv::Mat &thresholded)
50 {
51     int top = 0;
52     int left = 0;
53     int right = 0;
54     int bottom = 0;
55     for (int i = 0; i < thresholded.rows; i++)
56     {
57         for (int j = 0; j < thresholded.cols; j++)
58         {
59             if (thresholded.at<unsigned char>(i , j) == 255)
60             {
61                 top = i;
62                 break;
63             }
64         }
65         if (top)
66             break;
67     }
68 
69     for (int i = thresholded.rows - 1; i > 0; i--)
70     {
71         for (int j = 0; j < thresholded.cols; j++)
72         {
73             if (thresholded.at<unsigned char>(i , j) == 255)
74             {
75                 bottom = i;
76                 break;
77             }
78         }
79         if (bottom)
80             break;
81     }
82 
83     for (int i = 0; i < thresholded.cols; i++)
84     {
85         for (int j = 0; j < thresholded.rows; j++)
86         {
87             if (thresholded.at<unsigned char>(j , i) == 255)
88             {
89                 left = i;
90                 break;
91             }
92         }
93         if (left)
94             break;
95     }
96 
97     for (int i = thresholded.cols - 1; i > 0; i--)
98     {
99         for (int j = 0; j < thresholded.rows; j++)
100         {
101             if (thresholded.at<unsigned char>(j, i) == 255)
102             {
103                 right = i;
104                 break;
105             }
106         }
107         if (right)
108             break;
109     }
110     thresholded = thresholded (cv::Rect(left, top, right - left, bottom - top));
111 }
112 
113 int
estimate_kernel_size(const cv::Mat & image)114 CVImageDeblurring::estimate_kernel_size (const cv::Mat &image)
115 {
116     int kernel_size = 0;
117     cv::Mat thresholded;
118     cv::Mat dst;
119     cv::Laplacian (image, dst, -1, 3, 1, 0, cv::BORDER_CONSTANT);
120     dst.convertTo (dst, CV_32FC1);
121     cv::filter2D (dst, thresholded, -1, dst, cv::Point(-1, -1), 0, cv::BORDER_CONSTANT);
122 
123     for (int i = 0; i < 10; i++)
124     {
125         cv::Mat thresholded_new;
126         double min_val;
127         double max_val;
128         cv::minMaxLoc (thresholded, &min_val, &max_val);
129         cv::threshold (thresholded, thresholded, round(max_val / 3.5), 255, cv::THRESH_BINARY);
130         thresholded.convertTo (thresholded, CV_8UC1);
131         crop_border (thresholded);
132         if (thresholded.rows < 3)
133         {
134             break;
135         }
136         int filter_size = (int)(std::max(3, ((thresholded.rows + thresholded.cols) / 2) / 10));
137         if (!(filter_size & 1))
138         {
139             filter_size++;
140         }
141         cv::Mat filter = cv::Mat::ones (filter_size, filter_size, CV_32FC1) / (float)(filter_size * filter_size - 1);
142         filter.at<float> (filter_size / 2, filter_size / 2) = 0;
143         cv::filter2D (thresholded, thresholded_new, -1, filter, cv::Point(-1, -1), 0, cv::BORDER_CONSTANT);
144         kernel_size = (thresholded_new.rows + thresholded_new.cols) / 2;
145         if (!(kernel_size & 1))
146         {
147             kernel_size++;
148         }
149         thresholded = thresholded_new.clone();
150     }
151     return kernel_size;
152 }
153 
154 void
blind_deblurring(const cv::Mat & blurred,cv::Mat & deblurred,cv::Mat & kernel,int kernel_size,float noise_power,bool use_edgetaper)155 CVImageDeblurring::blind_deblurring (const cv::Mat &blurred, cv::Mat &deblurred, cv::Mat &kernel, int kernel_size, float noise_power, bool use_edgetaper)
156 {
157     cv::Mat gray_blurred;
158     cv::cvtColor (blurred, gray_blurred, CV_BGR2GRAY);
159     if (noise_power < 0)
160     {
161         cv::Mat median_blurred;
162         medianBlur (gray_blurred, median_blurred, 3);
163         noise_power = 1.0f / _helper->get_snr (gray_blurred, median_blurred);
164         XCAM_LOG_DEBUG ("estimated inv snr %f", noise_power);
165     }
166     if (kernel_size < 0)
167     {
168         kernel_size = estimate_kernel_size (gray_blurred);
169         XCAM_LOG_DEBUG ("estimated kernel size %d", kernel_size);
170     }
171     if (use_edgetaper) {
172         XCAM_LOG_DEBUG ("edgetaper will be used");
173     }
174     else {
175         XCAM_LOG_DEBUG ("edgetaper will not be used");
176     }
177     std::vector<cv::Mat> blurred_rgb (3);
178     cv::split (blurred, blurred_rgb);
179     std::vector<cv::Mat> deblurred_rgb (3);
180     cv::Mat result_deblurred;
181     cv::Mat result_kernel;
182     blind_deblurring_one_channel (gray_blurred, result_kernel, kernel_size, noise_power);
183     for (int i = 0; i < 3; i++)
184     {
185         cv::Mat input;
186         if (use_edgetaper)
187         {
188             _edgetaper->edgetaper (blurred_rgb[i], result_kernel, input);
189         }
190         else
191         {
192             input = blurred_rgb[i].clone ();
193         }
194         _wiener->wiener_filter (input, result_kernel, deblurred_rgb[i], noise_power);
195         _helper->apply_constraints (deblurred_rgb[i], 0);
196     }
197     cv::merge (deblurred_rgb, result_deblurred);
198     result_deblurred.convertTo (result_deblurred, CV_8UC3);
199     fastNlMeansDenoisingColored (result_deblurred, deblurred, 3, 3, 7, 21);
200     kernel = result_kernel.clone ();
201 }
202 
203 void
blind_deblurring_one_channel(const cv::Mat & blurred,cv::Mat & kernel,int kernel_size,float noise_power)204 CVImageDeblurring::blind_deblurring_one_channel (const cv::Mat &blurred, cv::Mat &kernel, int kernel_size, float noise_power)
205 {
206     cv::Mat kernel_current = cv::Mat::zeros (kernel_size, kernel_size, CV_32FC1);
207     cv::Mat deblurred_current = _helper->erosion (blurred, 2, 0);
208     float sigmar = 20;
209     for (int i = 0; i < _config.iterations; i++)
210     {
211         cv::Mat sharpened = _sharp->sharp_image_gray (deblurred_current, sigmar);
212         _wiener->wiener_filter (blurred, sharpened.clone (), kernel_current, noise_power);
213         kernel_current = kernel_current (cv::Rect (0, 0, kernel_size, kernel_size));
214         double min_val;
215         double max_val;
216         cv::minMaxLoc (kernel_current, &min_val, &max_val);
217         _helper->apply_constraints (kernel_current, (float)max_val / 20);
218         _helper->normalize_weights (kernel_current);
219         _wiener->wiener_filter (blurred, kernel_current.clone(), deblurred_current, noise_power);
220         _helper->apply_constraints (deblurred_current, 0);
221         sigmar *= 0.9;
222     }
223     kernel = kernel_current.clone ();
224 }
225 
226 }
227