1 /*M///////////////////////////////////////////////////////////////////////////////////////
2 //
3 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
4 //
5 //  By downloading, copying, installing or using the software you agree to this license.
6 //  If you do not agree to this license, do not download, install,
7 //  copy or use the software.
8 //
9 //
10 //                           License Agreement
11 //                For Open Source Computer Vision Library
12 //
13 // Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
14 // Copyright (C) 2009, Willow Garage Inc., all rights reserved.
15 // Third party copyrights are property of their respective owners.
16 //
17 // Redistribution and use in source and binary forms, with or without modification,
18 // are permitted provided that the following conditions are met:
19 //
20 //   * Redistribution's of source code must retain the above copyright notice,
21 //     this list of conditions and the following disclaimer.
22 //
23 //   * Redistribution's in binary form must reproduce the above copyright notice,
24 //     this list of conditions and the following disclaimer in the documentation
25 //     and/or other materials provided with the distribution.
26 //
27 //   * The name of the copyright holders may not be used to endorse or promote products
28 //     derived from this software without specific prior written permission.
29 //
30 // This software is provided by the copyright holders and contributors "as is" and
31 // any express or implied warranties, including, but not limited to, the implied
32 // warranties of merchantability and fitness for a particular purpose are disclaimed.
33 // In no event shall the Intel Corporation or contributors be liable for any direct,
34 // indirect, incidental, special, exemplary, or consequential damages
35 // (including, but not limited to, procurement of substitute goods or services;
36 // loss of use, data, or profits; or business interruption) however caused
37 // and on any theory of liability, whether in contract, strict liability,
38 // or tort (including negligence or otherwise) arising in any way out of
39 // the use of this software, even if advised of the possibility of such damage.
40 //
41 //M*/
42 
43 // S. Farsiu , D. Robinson, M. Elad, P. Milanfar. Fast and robust multiframe super resolution.
44 // Dennis Mitzel, Thomas Pock, Thomas Schoenemann, Daniel Cremers. Video Super Resolution using Duality Based TV-L1 Optical Flow.
45 
46 #include "precomp.hpp"
47 #include "opencl_kernels_superres.hpp"
48 
49 using namespace cv;
50 using namespace cv::superres;
51 using namespace cv::superres::detail;
52 
53 namespace
54 {
55 #ifdef HAVE_OPENCL
56 
ocl_calcRelativeMotions(InputArrayOfArrays _forwardMotions,InputArrayOfArrays _backwardMotions,OutputArrayOfArrays _relForwardMotions,OutputArrayOfArrays _relBackwardMotions,int baseIdx,const Size & size)57     bool ocl_calcRelativeMotions(InputArrayOfArrays _forwardMotions, InputArrayOfArrays _backwardMotions,
58                                  OutputArrayOfArrays _relForwardMotions, OutputArrayOfArrays _relBackwardMotions,
59                                  int baseIdx, const Size & size)
60     {
61         std::vector<UMat> & forwardMotions = *(std::vector<UMat> *)_forwardMotions.getObj(),
62                 & backwardMotions = *(std::vector<UMat> *)_backwardMotions.getObj(),
63                 & relForwardMotions = *(std::vector<UMat> *)_relForwardMotions.getObj(),
64                 & relBackwardMotions = *(std::vector<UMat> *)_relBackwardMotions.getObj();
65 
66         const int count = static_cast<int>(forwardMotions.size());
67 
68         relForwardMotions.resize(count);
69         relForwardMotions[baseIdx].create(size, CV_32FC2);
70         relForwardMotions[baseIdx].setTo(Scalar::all(0));
71 
72         relBackwardMotions.resize(count);
73         relBackwardMotions[baseIdx].create(size, CV_32FC2);
74         relBackwardMotions[baseIdx].setTo(Scalar::all(0));
75 
76         for (int i = baseIdx - 1; i >= 0; --i)
77         {
78             add(relForwardMotions[i + 1], forwardMotions[i], relForwardMotions[i]);
79             add(relBackwardMotions[i + 1], backwardMotions[i + 1], relBackwardMotions[i]);
80         }
81 
82         for (int i = baseIdx + 1; i < count; ++i)
83         {
84             add(relForwardMotions[i - 1], backwardMotions[i], relForwardMotions[i]);
85             add(relBackwardMotions[i - 1], forwardMotions[i - 1], relBackwardMotions[i]);
86         }
87 
88         return true;
89     }
90 
91 #endif
92 
calcRelativeMotions(InputArrayOfArrays _forwardMotions,InputArrayOfArrays _backwardMotions,OutputArrayOfArrays _relForwardMotions,OutputArrayOfArrays _relBackwardMotions,int baseIdx,const Size & size)93     void calcRelativeMotions(InputArrayOfArrays _forwardMotions, InputArrayOfArrays _backwardMotions,
94                              OutputArrayOfArrays _relForwardMotions, OutputArrayOfArrays _relBackwardMotions,
95                              int baseIdx, const Size & size)
96     {
97         CV_OCL_RUN(_forwardMotions.isUMatVector() && _backwardMotions.isUMatVector() &&
98                    _relForwardMotions.isUMatVector() && _relBackwardMotions.isUMatVector(),
99                    ocl_calcRelativeMotions(_forwardMotions, _backwardMotions, _relForwardMotions,
100                                            _relBackwardMotions, baseIdx, size))
101 
102         std::vector<Mat> & forwardMotions = *(std::vector<Mat> *)_forwardMotions.getObj(),
103                 & backwardMotions = *(std::vector<Mat> *)_backwardMotions.getObj(),
104                 & relForwardMotions = *(std::vector<Mat> *)_relForwardMotions.getObj(),
105                 & relBackwardMotions = *(std::vector<Mat> *)_relBackwardMotions.getObj();
106 
107         const int count = static_cast<int>(forwardMotions.size());
108 
109         relForwardMotions.resize(count);
110         relForwardMotions[baseIdx].create(size, CV_32FC2);
111         relForwardMotions[baseIdx].setTo(Scalar::all(0));
112 
113         relBackwardMotions.resize(count);
114         relBackwardMotions[baseIdx].create(size, CV_32FC2);
115         relBackwardMotions[baseIdx].setTo(Scalar::all(0));
116 
117         for (int i = baseIdx - 1; i >= 0; --i)
118         {
119             add(relForwardMotions[i + 1], forwardMotions[i], relForwardMotions[i]);
120             add(relBackwardMotions[i + 1], backwardMotions[i + 1], relBackwardMotions[i]);
121         }
122 
123         for (int i = baseIdx + 1; i < count; ++i)
124         {
125             add(relForwardMotions[i - 1], backwardMotions[i], relForwardMotions[i]);
126             add(relBackwardMotions[i - 1], forwardMotions[i - 1], relBackwardMotions[i]);
127         }
128     }
129 #ifdef HAVE_OPENCL
130 
ocl_upscaleMotions(InputArrayOfArrays _lowResMotions,OutputArrayOfArrays _highResMotions,int scale)131     bool ocl_upscaleMotions(InputArrayOfArrays _lowResMotions, OutputArrayOfArrays _highResMotions, int scale)
132     {
133         std::vector<UMat> & lowResMotions = *(std::vector<UMat> *)_lowResMotions.getObj(),
134                 & highResMotions = *(std::vector<UMat> *)_highResMotions.getObj();
135 
136         highResMotions.resize(lowResMotions.size());
137 
138         for (size_t i = 0; i < lowResMotions.size(); ++i)
139         {
140             resize(lowResMotions[i], highResMotions[i], Size(), scale, scale, INTER_LINEAR); // TODO
141             multiply(highResMotions[i], Scalar::all(scale), highResMotions[i]);
142         }
143 
144         return true;
145     }
146 
147 #endif
148 
upscaleMotions(InputArrayOfArrays _lowResMotions,OutputArrayOfArrays _highResMotions,int scale)149     void upscaleMotions(InputArrayOfArrays _lowResMotions, OutputArrayOfArrays _highResMotions, int scale)
150     {
151         CV_OCL_RUN(_lowResMotions.isUMatVector() && _highResMotions.isUMatVector(),
152                    ocl_upscaleMotions(_lowResMotions, _highResMotions, scale))
153 
154         std::vector<Mat> & lowResMotions = *(std::vector<Mat> *)_lowResMotions.getObj(),
155                 & highResMotions = *(std::vector<Mat> *)_highResMotions.getObj();
156 
157         highResMotions.resize(lowResMotions.size());
158 
159         for (size_t i = 0; i < lowResMotions.size(); ++i)
160         {
161             resize(lowResMotions[i], highResMotions[i], Size(), scale, scale, INTER_CUBIC);
162             multiply(highResMotions[i], Scalar::all(scale), highResMotions[i]);
163         }
164     }
165 
166 #ifdef HAVE_OPENCL
167 
ocl_buildMotionMaps(InputArray _forwardMotion,InputArray _backwardMotion,OutputArray _forwardMap,OutputArray _backwardMap)168     bool ocl_buildMotionMaps(InputArray _forwardMotion, InputArray _backwardMotion,
169                              OutputArray _forwardMap, OutputArray _backwardMap)
170     {
171         ocl::Kernel k("buildMotionMaps", ocl::superres::superres_btvl1_oclsrc);
172         if (k.empty())
173             return false;
174 
175         UMat forwardMotion = _forwardMotion.getUMat(), backwardMotion = _backwardMotion.getUMat();
176         Size size = forwardMotion.size();
177 
178         _forwardMap.create(size, CV_32FC2);
179         _backwardMap.create(size, CV_32FC2);
180 
181         UMat forwardMap = _forwardMap.getUMat(), backwardMap = _backwardMap.getUMat();
182 
183         k.args(ocl::KernelArg::ReadOnlyNoSize(forwardMotion),
184                ocl::KernelArg::ReadOnlyNoSize(backwardMotion),
185                ocl::KernelArg::WriteOnlyNoSize(forwardMap),
186                ocl::KernelArg::WriteOnly(backwardMap));
187 
188         size_t globalsize[2] = { size.width, size.height };
189         return k.run(2, globalsize, NULL, false);
190     }
191 
192 #endif
193 
buildMotionMaps(InputArray _forwardMotion,InputArray _backwardMotion,OutputArray _forwardMap,OutputArray _backwardMap)194     void buildMotionMaps(InputArray _forwardMotion, InputArray _backwardMotion,
195                          OutputArray _forwardMap, OutputArray _backwardMap)
196     {
197         CV_OCL_RUN(_forwardMap.isUMat() && _backwardMap.isUMat(),
198                    ocl_buildMotionMaps(_forwardMotion, _backwardMotion, _forwardMap,
199                                        _backwardMap));
200 
201         Mat forwardMotion = _forwardMotion.getMat(), backwardMotion = _backwardMotion.getMat();
202 
203         _forwardMap.create(forwardMotion.size(), CV_32FC2);
204         _backwardMap.create(forwardMotion.size(), CV_32FC2);
205 
206         Mat forwardMap = _forwardMap.getMat(), backwardMap = _backwardMap.getMat();
207 
208         for (int y = 0; y < forwardMotion.rows; ++y)
209         {
210             const Point2f* forwardMotionRow = forwardMotion.ptr<Point2f>(y);
211             const Point2f* backwardMotionRow = backwardMotion.ptr<Point2f>(y);
212             Point2f* forwardMapRow = forwardMap.ptr<Point2f>(y);
213             Point2f* backwardMapRow = backwardMap.ptr<Point2f>(y);
214 
215             for (int x = 0; x < forwardMotion.cols; ++x)
216             {
217                 Point2f base(static_cast<float>(x), static_cast<float>(y));
218 
219                 forwardMapRow[x] = base + backwardMotionRow[x];
220                 backwardMapRow[x] = base + forwardMotionRow[x];
221             }
222         }
223     }
224 
225     template <typename T>
upscaleImpl(InputArray _src,OutputArray _dst,int scale)226     void upscaleImpl(InputArray _src, OutputArray _dst, int scale)
227     {
228         Mat src = _src.getMat();
229         _dst.create(src.rows * scale, src.cols * scale, src.type());
230         _dst.setTo(Scalar::all(0));
231         Mat dst = _dst.getMat();
232 
233         for (int y = 0, Y = 0; y < src.rows; ++y, Y += scale)
234         {
235             const T * const srcRow = src.ptr<T>(y);
236             T * const dstRow = dst.ptr<T>(Y);
237 
238             for (int x = 0, X = 0; x < src.cols; ++x, X += scale)
239                 dstRow[X] = srcRow[x];
240         }
241     }
242 
243 #ifdef HAVE_OPENCL
244 
ocl_upscale(InputArray _src,OutputArray _dst,int scale)245     static bool ocl_upscale(InputArray _src, OutputArray _dst, int scale)
246     {
247         int type = _src.type(), cn = CV_MAT_CN(type);
248         ocl::Kernel k("upscale", ocl::superres::superres_btvl1_oclsrc,
249                       format("-D cn=%d", cn));
250         if (k.empty())
251             return false;
252 
253         UMat src = _src.getUMat();
254         _dst.create(src.rows * scale, src.cols * scale, type);
255         _dst.setTo(Scalar::all(0));
256         UMat dst = _dst.getUMat();
257 
258         k.args(ocl::KernelArg::ReadOnly(src),
259                ocl::KernelArg::ReadWriteNoSize(dst), scale);
260 
261         size_t globalsize[2] = { src.cols, src.rows };
262         return k.run(2, globalsize, NULL, false);
263     }
264 
265 #endif
266 
267     typedef struct _Point4f { float ar[4]; } Point4f;
268 
upscale(InputArray _src,OutputArray _dst,int scale)269     void upscale(InputArray _src, OutputArray _dst, int scale)
270     {
271         int cn = _src.channels();
272         CV_Assert( cn == 1 || cn == 3 || cn == 4 );
273 
274         CV_OCL_RUN(_dst.isUMat(),
275                    ocl_upscale(_src, _dst, scale))
276 
277         typedef void (*func_t)(InputArray src, OutputArray dst, int scale);
278         static const func_t funcs[] =
279         {
280             0, upscaleImpl<float>, 0, upscaleImpl<Point3f>, upscaleImpl<Point4f>
281         };
282 
283         const func_t func = funcs[cn];
284         CV_Assert(func != 0);
285         func(_src, _dst, scale);
286     }
287 
diffSign(float a,float b)288     inline float diffSign(float a, float b)
289     {
290         return a > b ? 1.0f : a < b ? -1.0f : 0.0f;
291     }
292 
diffSign(Point3f a,Point3f b)293     Point3f diffSign(Point3f a, Point3f b)
294     {
295         return Point3f(
296             a.x > b.x ? 1.0f : a.x < b.x ? -1.0f : 0.0f,
297             a.y > b.y ? 1.0f : a.y < b.y ? -1.0f : 0.0f,
298             a.z > b.z ? 1.0f : a.z < b.z ? -1.0f : 0.0f
299         );
300     }
301 
302 #ifdef HAVE_OPENCL
303 
ocl_diffSign(InputArray _src1,OutputArray _src2,OutputArray _dst)304     static bool ocl_diffSign(InputArray _src1, OutputArray _src2, OutputArray _dst)
305     {
306         ocl::Kernel k("diffSign", ocl::superres::superres_btvl1_oclsrc);
307         if (k.empty())
308             return false;
309 
310         UMat src1 = _src1.getUMat(), src2 = _src2.getUMat();
311         _dst.create(src1.size(), src1.type());
312         UMat dst = _dst.getUMat();
313 
314         int cn = src1.channels();
315         k.args(ocl::KernelArg::ReadOnlyNoSize(src1),
316                ocl::KernelArg::ReadOnlyNoSize(src2),
317                ocl::KernelArg::WriteOnly(dst, cn));
318 
319         size_t globalsize[2] = { src1.cols * cn, src1.rows };
320         return k.run(2, globalsize, NULL, false);
321     }
322 
323 #endif
324 
diffSign(InputArray _src1,OutputArray _src2,OutputArray _dst)325     void diffSign(InputArray _src1, OutputArray _src2, OutputArray _dst)
326     {
327         CV_OCL_RUN(_dst.isUMat(),
328                    ocl_diffSign(_src1, _src2, _dst))
329 
330         Mat src1 = _src1.getMat(), src2 = _src2.getMat();
331         _dst.create(src1.size(), src1.type());
332         Mat dst = _dst.getMat();
333 
334         const int count = src1.cols * src1.channels();
335 
336         for (int y = 0; y < src1.rows; ++y)
337         {
338             const float * const src1Ptr = src1.ptr<float>(y);
339             const float * const src2Ptr = src2.ptr<float>(y);
340             float* dstPtr = dst.ptr<float>(y);
341 
342             for (int x = 0; x < count; ++x)
343                 dstPtr[x] = diffSign(src1Ptr[x], src2Ptr[x]);
344         }
345     }
346 
calcBtvWeights(int btvKernelSize,double alpha,std::vector<float> & btvWeights)347     void calcBtvWeights(int btvKernelSize, double alpha, std::vector<float>& btvWeights)
348     {
349         const size_t size = btvKernelSize * btvKernelSize;
350 
351         btvWeights.resize(size);
352 
353         const int ksize = (btvKernelSize - 1) / 2;
354         const float alpha_f = static_cast<float>(alpha);
355 
356         for (int m = 0, ind = 0; m <= ksize; ++m)
357         {
358             for (int l = ksize; l + m >= 0; --l, ++ind)
359                 btvWeights[ind] = pow(alpha_f, std::abs(m) + std::abs(l));
360         }
361     }
362 
363     template <typename T>
364     struct BtvRegularizationBody : ParallelLoopBody
365     {
366         void operator ()(const Range& range) const;
367 
368         Mat src;
369         mutable Mat dst;
370         int ksize;
371         const float* btvWeights;
372     };
373 
374     template <typename T>
operator ()(const Range & range) const375     void BtvRegularizationBody<T>::operator ()(const Range& range) const
376     {
377         for (int i = range.start; i < range.end; ++i)
378         {
379             const T * const srcRow = src.ptr<T>(i);
380             T * const dstRow = dst.ptr<T>(i);
381 
382             for(int j = ksize; j < src.cols - ksize; ++j)
383             {
384                 const T srcVal = srcRow[j];
385 
386                 for (int m = 0, ind = 0; m <= ksize; ++m)
387                 {
388                     const T* srcRow2 = src.ptr<T>(i - m);
389                     const T* srcRow3 = src.ptr<T>(i + m);
390 
391                     for (int l = ksize; l + m >= 0; --l, ++ind)
392                         dstRow[j] += btvWeights[ind] * (diffSign(srcVal, srcRow3[j + l])
393                                                         - diffSign(srcRow2[j - l], srcVal));
394                 }
395             }
396         }
397     }
398 
399     template <typename T>
calcBtvRegularizationImpl(InputArray _src,OutputArray _dst,int btvKernelSize,const std::vector<float> & btvWeights)400     void calcBtvRegularizationImpl(InputArray _src, OutputArray _dst, int btvKernelSize, const std::vector<float>& btvWeights)
401     {
402         Mat src = _src.getMat();
403         _dst.create(src.size(), src.type());
404         _dst.setTo(Scalar::all(0));
405         Mat dst = _dst.getMat();
406 
407         const int ksize = (btvKernelSize - 1) / 2;
408 
409         BtvRegularizationBody<T> body;
410 
411         body.src = src;
412         body.dst = dst;
413         body.ksize = ksize;
414         body.btvWeights = &btvWeights[0];
415 
416         parallel_for_(Range(ksize, src.rows - ksize), body);
417     }
418 
419 #ifdef HAVE_OPENCL
420 
ocl_calcBtvRegularization(InputArray _src,OutputArray _dst,int btvKernelSize,const UMat & ubtvWeights)421     static bool ocl_calcBtvRegularization(InputArray _src, OutputArray _dst, int btvKernelSize, const UMat & ubtvWeights)
422     {
423         int cn = _src.channels();
424         ocl::Kernel k("calcBtvRegularization", ocl::superres::superres_btvl1_oclsrc,
425                       format("-D cn=%d", cn));
426         if (k.empty())
427             return false;
428 
429         UMat src = _src.getUMat();
430         _dst.create(src.size(), src.type());
431         _dst.setTo(Scalar::all(0));
432         UMat dst = _dst.getUMat();
433 
434         const int ksize = (btvKernelSize - 1) / 2;
435 
436         k.args(ocl::KernelArg::ReadOnlyNoSize(src), ocl::KernelArg::WriteOnly(dst),
437               ksize, ocl::KernelArg::PtrReadOnly(ubtvWeights));
438 
439         size_t globalsize[2] = { src.cols, src.rows };
440         return k.run(2, globalsize, NULL, false);
441     }
442 
443 #endif
444 
calcBtvRegularization(InputArray _src,OutputArray _dst,int btvKernelSize,const std::vector<float> & btvWeights,const UMat & ubtvWeights)445     void calcBtvRegularization(InputArray _src, OutputArray _dst, int btvKernelSize,
446                                const std::vector<float>& btvWeights, const UMat & ubtvWeights)
447     {
448         CV_OCL_RUN(_dst.isUMat(),
449                    ocl_calcBtvRegularization(_src, _dst, btvKernelSize, ubtvWeights))
450         (void)ubtvWeights;
451 
452         typedef void (*func_t)(InputArray _src, OutputArray _dst, int btvKernelSize, const std::vector<float>& btvWeights);
453         static const func_t funcs[] =
454         {
455             0, calcBtvRegularizationImpl<float>, 0, calcBtvRegularizationImpl<Point3f>, 0
456         };
457 
458         const func_t func = funcs[_src.channels()];
459         CV_Assert(func != 0);
460         func(_src, _dst, btvKernelSize, btvWeights);
461     }
462 
463     class BTVL1_Base : public cv::superres::SuperResolution
464     {
465     public:
466         BTVL1_Base();
467 
468         void process(InputArrayOfArrays src, OutputArray dst, InputArrayOfArrays forwardMotions,
469                      InputArrayOfArrays backwardMotions, int baseIdx);
470 
471         void collectGarbage();
472 
473         CV_IMPL_PROPERTY(int, Scale, scale_)
474         CV_IMPL_PROPERTY(int, Iterations, iterations_)
475         CV_IMPL_PROPERTY(double, Tau, tau_)
476         CV_IMPL_PROPERTY(double, Labmda, lambda_)
477         CV_IMPL_PROPERTY(double, Alpha, alpha_)
478         CV_IMPL_PROPERTY(int, KernelSize, btvKernelSize_)
479         CV_IMPL_PROPERTY(int, BlurKernelSize, blurKernelSize_)
480         CV_IMPL_PROPERTY(double, BlurSigma, blurSigma_)
481         CV_IMPL_PROPERTY(int, TemporalAreaRadius, temporalAreaRadius_)
482         CV_IMPL_PROPERTY_S(Ptr<cv::superres::DenseOpticalFlowExt>, OpticalFlow, opticalFlow_)
483 
484     protected:
485         int scale_;
486         int iterations_;
487         double tau_;
488         double lambda_;
489         double alpha_;
490         int btvKernelSize_;
491         int blurKernelSize_;
492         double blurSigma_;
493         int temporalAreaRadius_; // not used in some implementations
494         Ptr<cv::superres::DenseOpticalFlowExt> opticalFlow_;
495 
496     private:
497         bool ocl_process(InputArrayOfArrays src, OutputArray dst, InputArrayOfArrays forwardMotions,
498                          InputArrayOfArrays backwardMotions, int baseIdx);
499 
500         //Ptr<FilterEngine> filter_;
501         int curBlurKernelSize_;
502         double curBlurSigma_;
503         int curSrcType_;
504 
505         std::vector<float> btvWeights_;
506         UMat ubtvWeights_;
507 
508         int curBtvKernelSize_;
509         double curAlpha_;
510 
511         // Mat
512         std::vector<Mat> lowResForwardMotions_;
513         std::vector<Mat> lowResBackwardMotions_;
514 
515         std::vector<Mat> highResForwardMotions_;
516         std::vector<Mat> highResBackwardMotions_;
517 
518         std::vector<Mat> forwardMaps_;
519         std::vector<Mat> backwardMaps_;
520 
521         Mat highRes_;
522 
523         Mat diffTerm_, regTerm_;
524         Mat a_, b_, c_;
525 
526 #ifdef HAVE_OPENCL
527         // UMat
528         std::vector<UMat> ulowResForwardMotions_;
529         std::vector<UMat> ulowResBackwardMotions_;
530 
531         std::vector<UMat> uhighResForwardMotions_;
532         std::vector<UMat> uhighResBackwardMotions_;
533 
534         std::vector<UMat> uforwardMaps_;
535         std::vector<UMat> ubackwardMaps_;
536 
537         UMat uhighRes_;
538 
539         UMat udiffTerm_, uregTerm_;
540         UMat ua_, ub_, uc_;
541 #endif
542     };
543 
BTVL1_Base()544     BTVL1_Base::BTVL1_Base()
545     {
546         scale_ = 4;
547         iterations_ = 180;
548         lambda_ = 0.03;
549         tau_ = 1.3;
550         alpha_ = 0.7;
551         btvKernelSize_ = 7;
552         blurKernelSize_ = 5;
553         blurSigma_ = 0.0;
554         temporalAreaRadius_ = 0;
555         opticalFlow_ = createOptFlow_Farneback();
556 
557         curBlurKernelSize_ = -1;
558         curBlurSigma_ = -1.0;
559         curSrcType_ = -1;
560 
561         curBtvKernelSize_ = -1;
562         curAlpha_ = -1.0;
563     }
564 
565 #ifdef HAVE_OPENCL
566 
ocl_process(InputArrayOfArrays _src,OutputArray _dst,InputArrayOfArrays _forwardMotions,InputArrayOfArrays _backwardMotions,int baseIdx)567     bool BTVL1_Base::ocl_process(InputArrayOfArrays _src, OutputArray _dst, InputArrayOfArrays _forwardMotions,
568                                  InputArrayOfArrays _backwardMotions, int baseIdx)
569     {
570         std::vector<UMat> & src = *(std::vector<UMat> *)_src.getObj(),
571                 & forwardMotions = *(std::vector<UMat> *)_forwardMotions.getObj(),
572                 & backwardMotions = *(std::vector<UMat> *)_backwardMotions.getObj();
573 
574         // update blur filter and btv weights
575         if (blurKernelSize_ != curBlurKernelSize_ || blurSigma_ != curBlurSigma_ || src[0].type() != curSrcType_)
576         {
577             //filter_ = createGaussianFilter(src[0].type(), Size(blurKernelSize_, blurKernelSize_), blurSigma_);
578             curBlurKernelSize_ = blurKernelSize_;
579             curBlurSigma_ = blurSigma_;
580             curSrcType_ = src[0].type();
581         }
582 
583         if (btvWeights_.empty() || btvKernelSize_ != curBtvKernelSize_ || alpha_ != curAlpha_)
584         {
585             calcBtvWeights(btvKernelSize_, alpha_, btvWeights_);
586             Mat(btvWeights_, true).copyTo(ubtvWeights_);
587 
588             curBtvKernelSize_ = btvKernelSize_;
589             curAlpha_ = alpha_;
590         }
591 
592         // calc high res motions
593         calcRelativeMotions(forwardMotions, backwardMotions, ulowResForwardMotions_, ulowResBackwardMotions_, baseIdx, src[0].size());
594 
595         upscaleMotions(ulowResForwardMotions_, uhighResForwardMotions_, scale_);
596         upscaleMotions(ulowResBackwardMotions_, uhighResBackwardMotions_, scale_);
597 
598         uforwardMaps_.resize(uhighResForwardMotions_.size());
599         ubackwardMaps_.resize(uhighResForwardMotions_.size());
600         for (size_t i = 0; i < uhighResForwardMotions_.size(); ++i)
601             buildMotionMaps(uhighResForwardMotions_[i], uhighResBackwardMotions_[i], uforwardMaps_[i], ubackwardMaps_[i]);
602 
603         // initial estimation
604         const Size lowResSize = src[0].size();
605         const Size highResSize(lowResSize.width * scale_, lowResSize.height * scale_);
606 
607         resize(src[baseIdx], uhighRes_, highResSize, 0, 0, INTER_LINEAR); // TODO
608 
609         // iterations
610         udiffTerm_.create(highResSize, uhighRes_.type());
611         ua_.create(highResSize, uhighRes_.type());
612         ub_.create(highResSize, uhighRes_.type());
613         uc_.create(lowResSize, uhighRes_.type());
614 
615         for (int i = 0; i < iterations_; ++i)
616         {
617             udiffTerm_.setTo(Scalar::all(0));
618 
619             for (size_t k = 0; k < src.size(); ++k)
620             {
621                 // a = M * Ih
622                 remap(uhighRes_, ua_, ubackwardMaps_[k], noArray(), INTER_NEAREST);
623                 // b = HM * Ih
624                 GaussianBlur(ua_, ub_, Size(blurKernelSize_, blurKernelSize_), blurSigma_);
625                 // c = DHM * Ih
626                 resize(ub_, uc_, lowResSize, 0, 0, INTER_NEAREST);
627 
628                 diffSign(src[k], uc_, uc_);
629 
630                 // a = Dt * diff
631                 upscale(uc_, ua_, scale_);
632 
633                 // b = HtDt * diff
634                 GaussianBlur(ua_, ub_, Size(blurKernelSize_, blurKernelSize_), blurSigma_);
635                 // a = MtHtDt * diff
636                 remap(ub_, ua_, uforwardMaps_[k], noArray(), INTER_NEAREST);
637 
638                 add(udiffTerm_, ua_, udiffTerm_);
639             }
640 
641             if (lambda_ > 0)
642             {
643                 calcBtvRegularization(uhighRes_, uregTerm_, btvKernelSize_, btvWeights_, ubtvWeights_);
644                 addWeighted(udiffTerm_, 1.0, uregTerm_, -lambda_, 0.0, udiffTerm_);
645             }
646 
647             addWeighted(uhighRes_, 1.0, udiffTerm_, tau_, 0.0, uhighRes_);
648         }
649 
650         Rect inner(btvKernelSize_, btvKernelSize_, uhighRes_.cols - 2 * btvKernelSize_, uhighRes_.rows - 2 * btvKernelSize_);
651         uhighRes_(inner).copyTo(_dst);
652 
653         return true;
654     }
655 
656 #endif
657 
process(InputArrayOfArrays _src,OutputArray _dst,InputArrayOfArrays _forwardMotions,InputArrayOfArrays _backwardMotions,int baseIdx)658     void BTVL1_Base::process(InputArrayOfArrays _src, OutputArray _dst, InputArrayOfArrays _forwardMotions,
659                              InputArrayOfArrays _backwardMotions, int baseIdx)
660     {
661         CV_Assert( scale_ > 1 );
662         CV_Assert( iterations_ > 0 );
663         CV_Assert( tau_ > 0.0 );
664         CV_Assert( alpha_ > 0.0 );
665         CV_Assert( btvKernelSize_ > 0 );
666         CV_Assert( blurKernelSize_ > 0 );
667         CV_Assert( blurSigma_ >= 0.0 );
668 
669         CV_OCL_RUN(_src.isUMatVector() && _dst.isUMat() && _forwardMotions.isUMatVector() &&
670                    _backwardMotions.isUMatVector(),
671                    ocl_process(_src, _dst, _forwardMotions, _backwardMotions, baseIdx))
672 
673         std::vector<Mat> & src = *(std::vector<Mat> *)_src.getObj(),
674                 & forwardMotions = *(std::vector<Mat> *)_forwardMotions.getObj(),
675                 & backwardMotions = *(std::vector<Mat> *)_backwardMotions.getObj();
676 
677         // update blur filter and btv weights
678         if (blurKernelSize_ != curBlurKernelSize_ || blurSigma_ != curBlurSigma_ || src[0].type() != curSrcType_)
679         {
680             //filter_ = createGaussianFilter(src[0].type(), Size(blurKernelSize_, blurKernelSize_), blurSigma_);
681             curBlurKernelSize_ = blurKernelSize_;
682             curBlurSigma_ = blurSigma_;
683             curSrcType_ = src[0].type();
684         }
685 
686         if (btvWeights_.empty() || btvKernelSize_ != curBtvKernelSize_ || alpha_ != curAlpha_)
687         {
688             calcBtvWeights(btvKernelSize_, alpha_, btvWeights_);
689             curBtvKernelSize_ = btvKernelSize_;
690             curAlpha_ = alpha_;
691         }
692 
693         // calc high res motions
694         calcRelativeMotions(forwardMotions, backwardMotions, lowResForwardMotions_, lowResBackwardMotions_, baseIdx, src[0].size());
695 
696         upscaleMotions(lowResForwardMotions_, highResForwardMotions_, scale_);
697         upscaleMotions(lowResBackwardMotions_, highResBackwardMotions_, scale_);
698 
699         forwardMaps_.resize(highResForwardMotions_.size());
700         backwardMaps_.resize(highResForwardMotions_.size());
701         for (size_t i = 0; i < highResForwardMotions_.size(); ++i)
702             buildMotionMaps(highResForwardMotions_[i], highResBackwardMotions_[i], forwardMaps_[i], backwardMaps_[i]);
703 
704         // initial estimation
705         const Size lowResSize = src[0].size();
706         const Size highResSize(lowResSize.width * scale_, lowResSize.height * scale_);
707 
708         resize(src[baseIdx], highRes_, highResSize, 0, 0, INTER_CUBIC);
709 
710         // iterations
711         diffTerm_.create(highResSize, highRes_.type());
712         a_.create(highResSize, highRes_.type());
713         b_.create(highResSize, highRes_.type());
714         c_.create(lowResSize, highRes_.type());
715 
716         for (int i = 0; i < iterations_; ++i)
717         {
718             diffTerm_.setTo(Scalar::all(0));
719 
720             for (size_t k = 0; k < src.size(); ++k)
721             {
722                 // a = M * Ih
723                 remap(highRes_, a_, backwardMaps_[k], noArray(), INTER_NEAREST);
724                 // b = HM * Ih
725                 GaussianBlur(a_, b_, Size(blurKernelSize_, blurKernelSize_), blurSigma_);
726                 // c = DHM * Ih
727                 resize(b_, c_, lowResSize, 0, 0, INTER_NEAREST);
728 
729                 diffSign(src[k], c_, c_);
730 
731                 // a = Dt * diff
732                 upscale(c_, a_, scale_);
733                 // b = HtDt * diff
734                 GaussianBlur(a_, b_, Size(blurKernelSize_, blurKernelSize_), blurSigma_);
735                 // a = MtHtDt * diff
736                 remap(b_, a_, forwardMaps_[k], noArray(), INTER_NEAREST);
737 
738                 add(diffTerm_, a_, diffTerm_);
739             }
740 
741             if (lambda_ > 0)
742             {
743                 calcBtvRegularization(highRes_, regTerm_, btvKernelSize_, btvWeights_, ubtvWeights_);
744                 addWeighted(diffTerm_, 1.0, regTerm_, -lambda_, 0.0, diffTerm_);
745             }
746 
747             addWeighted(highRes_, 1.0, diffTerm_, tau_, 0.0, highRes_);
748         }
749 
750         Rect inner(btvKernelSize_, btvKernelSize_, highRes_.cols - 2 * btvKernelSize_, highRes_.rows - 2 * btvKernelSize_);
751         highRes_(inner).copyTo(_dst);
752     }
753 
collectGarbage()754     void BTVL1_Base::collectGarbage()
755     {
756         // Mat
757         lowResForwardMotions_.clear();
758         lowResBackwardMotions_.clear();
759 
760         highResForwardMotions_.clear();
761         highResBackwardMotions_.clear();
762 
763         forwardMaps_.clear();
764         backwardMaps_.clear();
765 
766         highRes_.release();
767 
768         diffTerm_.release();
769         regTerm_.release();
770         a_.release();
771         b_.release();
772         c_.release();
773 
774 #ifdef HAVE_OPENCL
775         // UMat
776         ulowResForwardMotions_.clear();
777         ulowResBackwardMotions_.clear();
778 
779         uhighResForwardMotions_.clear();
780         uhighResBackwardMotions_.clear();
781 
782         uforwardMaps_.clear();
783         ubackwardMaps_.clear();
784 
785         uhighRes_.release();
786 
787         udiffTerm_.release();
788         uregTerm_.release();
789         ua_.release();
790         ub_.release();
791         uc_.release();
792 #endif
793     }
794 
795 ////////////////////////////////////////////////////////////////////
796 
797     class BTVL1 : public BTVL1_Base
798     {
799     public:
800         BTVL1();
801 
802         void collectGarbage();
803 
804     protected:
805         void initImpl(Ptr<FrameSource>& frameSource);
806         bool ocl_initImpl(Ptr<FrameSource>& frameSource);
807 
808         void processImpl(Ptr<FrameSource>& frameSource, OutputArray output);
809         bool ocl_processImpl(Ptr<FrameSource>& frameSource, OutputArray output);
810 
811     private:
812         void readNextFrame(Ptr<FrameSource>& frameSource);
813         bool ocl_readNextFrame(Ptr<FrameSource>& frameSource);
814 
815         void processFrame(int idx);
816         bool ocl_processFrame(int idx);
817 
818         int storePos_;
819         int procPos_;
820         int outPos_;
821 
822         // Mat
823         Mat curFrame_;
824         Mat prevFrame_;
825 
826         std::vector<Mat> frames_;
827         std::vector<Mat> forwardMotions_;
828         std::vector<Mat> backwardMotions_;
829         std::vector<Mat> outputs_;
830 
831         std::vector<Mat> srcFrames_;
832         std::vector<Mat> srcForwardMotions_;
833         std::vector<Mat> srcBackwardMotions_;
834         Mat finalOutput_;
835 
836 #ifdef HAVE_OPENCL
837         // UMat
838         UMat ucurFrame_;
839         UMat uprevFrame_;
840 
841         std::vector<UMat> uframes_;
842         std::vector<UMat> uforwardMotions_;
843         std::vector<UMat> ubackwardMotions_;
844         std::vector<UMat> uoutputs_;
845 
846         std::vector<UMat> usrcFrames_;
847         std::vector<UMat> usrcForwardMotions_;
848         std::vector<UMat> usrcBackwardMotions_;
849 #endif
850     };
851 
BTVL1()852     BTVL1::BTVL1()
853     {
854         temporalAreaRadius_ = 4;
855     }
856 
collectGarbage()857     void BTVL1::collectGarbage()
858     {
859         // Mat
860         curFrame_.release();
861         prevFrame_.release();
862 
863         frames_.clear();
864         forwardMotions_.clear();
865         backwardMotions_.clear();
866         outputs_.clear();
867 
868         srcFrames_.clear();
869         srcForwardMotions_.clear();
870         srcBackwardMotions_.clear();
871         finalOutput_.release();
872 
873 #ifdef HAVE_OPENCL
874         // UMat
875         ucurFrame_.release();
876         uprevFrame_.release();
877 
878         uframes_.clear();
879         uforwardMotions_.clear();
880         ubackwardMotions_.clear();
881         uoutputs_.clear();
882 
883         usrcFrames_.clear();
884         usrcForwardMotions_.clear();
885         usrcBackwardMotions_.clear();
886 #endif
887 
888         SuperResolution::collectGarbage();
889         BTVL1_Base::collectGarbage();
890     }
891 
892 #ifdef HAVE_OPENCL
893 
ocl_initImpl(Ptr<FrameSource> & frameSource)894     bool BTVL1::ocl_initImpl(Ptr<FrameSource>& frameSource)
895     {
896         const int cacheSize = 2 * temporalAreaRadius_ + 1;
897 
898         uframes_.resize(cacheSize);
899         uforwardMotions_.resize(cacheSize);
900         ubackwardMotions_.resize(cacheSize);
901         uoutputs_.resize(cacheSize);
902 
903         storePos_ = -1;
904 
905         for (int t = -temporalAreaRadius_; t <= temporalAreaRadius_; ++t)
906             readNextFrame(frameSource);
907 
908         for (int i = 0; i <= temporalAreaRadius_; ++i)
909             processFrame(i);
910 
911         procPos_ = temporalAreaRadius_;
912         outPos_ = -1;
913 
914         return true;
915     }
916 
917 #endif
918 
initImpl(Ptr<FrameSource> & frameSource)919     void BTVL1::initImpl(Ptr<FrameSource>& frameSource)
920     {
921         const int cacheSize = 2 * temporalAreaRadius_ + 1;
922 
923         frames_.resize(cacheSize);
924         forwardMotions_.resize(cacheSize);
925         backwardMotions_.resize(cacheSize);
926         outputs_.resize(cacheSize);
927 
928         CV_OCL_RUN(isUmat_,
929                    ocl_initImpl(frameSource))
930 
931         storePos_ = -1;
932 
933         for (int t = -temporalAreaRadius_; t <= temporalAreaRadius_; ++t)
934             readNextFrame(frameSource);
935 
936         for (int i = 0; i <= temporalAreaRadius_; ++i)
937             processFrame(i);
938 
939         procPos_ = temporalAreaRadius_;
940         outPos_ = -1;
941     }
942 
943 #ifdef HAVE_OPENCL
944 
ocl_processImpl(Ptr<FrameSource> &,OutputArray _output)945     bool BTVL1::ocl_processImpl(Ptr<FrameSource>& /*frameSource*/, OutputArray _output)
946     {
947         const UMat& curOutput = at(outPos_, uoutputs_);
948         curOutput.convertTo(_output, CV_8U);
949 
950         return true;
951     }
952 
953 #endif
954 
processImpl(Ptr<FrameSource> & frameSource,OutputArray _output)955     void BTVL1::processImpl(Ptr<FrameSource>& frameSource, OutputArray _output)
956     {
957         if (outPos_ >= storePos_)
958         {
959             _output.release();
960             return;
961         }
962 
963         readNextFrame(frameSource);
964 
965         if (procPos_ < storePos_)
966         {
967             ++procPos_;
968             processFrame(procPos_);
969         }
970         ++outPos_;
971 
972         CV_OCL_RUN(isUmat_,
973                    ocl_processImpl(frameSource, _output))
974 
975         const Mat& curOutput = at(outPos_, outputs_);
976 
977         if (_output.kind() < _InputArray::OPENGL_BUFFER || _output.isUMat())
978             curOutput.convertTo(_output, CV_8U);
979         else
980         {
981             curOutput.convertTo(finalOutput_, CV_8U);
982             arrCopy(finalOutput_, _output);
983         }
984     }
985 
986 #ifdef HAVE_OPENCL
987 
ocl_readNextFrame(Ptr<FrameSource> &)988     bool BTVL1::ocl_readNextFrame(Ptr<FrameSource>& /*frameSource*/)
989     {
990         ucurFrame_.convertTo(at(storePos_, uframes_), CV_32F);
991 
992         if (storePos_ > 0)
993         {
994             opticalFlow_->calc(uprevFrame_, ucurFrame_, at(storePos_ - 1, uforwardMotions_));
995             opticalFlow_->calc(ucurFrame_, uprevFrame_, at(storePos_, ubackwardMotions_));
996         }
997 
998         ucurFrame_.copyTo(uprevFrame_);
999         return true;
1000     }
1001 
1002 #endif
1003 
readNextFrame(Ptr<FrameSource> & frameSource)1004     void BTVL1::readNextFrame(Ptr<FrameSource>& frameSource)
1005     {
1006         frameSource->nextFrame(curFrame_);
1007         if (curFrame_.empty())
1008             return;
1009 
1010 #ifdef HAVE_OPENCL
1011         if (isUmat_)
1012             curFrame_.copyTo(ucurFrame_);
1013 #endif
1014         ++storePos_;
1015 
1016         CV_OCL_RUN(isUmat_,
1017                    ocl_readNextFrame(frameSource))
1018 
1019         curFrame_.convertTo(at(storePos_, frames_), CV_32F);
1020 
1021         if (storePos_ > 0)
1022         {
1023             opticalFlow_->calc(prevFrame_, curFrame_, at(storePos_ - 1, forwardMotions_));
1024             opticalFlow_->calc(curFrame_, prevFrame_, at(storePos_, backwardMotions_));
1025         }
1026 
1027         curFrame_.copyTo(prevFrame_);
1028     }
1029 
1030 #ifdef HAVE_OPENCL
1031 
ocl_processFrame(int idx)1032     bool BTVL1::ocl_processFrame(int idx)
1033     {
1034         const int startIdx = std::max(idx - temporalAreaRadius_, 0);
1035         const int procIdx = idx;
1036         const int endIdx = std::min(startIdx + 2 * temporalAreaRadius_, storePos_);
1037 
1038         const int count = endIdx - startIdx + 1;
1039 
1040         usrcFrames_.resize(count);
1041         usrcForwardMotions_.resize(count);
1042         usrcBackwardMotions_.resize(count);
1043 
1044         int baseIdx = -1;
1045 
1046         for (int i = startIdx, k = 0; i <= endIdx; ++i, ++k)
1047         {
1048             if (i == procIdx)
1049                 baseIdx = k;
1050 
1051             usrcFrames_[k] = at(i, uframes_);
1052 
1053             if (i < endIdx)
1054                 usrcForwardMotions_[k] = at(i, uforwardMotions_);
1055             if (i > startIdx)
1056                 usrcBackwardMotions_[k] = at(i, ubackwardMotions_);
1057         }
1058 
1059         process(usrcFrames_, at(idx, uoutputs_), usrcForwardMotions_, usrcBackwardMotions_, baseIdx);
1060 
1061         return true;
1062     }
1063 
1064 #endif
1065 
processFrame(int idx)1066     void BTVL1::processFrame(int idx)
1067     {
1068         CV_OCL_RUN(isUmat_,
1069                    ocl_processFrame(idx))
1070 
1071         const int startIdx = std::max(idx - temporalAreaRadius_, 0);
1072         const int procIdx = idx;
1073         const int endIdx = std::min(startIdx + 2 * temporalAreaRadius_, storePos_);
1074 
1075         const int count = endIdx - startIdx + 1;
1076 
1077         srcFrames_.resize(count);
1078         srcForwardMotions_.resize(count);
1079         srcBackwardMotions_.resize(count);
1080 
1081         int baseIdx = -1;
1082 
1083         for (int i = startIdx, k = 0; i <= endIdx; ++i, ++k)
1084         {
1085             if (i == procIdx)
1086                 baseIdx = k;
1087 
1088             srcFrames_[k] = at(i, frames_);
1089 
1090             if (i < endIdx)
1091                 srcForwardMotions_[k] = at(i, forwardMotions_);
1092             if (i > startIdx)
1093                 srcBackwardMotions_[k] = at(i, backwardMotions_);
1094         }
1095 
1096         process(srcFrames_, at(idx, outputs_), srcForwardMotions_, srcBackwardMotions_, baseIdx);
1097     }
1098 }
1099 
createSuperResolution_BTVL1()1100 Ptr<cv::superres::SuperResolution> cv::superres::createSuperResolution_BTVL1()
1101 {
1102     return makePtr<BTVL1>();
1103 }
1104