1 /*
2  * cl_post_image_processor.cpp - CL post image processor
3  *
4  *  Copyright (c) 2015 Intel Corporation
5  *
6  * Licensed under the Apache License, Version 2.0 (the "License");
7  * you may not use this file except in compliance with the License.
8  * You may obtain a copy of the License at
9  *
10  *      http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  *
18  * Author: Wind Yuan <feng.yuan@intel.com>
19  * Author: Yinhang Liu <yinhangx.liu@intel.com>
20  */
21 
22 #include "cl_post_image_processor.h"
23 #include "cl_context.h"
24 
25 #include "cl_tnr_handler.h"
26 #include "cl_retinex_handler.h"
27 #include "cl_defog_dcp_handler.h"
28 #include "cl_wavelet_denoise_handler.h"
29 #include "cl_newwavelet_denoise_handler.h"
30 #include "cl_3d_denoise_handler.h"
31 #include "cl_image_scaler.h"
32 #include "cl_wire_frame_handler.h"
33 #include "cl_csc_handler.h"
34 #include "cl_image_warp_handler.h"
35 #include "cl_image_360_stitch.h"
36 #include "cl_video_stabilizer.h"
37 
38 #define XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE 6
39 #define XCAM_CL_POST_IMAGE_MAX_POOL_SIZE 12
40 
41 namespace XCam {
42 
CLPostImageProcessor()43 CLPostImageProcessor::CLPostImageProcessor ()
44     : CLImageProcessor ("CLPostImageProcessor")
45     , _output_fourcc (V4L2_PIX_FMT_NV12)
46     , _out_sample_type (OutSampleYuv)
47     , _scaler_factor (1.0)
48     , _tnr_mode (TnrYuv)
49     , _defog_mode (CLPostImageProcessor::DefogDisabled)
50     , _wavelet_basis (CL_WAVELET_DISABLED)
51     , _wavelet_channel (CL_IMAGE_CHANNEL_UV)
52     , _wavelet_bayes_shrink (false)
53     , _3d_denoise_mode (CLPostImageProcessor::Denoise3DDisabled)
54     , _3d_denoise_ref_count (3)
55     , _enable_scaler (false)
56     , _enable_wireframe (false)
57     , _enable_image_warp (false)
58     , _enable_stitch (false)
59     , _stitch_enable_seam (false)
60     , _stitch_fisheye_map (false)
61     , _stitch_lsc (false)
62     , _stitch_fm_ocl (false)
63     , _stitch_scale_mode (CLBlenderScaleLocal)
64     , _stitch_width (0)
65     , _stitch_height (0)
66     , _stitch_res_mode (0)
67     , _surround_mode (SphereView)
68 {
69     XCAM_LOG_DEBUG ("CLPostImageProcessor constructed");
70 }
71 
~CLPostImageProcessor()72 CLPostImageProcessor::~CLPostImageProcessor ()
73 {
74     XCAM_LOG_DEBUG ("CLPostImageProcessor destructed");
75 }
76 
77 bool
set_output_format(uint32_t fourcc)78 CLPostImageProcessor::set_output_format (uint32_t fourcc)
79 {
80     switch (fourcc) {
81     case XCAM_PIX_FMT_RGBA64:
82     case V4L2_PIX_FMT_XBGR32:
83     case V4L2_PIX_FMT_ABGR32:
84     case V4L2_PIX_FMT_BGR32:
85     case V4L2_PIX_FMT_RGB32:
86     case V4L2_PIX_FMT_ARGB32:
87     case V4L2_PIX_FMT_XRGB32:
88         _out_sample_type = OutSampleRGB;
89         break;
90     case V4L2_PIX_FMT_NV12:
91         _out_sample_type = OutSampleYuv;
92         break;
93     default:
94         XCAM_LOG_WARNING (
95             "cl post processor doesn't support output format: %s",
96             xcam_fourcc_to_string(fourcc));
97         return false;
98     }
99 
100     _output_fourcc = fourcc;
101     return true;
102 }
103 
104 void
set_stats_callback(const SmartPtr<StatsCallback> & callback)105 CLPostImageProcessor::set_stats_callback (const SmartPtr<StatsCallback> &callback)
106 {
107     XCAM_ASSERT (callback.ptr ());
108     _stats_callback = callback;
109 }
110 
111 bool
set_scaler_factor(const double factor)112 CLPostImageProcessor::set_scaler_factor (const double factor)
113 {
114     _scaler_factor = factor;
115 
116     return true;
117 }
118 
119 bool
can_process_result(SmartPtr<X3aResult> & result)120 CLPostImageProcessor::can_process_result (SmartPtr < X3aResult > & result)
121 {
122     if (!result.ptr ())
123         return false;
124 
125     switch (result->get_type ()) {
126     case XCAM_3A_RESULT_TEMPORAL_NOISE_REDUCTION_YUV:
127     case XCAM_3A_RESULT_3D_NOISE_REDUCTION:
128     case XCAM_3A_RESULT_WAVELET_NOISE_REDUCTION:
129     case XCAM_3A_RESULT_FACE_DETECTION:
130     case XCAM_3A_RESULT_DVS:
131         return true;
132     default:
133         return false;
134     }
135 
136     return false;
137 }
138 
139 XCamReturn
apply_3a_results(X3aResultList & results)140 CLPostImageProcessor::apply_3a_results (X3aResultList &results)
141 {
142     XCamReturn ret = XCAM_RETURN_NO_ERROR;
143 
144     for (X3aResultList::iterator iter = results.begin (); iter != results.end (); ++iter)
145     {
146         SmartPtr<X3aResult> &result = *iter;
147         ret = apply_3a_result (result);
148         if (ret != XCAM_RETURN_NO_ERROR)
149             break;
150     }
151 
152     return ret;
153 }
154 
155 XCamReturn
apply_3a_result(SmartPtr<X3aResult> & result)156 CLPostImageProcessor::apply_3a_result (SmartPtr<X3aResult> &result)
157 {
158     STREAM_LOCK;
159 
160     if (!result.ptr ())
161         return XCAM_RETURN_BYPASS;
162 
163     uint32_t res_type = result->get_type ();
164 
165     switch (res_type) {
166     case XCAM_3A_RESULT_TEMPORAL_NOISE_REDUCTION_YUV: {
167         SmartPtr<X3aTemporalNoiseReduction> tnr_res = result.dynamic_cast_ptr<X3aTemporalNoiseReduction> ();
168         XCAM_ASSERT (tnr_res.ptr ());
169         if (_tnr.ptr ()) {
170             if (_defog_mode != CLPostImageProcessor::DefogDisabled) {
171                 XCam3aResultTemporalNoiseReduction config;
172                 xcam_mem_clear (config);
173                 // isp processor
174                 // config.gain = 0.12;
175 
176                 // cl processor
177                 config.gain = 0.22;
178 
179                 config.threshold [0] = 0.00081;
180                 config.threshold [1] = 0.00072;
181                 _tnr->set_yuv_config (config);
182             } else {
183                 _tnr->set_yuv_config (tnr_res->get_standard_result ());
184             }
185         }
186         break;
187     }
188     case XCAM_3A_RESULT_3D_NOISE_REDUCTION: {
189         SmartPtr<X3aTemporalNoiseReduction> nr_res = result.dynamic_cast_ptr<X3aTemporalNoiseReduction> ();
190         XCAM_ASSERT (nr_res.ptr ());
191         if (_3d_denoise.ptr ()) {
192             _3d_denoise->set_denoise_config (nr_res->get_standard_result ());
193         }
194         break;
195     }
196     case XCAM_3A_RESULT_WAVELET_NOISE_REDUCTION: {
197         SmartPtr<X3aWaveletNoiseReduction> wavelet_res = result.dynamic_cast_ptr<X3aWaveletNoiseReduction> ();
198         XCAM_ASSERT (wavelet_res.ptr ());
199         if (_wavelet.ptr()) {
200             _wavelet->set_denoise_config (wavelet_res->get_standard_result ());
201         }
202         if (_newwavelet.ptr()) {
203             _newwavelet->set_denoise_config (wavelet_res->get_standard_result ());
204         }
205         break;
206     }
207     case XCAM_3A_RESULT_FACE_DETECTION: {
208         SmartPtr<X3aFaceDetectionResult> fd_res = result.dynamic_cast_ptr<X3aFaceDetectionResult> ();
209         XCAM_ASSERT (fd_res.ptr ());
210         if (_wireframe.ptr ()) {
211             _wireframe->set_wire_frame_config (fd_res->get_standard_result_ptr (), get_scaler_factor ());
212         }
213         break;
214     }
215     case XCAM_3A_RESULT_DVS: {
216         SmartPtr<X3aDVSResult> dvs_res = result.dynamic_cast_ptr<X3aDVSResult> ();
217         XCAM_ASSERT (dvs_res.ptr ());
218         if (_image_warp.ptr ()) {
219             _image_warp->set_warp_config (dvs_res->get_standard_result ());
220         }
221         break;
222     }
223     default:
224         XCAM_LOG_WARNING ("CLPostImageProcessor unknown 3a result: %d", res_type);
225         break;
226     }
227 
228     return XCAM_RETURN_NO_ERROR;
229 }
230 
231 
232 XCamReturn
create_handlers()233 CLPostImageProcessor::create_handlers ()
234 {
235     SmartPtr<CLImageHandler> image_handler;
236     SmartPtr<CLContext> context = get_cl_context ();
237 
238     XCAM_ASSERT (context.ptr ());
239 
240     /* defog: retinex */
241     image_handler = create_cl_retinex_image_handler (context);
242     _retinex = image_handler.dynamic_cast_ptr<CLRetinexImageHandler> ();
243     XCAM_FAIL_RETURN (
244         WARNING,
245         _retinex.ptr (),
246         XCAM_RETURN_ERROR_CL,
247         "CLPostImageProcessor create retinex handler failed");
248     _retinex->enable_handler (_defog_mode == CLPostImageProcessor::DefogRetinex);
249     image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
250     image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
251     add_handler (image_handler);
252 
253     /* defog: dark channel prior */
254     image_handler = create_cl_defog_dcp_image_handler (context);
255     _defog_dcp = image_handler.dynamic_cast_ptr<CLDefogDcpImageHandler> ();
256     XCAM_FAIL_RETURN (
257         WARNING,
258         _defog_dcp.ptr (),
259         XCAM_RETURN_ERROR_CL,
260         "CLPostImageProcessor create defog handler failed");
261     _defog_dcp->enable_handler (_defog_mode == CLPostImageProcessor::DefogDarkChannelPrior);
262     image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
263     image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
264     add_handler (image_handler);
265 
266     /* Temporal Noise Reduction */
267     if (_defog_mode != CLPostImageProcessor::DefogDisabled) {
268         switch (_tnr_mode) {
269         case TnrYuv: {
270             image_handler = create_cl_tnr_image_handler (context, CL_TNR_TYPE_YUV);
271             _tnr = image_handler.dynamic_cast_ptr<CLTnrImageHandler> ();
272             XCAM_FAIL_RETURN (
273                 WARNING,
274                 _tnr.ptr (),
275                 XCAM_RETURN_ERROR_CL,
276                 "CLPostImageProcessor create tnr handler failed");
277             image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
278             image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
279             add_handler (image_handler);
280             break;
281         }
282         case TnrDisable:
283             XCAM_LOG_DEBUG ("CLPostImageProcessor disable tnr");
284             break;
285         default:
286             XCAM_LOG_WARNING ("CLPostImageProcessor unknown tnr mode (%d)", _tnr_mode);
287             break;
288         }
289     }
290 
291     /* wavelet denoise */
292     switch (_wavelet_basis) {
293     case CL_WAVELET_HAT: {
294         image_handler = create_cl_wavelet_denoise_image_handler (context, _wavelet_channel);
295         _wavelet = image_handler.dynamic_cast_ptr<CLWaveletDenoiseImageHandler> ();
296         XCAM_FAIL_RETURN (
297             WARNING,
298             _wavelet.ptr (),
299             XCAM_RETURN_ERROR_CL,
300             "CLPostImageProcessor create wavelet denoise handler failed");
301         _wavelet->enable_handler (true);
302         image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
303         image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
304         add_handler (image_handler);
305         break;
306     }
307     case CL_WAVELET_HAAR: {
308         image_handler = create_cl_newwavelet_denoise_image_handler (context, _wavelet_channel, _wavelet_bayes_shrink);
309         _newwavelet = image_handler.dynamic_cast_ptr<CLNewWaveletDenoiseImageHandler> ();
310         XCAM_FAIL_RETURN (
311             WARNING,
312             _newwavelet.ptr (),
313             XCAM_RETURN_ERROR_CL,
314             "CLPostImageProcessor create new wavelet denoise handler failed");
315         _newwavelet->enable_handler (true);
316         image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
317         image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
318         add_handler (image_handler);
319         break;
320     }
321     case CL_WAVELET_DISABLED:
322     default :
323         XCAM_LOG_DEBUG ("unknown or disable wavelet (%d)", _wavelet_basis);
324         break;
325     }
326 
327     /* 3D noise reduction */
328     if (_3d_denoise_mode != CLPostImageProcessor::Denoise3DDisabled) {
329         uint32_t denoise_channel = CL_IMAGE_CHANNEL_UV;
330 
331         if (_3d_denoise_mode == CLPostImageProcessor::Denoise3DUV) {
332             denoise_channel = CL_IMAGE_CHANNEL_UV;
333         } else if (_3d_denoise_mode == CLPostImageProcessor::Denoise3DYuv) {
334             denoise_channel = CL_IMAGE_CHANNEL_Y | CL_IMAGE_CHANNEL_UV;
335         }
336 
337         image_handler = create_cl_3d_denoise_image_handler (context, denoise_channel, _3d_denoise_ref_count);
338         _3d_denoise = image_handler.dynamic_cast_ptr<CL3DDenoiseImageHandler> ();
339         XCAM_FAIL_RETURN (
340             WARNING,
341             _3d_denoise.ptr (),
342             XCAM_RETURN_ERROR_CL,
343             "CL3aImageProcessor create 3D noise reduction handler failed");
344         image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
345         image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
346         image_handler->enable_handler (true);
347         add_handler (image_handler);
348     }
349 
350     /* image scaler */
351     image_handler = create_cl_image_scaler_handler (context, V4L2_PIX_FMT_NV12);
352     _scaler = image_handler.dynamic_cast_ptr<CLImageScaler> ();
353     XCAM_FAIL_RETURN (
354         WARNING,
355         _scaler.ptr (),
356         XCAM_RETURN_ERROR_CL,
357         "CLPostImageProcessor create scaler handler failed");
358     _scaler->set_scaler_factor (_scaler_factor, _scaler_factor);
359     _scaler->set_buffer_callback (_stats_callback);
360     image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
361     image_handler->enable_handler (_enable_scaler);
362     add_handler (image_handler);
363 
364     /* wire frame */
365     image_handler = create_cl_wire_frame_image_handler (context);
366     _wireframe = image_handler.dynamic_cast_ptr<CLWireFrameImageHandler> ();
367     XCAM_FAIL_RETURN (
368         WARNING,
369         _wireframe.ptr (),
370         XCAM_RETURN_ERROR_CL,
371         "CLPostImageProcessor create wire frame handler failed");
372     _wireframe->enable_handler (_enable_wireframe);
373     image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
374     image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
375     add_handler (image_handler);
376 
377     /* image warp */
378     image_handler = create_cl_image_warp_handler (context);
379     _image_warp = image_handler.dynamic_cast_ptr<CLImageWarpHandler> ();
380     XCAM_FAIL_RETURN (
381         WARNING,
382         _image_warp.ptr (),
383         XCAM_RETURN_ERROR_CL,
384         "CLPostImageProcessor create image warp handler failed");
385     _image_warp->enable_handler (_enable_image_warp);
386     image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
387     image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
388     add_handler (image_handler);
389 
390     /* video stabilization */
391     image_handler = create_cl_video_stab_handler (context);
392     _video_stab = image_handler.dynamic_cast_ptr<CLVideoStabilizer> ();
393     XCAM_FAIL_RETURN (
394         WARNING,
395         _video_stab.ptr (),
396         XCAM_RETURN_ERROR_CL,
397         "CLPostImageProcessor create video stabilizer failed");
398     _video_stab->enable_handler (false);
399     image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
400     image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
401     add_handler (image_handler);
402 
403     /* image stitch */
404     image_handler =
405         create_image_360_stitch (context, _stitch_enable_seam, _stitch_scale_mode,
406                                  _stitch_fisheye_map, _stitch_lsc, (SurroundMode) _surround_mode, (StitchResMode) _stitch_res_mode);
407     _stitch = image_handler.dynamic_cast_ptr<CLImage360Stitch> ();
408     XCAM_FAIL_RETURN (
409         WARNING,
410         _stitch.ptr (),
411         XCAM_RETURN_ERROR_CL,
412         "CLPostImageProcessor create image stitch handler failed");
413     _stitch->set_output_size (_stitch_width, _stitch_height);
414 #if HAVE_OPENCV
415     _stitch->set_feature_match_ocl (_stitch_fm_ocl);
416 #endif
417     image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
418     image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
419     image_handler->enable_handler (_enable_stitch);
420     add_handler (image_handler);
421 
422     /* csc (nv12torgba) */
423     image_handler = create_cl_csc_image_handler (context, CL_CSC_TYPE_NV12TORGBA);
424     _csc = image_handler.dynamic_cast_ptr<CLCscImageHandler> ();
425     XCAM_FAIL_RETURN (
426         WARNING,
427         _csc .ptr (),
428         XCAM_RETURN_ERROR_CL,
429         "CLPostImageProcessor create csc handler failed");
430     _csc->enable_handler (_out_sample_type == OutSampleRGB);
431     _csc->set_output_format (_output_fourcc);
432     image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
433     image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
434     add_handler (image_handler);
435 
436     return XCAM_RETURN_NO_ERROR;
437 }
438 
439 bool
set_tnr(CLTnrMode mode)440 CLPostImageProcessor::set_tnr (CLTnrMode mode)
441 {
442     _tnr_mode = mode;
443 
444     STREAM_LOCK;
445 
446     return true;
447 }
448 
449 bool
set_defog_mode(CLDefogMode mode)450 CLPostImageProcessor::set_defog_mode (CLDefogMode mode)
451 {
452     _defog_mode = mode;
453 
454     STREAM_LOCK;
455 
456     return true;
457 }
458 
459 bool
set_wavelet(CLWaveletBasis basis,uint32_t channel,bool bayes_shrink)460 CLPostImageProcessor::set_wavelet (CLWaveletBasis basis, uint32_t channel, bool bayes_shrink)
461 {
462     _wavelet_basis = basis;
463     _wavelet_channel = (CLImageChannel) channel;
464     _wavelet_bayes_shrink = bayes_shrink;
465 
466     STREAM_LOCK;
467 
468     return true;
469 }
470 
471 bool
set_3ddenoise_mode(CL3DDenoiseMode mode,uint8_t ref_frame_count)472 CLPostImageProcessor::set_3ddenoise_mode (CL3DDenoiseMode mode, uint8_t ref_frame_count)
473 {
474     _3d_denoise_mode = mode;
475     _3d_denoise_ref_count = ref_frame_count;
476 
477     STREAM_LOCK;
478 
479     return true;
480 }
481 
482 bool
set_scaler(bool enable)483 CLPostImageProcessor::set_scaler (bool enable)
484 {
485     _enable_scaler = enable;
486 
487     STREAM_LOCK;
488 
489     return true;
490 }
491 
492 bool
set_wireframe(bool enable)493 CLPostImageProcessor::set_wireframe (bool enable)
494 {
495     _enable_wireframe = enable;
496 
497     STREAM_LOCK;
498 
499     return true;
500 }
501 
502 bool
set_image_warp(bool enable)503 CLPostImageProcessor::set_image_warp (bool enable)
504 {
505     _enable_image_warp = enable;
506 
507     STREAM_LOCK;
508 
509     return true;
510 }
511 
512 bool
set_image_stitch(bool enable_stitch,bool enable_seam,CLBlenderScaleMode scale_mode,bool enable_fisheye_map,bool lsc,bool fm_ocl,uint32_t stitch_width,uint32_t stitch_height,uint32_t res_mode)513 CLPostImageProcessor::set_image_stitch (
514     bool enable_stitch, bool enable_seam, CLBlenderScaleMode scale_mode, bool enable_fisheye_map,
515     bool lsc, bool fm_ocl, uint32_t stitch_width, uint32_t stitch_height, uint32_t res_mode)
516 {
517     XCAM_ASSERT (scale_mode < CLBlenderScaleMax);
518 
519     _enable_stitch = enable_stitch;
520     if (enable_stitch)
521         _stitch_enable_seam = enable_seam;
522     else
523         _stitch_enable_seam = false;
524 
525     _stitch_scale_mode = scale_mode;
526     _stitch_fisheye_map = enable_fisheye_map;
527     _stitch_lsc = lsc;
528     _stitch_width = stitch_width;
529     _stitch_height = stitch_height;
530     _stitch_res_mode = res_mode;
531 
532 #if HAVE_OPENCV
533     _stitch_fm_ocl = fm_ocl;
534 #else
535     XCAM_UNUSED (fm_ocl);
536 #endif
537 
538     STREAM_LOCK;
539 
540     return true;
541 }
542 
543 };
544