1 /*
2 * cl_post_image_processor.cpp - CL post image processor
3 *
4 * Copyright (c) 2015 Intel Corporation
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * Author: Wind Yuan <feng.yuan@intel.com>
19 * Author: Yinhang Liu <yinhangx.liu@intel.com>
20 */
21
22 #include "cl_post_image_processor.h"
23 #include "cl_context.h"
24
25 #include "cl_tnr_handler.h"
26 #include "cl_retinex_handler.h"
27 #include "cl_defog_dcp_handler.h"
28 #include "cl_wavelet_denoise_handler.h"
29 #include "cl_newwavelet_denoise_handler.h"
30 #include "cl_3d_denoise_handler.h"
31 #include "cl_image_scaler.h"
32 #include "cl_wire_frame_handler.h"
33 #include "cl_csc_handler.h"
34 #include "cl_image_warp_handler.h"
35 #include "cl_image_360_stitch.h"
36 #include "cl_video_stabilizer.h"
37
38 #define XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE 6
39 #define XCAM_CL_POST_IMAGE_MAX_POOL_SIZE 12
40
41 namespace XCam {
42
CLPostImageProcessor()43 CLPostImageProcessor::CLPostImageProcessor ()
44 : CLImageProcessor ("CLPostImageProcessor")
45 , _output_fourcc (V4L2_PIX_FMT_NV12)
46 , _out_sample_type (OutSampleYuv)
47 , _scaler_factor (1.0)
48 , _tnr_mode (TnrYuv)
49 , _defog_mode (CLPostImageProcessor::DefogDisabled)
50 , _wavelet_basis (CL_WAVELET_DISABLED)
51 , _wavelet_channel (CL_IMAGE_CHANNEL_UV)
52 , _wavelet_bayes_shrink (false)
53 , _3d_denoise_mode (CLPostImageProcessor::Denoise3DDisabled)
54 , _3d_denoise_ref_count (3)
55 , _enable_scaler (false)
56 , _enable_wireframe (false)
57 , _enable_image_warp (false)
58 , _enable_stitch (false)
59 , _stitch_enable_seam (false)
60 , _stitch_fisheye_map (false)
61 , _stitch_lsc (false)
62 , _stitch_fm_ocl (false)
63 , _stitch_scale_mode (CLBlenderScaleLocal)
64 , _stitch_width (0)
65 , _stitch_height (0)
66 , _stitch_res_mode (0)
67 , _surround_mode (SphereView)
68 {
69 XCAM_LOG_DEBUG ("CLPostImageProcessor constructed");
70 }
71
~CLPostImageProcessor()72 CLPostImageProcessor::~CLPostImageProcessor ()
73 {
74 XCAM_LOG_DEBUG ("CLPostImageProcessor destructed");
75 }
76
77 bool
set_output_format(uint32_t fourcc)78 CLPostImageProcessor::set_output_format (uint32_t fourcc)
79 {
80 switch (fourcc) {
81 case XCAM_PIX_FMT_RGBA64:
82 case V4L2_PIX_FMT_XBGR32:
83 case V4L2_PIX_FMT_ABGR32:
84 case V4L2_PIX_FMT_BGR32:
85 case V4L2_PIX_FMT_RGB32:
86 case V4L2_PIX_FMT_ARGB32:
87 case V4L2_PIX_FMT_XRGB32:
88 _out_sample_type = OutSampleRGB;
89 break;
90 case V4L2_PIX_FMT_NV12:
91 _out_sample_type = OutSampleYuv;
92 break;
93 default:
94 XCAM_LOG_WARNING (
95 "cl post processor doesn't support output format: %s",
96 xcam_fourcc_to_string(fourcc));
97 return false;
98 }
99
100 _output_fourcc = fourcc;
101 return true;
102 }
103
104 void
set_stats_callback(const SmartPtr<StatsCallback> & callback)105 CLPostImageProcessor::set_stats_callback (const SmartPtr<StatsCallback> &callback)
106 {
107 XCAM_ASSERT (callback.ptr ());
108 _stats_callback = callback;
109 }
110
111 bool
set_scaler_factor(const double factor)112 CLPostImageProcessor::set_scaler_factor (const double factor)
113 {
114 _scaler_factor = factor;
115
116 return true;
117 }
118
119 bool
can_process_result(SmartPtr<X3aResult> & result)120 CLPostImageProcessor::can_process_result (SmartPtr < X3aResult > & result)
121 {
122 if (!result.ptr ())
123 return false;
124
125 switch (result->get_type ()) {
126 case XCAM_3A_RESULT_TEMPORAL_NOISE_REDUCTION_YUV:
127 case XCAM_3A_RESULT_3D_NOISE_REDUCTION:
128 case XCAM_3A_RESULT_WAVELET_NOISE_REDUCTION:
129 case XCAM_3A_RESULT_FACE_DETECTION:
130 case XCAM_3A_RESULT_DVS:
131 return true;
132 default:
133 return false;
134 }
135
136 return false;
137 }
138
139 XCamReturn
apply_3a_results(X3aResultList & results)140 CLPostImageProcessor::apply_3a_results (X3aResultList &results)
141 {
142 XCamReturn ret = XCAM_RETURN_NO_ERROR;
143
144 for (X3aResultList::iterator iter = results.begin (); iter != results.end (); ++iter)
145 {
146 SmartPtr<X3aResult> &result = *iter;
147 ret = apply_3a_result (result);
148 if (ret != XCAM_RETURN_NO_ERROR)
149 break;
150 }
151
152 return ret;
153 }
154
155 XCamReturn
apply_3a_result(SmartPtr<X3aResult> & result)156 CLPostImageProcessor::apply_3a_result (SmartPtr<X3aResult> &result)
157 {
158 STREAM_LOCK;
159
160 if (!result.ptr ())
161 return XCAM_RETURN_BYPASS;
162
163 uint32_t res_type = result->get_type ();
164
165 switch (res_type) {
166 case XCAM_3A_RESULT_TEMPORAL_NOISE_REDUCTION_YUV: {
167 SmartPtr<X3aTemporalNoiseReduction> tnr_res = result.dynamic_cast_ptr<X3aTemporalNoiseReduction> ();
168 XCAM_ASSERT (tnr_res.ptr ());
169 if (_tnr.ptr ()) {
170 if (_defog_mode != CLPostImageProcessor::DefogDisabled) {
171 XCam3aResultTemporalNoiseReduction config;
172 xcam_mem_clear (config);
173 // isp processor
174 // config.gain = 0.12;
175
176 // cl processor
177 config.gain = 0.22;
178
179 config.threshold [0] = 0.00081;
180 config.threshold [1] = 0.00072;
181 _tnr->set_yuv_config (config);
182 } else {
183 _tnr->set_yuv_config (tnr_res->get_standard_result ());
184 }
185 }
186 break;
187 }
188 case XCAM_3A_RESULT_3D_NOISE_REDUCTION: {
189 SmartPtr<X3aTemporalNoiseReduction> nr_res = result.dynamic_cast_ptr<X3aTemporalNoiseReduction> ();
190 XCAM_ASSERT (nr_res.ptr ());
191 if (_3d_denoise.ptr ()) {
192 _3d_denoise->set_denoise_config (nr_res->get_standard_result ());
193 }
194 break;
195 }
196 case XCAM_3A_RESULT_WAVELET_NOISE_REDUCTION: {
197 SmartPtr<X3aWaveletNoiseReduction> wavelet_res = result.dynamic_cast_ptr<X3aWaveletNoiseReduction> ();
198 XCAM_ASSERT (wavelet_res.ptr ());
199 if (_wavelet.ptr()) {
200 _wavelet->set_denoise_config (wavelet_res->get_standard_result ());
201 }
202 if (_newwavelet.ptr()) {
203 _newwavelet->set_denoise_config (wavelet_res->get_standard_result ());
204 }
205 break;
206 }
207 case XCAM_3A_RESULT_FACE_DETECTION: {
208 SmartPtr<X3aFaceDetectionResult> fd_res = result.dynamic_cast_ptr<X3aFaceDetectionResult> ();
209 XCAM_ASSERT (fd_res.ptr ());
210 if (_wireframe.ptr ()) {
211 _wireframe->set_wire_frame_config (fd_res->get_standard_result_ptr (), get_scaler_factor ());
212 }
213 break;
214 }
215 case XCAM_3A_RESULT_DVS: {
216 SmartPtr<X3aDVSResult> dvs_res = result.dynamic_cast_ptr<X3aDVSResult> ();
217 XCAM_ASSERT (dvs_res.ptr ());
218 if (_image_warp.ptr ()) {
219 _image_warp->set_warp_config (dvs_res->get_standard_result ());
220 }
221 break;
222 }
223 default:
224 XCAM_LOG_WARNING ("CLPostImageProcessor unknown 3a result: %d", res_type);
225 break;
226 }
227
228 return XCAM_RETURN_NO_ERROR;
229 }
230
231
232 XCamReturn
create_handlers()233 CLPostImageProcessor::create_handlers ()
234 {
235 SmartPtr<CLImageHandler> image_handler;
236 SmartPtr<CLContext> context = get_cl_context ();
237
238 XCAM_ASSERT (context.ptr ());
239
240 /* defog: retinex */
241 image_handler = create_cl_retinex_image_handler (context);
242 _retinex = image_handler.dynamic_cast_ptr<CLRetinexImageHandler> ();
243 XCAM_FAIL_RETURN (
244 WARNING,
245 _retinex.ptr (),
246 XCAM_RETURN_ERROR_CL,
247 "CLPostImageProcessor create retinex handler failed");
248 _retinex->enable_handler (_defog_mode == CLPostImageProcessor::DefogRetinex);
249 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
250 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
251 add_handler (image_handler);
252
253 /* defog: dark channel prior */
254 image_handler = create_cl_defog_dcp_image_handler (context);
255 _defog_dcp = image_handler.dynamic_cast_ptr<CLDefogDcpImageHandler> ();
256 XCAM_FAIL_RETURN (
257 WARNING,
258 _defog_dcp.ptr (),
259 XCAM_RETURN_ERROR_CL,
260 "CLPostImageProcessor create defog handler failed");
261 _defog_dcp->enable_handler (_defog_mode == CLPostImageProcessor::DefogDarkChannelPrior);
262 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
263 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
264 add_handler (image_handler);
265
266 /* Temporal Noise Reduction */
267 if (_defog_mode != CLPostImageProcessor::DefogDisabled) {
268 switch (_tnr_mode) {
269 case TnrYuv: {
270 image_handler = create_cl_tnr_image_handler (context, CL_TNR_TYPE_YUV);
271 _tnr = image_handler.dynamic_cast_ptr<CLTnrImageHandler> ();
272 XCAM_FAIL_RETURN (
273 WARNING,
274 _tnr.ptr (),
275 XCAM_RETURN_ERROR_CL,
276 "CLPostImageProcessor create tnr handler failed");
277 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
278 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
279 add_handler (image_handler);
280 break;
281 }
282 case TnrDisable:
283 XCAM_LOG_DEBUG ("CLPostImageProcessor disable tnr");
284 break;
285 default:
286 XCAM_LOG_WARNING ("CLPostImageProcessor unknown tnr mode (%d)", _tnr_mode);
287 break;
288 }
289 }
290
291 /* wavelet denoise */
292 switch (_wavelet_basis) {
293 case CL_WAVELET_HAT: {
294 image_handler = create_cl_wavelet_denoise_image_handler (context, _wavelet_channel);
295 _wavelet = image_handler.dynamic_cast_ptr<CLWaveletDenoiseImageHandler> ();
296 XCAM_FAIL_RETURN (
297 WARNING,
298 _wavelet.ptr (),
299 XCAM_RETURN_ERROR_CL,
300 "CLPostImageProcessor create wavelet denoise handler failed");
301 _wavelet->enable_handler (true);
302 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
303 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
304 add_handler (image_handler);
305 break;
306 }
307 case CL_WAVELET_HAAR: {
308 image_handler = create_cl_newwavelet_denoise_image_handler (context, _wavelet_channel, _wavelet_bayes_shrink);
309 _newwavelet = image_handler.dynamic_cast_ptr<CLNewWaveletDenoiseImageHandler> ();
310 XCAM_FAIL_RETURN (
311 WARNING,
312 _newwavelet.ptr (),
313 XCAM_RETURN_ERROR_CL,
314 "CLPostImageProcessor create new wavelet denoise handler failed");
315 _newwavelet->enable_handler (true);
316 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
317 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
318 add_handler (image_handler);
319 break;
320 }
321 case CL_WAVELET_DISABLED:
322 default :
323 XCAM_LOG_DEBUG ("unknown or disable wavelet (%d)", _wavelet_basis);
324 break;
325 }
326
327 /* 3D noise reduction */
328 if (_3d_denoise_mode != CLPostImageProcessor::Denoise3DDisabled) {
329 uint32_t denoise_channel = CL_IMAGE_CHANNEL_UV;
330
331 if (_3d_denoise_mode == CLPostImageProcessor::Denoise3DUV) {
332 denoise_channel = CL_IMAGE_CHANNEL_UV;
333 } else if (_3d_denoise_mode == CLPostImageProcessor::Denoise3DYuv) {
334 denoise_channel = CL_IMAGE_CHANNEL_Y | CL_IMAGE_CHANNEL_UV;
335 }
336
337 image_handler = create_cl_3d_denoise_image_handler (context, denoise_channel, _3d_denoise_ref_count);
338 _3d_denoise = image_handler.dynamic_cast_ptr<CL3DDenoiseImageHandler> ();
339 XCAM_FAIL_RETURN (
340 WARNING,
341 _3d_denoise.ptr (),
342 XCAM_RETURN_ERROR_CL,
343 "CL3aImageProcessor create 3D noise reduction handler failed");
344 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
345 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
346 image_handler->enable_handler (true);
347 add_handler (image_handler);
348 }
349
350 /* image scaler */
351 image_handler = create_cl_image_scaler_handler (context, V4L2_PIX_FMT_NV12);
352 _scaler = image_handler.dynamic_cast_ptr<CLImageScaler> ();
353 XCAM_FAIL_RETURN (
354 WARNING,
355 _scaler.ptr (),
356 XCAM_RETURN_ERROR_CL,
357 "CLPostImageProcessor create scaler handler failed");
358 _scaler->set_scaler_factor (_scaler_factor, _scaler_factor);
359 _scaler->set_buffer_callback (_stats_callback);
360 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
361 image_handler->enable_handler (_enable_scaler);
362 add_handler (image_handler);
363
364 /* wire frame */
365 image_handler = create_cl_wire_frame_image_handler (context);
366 _wireframe = image_handler.dynamic_cast_ptr<CLWireFrameImageHandler> ();
367 XCAM_FAIL_RETURN (
368 WARNING,
369 _wireframe.ptr (),
370 XCAM_RETURN_ERROR_CL,
371 "CLPostImageProcessor create wire frame handler failed");
372 _wireframe->enable_handler (_enable_wireframe);
373 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
374 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
375 add_handler (image_handler);
376
377 /* image warp */
378 image_handler = create_cl_image_warp_handler (context);
379 _image_warp = image_handler.dynamic_cast_ptr<CLImageWarpHandler> ();
380 XCAM_FAIL_RETURN (
381 WARNING,
382 _image_warp.ptr (),
383 XCAM_RETURN_ERROR_CL,
384 "CLPostImageProcessor create image warp handler failed");
385 _image_warp->enable_handler (_enable_image_warp);
386 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
387 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
388 add_handler (image_handler);
389
390 /* video stabilization */
391 image_handler = create_cl_video_stab_handler (context);
392 _video_stab = image_handler.dynamic_cast_ptr<CLVideoStabilizer> ();
393 XCAM_FAIL_RETURN (
394 WARNING,
395 _video_stab.ptr (),
396 XCAM_RETURN_ERROR_CL,
397 "CLPostImageProcessor create video stabilizer failed");
398 _video_stab->enable_handler (false);
399 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
400 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
401 add_handler (image_handler);
402
403 /* image stitch */
404 image_handler =
405 create_image_360_stitch (context, _stitch_enable_seam, _stitch_scale_mode,
406 _stitch_fisheye_map, _stitch_lsc, (SurroundMode) _surround_mode, (StitchResMode) _stitch_res_mode);
407 _stitch = image_handler.dynamic_cast_ptr<CLImage360Stitch> ();
408 XCAM_FAIL_RETURN (
409 WARNING,
410 _stitch.ptr (),
411 XCAM_RETURN_ERROR_CL,
412 "CLPostImageProcessor create image stitch handler failed");
413 _stitch->set_output_size (_stitch_width, _stitch_height);
414 #if HAVE_OPENCV
415 _stitch->set_feature_match_ocl (_stitch_fm_ocl);
416 #endif
417 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
418 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
419 image_handler->enable_handler (_enable_stitch);
420 add_handler (image_handler);
421
422 /* csc (nv12torgba) */
423 image_handler = create_cl_csc_image_handler (context, CL_CSC_TYPE_NV12TORGBA);
424 _csc = image_handler.dynamic_cast_ptr<CLCscImageHandler> ();
425 XCAM_FAIL_RETURN (
426 WARNING,
427 _csc .ptr (),
428 XCAM_RETURN_ERROR_CL,
429 "CLPostImageProcessor create csc handler failed");
430 _csc->enable_handler (_out_sample_type == OutSampleRGB);
431 _csc->set_output_format (_output_fourcc);
432 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
433 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
434 add_handler (image_handler);
435
436 return XCAM_RETURN_NO_ERROR;
437 }
438
439 bool
set_tnr(CLTnrMode mode)440 CLPostImageProcessor::set_tnr (CLTnrMode mode)
441 {
442 _tnr_mode = mode;
443
444 STREAM_LOCK;
445
446 return true;
447 }
448
449 bool
set_defog_mode(CLDefogMode mode)450 CLPostImageProcessor::set_defog_mode (CLDefogMode mode)
451 {
452 _defog_mode = mode;
453
454 STREAM_LOCK;
455
456 return true;
457 }
458
459 bool
set_wavelet(CLWaveletBasis basis,uint32_t channel,bool bayes_shrink)460 CLPostImageProcessor::set_wavelet (CLWaveletBasis basis, uint32_t channel, bool bayes_shrink)
461 {
462 _wavelet_basis = basis;
463 _wavelet_channel = (CLImageChannel) channel;
464 _wavelet_bayes_shrink = bayes_shrink;
465
466 STREAM_LOCK;
467
468 return true;
469 }
470
471 bool
set_3ddenoise_mode(CL3DDenoiseMode mode,uint8_t ref_frame_count)472 CLPostImageProcessor::set_3ddenoise_mode (CL3DDenoiseMode mode, uint8_t ref_frame_count)
473 {
474 _3d_denoise_mode = mode;
475 _3d_denoise_ref_count = ref_frame_count;
476
477 STREAM_LOCK;
478
479 return true;
480 }
481
482 bool
set_scaler(bool enable)483 CLPostImageProcessor::set_scaler (bool enable)
484 {
485 _enable_scaler = enable;
486
487 STREAM_LOCK;
488
489 return true;
490 }
491
492 bool
set_wireframe(bool enable)493 CLPostImageProcessor::set_wireframe (bool enable)
494 {
495 _enable_wireframe = enable;
496
497 STREAM_LOCK;
498
499 return true;
500 }
501
502 bool
set_image_warp(bool enable)503 CLPostImageProcessor::set_image_warp (bool enable)
504 {
505 _enable_image_warp = enable;
506
507 STREAM_LOCK;
508
509 return true;
510 }
511
512 bool
set_image_stitch(bool enable_stitch,bool enable_seam,CLBlenderScaleMode scale_mode,bool enable_fisheye_map,bool lsc,bool fm_ocl,uint32_t stitch_width,uint32_t stitch_height,uint32_t res_mode)513 CLPostImageProcessor::set_image_stitch (
514 bool enable_stitch, bool enable_seam, CLBlenderScaleMode scale_mode, bool enable_fisheye_map,
515 bool lsc, bool fm_ocl, uint32_t stitch_width, uint32_t stitch_height, uint32_t res_mode)
516 {
517 XCAM_ASSERT (scale_mode < CLBlenderScaleMax);
518
519 _enable_stitch = enable_stitch;
520 if (enable_stitch)
521 _stitch_enable_seam = enable_seam;
522 else
523 _stitch_enable_seam = false;
524
525 _stitch_scale_mode = scale_mode;
526 _stitch_fisheye_map = enable_fisheye_map;
527 _stitch_lsc = lsc;
528 _stitch_width = stitch_width;
529 _stitch_height = stitch_height;
530 _stitch_res_mode = res_mode;
531
532 #if HAVE_OPENCV
533 _stitch_fm_ocl = fm_ocl;
534 #else
535 XCAM_UNUSED (fm_ocl);
536 #endif
537
538 STREAM_LOCK;
539
540 return true;
541 }
542
543 };
544