1 /*
2 * cl_image_360_stitch.cpp - CL Image 360 stitch
3 *
4 * Copyright (c) 2016 Intel Corporation
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * Author: Wind Yuan <feng.yuan@intel.com>
19 */
20
21 #include "cl_utils.h"
22 #include "cl_image_360_stitch.h"
23 #if HAVE_OPENCV
24 #include "cv_feature_match.h"
25 #endif
26
27 #define XCAM_BLENDER_GLOBAL_SCALE_EXT_WIDTH 64
28
29 #define STITCH_CHECK(ret, msg, ...) \
30 if ((ret) != XCAM_RETURN_NO_ERROR) { \
31 XCAM_LOG_WARNING (msg, ## __VA_ARGS__); \
32 return ret; \
33 }
34
35 namespace XCam {
36
CLBlenderGlobalScaleKernel(const SmartPtr<CLContext> & context,SmartPtr<CLImage360Stitch> & stitch,bool is_uv)37 CLBlenderGlobalScaleKernel::CLBlenderGlobalScaleKernel (
38 const SmartPtr<CLContext> &context, SmartPtr<CLImage360Stitch> &stitch, bool is_uv)
39 : CLBlenderScaleKernel (context, is_uv)
40 , _stitch (stitch)
41 {
42 }
43
44 SmartPtr<CLImage>
get_input_image()45 CLBlenderGlobalScaleKernel::get_input_image () {
46 SmartPtr<CLContext> context = get_context ();
47 SmartPtr<VideoBuffer> input = _stitch->get_global_scale_input ();
48
49 CLImageDesc cl_desc;
50 SmartPtr<CLImage> cl_image;
51 const VideoBufferInfo &buf_info = input->get_video_info ();
52
53 cl_desc.format.image_channel_data_type = CL_UNORM_INT8;
54 if (_is_uv) {
55 cl_desc.format.image_channel_order = CL_RG;
56 cl_desc.width = buf_info.width / 2;
57 cl_desc.height = buf_info.height / 2;
58 cl_desc.row_pitch = buf_info.strides[1];
59 cl_image = convert_to_climage (context, input, cl_desc, buf_info.offsets[1]);
60 } else {
61 cl_desc.format.image_channel_order = CL_R;
62 cl_desc.width = buf_info.width;
63 cl_desc.height = buf_info.height;
64 cl_desc.row_pitch = buf_info.strides[0];
65 cl_image = convert_to_climage (context, input, cl_desc, buf_info.offsets[0]);
66 }
67
68 return cl_image;
69 }
70
71 SmartPtr<CLImage>
get_output_image()72 CLBlenderGlobalScaleKernel::get_output_image () {
73 SmartPtr<CLContext> context = get_context ();
74 SmartPtr<VideoBuffer> output = _stitch->get_global_scale_output ();
75
76 CLImageDesc cl_desc;
77 SmartPtr<CLImage> cl_image;
78 const VideoBufferInfo &buf_info = output->get_video_info ();
79
80 cl_desc.format.image_channel_data_type = CL_UNSIGNED_INT16;
81 cl_desc.format.image_channel_order = CL_RGBA;
82 if (_is_uv) {
83 cl_desc.width = buf_info.width / 8;
84 cl_desc.height = buf_info.height / 2;
85 cl_desc.row_pitch = buf_info.strides[1];
86 cl_image = convert_to_climage (context, output, cl_desc, buf_info.offsets[1]);
87 } else {
88 cl_desc.width = buf_info.width / 8;
89 cl_desc.height = buf_info.height;
90 cl_desc.row_pitch = buf_info.strides[0];
91 cl_image = convert_to_climage (context, output, cl_desc, buf_info.offsets[0]);
92 }
93
94 return cl_image;
95 }
96
97 bool
get_output_info(uint32_t & out_width,uint32_t & out_height,int & out_offset_x)98 CLBlenderGlobalScaleKernel::get_output_info (
99 uint32_t &out_width, uint32_t &out_height, int &out_offset_x)
100 {
101 SmartPtr<VideoBuffer> output = _stitch->get_global_scale_output ();
102 const VideoBufferInfo &output_info = output->get_video_info ();
103
104 out_width = output_info.width / 8;
105 out_height = _is_uv ? output_info.height / 2 : output_info.height;
106 out_offset_x = 0;
107
108 return true;
109 }
110
111 #if HAVE_OPENCV
112 static CVFMConfig
get_fm_default_config(StitchResMode res_mode)113 get_fm_default_config (StitchResMode res_mode)
114 {
115 CVFMConfig config;
116
117 switch (res_mode) {
118 case StitchRes1080P: {
119 config.sitch_min_width = 56;
120 config.min_corners = 8;
121 config.offset_factor = 0.8f;
122 config.delta_mean_offset = 5.0f;
123 config.recur_offset_error = 8.0f;
124 config.max_adjusted_offset = 12.0f;
125 config.max_valid_offset_y = 8.0f;
126 config.max_track_error = 24.0f;
127
128 break;
129 }
130 case StitchRes1080P4: {
131 config.sitch_min_width = 128;
132 config.min_corners = 4;
133 config.offset_factor = 0.8f;
134 config.delta_mean_offset = 24.0f;
135 config.recur_offset_error = 12.0f;
136 config.max_adjusted_offset = 24.0f;
137 config.max_valid_offset_y = 64.0f;
138 config.max_track_error = 32.0f;
139
140 break;
141 }
142 case StitchRes4K: {
143 config.sitch_min_width = 160;
144 config.min_corners = 8;
145 config.offset_factor = 0.8f;
146 config.delta_mean_offset = 5.0f;
147 config.recur_offset_error = 8.0f;
148 config.max_adjusted_offset = 12.0f;
149 config.max_valid_offset_y = 8.0f;
150 config.max_track_error = 24.0f;
151
152 break;
153 }
154 default:
155 XCAM_LOG_DEBUG ("unknown reslution mode (%d)", res_mode);
156 break;
157 }
158
159 return config;
160 }
161 #endif
162
163 static StitchInfo
get_default_stitch_info(StitchResMode res_mode)164 get_default_stitch_info (StitchResMode res_mode)
165 {
166 StitchInfo stitch_info;
167
168 switch (res_mode) {
169 case StitchRes1080P: {
170 stitch_info.merge_width[0] = 56;
171 stitch_info.merge_width[1] = 56;
172
173 stitch_info.crop[0].left = 96;
174 stitch_info.crop[0].right = 96;
175 stitch_info.crop[0].top = 0;
176 stitch_info.crop[0].bottom = 0;
177 stitch_info.crop[1].left = 96;
178 stitch_info.crop[1].right = 96;
179 stitch_info.crop[1].top = 0;
180 stitch_info.crop[1].bottom = 0;
181
182 stitch_info.fisheye_info[0].center_x = 480.0f;
183 stitch_info.fisheye_info[0].center_y = 480.0f;
184 stitch_info.fisheye_info[0].wide_angle = 202.8f;
185 stitch_info.fisheye_info[0].radius = 480.0f;
186 stitch_info.fisheye_info[0].rotate_angle = -90.0f;
187 stitch_info.fisheye_info[1].center_x = 1440.0f;
188 stitch_info.fisheye_info[1].center_y = 480.0f;
189 stitch_info.fisheye_info[1].wide_angle = 202.8f;
190 stitch_info.fisheye_info[1].radius = 480.0f;
191 stitch_info.fisheye_info[1].rotate_angle = 89.4f;
192 break;
193 }
194 case StitchRes1080P4: {
195 stitch_info.merge_width[0] = 288;
196 stitch_info.merge_width[1] = 288;
197 stitch_info.merge_width[2] = 288;
198 stitch_info.merge_width[3] = 288;
199
200 stitch_info.crop[0].left = 0;
201 stitch_info.crop[0].right = 0;
202 stitch_info.crop[0].top = 0;
203 stitch_info.crop[0].bottom = 0;
204 stitch_info.crop[1].left = 0;
205 stitch_info.crop[1].right = 0;
206 stitch_info.crop[1].top = 0;
207 stitch_info.crop[1].bottom = 0;
208 stitch_info.crop[2].left = 0;
209 stitch_info.crop[2].right = 0;
210 stitch_info.crop[2].top = 0;
211 stitch_info.crop[2].bottom = 0;
212 stitch_info.crop[3].left = 0;
213 stitch_info.crop[3].right = 0;
214 stitch_info.crop[3].top = 0;
215 stitch_info.crop[3].bottom = 0;
216
217 stitch_info.fisheye_info[0].center_x = 640.0f;
218 stitch_info.fisheye_info[0].center_y = 400.0f;
219 stitch_info.fisheye_info[0].wide_angle = 120.0f;
220 stitch_info.fisheye_info[0].radius = 640.0f;
221 stitch_info.fisheye_info[0].rotate_angle = 0.0f;
222 stitch_info.fisheye_info[1].center_x = 640.0f;
223 stitch_info.fisheye_info[1].center_y = 400.0f;
224 stitch_info.fisheye_info[1].wide_angle = 120.0f;
225 stitch_info.fisheye_info[1].radius = 640.0f;
226 stitch_info.fisheye_info[1].rotate_angle = 0.0f;
227 stitch_info.fisheye_info[2].center_x = 640.0f;
228 stitch_info.fisheye_info[2].center_y = 400.0f;
229 stitch_info.fisheye_info[2].wide_angle = 120.0f;
230 stitch_info.fisheye_info[2].radius = 640.0f;
231 stitch_info.fisheye_info[2].rotate_angle = 0.0f;
232 stitch_info.fisheye_info[3].center_x = 640.0f;
233 stitch_info.fisheye_info[3].center_y = 400.0f;
234 stitch_info.fisheye_info[3].wide_angle = 120.0f;
235 stitch_info.fisheye_info[3].radius = 640.0f;
236 stitch_info.fisheye_info[3].rotate_angle = 0.0f;
237 break;
238 }
239 case StitchRes4K: {
240 stitch_info.merge_width[0] = 160;
241 stitch_info.merge_width[1] = 160;
242
243 stitch_info.crop[0].left = 64;
244 stitch_info.crop[0].right = 64;
245 stitch_info.crop[0].top = 0;
246 stitch_info.crop[0].bottom = 0;
247 stitch_info.crop[1].left = 64;
248 stitch_info.crop[1].right = 64;
249 stitch_info.crop[1].top = 0;
250 stitch_info.crop[1].bottom = 0;
251
252 stitch_info.fisheye_info[0].center_x = 1024.0f;
253 stitch_info.fisheye_info[0].center_y = 1024.0f;
254 stitch_info.fisheye_info[0].wide_angle = 195.0f;
255 stitch_info.fisheye_info[0].radius = 1040.0f;
256 stitch_info.fisheye_info[0].rotate_angle = 0.0f;
257
258 stitch_info.fisheye_info[1].center_x = 3072.0f;
259 stitch_info.fisheye_info[1].center_y = 1016.0f;
260 stitch_info.fisheye_info[1].wide_angle = 192.0f;
261 stitch_info.fisheye_info[1].radius = 1040.0f;
262 stitch_info.fisheye_info[1].rotate_angle = 0.4f;
263 break;
264 }
265 default:
266 XCAM_LOG_DEBUG ("unknown reslution mode (%d)", res_mode);
267 break;
268 }
269
270 return stitch_info;
271 }
272
CLImage360Stitch(const SmartPtr<CLContext> & context,CLBlenderScaleMode scale_mode,SurroundMode surround_mode,StitchResMode res_mode,int fisheye_num,bool all_in_one_img)273 CLImage360Stitch::CLImage360Stitch (
274 const SmartPtr<CLContext> &context, CLBlenderScaleMode scale_mode, SurroundMode surround_mode,
275 StitchResMode res_mode, int fisheye_num, bool all_in_one_img)
276 : CLMultiImageHandler (context, "CLImage360Stitch")
277 , _context (context)
278 , _output_width (0)
279 , _output_height (0)
280 , _scale_mode (scale_mode)
281 , _surround_mode (surround_mode)
282 , _res_mode (res_mode)
283 , _is_stitch_inited (false)
284 , _fisheye_num (fisheye_num)
285 , _all_in_one_img (all_in_one_img)
286 {
287 #if HAVE_OPENCV
288 for (int i = 0; i < fisheye_num; i++) {
289 _feature_match[i] = new CVFeatureMatch ();
290 XCAM_ASSERT (_feature_match[i].ptr ());
291 _feature_match[i]->set_config (get_fm_default_config (res_mode));
292 _feature_match[i]->set_fm_index (i);
293 }
294 #endif
295 }
296
297 bool
set_stitch_info(StitchInfo stitch_info)298 CLImage360Stitch::set_stitch_info (StitchInfo stitch_info)
299 {
300 if (_is_stitch_inited) {
301 XCAM_LOG_WARNING ("stitching info was initialized and can't be set twice");
302 return false;
303 }
304
305 for (int index = 0; index < _fisheye_num; ++index) {
306 _fisheye[index].handler->set_fisheye_info (stitch_info.fisheye_info[index]);
307 }
308
309 _stitch_info = stitch_info;
310 _is_stitch_inited = true;
311
312 return true;
313 }
314
315 StitchInfo
get_stitch_info()316 CLImage360Stitch::get_stitch_info ()
317 {
318 if (!_is_stitch_inited) {
319 XCAM_LOG_WARNING ("stitch-info was not initialized, return default parameters");
320 return get_default_stitch_info (_res_mode);
321 }
322
323 return _stitch_info;
324 }
325
326 bool
set_fisheye_handler(SmartPtr<CLFisheyeHandler> fisheye,int index)327 CLImage360Stitch::set_fisheye_handler (SmartPtr<CLFisheyeHandler> fisheye, int index)
328 {
329 XCAM_ASSERT (index < _fisheye_num);
330
331 _fisheye[index].handler = fisheye;
332 SmartPtr<CLImageHandler> handler = fisheye;
333 return add_image_handler (handler);
334 }
335
336 bool
set_blender(SmartPtr<CLBlender> blender,int idx)337 CLImage360Stitch::set_blender (SmartPtr<CLBlender> blender, int idx)
338 {
339 _blender[idx] = blender;
340
341 SmartPtr<CLImageHandler> handler = blender;
342 return add_image_handler (handler);
343 }
344
345 void
set_fisheye_intrinsic(IntrinsicParameter intrinsic_param,int index)346 CLImage360Stitch::set_fisheye_intrinsic (IntrinsicParameter intrinsic_param, int index)
347 {
348 _fisheye[index].handler->set_intrinsic_param(intrinsic_param);
349 }
350
351 void
set_fisheye_extrinsic(ExtrinsicParameter extrinsic_param,int index)352 CLImage360Stitch::set_fisheye_extrinsic (ExtrinsicParameter extrinsic_param, int index)
353 {
354 _fisheye[index].handler->set_extrinsic_param(extrinsic_param);
355 }
356
357 const BowlDataConfig &
get_fisheye_bowl_config(int index)358 CLImage360Stitch::get_fisheye_bowl_config (int index)
359 {
360 XCAM_ASSERT (index < _fisheye_num);
361 return _fisheye[index].handler->get_bowl_config ();
362 }
363
364 bool
set_image_overlap(const int idx,const Rect & overlap0,const Rect & overlap1)365 CLImage360Stitch::set_image_overlap (const int idx, const Rect &overlap0, const Rect &overlap1)
366 {
367 XCAM_ASSERT (idx < _fisheye_num);
368 _overlaps[idx][0] = overlap0;
369 _overlaps[idx][1] = overlap1;
370 return true;
371 }
372
373 void
set_feature_match_ocl(bool fm_ocl)374 CLImage360Stitch::set_feature_match_ocl (bool fm_ocl)
375 {
376 #if HAVE_OPENCV
377 for (int i = 0; i < _fisheye_num; i++) {
378 _feature_match[i]->set_ocl (fm_ocl);
379 }
380 #else
381 XCAM_UNUSED (fm_ocl);
382 XCAM_LOG_WARNING ("non-OpenCV mode, failed to set ocl for feature match");
383 #endif
384 }
385
386 #if HAVE_OPENCV
387 void
set_feature_match_config(const int idx,CVFMConfig config)388 CLImage360Stitch::set_feature_match_config (const int idx, CVFMConfig config)
389 {
390 _feature_match[idx]->set_config (config);
391 }
392
393 CVFMConfig
get_feature_match_config(const int idx)394 CLImage360Stitch::get_feature_match_config (const int idx)
395 {
396 return _feature_match[idx]->get_config ();
397 }
398 #endif
399
400 void
calc_fisheye_initial_info(SmartPtr<VideoBuffer> & output)401 CLImage360Stitch::calc_fisheye_initial_info (SmartPtr<VideoBuffer> &output)
402 {
403 const VideoBufferInfo &out_info = output->get_video_info ();
404
405 if(_surround_mode == SphereView) {
406 uint32_t fisheye_width_sum = out_info.width;
407 for (int i = 0; i < _fisheye_num; i++) {
408 fisheye_width_sum += _stitch_info.merge_width[i] + _stitch_info.crop[i].left + _stitch_info.crop[i].right;
409 }
410 _fisheye[0].width = fisheye_width_sum / _fisheye_num;
411 _fisheye[0].width = XCAM_ALIGN_UP (_fisheye[0].width, 16);
412 _fisheye[0].height = out_info.height + _stitch_info.crop[0].top + _stitch_info.crop[0].bottom;
413 XCAM_LOG_INFO (
414 "fisheye correction output size width:%d height:%d",
415 _fisheye[0].width, _fisheye[0].height);
416
417 for (int i = 1; i < _fisheye_num; i++) {
418 _fisheye[i].width = _fisheye[0].width;
419 _fisheye[i].height = _fisheye[0].height;
420 }
421
422 float max_dst_longitude, max_dst_latitude;
423 for (int i = 0; i < _fisheye_num; ++i) {
424 max_dst_latitude = (_stitch_info.fisheye_info[i].wide_angle > 180.0f) ?
425 180.0f : _stitch_info.fisheye_info[i].wide_angle;
426 max_dst_longitude = max_dst_latitude * _fisheye[i].width / _fisheye[i].height;
427
428 _fisheye[i].handler->set_dst_range (max_dst_longitude, max_dst_latitude);
429 _fisheye[i].handler->set_output_size (_fisheye[i].width, _fisheye[i].height);
430 }
431 } else {
432 _fisheye[0].height = out_info.height + _stitch_info.crop[0].top + _stitch_info.crop[0].bottom;
433
434 float view_angle[XCAM_STITCH_FISHEYE_MAX_NUM];
435
436 view_angle[0] = 68.0f;
437 _fisheye[0].width = view_angle[0] / 360.0f * out_info.width;
438 _fisheye[0].width = XCAM_ALIGN_UP (_fisheye[0].width, 32);
439
440 view_angle[1] = 152.0f;
441 _fisheye[1].width = view_angle[1] / 360.0f * out_info.width;
442 _fisheye[1].width = XCAM_ALIGN_UP (_fisheye[1].width, 32);
443
444 view_angle[2] = 68.0f;
445 _fisheye[2].width = view_angle[2] / 360.0f * out_info.width;
446 _fisheye[2].width = XCAM_ALIGN_UP (_fisheye[2].width, 32);
447
448 view_angle[3] = 152.0f;
449 _fisheye[3].width = view_angle[3] / 360.0f * out_info.width;
450 _fisheye[3].width = XCAM_ALIGN_UP (_fisheye[3].width, 32);
451
452 XCAM_LOG_INFO (
453 "fisheye correction output size width:%d height:%d",
454 _fisheye[0].width, _fisheye[0].height);
455
456 BowlDataConfig bowl_data_config[XCAM_STITCH_FISHEYE_MAX_NUM];
457
458 bowl_data_config[0].angle_start = -view_angle[0] / 2;
459 bowl_data_config[0].angle_end = view_angle[0] / 2;
460
461 for (int i = 1; i < _fisheye_num; i++) {
462 _fisheye[i].height = _fisheye[0].height;
463 float angle_center = 360.0f / _fisheye_num * i;
464 bowl_data_config[i].angle_start = angle_center - view_angle[i] / 2;
465 bowl_data_config[i].angle_end = angle_center + view_angle[i] / 2;
466 }
467
468 for(int i = 0; i < _fisheye_num; i++) {
469 _fisheye[i].handler->set_bowl_config(bowl_data_config[i]);
470 _fisheye[i].handler->set_output_size (_fisheye[i].width, _fisheye[i].height);
471 }
472
473 for(int i = 0; i < _fisheye_num; i++) {
474 _stitch_info.merge_width[i] = XCAM_ALIGN_UP((uint32_t)(20.0f / 360.0f * out_info.width), 32);
475 }
476 }
477 }
478
479 void
update_image_overlap()480 CLImage360Stitch::update_image_overlap ()
481 {
482 static bool is_merge_info_inited = false;
483 if (!is_merge_info_inited) {
484 int idx_next = 1;
485 for (int i = 0; i < _fisheye_num; i++) {
486 idx_next = (i == (_fisheye_num - 1)) ? 0 : (i + 1);
487
488 _img_merge_info[i].left.pos_x = _stitch_info.crop[i].left;
489 _img_merge_info[i].left.pos_y = _stitch_info.crop[i].top;
490 _img_merge_info[i].left.width = _stitch_info.merge_width[i];
491 _img_merge_info[i].left.height = _fisheye[i].height - _stitch_info.crop[i].top
492 - _stitch_info.crop[i].bottom;
493
494 _img_merge_info[i].right.pos_x = _fisheye[i].width - _stitch_info.crop[i].right
495 - _stitch_info.merge_width[idx_next];
496 _img_merge_info[i].right.pos_y = _stitch_info.crop[i].top;
497 _img_merge_info[i].right.width = _stitch_info.merge_width[idx_next];
498 _img_merge_info[i].right.height = _fisheye[i].height - _stitch_info.crop[i].top
499 - _stitch_info.crop[i].bottom;
500 }
501
502 is_merge_info_inited = true;
503 }
504
505 for (int i = 0; i < _fisheye_num; i++) {
506 set_image_overlap (i, _img_merge_info[i].left, _img_merge_info[i].right);
507 }
508 }
509
510 XCamReturn
prepare_buffer_pool_video_info(const VideoBufferInfo & input,VideoBufferInfo & output)511 CLImage360Stitch::prepare_buffer_pool_video_info (
512 const VideoBufferInfo &input, VideoBufferInfo &output)
513 {
514 if (_output_width == 0 || _output_height == 0) {
515 XCAM_LOG_ERROR ("incorrect output size: width:%d height:%d", _output_width, _output_height);
516 return XCAM_RETURN_ERROR_PARAM;
517 }
518
519 // aligned at least XCAM_CL_BLENDER_ALIGNMENT_X
520 uint32_t aligned_width = XCAM_MAX (16, XCAM_CL_BLENDER_ALIGNMENT_X);
521 output.init (
522 input.format, _output_width, _output_height,
523 XCAM_ALIGN_UP(_output_width, aligned_width), XCAM_ALIGN_UP(_output_height, 16));
524
525 return XCAM_RETURN_NO_ERROR;
526 }
527
528 XCamReturn
ensure_fisheye_parameters(SmartPtr<VideoBuffer> & input,SmartPtr<VideoBuffer> & output)529 CLImage360Stitch::ensure_fisheye_parameters (
530 SmartPtr<VideoBuffer> &input, SmartPtr<VideoBuffer> &output)
531 {
532 static bool is_fisheye_inited = false;
533
534 if (!is_fisheye_inited) {
535 calc_fisheye_initial_info (output);
536 is_fisheye_inited = true;
537 }
538
539 SmartPtr<VideoBuffer> pre_buf;
540 SmartPtr<VideoBuffer> cur_buf = input;
541 for (int i = 0; i < _fisheye_num; i++) {
542 if (!_fisheye[i].pool.ptr ())
543 create_buffer_pool (_fisheye[i].pool, _fisheye[i].width, _fisheye[i].height);
544
545 _fisheye[i].buf = _fisheye[i].pool->get_buffer (_fisheye[i].pool);
546 XCAM_ASSERT (_fisheye[i].buf.ptr ());
547
548 XCamReturn ret = ensure_handler_parameters (_fisheye[i].handler, cur_buf, _fisheye[i].buf);
549 STITCH_CHECK (ret, "execute fisheye prepare_parameters failed");
550
551 if (!_all_in_one_img) {
552 pre_buf = cur_buf;
553 cur_buf = cur_buf->find_typed_attach<VideoBuffer> ();
554 if (!cur_buf.ptr () && (i != (_fisheye_num - 1))) {
555 XCAM_LOG_ERROR ("conflicting attached buffers and fisheye number");
556 return XCAM_RETURN_ERROR_PARAM;
557 }
558 pre_buf->detach_buffer (cur_buf);
559 }
560 }
561
562 return XCAM_RETURN_NO_ERROR;
563 }
564
565 XCamReturn
prepare_global_scale_blender_parameters(SmartPtr<VideoBuffer> & input0,SmartPtr<VideoBuffer> & input1,SmartPtr<VideoBuffer> & output,int idx,int idx_next,int & cur_start_pos)566 CLImage360Stitch::prepare_global_scale_blender_parameters (
567 SmartPtr<VideoBuffer> &input0, SmartPtr<VideoBuffer> &input1, SmartPtr<VideoBuffer> &output,
568 int idx, int idx_next, int &cur_start_pos)
569 {
570 const VideoBufferInfo &in0_info = input0->get_video_info ();
571 const VideoBufferInfo &in1_info = input1->get_video_info ();
572 const VideoBufferInfo &out_info = output->get_video_info ();
573
574 XCAM_ASSERT (in0_info.height == in1_info.height);
575 XCAM_ASSERT (in0_info.width <= out_info.width && in1_info.width <= out_info.width);
576
577 Rect left_lap = get_image_overlap (idx, 1);
578 Rect right_lap = get_image_overlap (idx_next, 0);
579
580 int left_img_mid = XCAM_ALIGN_DOWN (in0_info.width / 2, XCAM_CL_BLENDER_ALIGNMENT_X);
581 int right_img_mid = XCAM_ALIGN_DOWN (in1_info.width / 2, XCAM_CL_BLENDER_ALIGNMENT_X);
582
583 int32_t prev_pos;
584 prev_pos = left_lap.pos_x;
585 left_lap.pos_x = XCAM_ALIGN_AROUND (left_lap.pos_x, XCAM_CL_BLENDER_ALIGNMENT_X);
586 left_lap.width = XCAM_ALIGN_UP (left_lap.width, XCAM_CL_BLENDER_ALIGNMENT_X);
587 right_lap.pos_x += left_lap.pos_x - prev_pos;
588 right_lap.pos_x = XCAM_ALIGN_AROUND (right_lap.pos_x, XCAM_CL_BLENDER_ALIGNMENT_X);
589 right_lap.width = left_lap.width;
590
591 Rect area;
592 area.pos_y = left_lap.pos_y;
593 area.height = left_lap.height;
594 area.pos_x = left_img_mid;
595 area.width = left_lap.pos_x + left_lap.width - left_img_mid;
596 _blender[idx]->set_input_valid_area (area, 0);
597
598 area.pos_y = right_lap.pos_y;
599 area.height = right_lap.height;
600 area.pos_x = right_lap.pos_x;
601 area.width = right_img_mid - right_lap.pos_x;
602 _blender[idx]->set_input_valid_area (area, 1);
603
604 Rect out_merge_window;
605 out_merge_window.width = left_lap.width;
606 out_merge_window.pos_x = cur_start_pos + (left_lap.pos_x - left_img_mid);
607 out_merge_window.pos_y = 0;
608 out_merge_window.height = out_info.height;
609 _blender[idx]->set_merge_window (out_merge_window);
610
611 _blender[idx]->set_input_merge_area (left_lap, 0);
612 _blender[idx]->set_input_merge_area (right_lap, 1);
613
614 cur_start_pos += left_lap.pos_x - left_img_mid + right_img_mid - right_lap.pos_x;
615 return XCAM_RETURN_NO_ERROR;
616 }
617
618 XCamReturn
prepare_local_scale_blender_parameters(SmartPtr<VideoBuffer> & input0,SmartPtr<VideoBuffer> & input1,SmartPtr<VideoBuffer> & output,int idx,int idx_next)619 CLImage360Stitch::prepare_local_scale_blender_parameters (
620 SmartPtr<VideoBuffer> &input0, SmartPtr<VideoBuffer> &input1, SmartPtr<VideoBuffer> &output, int idx, int idx_next)
621 {
622 const VideoBufferInfo &in0_info = input0->get_video_info ();
623 const VideoBufferInfo &in1_info = input1->get_video_info ();
624 const VideoBufferInfo &out_info = output->get_video_info ();
625
626 XCAM_ASSERT (in0_info.height == in1_info.height);
627 XCAM_ASSERT (in0_info.width <= out_info.width && in1_info.width <= out_info.width);
628
629 Rect left_lap = get_image_overlap (idx, 1);
630 Rect right_lap = get_image_overlap (idx_next, 0);
631
632 int left_img_mid = XCAM_ALIGN_DOWN (in0_info.width / 2, XCAM_CL_BLENDER_ALIGNMENT_X);
633 int right_img_mid = XCAM_ALIGN_DOWN (in1_info.width / 2, XCAM_CL_BLENDER_ALIGNMENT_X);
634 int cur_start_pos = XCAM_ALIGN_DOWN (out_info.width / _fisheye_num * idx, XCAM_CL_BLENDER_ALIGNMENT_X);
635 int merge_std_width = XCAM_ALIGN_DOWN (out_info.width / _fisheye_num, XCAM_CL_BLENDER_ALIGNMENT_X);
636
637 int32_t prev_pos;
638 prev_pos = left_lap.pos_x;
639 left_lap.pos_x = XCAM_ALIGN_AROUND (left_lap.pos_x, XCAM_CL_BLENDER_ALIGNMENT_X);
640 left_lap.width = XCAM_ALIGN_UP (left_lap.width, XCAM_CL_BLENDER_ALIGNMENT_X);
641 right_lap.pos_x += left_lap.pos_x - prev_pos;
642 right_lap.pos_x = XCAM_ALIGN_AROUND (right_lap.pos_x, XCAM_CL_BLENDER_ALIGNMENT_X);
643 right_lap.width = left_lap.width;
644
645 Rect area;
646 area.pos_y = left_lap.pos_y;
647 area.height = left_lap.height;
648 area.pos_x = left_img_mid;
649 area.width = left_lap.pos_x + left_lap.width - left_img_mid;
650 _blender[idx]->set_input_valid_area (area, 0);
651
652 area.pos_y = right_lap.pos_y;
653 area.height = right_lap.height;
654 area.pos_x = right_lap.pos_x;
655 area.width = right_img_mid - right_lap.pos_x;
656 _blender[idx]->set_input_valid_area (area, 1);
657
658 Rect out_merge_window;
659 int delta_width = merge_std_width - (right_img_mid - right_lap.pos_x) - (left_lap.pos_x - left_img_mid);
660 out_merge_window.width = left_lap.width + delta_width;
661 out_merge_window.pos_x = cur_start_pos + (left_lap.pos_x - left_img_mid);
662 out_merge_window.pos_y = 0;
663 out_merge_window.height = out_info.height;
664 _blender[idx]->set_merge_window (out_merge_window);
665
666 _blender[idx]->set_input_merge_area (left_lap, 0);
667 _blender[idx]->set_input_merge_area (right_lap, 1);
668
669 return XCAM_RETURN_NO_ERROR;
670 }
671
672 bool
create_buffer_pool(SmartPtr<BufferPool> & buf_pool,uint32_t width,uint32_t height)673 CLImage360Stitch::create_buffer_pool (SmartPtr<BufferPool> &buf_pool, uint32_t width, uint32_t height)
674 {
675 VideoBufferInfo buf_info;
676 width = XCAM_ALIGN_UP (width, 16);
677 buf_info.init (V4L2_PIX_FMT_NV12, width, height,
678 XCAM_ALIGN_UP (width, 16), XCAM_ALIGN_UP (height, 16));
679
680 buf_pool = new CLVideoBufferPool ();
681 XCAM_ASSERT (buf_pool.ptr ());
682 buf_pool->set_video_info (buf_info);
683 if (!buf_pool->reserve (6)) {
684 XCAM_LOG_ERROR ("CLImage360Stitch init buffer pool failed");
685 return false;
686 }
687
688 return true;
689 }
690
691 XCamReturn
reset_buffer_info(SmartPtr<VideoBuffer> & input)692 CLImage360Stitch::reset_buffer_info (SmartPtr<VideoBuffer> &input)
693 {
694 VideoBufferInfo reset_info;
695 const VideoBufferInfo &buf_info = input->get_video_info ();
696
697 uint32_t reset_width = 0;
698 for (int i = 0; i < _fisheye_num; i++) {
699 Rect img_left = get_image_overlap (i, 0);
700 Rect img_right = get_image_overlap (i, 1);
701
702 reset_width += img_right.pos_x - img_left.pos_x;
703 }
704
705 reset_width = XCAM_ALIGN_UP (reset_width, XCAM_CL_BLENDER_ALIGNMENT_X);
706 reset_info.init (buf_info.format, reset_width, buf_info.height,
707 buf_info.aligned_width, buf_info.aligned_height);
708
709 input->set_video_info (reset_info);
710 return XCAM_RETURN_NO_ERROR;
711 }
712
713 XCamReturn
prepare_parameters(SmartPtr<VideoBuffer> & input,SmartPtr<VideoBuffer> & output)714 CLImage360Stitch::prepare_parameters (SmartPtr<VideoBuffer> &input, SmartPtr<VideoBuffer> &output)
715 {
716 XCamReturn ret = XCAM_RETURN_NO_ERROR;
717 if (!_is_stitch_inited)
718 set_stitch_info (get_default_stitch_info (_res_mode));
719
720 ret = ensure_fisheye_parameters (input, output);
721 STITCH_CHECK (ret, "ensure fisheye parameters failed");
722
723 update_image_overlap ();
724 if (_scale_mode == CLBlenderScaleLocal) {
725 int idx_next = 1;
726 for (int i = 0; i < _fisheye_num; i++) {
727 idx_next = (i == (_fisheye_num - 1)) ? 0 : (i + 1);
728
729 ret = prepare_local_scale_blender_parameters (
730 _fisheye[i].buf, _fisheye[idx_next].buf, output, i, idx_next);
731 STITCH_CHECK (ret, "prepare local scale blender parameters failed");
732
733 _fisheye[i].buf->attach_buffer (_fisheye[idx_next].buf);
734 ret = ensure_handler_parameters (_blender[i], _fisheye[i].buf, output);
735 STITCH_CHECK (ret, "blender: execute ensure_parameters failed");
736 _fisheye[i].buf->detach_buffer (_fisheye[idx_next].buf);
737 }
738 } else { //global scale
739 const VideoBufferInfo &buf_info = output->get_video_info ();
740 if (!_scale_buf_pool.ptr ())
741 create_buffer_pool (_scale_buf_pool, buf_info.width + XCAM_BLENDER_GLOBAL_SCALE_EXT_WIDTH, buf_info.height);
742 SmartPtr<VideoBuffer> scale_input = _scale_buf_pool->get_buffer (_scale_buf_pool);
743 XCAM_ASSERT (scale_input.ptr ());
744
745 int idx_next = 1;
746 int cur_start_pos = 0;
747 for (int i = 0; i < _fisheye_num; i++) {
748 idx_next = (i == (_fisheye_num - 1)) ? 0 : (i + 1);
749
750 ret = prepare_global_scale_blender_parameters (
751 _fisheye[i].buf, _fisheye[idx_next].buf, scale_input, i, idx_next, cur_start_pos);
752 STITCH_CHECK (ret, "prepare global scale blender parameters failed");
753
754 _fisheye[i].buf->attach_buffer (_fisheye[idx_next].buf);
755 ret = ensure_handler_parameters (_blender[i], _fisheye[i].buf, scale_input);
756 STITCH_CHECK (ret, "blender: execute ensure_parameters failed");
757 _fisheye[i].buf->detach_buffer (_fisheye[idx_next].buf);
758 }
759
760 reset_buffer_info (scale_input);
761 _scale_global_input = scale_input;
762 _scale_global_output = output;
763 }
764
765 return XCAM_RETURN_NO_ERROR;
766 }
767
768 XCamReturn
execute_done(SmartPtr<VideoBuffer> & output)769 CLImage360Stitch::execute_done (SmartPtr<VideoBuffer> &output)
770 {
771 #if HAVE_OPENCV
772 for (int i = 0; i < _fisheye_num; i++) {
773 if (!_feature_match[i]->is_ocl_path ()) {
774 get_context ()->finish ();
775 break;
776 }
777 }
778 #endif
779
780 _scale_global_input.release ();
781 _scale_global_output.release ();
782
783 return CLMultiImageHandler::execute_done (output);
784 }
785
786 static void
convert_to_stitch_rect(Rect xcam_rect,Rect & stitch_rect)787 convert_to_stitch_rect (Rect xcam_rect, Rect &stitch_rect)
788 {
789 stitch_rect.pos_x = xcam_rect.pos_x;
790 stitch_rect.pos_y = xcam_rect.pos_y + xcam_rect.height / 3;
791 stitch_rect.width = xcam_rect.width;
792 stitch_rect.height = xcam_rect.height / 3;
793 }
794
795 static void
convert_to_xcam_rect(Rect stitch_rect,Rect & xcam_rect)796 convert_to_xcam_rect (Rect stitch_rect, Rect &xcam_rect)
797 {
798 xcam_rect.pos_x = stitch_rect.pos_x;
799 xcam_rect.width = stitch_rect.width;
800 }
801
802
803 XCamReturn
sub_handler_execute_done(SmartPtr<CLImageHandler> & handler)804 CLImage360Stitch::sub_handler_execute_done (SmartPtr<CLImageHandler> &handler)
805 {
806 #if HAVE_OPENCV
807 XCAM_ASSERT (handler.ptr ());
808
809 if (handler.ptr () == _fisheye[_fisheye_num - 1].handler.ptr ()) {
810 int idx_next = 1;
811 Rect crop_left, crop_right;
812
813 for (int i = 0; i < _fisheye_num; i++) {
814 idx_next = (i == (_fisheye_num - 1)) ? 0 : (i + 1);
815
816 convert_to_stitch_rect (_img_merge_info[i].right, crop_left);
817 convert_to_stitch_rect (_img_merge_info[idx_next].left, crop_right);
818
819 _feature_match[i]->optical_flow_feature_match (
820 _fisheye[i].buf, _fisheye[idx_next].buf, crop_left, crop_right, _fisheye[i].width);
821
822 convert_to_xcam_rect (crop_left, _img_merge_info[i].right);
823 convert_to_xcam_rect (crop_right, _img_merge_info[idx_next].left);
824 }
825 }
826 #else
827 XCAM_UNUSED (handler);
828 #endif
829
830 return XCAM_RETURN_NO_ERROR;
831 }
832
833 static SmartPtr<CLImageKernel>
create_blender_global_scale_kernel(const SmartPtr<CLContext> & context,SmartPtr<CLImage360Stitch> & stitch,bool is_uv)834 create_blender_global_scale_kernel (
835 const SmartPtr<CLContext> &context,
836 SmartPtr<CLImage360Stitch> &stitch,
837 bool is_uv)
838 {
839 char transform_option[1024];
840 snprintf (transform_option, sizeof(transform_option), "-DPYRAMID_UV=%d", is_uv ? 1 : 0);
841
842 static const XCamKernelInfo &kernel_info = {
843 "kernel_pyramid_scale",
844 #include "kernel_gauss_lap_pyramid.clx"
845 , 0
846 };
847
848 SmartPtr<CLImageKernel> kernel;
849 kernel = new CLBlenderGlobalScaleKernel (context, stitch, is_uv);
850 XCAM_ASSERT (kernel.ptr ());
851 XCAM_FAIL_RETURN (
852 ERROR,
853 kernel->build_kernel (kernel_info, transform_option) == XCAM_RETURN_NO_ERROR,
854 NULL,
855 "load blender global scaling kernel(%s) failed", is_uv ? "UV" : "Y");
856
857 return kernel;
858 }
859
860 SmartPtr<CLImageHandler>
create_image_360_stitch(const SmartPtr<CLContext> & context,bool need_seam,CLBlenderScaleMode scale_mode,bool fisheye_map,bool need_lsc,SurroundMode surround_mode,StitchResMode res_mode,int fisheye_num,bool all_in_one_img)861 create_image_360_stitch (
862 const SmartPtr<CLContext> &context, bool need_seam,
863 CLBlenderScaleMode scale_mode, bool fisheye_map, bool need_lsc, SurroundMode surround_mode,
864 StitchResMode res_mode, int fisheye_num, bool all_in_one_img)
865 {
866 const int layer = 2;
867 const bool need_uv = true;
868 SmartPtr<CLFisheyeHandler> fisheye;
869 SmartPtr<CLBlender> blender;
870 SmartPtr<CLImage360Stitch> stitch = new CLImage360Stitch (
871 context, scale_mode, surround_mode, res_mode, fisheye_num, all_in_one_img);
872 XCAM_ASSERT (stitch.ptr ());
873
874 for (int index = 0; index < fisheye_num; ++index) {
875 fisheye = create_fisheye_handler (context, surround_mode, fisheye_map, need_lsc).dynamic_cast_ptr<CLFisheyeHandler> ();
876 XCAM_FAIL_RETURN (ERROR, fisheye.ptr (), NULL, "image_360_stitch create fisheye handler failed");
877 fisheye->disable_buf_pool (true);
878 stitch->set_fisheye_handler (fisheye, index);
879 }
880
881 for (int index = 0; index < fisheye_num; ++index) {
882 blender = create_pyramid_blender (context, layer, need_uv, need_seam, scale_mode).dynamic_cast_ptr<CLBlender> ();
883 XCAM_FAIL_RETURN (ERROR, blender.ptr (), NULL, "image_360_stitch create blender failed");
884 blender->disable_buf_pool (true);
885 stitch->set_blender (blender, index);
886 }
887
888 if (scale_mode == CLBlenderScaleGlobal) {
889 int max_plane = need_uv ? 2 : 1;
890 bool uv_status[2] = {false, true};
891 for (int plane = 0; plane < max_plane; ++plane) {
892 SmartPtr<CLImageKernel> kernel = create_blender_global_scale_kernel (context, stitch, uv_status[plane]);
893 XCAM_FAIL_RETURN (ERROR, kernel.ptr (), NULL, "create blender global scaling kernel failed");
894 stitch->add_kernel (kernel);
895 }
896 }
897
898 return stitch;
899 }
900
901 }
902
903