1 #include <string>
2 #include <iostream>
3 #include <fstream>
4 #include <sstream>
5 #include <stdexcept>
6 #include "opencv2/core.hpp"
7 #include <opencv2/core/utility.hpp>
8 #include "opencv2/video.hpp"
9 #include "opencv2/imgproc.hpp"
10 #include "opencv2/videoio.hpp"
11 #include "opencv2/highgui.hpp"
12 #include "opencv2/videostab.hpp"
13 #include "opencv2/opencv_modules.hpp"
14
15 #define arg(name) cmd.get<string>(name)
16 #define argb(name) cmd.get<bool>(name)
17 #define argi(name) cmd.get<int>(name)
18 #define argf(name) cmd.get<float>(name)
19 #define argd(name) cmd.get<double>(name)
20
21 using namespace std;
22 using namespace cv;
23 using namespace cv::videostab;
24
25 Ptr<IFrameSource> stabilizedFrames;
26 string saveMotionsPath;
27 double outputFps;
28 string outputPath;
29 bool quietMode;
30
31 void run();
32 void saveMotionsIfNecessary();
33 void printHelp();
34 MotionModel motionModel(const string &str);
35
36
run()37 void run()
38 {
39 VideoWriter writer;
40 Mat stabilizedFrame;
41 int nframes = 0;
42
43 // for each stabilized frame
44 while (!(stabilizedFrame = stabilizedFrames->nextFrame()).empty())
45 {
46 nframes++;
47
48 // init writer (once) and save stabilized frame
49 if (!outputPath.empty())
50 {
51 if (!writer.isOpened())
52 writer.open(outputPath, VideoWriter::fourcc('X','V','I','D'),
53 outputFps, stabilizedFrame.size());
54 writer << stabilizedFrame;
55 }
56
57 // show stabilized frame
58 if (!quietMode)
59 {
60 imshow("stabilizedFrame", stabilizedFrame);
61 char key = static_cast<char>(waitKey(3));
62 if (key == 27) { cout << endl; break; }
63 }
64 }
65
66 cout << "processed frames: " << nframes << endl
67 << "finished\n";
68 }
69
70
printHelp()71 void printHelp()
72 {
73 cout << "OpenCV video stabilizer.\n"
74 "Usage: videostab <file_path> [arguments]\n\n"
75 "Arguments:\n"
76 " -m, --model=(transl|transl_and_scale|rigid|similarity|affine|homography)\n"
77 " Set motion model. The default is affine.\n"
78 " -lp, --lin-prog-motion-est=(yes|no)\n"
79 " Turn on/off LP based motion estimation. The default is no.\n"
80 " --subset=(<int_number>|auto)\n"
81 " Number of random samples per one motion hypothesis. The default is auto.\n"
82 " --thresh=(<float_number>|auto)\n"
83 " Maximum error to classify match as inlier. The default is auto.\n"
84 " --outlier-ratio=<float_number>\n"
85 " Motion estimation outlier ratio hypothesis. The default is 0.5.\n"
86 " --min-inlier-ratio=<float_number>\n"
87 " Minimum inlier ratio to decide if estimated motion is OK. The default is 0.1.\n"
88 " --nkps=<int_number>\n"
89 " Number of keypoints to find in each frame. The default is 1000.\n"
90 " --local-outlier-rejection=(yes|no)\n"
91 " Perform local outlier rejection. The default is no.\n\n"
92 " -sm, --save-motions=(<file_path>|no)\n"
93 " Save estimated motions into file. The default is no.\n"
94 " -lm, --load-motions=(<file_path>|no)\n"
95 " Load motions from file. The default is no.\n\n"
96 " -r, --radius=<int_number>\n"
97 " Set sliding window radius. The default is 15.\n"
98 " --stdev=(<float_number>|auto)\n"
99 " Set smoothing weights standard deviation. The default is auto\n"
100 " (i.e. sqrt(radius)).\n"
101 " -lps, --lin-prog-stab=(yes|no)\n"
102 " Turn on/off linear programming based stabilization method.\n"
103 " --lps-trim-ratio=(<float_number>|auto)\n"
104 " Trimming ratio used in linear programming based method.\n"
105 " --lps-w1=(<float_number>|1)\n"
106 " 1st derivative weight. The default is 1.\n"
107 " --lps-w2=(<float_number>|10)\n"
108 " 2nd derivative weight. The default is 10.\n"
109 " --lps-w3=(<float_number>|100)\n"
110 " 3rd derivative weight. The default is 100.\n"
111 " --lps-w4=(<float_number>|100)\n"
112 " Non-translation motion components weight. The default is 100.\n\n"
113 " --deblur=(yes|no)\n"
114 " Do deblurring.\n"
115 " --deblur-sens=<float_number>\n"
116 " Set deblurring sensitivity (from 0 to +inf). The default is 0.1.\n\n"
117 " -t, --trim-ratio=<float_number>\n"
118 " Set trimming ratio (from 0 to 0.5). The default is 0.1.\n"
119 " -et, --est-trim=(yes|no)\n"
120 " Estimate trim ratio automatically. The default is yes.\n"
121 " -ic, --incl-constr=(yes|no)\n"
122 " Ensure the inclusion constraint is always satisfied. The default is no.\n\n"
123 " -bm, --border-mode=(replicate|reflect|const)\n"
124 " Set border extrapolation mode. The default is replicate.\n\n"
125 " --mosaic=(yes|no)\n"
126 " Do consistent mosaicing. The default is no.\n"
127 " --mosaic-stdev=<float_number>\n"
128 " Consistent mosaicing stdev threshold. The default is 10.0.\n\n"
129 " -mi, --motion-inpaint=(yes|no)\n"
130 " Do motion inpainting (requires CUDA support). The default is no.\n"
131 " --mi-dist-thresh=<float_number>\n"
132 " Estimated flow distance threshold for motion inpainting. The default is 5.0.\n\n"
133 " -ci, --color-inpaint=(no|average|ns|telea)\n"
134 " Do color inpainting. The defailt is no.\n"
135 " --ci-radius=<float_number>\n"
136 " Set color inpainting radius (for ns and telea options only).\n"
137 " The default is 2.0\n\n"
138 " -ws, --wobble-suppress=(yes|no)\n"
139 " Perform wobble suppression. The default is no.\n"
140 " --ws-lp=(yes|no)\n"
141 " Turn on/off LP based motion estimation. The default is no.\n"
142 " --ws-period=<int_number>\n"
143 " Set wobble suppression period. The default is 30.\n"
144 " --ws-model=(transl|transl_and_scale|rigid|similarity|affine|homography)\n"
145 " Set wobble suppression motion model (must have more DOF than motion \n"
146 " estimation model). The default is homography.\n"
147 " --ws-subset=(<int_number>|auto)\n"
148 " Number of random samples per one motion hypothesis. The default is auto.\n"
149 " --ws-thresh=(<float_number>|auto)\n"
150 " Maximum error to classify match as inlier. The default is auto.\n"
151 " --ws-outlier-ratio=<float_number>\n"
152 " Motion estimation outlier ratio hypothesis. The default is 0.5.\n"
153 " --ws-min-inlier-ratio=<float_number>\n"
154 " Minimum inlier ratio to decide if estimated motion is OK. The default is 0.1.\n"
155 " --ws-nkps=<int_number>\n"
156 " Number of keypoints to find in each frame. The default is 1000.\n"
157 " --ws-local-outlier-rejection=(yes|no)\n"
158 " Perform local outlier rejection. The default is no.\n\n"
159 " -sm2, --save-motions2=(<file_path>|no)\n"
160 " Save motions estimated for wobble suppression. The default is no.\n"
161 " -lm2, --load-motions2=(<file_path>|no)\n"
162 " Load motions for wobble suppression from file. The default is no.\n\n"
163 " -gpu=(yes|no)\n"
164 " Use CUDA optimization whenever possible. The default is no.\n\n"
165 " -o, --output=(no|<file_path>)\n"
166 " Set output file path explicitely. The default is stabilized.avi.\n"
167 " --fps=(<float_number>|auto)\n"
168 " Set output video FPS explicitely. By default the source FPS is used (auto).\n"
169 " -q, --quiet\n"
170 " Don't show output video frames.\n\n"
171 " -h, --help\n"
172 " Print help.\n\n"
173 "Note: some argument configurations lead to two passes, some to single pass.\n\n";
174 }
175
176 // motion estimator builders are for concise creation of motion estimators
177
178 class IMotionEstimatorBuilder
179 {
180 public:
~IMotionEstimatorBuilder()181 virtual ~IMotionEstimatorBuilder() {}
182 virtual Ptr<ImageMotionEstimatorBase> build() = 0;
183 protected:
IMotionEstimatorBuilder(CommandLineParser & command)184 IMotionEstimatorBuilder(CommandLineParser &command) : cmd(command) {}
185 CommandLineParser cmd;
186 };
187
188
189 class MotionEstimatorRansacL2Builder : public IMotionEstimatorBuilder
190 {
191 public:
MotionEstimatorRansacL2Builder(CommandLineParser & command,bool use_gpu,const string & _prefix="")192 MotionEstimatorRansacL2Builder(CommandLineParser &command, bool use_gpu, const string &_prefix = "")
193 : IMotionEstimatorBuilder(command), gpu(use_gpu), prefix(_prefix) {}
194
build()195 virtual Ptr<ImageMotionEstimatorBase> build()
196 {
197 Ptr<MotionEstimatorRansacL2> est = makePtr<MotionEstimatorRansacL2>(motionModel(arg(prefix + "model")));
198
199 RansacParams ransac = est->ransacParams();
200 if (arg(prefix + "subset") != "auto")
201 ransac.size = argi(prefix + "subset");
202 if (arg(prefix + "thresh") != "auto")
203 ransac.thresh = argf(prefix + "thresh");
204 ransac.eps = argf(prefix + "outlier-ratio");
205 est->setRansacParams(ransac);
206
207 est->setMinInlierRatio(argf(prefix + "min-inlier-ratio"));
208
209 Ptr<IOutlierRejector> outlierRejector = makePtr<NullOutlierRejector>();
210 if (arg(prefix + "local-outlier-rejection") == "yes")
211 {
212 Ptr<TranslationBasedLocalOutlierRejector> tblor = makePtr<TranslationBasedLocalOutlierRejector>();
213 RansacParams ransacParams = tblor->ransacParams();
214 if (arg(prefix + "thresh") != "auto")
215 ransacParams.thresh = argf(prefix + "thresh");
216 tblor->setRansacParams(ransacParams);
217 outlierRejector = tblor;
218 }
219
220 #if defined(HAVE_OPENCV_CUDAIMGPROC) && defined(HAVE_OPENCV_CUDAOPTFLOW)
221 if (gpu)
222 {
223 Ptr<KeypointBasedMotionEstimatorGpu> kbest = makePtr<KeypointBasedMotionEstimatorGpu>(est);
224 kbest->setOutlierRejector(outlierRejector);
225 return kbest;
226 }
227 #endif
228
229 Ptr<KeypointBasedMotionEstimator> kbest = makePtr<KeypointBasedMotionEstimator>(est);
230 kbest->setDetector(GFTTDetector::create(argi(prefix + "nkps")));
231 kbest->setOutlierRejector(outlierRejector);
232 return kbest;
233 }
234 private:
235 bool gpu;
236 string prefix;
237 };
238
239
240 class MotionEstimatorL1Builder : public IMotionEstimatorBuilder
241 {
242 public:
MotionEstimatorL1Builder(CommandLineParser & command,bool use_gpu,const string & _prefix="")243 MotionEstimatorL1Builder(CommandLineParser &command, bool use_gpu, const string &_prefix = "")
244 : IMotionEstimatorBuilder(command), gpu(use_gpu), prefix(_prefix) {}
245
build()246 virtual Ptr<ImageMotionEstimatorBase> build()
247 {
248 Ptr<MotionEstimatorL1> est = makePtr<MotionEstimatorL1>(motionModel(arg(prefix + "model")));
249
250 Ptr<IOutlierRejector> outlierRejector = makePtr<NullOutlierRejector>();
251 if (arg(prefix + "local-outlier-rejection") == "yes")
252 {
253 Ptr<TranslationBasedLocalOutlierRejector> tblor = makePtr<TranslationBasedLocalOutlierRejector>();
254 RansacParams ransacParams = tblor->ransacParams();
255 if (arg(prefix + "thresh") != "auto")
256 ransacParams.thresh = argf(prefix + "thresh");
257 tblor->setRansacParams(ransacParams);
258 outlierRejector = tblor;
259 }
260
261 #if defined(HAVE_OPENCV_CUDAIMGPROC) && defined(HAVE_OPENCV_CUDAOPTFLOW)
262 if (gpu)
263 {
264 Ptr<KeypointBasedMotionEstimatorGpu> kbest = makePtr<KeypointBasedMotionEstimatorGpu>(est);
265 kbest->setOutlierRejector(outlierRejector);
266 return kbest;
267 }
268 #endif
269
270 Ptr<KeypointBasedMotionEstimator> kbest = makePtr<KeypointBasedMotionEstimator>(est);
271 kbest->setDetector(GFTTDetector::create(argi(prefix + "nkps")));
272 kbest->setOutlierRejector(outlierRejector);
273 return kbest;
274 }
275 private:
276 bool gpu;
277 string prefix;
278 };
279
280
main(int argc,const char ** argv)281 int main(int argc, const char **argv)
282 {
283 try
284 {
285 const char *keys =
286 "{ @1 | | }"
287 "{ m model | affine | }"
288 "{ lp lin-prog-motion-est | no | }"
289 "{ subset | auto | }"
290 "{ thresh | auto | }"
291 "{ outlier-ratio | 0.5 | }"
292 "{ min-inlier-ratio | 0.1 | }"
293 "{ nkps | 1000 | }"
294 "{ extra-kps | 0 | }"
295 "{ local-outlier-rejection | no | }"
296 "{ sm save-motions | no | }"
297 "{ lm load-motions | no | }"
298 "{ r radius | 15 | }"
299 "{ stdev | auto | }"
300 "{ lps lin-prog-stab | no | }"
301 "{ lps-trim-ratio | auto | }"
302 "{ lps-w1 | 1 | }"
303 "{ lps-w2 | 10 | }"
304 "{ lps-w3 | 100 | }"
305 "{ lps-w4 | 100 | }"
306 "{ deblur | no | }"
307 "{ deblur-sens | 0.1 | }"
308 "{ et est-trim | yes | }"
309 "{ t trim-ratio | 0.1 | }"
310 "{ ic incl-constr | no | }"
311 "{ bm border-mode | replicate | }"
312 "{ mosaic | no | }"
313 "{ ms mosaic-stdev | 10.0 | }"
314 "{ mi motion-inpaint | no | }"
315 "{ mi-dist-thresh | 5.0 | }"
316 "{ ci color-inpaint | no | }"
317 "{ ci-radius | 2 | }"
318 "{ ws wobble-suppress | no | }"
319 "{ ws-period | 30 | }"
320 "{ ws-model | homography | }"
321 "{ ws-subset | auto | }"
322 "{ ws-thresh | auto | }"
323 "{ ws-outlier-ratio | 0.5 | }"
324 "{ ws-min-inlier-ratio | 0.1 | }"
325 "{ ws-nkps | 1000 | }"
326 "{ ws-extra-kps | 0 | }"
327 "{ ws-local-outlier-rejection | no | }"
328 "{ ws-lp | no | }"
329 "{ sm2 save-motions2 | no | }"
330 "{ lm2 load-motions2 | no | }"
331 "{ gpu | no | }"
332 "{ o output | stabilized.avi | }"
333 "{ fps | auto | }"
334 "{ q quiet | | }"
335 "{ h help | | }";
336 CommandLineParser cmd(argc, argv, keys);
337
338 // parse command arguments
339
340 if (argb("help"))
341 {
342 printHelp();
343 return 0;
344 }
345
346 if (arg("gpu") == "yes")
347 {
348 cout << "initializing GPU..."; cout.flush();
349 Mat hostTmp = Mat::zeros(1, 1, CV_32F);
350 cuda::GpuMat deviceTmp;
351 deviceTmp.upload(hostTmp);
352 cout << endl;
353 }
354
355 StabilizerBase *stabilizer = 0;
356
357 // check if source video is specified
358
359 string inputPath = arg(0);
360 if (inputPath.empty())
361 throw runtime_error("specify video file path");
362
363 // get source video parameters
364
365 Ptr<VideoFileSource> source = makePtr<VideoFileSource>(inputPath);
366 cout << "frame count (rough): " << source->count() << endl;
367 if (arg("fps") == "auto")
368 outputFps = source->fps();
369 else
370 outputFps = argd("fps");
371
372 // prepare motion estimation builders
373
374 Ptr<IMotionEstimatorBuilder> motionEstBuilder;
375 if (arg("lin-prog-motion-est") == "yes")
376 motionEstBuilder.reset(new MotionEstimatorL1Builder(cmd, arg("gpu") == "yes"));
377 else
378 motionEstBuilder.reset(new MotionEstimatorRansacL2Builder(cmd, arg("gpu") == "yes"));
379
380 Ptr<IMotionEstimatorBuilder> wsMotionEstBuilder;
381 if (arg("ws-lp") == "yes")
382 wsMotionEstBuilder.reset(new MotionEstimatorL1Builder(cmd, arg("gpu") == "yes", "ws-"));
383 else
384 wsMotionEstBuilder.reset(new MotionEstimatorRansacL2Builder(cmd, arg("gpu") == "yes", "ws-"));
385
386 // determine whether we must use one pass or two pass stabilizer
387 bool isTwoPass =
388 arg("est-trim") == "yes" || arg("wobble-suppress") == "yes" || arg("lin-prog-stab") == "yes";
389
390 if (isTwoPass)
391 {
392 // we must use two pass stabilizer
393
394 TwoPassStabilizer *twoPassStabilizer = new TwoPassStabilizer();
395 stabilizer = twoPassStabilizer;
396 twoPassStabilizer->setEstimateTrimRatio(arg("est-trim") == "yes");
397
398 // determine stabilization technique
399
400 if (arg("lin-prog-stab") == "yes")
401 {
402 Ptr<LpMotionStabilizer> stab = makePtr<LpMotionStabilizer>();
403 stab->setFrameSize(Size(source->width(), source->height()));
404 stab->setTrimRatio(arg("lps-trim-ratio") == "auto" ? argf("trim-ratio") : argf("lps-trim-ratio"));
405 stab->setWeight1(argf("lps-w1"));
406 stab->setWeight2(argf("lps-w2"));
407 stab->setWeight3(argf("lps-w3"));
408 stab->setWeight4(argf("lps-w4"));
409 twoPassStabilizer->setMotionStabilizer(stab);
410 }
411 else if (arg("stdev") == "auto")
412 twoPassStabilizer->setMotionStabilizer(makePtr<GaussianMotionFilter>(argi("radius")));
413 else
414 twoPassStabilizer->setMotionStabilizer(makePtr<GaussianMotionFilter>(argi("radius"), argf("stdev")));
415
416 // init wobble suppressor if necessary
417
418 if (arg("wobble-suppress") == "yes")
419 {
420 Ptr<MoreAccurateMotionWobbleSuppressorBase> ws = makePtr<MoreAccurateMotionWobbleSuppressor>();
421 if (arg("gpu") == "yes")
422 #ifdef HAVE_OPENCV_CUDAWARPING
423 ws = makePtr<MoreAccurateMotionWobbleSuppressorGpu>();
424 #else
425 throw runtime_error("OpenCV is built without CUDA support");
426 #endif
427
428 ws->setMotionEstimator(wsMotionEstBuilder->build());
429 ws->setPeriod(argi("ws-period"));
430 twoPassStabilizer->setWobbleSuppressor(ws);
431
432 MotionModel model = ws->motionEstimator()->motionModel();
433 if (arg("load-motions2") != "no")
434 {
435 ws->setMotionEstimator(makePtr<FromFileMotionReader>(arg("load-motions2")));
436 ws->motionEstimator()->setMotionModel(model);
437 }
438 if (arg("save-motions2") != "no")
439 {
440 ws->setMotionEstimator(makePtr<ToFileMotionWriter>(arg("save-motions2"), ws->motionEstimator()));
441 ws->motionEstimator()->setMotionModel(model);
442 }
443 }
444 }
445 else
446 {
447 // we must use one pass stabilizer
448
449 OnePassStabilizer *onePassStabilizer = new OnePassStabilizer();
450 stabilizer = onePassStabilizer;
451 if (arg("stdev") == "auto")
452 onePassStabilizer->setMotionFilter(makePtr<GaussianMotionFilter>(argi("radius")));
453 else
454 onePassStabilizer->setMotionFilter(makePtr<GaussianMotionFilter>(argi("radius"), argf("stdev")));
455 }
456
457 stabilizer->setFrameSource(source);
458 stabilizer->setMotionEstimator(motionEstBuilder->build());
459
460 // cast stabilizer to simple frame source interface to read stabilized frames
461 stabilizedFrames.reset(dynamic_cast<IFrameSource*>(stabilizer));
462
463 MotionModel model = stabilizer->motionEstimator()->motionModel();
464 if (arg("load-motions") != "no")
465 {
466 stabilizer->setMotionEstimator(makePtr<FromFileMotionReader>(arg("load-motions")));
467 stabilizer->motionEstimator()->setMotionModel(model);
468 }
469 if (arg("save-motions") != "no")
470 {
471 stabilizer->setMotionEstimator(makePtr<ToFileMotionWriter>(arg("save-motions"), stabilizer->motionEstimator()));
472 stabilizer->motionEstimator()->setMotionModel(model);
473 }
474
475 stabilizer->setRadius(argi("radius"));
476
477 // init deblurer
478 if (arg("deblur") == "yes")
479 {
480 Ptr<WeightingDeblurer> deblurer = makePtr<WeightingDeblurer>();
481 deblurer->setRadius(argi("radius"));
482 deblurer->setSensitivity(argf("deblur-sens"));
483 stabilizer->setDeblurer(deblurer);
484 }
485
486 // set up trimming paramters
487 stabilizer->setTrimRatio(argf("trim-ratio"));
488 stabilizer->setCorrectionForInclusion(arg("incl-constr") == "yes");
489
490 if (arg("border-mode") == "reflect")
491 stabilizer->setBorderMode(BORDER_REFLECT);
492 else if (arg("border-mode") == "replicate")
493 stabilizer->setBorderMode(BORDER_REPLICATE);
494 else if (arg("border-mode") == "const")
495 stabilizer->setBorderMode(BORDER_CONSTANT);
496 else
497 throw runtime_error("unknown border extrapolation mode: "
498 + cmd.get<string>("border-mode"));
499
500 // init inpainter
501 InpaintingPipeline *inpainters = new InpaintingPipeline();
502 Ptr<InpainterBase> inpainters_(inpainters);
503 if (arg("mosaic") == "yes")
504 {
505 Ptr<ConsistentMosaicInpainter> inp = makePtr<ConsistentMosaicInpainter>();
506 inp->setStdevThresh(argf("mosaic-stdev"));
507 inpainters->pushBack(inp);
508 }
509 if (arg("motion-inpaint") == "yes")
510 {
511 Ptr<MotionInpainter> inp = makePtr<MotionInpainter>();
512 inp->setDistThreshold(argf("mi-dist-thresh"));
513 inpainters->pushBack(inp);
514 }
515 if (arg("color-inpaint") == "average")
516 inpainters->pushBack(makePtr<ColorAverageInpainter>());
517 else if (arg("color-inpaint") == "ns")
518 inpainters->pushBack(makePtr<ColorInpainter>(int(INPAINT_NS), argd("ci-radius")));
519 else if (arg("color-inpaint") == "telea")
520 inpainters->pushBack(makePtr<ColorInpainter>(int(INPAINT_TELEA), argd("ci-radius")));
521 else if (arg("color-inpaint") != "no")
522 throw runtime_error("unknown color inpainting method: " + arg("color-inpaint"));
523 if (!inpainters->empty())
524 {
525 inpainters->setRadius(argi("radius"));
526 stabilizer->setInpainter(inpainters_);
527 }
528
529 if (arg("output") != "no")
530 outputPath = arg("output");
531
532 quietMode = argb("quiet");
533
534 run();
535 }
536 catch (const exception &e)
537 {
538 cout << "error: " << e.what() << endl;
539 stabilizedFrames.release();
540 return -1;
541 }
542 stabilizedFrames.release();
543 return 0;
544 }
545
546
motionModel(const string & str)547 MotionModel motionModel(const string &str)
548 {
549 if (str == "transl")
550 return MM_TRANSLATION;
551 if (str == "transl_and_scale")
552 return MM_TRANSLATION_AND_SCALE;
553 if (str == "rigid")
554 return MM_RIGID;
555 if (str == "similarity")
556 return MM_SIMILARITY;
557 if (str == "affine")
558 return MM_AFFINE;
559 if (str == "homography")
560 return MM_HOMOGRAPHY;
561 throw runtime_error("unknown motion model: " + str);
562 }
563