1 /*
2 * Copyright (c) 2012 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <assert.h>
12 #include <limits.h>
13 #include <math.h>
14
15 #include "./vpx_dsp_rtcd.h"
16 #include "vpx_dsp/vpx_dsp_common.h"
17 #include "vpx_scale/yv12config.h"
18 #include "vpx/vpx_integer.h"
19 #include "vp9/common/vp9_reconinter.h"
20 #include "vp9/encoder/vp9_context_tree.h"
21 #include "vp9/encoder/vp9_denoiser.h"
22 #include "vp9/encoder/vp9_encoder.h"
23
24 /* The VP9 denoiser is similar to that of the VP8 denoiser. While
25 * choosing the motion vectors / reference frames, the denoiser is run, and if
26 * it did not modify the signal to much, the denoised block is copied to the
27 * signal.
28 */
29
30 #ifdef OUTPUT_YUV_DENOISED
31 static void make_grayscale(YV12_BUFFER_CONFIG *yuv);
32 #endif
33
absdiff_thresh(BLOCK_SIZE bs,int increase_denoising)34 static int absdiff_thresh(BLOCK_SIZE bs, int increase_denoising) {
35 (void)bs;
36 return 3 + (increase_denoising ? 1 : 0);
37 }
38
delta_thresh(BLOCK_SIZE bs,int increase_denoising)39 static int delta_thresh(BLOCK_SIZE bs, int increase_denoising) {
40 (void)bs;
41 (void)increase_denoising;
42 return 4;
43 }
44
noise_motion_thresh(BLOCK_SIZE bs,int increase_denoising)45 static int noise_motion_thresh(BLOCK_SIZE bs, int increase_denoising) {
46 (void)bs;
47 (void)increase_denoising;
48 return 625;
49 }
50
sse_thresh(BLOCK_SIZE bs,int increase_denoising)51 static unsigned int sse_thresh(BLOCK_SIZE bs, int increase_denoising) {
52 return (1 << num_pels_log2_lookup[bs]) * (increase_denoising ? 60 : 40);
53 }
54
sse_diff_thresh(BLOCK_SIZE bs,int increase_denoising,int motion_magnitude)55 static int sse_diff_thresh(BLOCK_SIZE bs, int increase_denoising,
56 int motion_magnitude) {
57 if (motion_magnitude >
58 noise_motion_thresh(bs, increase_denoising)) {
59 return 0;
60 } else {
61 return (1 << num_pels_log2_lookup[bs]) * 20;
62 }
63 }
64
total_adj_weak_thresh(BLOCK_SIZE bs,int increase_denoising)65 static int total_adj_weak_thresh(BLOCK_SIZE bs, int increase_denoising) {
66 return (1 << num_pels_log2_lookup[bs]) * (increase_denoising ? 3 : 2);
67 }
68
69 // TODO(jackychen): If increase_denoising is enabled in the future,
70 // we might need to update the code for calculating 'total_adj' in
71 // case the C code is not bit-exact with corresponding sse2 code.
vp9_denoiser_filter_c(const uint8_t * sig,int sig_stride,const uint8_t * mc_avg,int mc_avg_stride,uint8_t * avg,int avg_stride,int increase_denoising,BLOCK_SIZE bs,int motion_magnitude)72 int vp9_denoiser_filter_c(const uint8_t *sig, int sig_stride,
73 const uint8_t *mc_avg,
74 int mc_avg_stride,
75 uint8_t *avg, int avg_stride,
76 int increase_denoising,
77 BLOCK_SIZE bs,
78 int motion_magnitude) {
79 int r, c;
80 const uint8_t *sig_start = sig;
81 const uint8_t *mc_avg_start = mc_avg;
82 uint8_t *avg_start = avg;
83 int diff, adj, absdiff, delta;
84 int adj_val[] = {3, 4, 6};
85 int total_adj = 0;
86 int shift_inc = 1;
87
88 // If motion_magnitude is small, making the denoiser more aggressive by
89 // increasing the adjustment for each level. Add another increment for
90 // blocks that are labeled for increase denoising.
91 if (motion_magnitude <= MOTION_MAGNITUDE_THRESHOLD) {
92 if (increase_denoising) {
93 shift_inc = 2;
94 }
95 adj_val[0] += shift_inc;
96 adj_val[1] += shift_inc;
97 adj_val[2] += shift_inc;
98 }
99
100 // First attempt to apply a strong temporal denoising filter.
101 for (r = 0; r < (4 << b_height_log2_lookup[bs]); ++r) {
102 for (c = 0; c < (4 << b_width_log2_lookup[bs]); ++c) {
103 diff = mc_avg[c] - sig[c];
104 absdiff = abs(diff);
105
106 if (absdiff <= absdiff_thresh(bs, increase_denoising)) {
107 avg[c] = mc_avg[c];
108 total_adj += diff;
109 } else {
110 switch (absdiff) {
111 case 4: case 5: case 6: case 7:
112 adj = adj_val[0];
113 break;
114 case 8: case 9: case 10: case 11:
115 case 12: case 13: case 14: case 15:
116 adj = adj_val[1];
117 break;
118 default:
119 adj = adj_val[2];
120 }
121 if (diff > 0) {
122 avg[c] = VPXMIN(UINT8_MAX, sig[c] + adj);
123 total_adj += adj;
124 } else {
125 avg[c] = VPXMAX(0, sig[c] - adj);
126 total_adj -= adj;
127 }
128 }
129 }
130 sig += sig_stride;
131 avg += avg_stride;
132 mc_avg += mc_avg_stride;
133 }
134
135 // If the strong filter did not modify the signal too much, we're all set.
136 if (abs(total_adj) <= total_adj_strong_thresh(bs, increase_denoising)) {
137 return FILTER_BLOCK;
138 }
139
140 // Otherwise, we try to dampen the filter if the delta is not too high.
141 delta = ((abs(total_adj) - total_adj_strong_thresh(bs, increase_denoising))
142 >> num_pels_log2_lookup[bs]) + 1;
143
144 if (delta >= delta_thresh(bs, increase_denoising)) {
145 return COPY_BLOCK;
146 }
147
148 mc_avg = mc_avg_start;
149 avg = avg_start;
150 sig = sig_start;
151 for (r = 0; r < (4 << b_height_log2_lookup[bs]); ++r) {
152 for (c = 0; c < (4 << b_width_log2_lookup[bs]); ++c) {
153 diff = mc_avg[c] - sig[c];
154 adj = abs(diff);
155 if (adj > delta) {
156 adj = delta;
157 }
158 if (diff > 0) {
159 // Diff positive means we made positive adjustment above
160 // (in first try/attempt), so now make negative adjustment to bring
161 // denoised signal down.
162 avg[c] = VPXMAX(0, avg[c] - adj);
163 total_adj -= adj;
164 } else {
165 // Diff negative means we made negative adjustment above
166 // (in first try/attempt), so now make positive adjustment to bring
167 // denoised signal up.
168 avg[c] = VPXMIN(UINT8_MAX, avg[c] + adj);
169 total_adj += adj;
170 }
171 }
172 sig += sig_stride;
173 avg += avg_stride;
174 mc_avg += mc_avg_stride;
175 }
176
177 // We can use the filter if it has been sufficiently dampened
178 if (abs(total_adj) <= total_adj_weak_thresh(bs, increase_denoising)) {
179 return FILTER_BLOCK;
180 }
181 return COPY_BLOCK;
182 }
183
block_start(uint8_t * framebuf,int stride,int mi_row,int mi_col)184 static uint8_t *block_start(uint8_t *framebuf, int stride,
185 int mi_row, int mi_col) {
186 return framebuf + (stride * mi_row * 8) + (mi_col * 8);
187 }
188
perform_motion_compensation(VP9_DENOISER * denoiser,MACROBLOCK * mb,BLOCK_SIZE bs,int increase_denoising,int mi_row,int mi_col,PICK_MODE_CONTEXT * ctx,int * motion_magnitude,int is_skin)189 static VP9_DENOISER_DECISION perform_motion_compensation(VP9_DENOISER *denoiser,
190 MACROBLOCK *mb,
191 BLOCK_SIZE bs,
192 int increase_denoising,
193 int mi_row,
194 int mi_col,
195 PICK_MODE_CONTEXT *ctx,
196 int *motion_magnitude,
197 int is_skin) {
198 int mv_col, mv_row;
199 int sse_diff = ctx->zeromv_sse - ctx->newmv_sse;
200 MV_REFERENCE_FRAME frame;
201 MACROBLOCKD *filter_mbd = &mb->e_mbd;
202 MB_MODE_INFO *mbmi = &filter_mbd->mi[0]->mbmi;
203 MB_MODE_INFO saved_mbmi;
204 int i, j;
205 struct buf_2d saved_dst[MAX_MB_PLANE];
206 struct buf_2d saved_pre[MAX_MB_PLANE][2]; // 2 pre buffers
207
208 mv_col = ctx->best_sse_mv.as_mv.col;
209 mv_row = ctx->best_sse_mv.as_mv.row;
210 *motion_magnitude = mv_row * mv_row + mv_col * mv_col;
211 frame = ctx->best_reference_frame;
212
213 saved_mbmi = *mbmi;
214
215 if (is_skin && *motion_magnitude > 16)
216 return COPY_BLOCK;
217
218 // If the best reference frame uses inter-prediction and there is enough of a
219 // difference in sum-squared-error, use it.
220 if (frame != INTRA_FRAME &&
221 sse_diff > sse_diff_thresh(bs, increase_denoising, *motion_magnitude)) {
222 mbmi->ref_frame[0] = ctx->best_reference_frame;
223 mbmi->mode = ctx->best_sse_inter_mode;
224 mbmi->mv[0] = ctx->best_sse_mv;
225 } else {
226 // Otherwise, use the zero reference frame.
227 frame = ctx->best_zeromv_reference_frame;
228
229 mbmi->ref_frame[0] = ctx->best_zeromv_reference_frame;
230 mbmi->mode = ZEROMV;
231 mbmi->mv[0].as_int = 0;
232
233 ctx->best_sse_inter_mode = ZEROMV;
234 ctx->best_sse_mv.as_int = 0;
235 ctx->newmv_sse = ctx->zeromv_sse;
236 }
237
238 if (ctx->newmv_sse > sse_thresh(bs, increase_denoising)) {
239 // Restore everything to its original state
240 *mbmi = saved_mbmi;
241 return COPY_BLOCK;
242 }
243 if (*motion_magnitude >
244 (noise_motion_thresh(bs, increase_denoising) << 3)) {
245 // Restore everything to its original state
246 *mbmi = saved_mbmi;
247 return COPY_BLOCK;
248 }
249
250 // We will restore these after motion compensation.
251 for (i = 0; i < MAX_MB_PLANE; ++i) {
252 for (j = 0; j < 2; ++j) {
253 saved_pre[i][j] = filter_mbd->plane[i].pre[j];
254 }
255 saved_dst[i] = filter_mbd->plane[i].dst;
256 }
257
258 // Set the pointers in the MACROBLOCKD to point to the buffers in the denoiser
259 // struct.
260 for (j = 0; j < 2; ++j) {
261 filter_mbd->plane[0].pre[j].buf =
262 block_start(denoiser->running_avg_y[frame].y_buffer,
263 denoiser->running_avg_y[frame].y_stride,
264 mi_row, mi_col);
265 filter_mbd->plane[0].pre[j].stride =
266 denoiser->running_avg_y[frame].y_stride;
267 filter_mbd->plane[1].pre[j].buf =
268 block_start(denoiser->running_avg_y[frame].u_buffer,
269 denoiser->running_avg_y[frame].uv_stride,
270 mi_row, mi_col);
271 filter_mbd->plane[1].pre[j].stride =
272 denoiser->running_avg_y[frame].uv_stride;
273 filter_mbd->plane[2].pre[j].buf =
274 block_start(denoiser->running_avg_y[frame].v_buffer,
275 denoiser->running_avg_y[frame].uv_stride,
276 mi_row, mi_col);
277 filter_mbd->plane[2].pre[j].stride =
278 denoiser->running_avg_y[frame].uv_stride;
279 }
280 filter_mbd->plane[0].dst.buf =
281 block_start(denoiser->mc_running_avg_y.y_buffer,
282 denoiser->mc_running_avg_y.y_stride,
283 mi_row, mi_col);
284 filter_mbd->plane[0].dst.stride = denoiser->mc_running_avg_y.y_stride;
285 filter_mbd->plane[1].dst.buf =
286 block_start(denoiser->mc_running_avg_y.u_buffer,
287 denoiser->mc_running_avg_y.uv_stride,
288 mi_row, mi_col);
289 filter_mbd->plane[1].dst.stride = denoiser->mc_running_avg_y.uv_stride;
290 filter_mbd->plane[2].dst.buf =
291 block_start(denoiser->mc_running_avg_y.v_buffer,
292 denoiser->mc_running_avg_y.uv_stride,
293 mi_row, mi_col);
294 filter_mbd->plane[2].dst.stride = denoiser->mc_running_avg_y.uv_stride;
295
296 vp9_build_inter_predictors_sby(filter_mbd, mv_row, mv_col, bs);
297
298 // Restore everything to its original state
299 *mbmi = saved_mbmi;
300 for (i = 0; i < MAX_MB_PLANE; ++i) {
301 for (j = 0; j < 2; ++j) {
302 filter_mbd->plane[i].pre[j] = saved_pre[i][j];
303 }
304 filter_mbd->plane[i].dst = saved_dst[i];
305 }
306
307 mv_row = ctx->best_sse_mv.as_mv.row;
308 mv_col = ctx->best_sse_mv.as_mv.col;
309
310 return FILTER_BLOCK;
311 }
312
vp9_denoiser_denoise(VP9_DENOISER * denoiser,MACROBLOCK * mb,int mi_row,int mi_col,BLOCK_SIZE bs,PICK_MODE_CONTEXT * ctx)313 void vp9_denoiser_denoise(VP9_DENOISER *denoiser, MACROBLOCK *mb,
314 int mi_row, int mi_col, BLOCK_SIZE bs,
315 PICK_MODE_CONTEXT *ctx) {
316 int motion_magnitude = 0;
317 VP9_DENOISER_DECISION decision = COPY_BLOCK;
318 YV12_BUFFER_CONFIG avg = denoiser->running_avg_y[INTRA_FRAME];
319 YV12_BUFFER_CONFIG mc_avg = denoiser->mc_running_avg_y;
320 uint8_t *avg_start = block_start(avg.y_buffer, avg.y_stride, mi_row, mi_col);
321 uint8_t *mc_avg_start = block_start(mc_avg.y_buffer, mc_avg.y_stride,
322 mi_row, mi_col);
323 struct buf_2d src = mb->plane[0].src;
324 int is_skin = 0;
325
326 if (bs <= BLOCK_16X16 && denoiser->denoising_on) {
327 // Take center pixel in block to determine is_skin.
328 const int y_width_shift = (4 << b_width_log2_lookup[bs]) >> 1;
329 const int y_height_shift = (4 << b_height_log2_lookup[bs]) >> 1;
330 const int uv_width_shift = y_width_shift >> 1;
331 const int uv_height_shift = y_height_shift >> 1;
332 const int stride = mb->plane[0].src.stride;
333 const int strideuv = mb->plane[1].src.stride;
334 const uint8_t ysource =
335 mb->plane[0].src.buf[y_height_shift * stride + y_width_shift];
336 const uint8_t usource =
337 mb->plane[1].src.buf[uv_height_shift * strideuv + uv_width_shift];
338 const uint8_t vsource =
339 mb->plane[2].src.buf[uv_height_shift * strideuv + uv_width_shift];
340 is_skin = vp9_skin_pixel(ysource, usource, vsource);
341 }
342
343 if (denoiser->denoising_on)
344 decision = perform_motion_compensation(denoiser, mb, bs,
345 denoiser->increase_denoising,
346 mi_row, mi_col, ctx,
347 &motion_magnitude,
348 is_skin);
349
350 if (decision == FILTER_BLOCK) {
351 decision = vp9_denoiser_filter(src.buf, src.stride,
352 mc_avg_start, mc_avg.y_stride,
353 avg_start, avg.y_stride,
354 0, bs, motion_magnitude);
355 }
356
357 if (decision == FILTER_BLOCK) {
358 vpx_convolve_copy(avg_start, avg.y_stride, src.buf, src.stride,
359 NULL, 0, NULL, 0,
360 num_4x4_blocks_wide_lookup[bs] << 2,
361 num_4x4_blocks_high_lookup[bs] << 2);
362 } else { // COPY_BLOCK
363 vpx_convolve_copy(src.buf, src.stride, avg_start, avg.y_stride,
364 NULL, 0, NULL, 0,
365 num_4x4_blocks_wide_lookup[bs] << 2,
366 num_4x4_blocks_high_lookup[bs] << 2);
367 }
368 }
369
copy_frame(YV12_BUFFER_CONFIG * const dest,const YV12_BUFFER_CONFIG * const src)370 static void copy_frame(YV12_BUFFER_CONFIG * const dest,
371 const YV12_BUFFER_CONFIG * const src) {
372 int r;
373 const uint8_t *srcbuf = src->y_buffer;
374 uint8_t *destbuf = dest->y_buffer;
375
376 assert(dest->y_width == src->y_width);
377 assert(dest->y_height == src->y_height);
378
379 for (r = 0; r < dest->y_height; ++r) {
380 memcpy(destbuf, srcbuf, dest->y_width);
381 destbuf += dest->y_stride;
382 srcbuf += src->y_stride;
383 }
384 }
385
swap_frame_buffer(YV12_BUFFER_CONFIG * const dest,YV12_BUFFER_CONFIG * const src)386 static void swap_frame_buffer(YV12_BUFFER_CONFIG * const dest,
387 YV12_BUFFER_CONFIG * const src) {
388 uint8_t *tmp_buf = dest->y_buffer;
389 assert(dest->y_width == src->y_width);
390 assert(dest->y_height == src->y_height);
391 dest->y_buffer = src->y_buffer;
392 src->y_buffer = tmp_buf;
393 }
394
vp9_denoiser_update_frame_info(VP9_DENOISER * denoiser,YV12_BUFFER_CONFIG src,FRAME_TYPE frame_type,int refresh_alt_ref_frame,int refresh_golden_frame,int refresh_last_frame,int resized)395 void vp9_denoiser_update_frame_info(VP9_DENOISER *denoiser,
396 YV12_BUFFER_CONFIG src,
397 FRAME_TYPE frame_type,
398 int refresh_alt_ref_frame,
399 int refresh_golden_frame,
400 int refresh_last_frame,
401 int resized) {
402 // Copy source into denoised reference buffers on KEY_FRAME or
403 // if the just encoded frame was resized.
404 if (frame_type == KEY_FRAME || resized != 0) {
405 int i;
406 // Start at 1 so as not to overwrite the INTRA_FRAME
407 for (i = 1; i < MAX_REF_FRAMES; ++i)
408 copy_frame(&denoiser->running_avg_y[i], &src);
409 return;
410 }
411
412 // If more than one refresh occurs, must copy frame buffer.
413 if ((refresh_alt_ref_frame + refresh_golden_frame + refresh_last_frame)
414 > 1) {
415 if (refresh_alt_ref_frame) {
416 copy_frame(&denoiser->running_avg_y[ALTREF_FRAME],
417 &denoiser->running_avg_y[INTRA_FRAME]);
418 }
419 if (refresh_golden_frame) {
420 copy_frame(&denoiser->running_avg_y[GOLDEN_FRAME],
421 &denoiser->running_avg_y[INTRA_FRAME]);
422 }
423 if (refresh_last_frame) {
424 copy_frame(&denoiser->running_avg_y[LAST_FRAME],
425 &denoiser->running_avg_y[INTRA_FRAME]);
426 }
427 } else {
428 if (refresh_alt_ref_frame) {
429 swap_frame_buffer(&denoiser->running_avg_y[ALTREF_FRAME],
430 &denoiser->running_avg_y[INTRA_FRAME]);
431 }
432 if (refresh_golden_frame) {
433 swap_frame_buffer(&denoiser->running_avg_y[GOLDEN_FRAME],
434 &denoiser->running_avg_y[INTRA_FRAME]);
435 }
436 if (refresh_last_frame) {
437 swap_frame_buffer(&denoiser->running_avg_y[LAST_FRAME],
438 &denoiser->running_avg_y[INTRA_FRAME]);
439 }
440 }
441 }
442
vp9_denoiser_reset_frame_stats(PICK_MODE_CONTEXT * ctx)443 void vp9_denoiser_reset_frame_stats(PICK_MODE_CONTEXT *ctx) {
444 ctx->zeromv_sse = UINT_MAX;
445 ctx->newmv_sse = UINT_MAX;
446 }
447
vp9_denoiser_update_frame_stats(MB_MODE_INFO * mbmi,unsigned int sse,PREDICTION_MODE mode,PICK_MODE_CONTEXT * ctx)448 void vp9_denoiser_update_frame_stats(MB_MODE_INFO *mbmi, unsigned int sse,
449 PREDICTION_MODE mode,
450 PICK_MODE_CONTEXT *ctx) {
451 // TODO(tkopp): Use both MVs if possible
452 if (mbmi->mv[0].as_int == 0 && sse < ctx->zeromv_sse) {
453 ctx->zeromv_sse = sse;
454 ctx->best_zeromv_reference_frame = mbmi->ref_frame[0];
455 }
456
457 if (mbmi->mv[0].as_int != 0 && sse < ctx->newmv_sse) {
458 ctx->newmv_sse = sse;
459 ctx->best_sse_inter_mode = mode;
460 ctx->best_sse_mv = mbmi->mv[0];
461 ctx->best_reference_frame = mbmi->ref_frame[0];
462 }
463 }
464
vp9_denoiser_alloc(VP9_DENOISER * denoiser,int width,int height,int ssx,int ssy,int use_highbitdepth,int border)465 int vp9_denoiser_alloc(VP9_DENOISER *denoiser, int width, int height,
466 int ssx, int ssy,
467 #if CONFIG_VP9_HIGHBITDEPTH
468 int use_highbitdepth,
469 #endif
470 int border) {
471 int i, fail;
472 const int legacy_byte_alignment = 0;
473 assert(denoiser != NULL);
474
475 for (i = 0; i < MAX_REF_FRAMES; ++i) {
476 fail = vpx_alloc_frame_buffer(&denoiser->running_avg_y[i], width, height,
477 ssx, ssy,
478 #if CONFIG_VP9_HIGHBITDEPTH
479 use_highbitdepth,
480 #endif
481 border, legacy_byte_alignment);
482 if (fail) {
483 vp9_denoiser_free(denoiser);
484 return 1;
485 }
486 #ifdef OUTPUT_YUV_DENOISED
487 make_grayscale(&denoiser->running_avg_y[i]);
488 #endif
489 }
490
491 fail = vpx_alloc_frame_buffer(&denoiser->mc_running_avg_y, width, height,
492 ssx, ssy,
493 #if CONFIG_VP9_HIGHBITDEPTH
494 use_highbitdepth,
495 #endif
496 border, legacy_byte_alignment);
497 if (fail) {
498 vp9_denoiser_free(denoiser);
499 return 1;
500 }
501
502 fail = vpx_alloc_frame_buffer(&denoiser->last_source, width, height,
503 ssx, ssy,
504 #if CONFIG_VP9_HIGHBITDEPTH
505 use_highbitdepth,
506 #endif
507 border, legacy_byte_alignment);
508 if (fail) {
509 vp9_denoiser_free(denoiser);
510 return 1;
511 }
512 #ifdef OUTPUT_YUV_DENOISED
513 make_grayscale(&denoiser->running_avg_y[i]);
514 #endif
515 denoiser->increase_denoising = 0;
516 denoiser->frame_buffer_initialized = 1;
517 vp9_denoiser_init_noise_estimate(denoiser, width, height);
518 return 0;
519 }
520
vp9_denoiser_init_noise_estimate(VP9_DENOISER * denoiser,int width,int height)521 void vp9_denoiser_init_noise_estimate(VP9_DENOISER *denoiser,
522 int width,
523 int height) {
524 // Denoiser is off by default, i.e., no denoising is performed.
525 // Noise level is measured periodically, and if observed to be above
526 // thresh_noise_estimate, then denoising is performed, i.e., denoising_on = 1.
527 denoiser->denoising_on = 0;
528 denoiser->noise_estimate = 0;
529 denoiser->noise_estimate_count = 0;
530 denoiser->thresh_noise_estimate = 20;
531 if (width * height >= 1920 * 1080) {
532 denoiser->thresh_noise_estimate = 70;
533 } else if (width * height >= 1280 * 720) {
534 denoiser->thresh_noise_estimate = 40;
535 }
536 }
537
vp9_denoiser_free(VP9_DENOISER * denoiser)538 void vp9_denoiser_free(VP9_DENOISER *denoiser) {
539 int i;
540 denoiser->frame_buffer_initialized = 0;
541 if (denoiser == NULL) {
542 return;
543 }
544 for (i = 0; i < MAX_REF_FRAMES; ++i) {
545 vpx_free_frame_buffer(&denoiser->running_avg_y[i]);
546 }
547 vpx_free_frame_buffer(&denoiser->mc_running_avg_y);
548 vpx_free_frame_buffer(&denoiser->last_source);
549 }
550
vp9_denoiser_update_noise_estimate(VP9_COMP * const cpi)551 void vp9_denoiser_update_noise_estimate(VP9_COMP *const cpi) {
552 const VP9_COMMON *const cm = &cpi->common;
553 CYCLIC_REFRESH *const cr = cpi->cyclic_refresh;
554 int frame_period = 10;
555 int thresh_consec_zeromv = 8;
556 unsigned int thresh_sum_diff = 128;
557 int num_frames_estimate = 20;
558 int min_blocks_estimate = cm->mi_rows * cm->mi_cols >> 7;
559 // Estimate of noise level every frame_period frames.
560 // Estimate is between current source and last source.
561 if (cm->current_video_frame % frame_period != 0 ||
562 cpi->denoiser.last_source.y_buffer == NULL) {
563 copy_frame(&cpi->denoiser.last_source, cpi->Source);
564 return;
565 } else {
566 int num_samples = 0;
567 uint64_t avg_est = 0;
568 int bsize = BLOCK_16X16;
569 static const unsigned char const_source[16] = {
570 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
571 128, 128};
572 // Loop over sub-sample of 16x16 blocks of frame, and for blocks that have
573 // been encoded as zero/small mv at least x consecutive frames, compute
574 // the variance to update estimate of noise in the source.
575 const uint8_t *src_y = cpi->Source->y_buffer;
576 const int src_ystride = cpi->Source->y_stride;
577 const uint8_t *last_src_y = cpi->denoiser.last_source.y_buffer;
578 const int last_src_ystride = cpi->denoiser.last_source.y_stride;
579 const uint8_t *src_u = cpi->Source->u_buffer;
580 const uint8_t *src_v = cpi->Source->v_buffer;
581 const int src_uvstride = cpi->Source->uv_stride;
582 const int y_width_shift = (4 << b_width_log2_lookup[bsize]) >> 1;
583 const int y_height_shift = (4 << b_height_log2_lookup[bsize]) >> 1;
584 const int uv_width_shift = y_width_shift >> 1;
585 const int uv_height_shift = y_height_shift >> 1;
586 int mi_row, mi_col;
587 for (mi_row = 0; mi_row < cm->mi_rows; mi_row ++) {
588 for (mi_col = 0; mi_col < cm->mi_cols; mi_col ++) {
589 // 16x16 blocks, 1/4 sample of frame.
590 if (mi_row % 4 == 0 && mi_col % 4 == 0) {
591 int bl_index = mi_row * cm->mi_cols + mi_col;
592 int bl_index1 = bl_index + 1;
593 int bl_index2 = bl_index + cm->mi_cols;
594 int bl_index3 = bl_index2 + 1;
595 // Only consider blocks that are likely steady background. i.e, have
596 // been encoded as zero/low motion x (= thresh_consec_zeromv) frames
597 // in a row. consec_zero_mv[] defined for 8x8 blocks, so consider all
598 // 4 sub-blocks for 16x16 block. Also, avoid skin blocks.
599 const uint8_t ysource =
600 src_y[y_height_shift * src_ystride + y_width_shift];
601 const uint8_t usource =
602 src_u[uv_height_shift * src_uvstride + uv_width_shift];
603 const uint8_t vsource =
604 src_v[uv_height_shift * src_uvstride + uv_width_shift];
605 int is_skin = vp9_skin_pixel(ysource, usource, vsource);
606 if (cr->consec_zero_mv[bl_index] > thresh_consec_zeromv &&
607 cr->consec_zero_mv[bl_index1] > thresh_consec_zeromv &&
608 cr->consec_zero_mv[bl_index2] > thresh_consec_zeromv &&
609 cr->consec_zero_mv[bl_index3] > thresh_consec_zeromv &&
610 !is_skin) {
611 // Compute variance.
612 unsigned int sse;
613 unsigned int variance = cpi->fn_ptr[bsize].vf(src_y,
614 src_ystride,
615 last_src_y,
616 last_src_ystride,
617 &sse);
618 // Only consider this block as valid for noise measurement if the
619 // average term (sse - variance = N * avg^{2}, N = 16X16) of the
620 // temporal residual is small (avoid effects from lighting change).
621 if ((sse - variance) < thresh_sum_diff) {
622 unsigned int sse2;
623 const unsigned int spatial_variance =
624 cpi->fn_ptr[bsize].vf(src_y, src_ystride, const_source,
625 0, &sse2);
626 avg_est += variance / (10 + spatial_variance);
627 num_samples++;
628 }
629 }
630 }
631 src_y += 8;
632 last_src_y += 8;
633 src_u += 4;
634 src_v += 4;
635 }
636 src_y += (src_ystride << 3) - (cm->mi_cols << 3);
637 last_src_y += (last_src_ystride << 3) - (cm->mi_cols << 3);
638 src_u += (src_uvstride << 2) - (cm->mi_cols << 2);
639 src_v += (src_uvstride << 2) - (cm->mi_cols << 2);
640 }
641 // Update noise estimate if we have at a minimum number of block samples,
642 // and avg_est > 0 (avg_est == 0 can happen if the application inputs
643 // duplicate frames).
644 if (num_samples > min_blocks_estimate && avg_est > 0) {
645 // Normalize.
646 avg_est = (avg_est << 8) / num_samples;
647 // Update noise estimate.
648 cpi->denoiser.noise_estimate = (3 * cpi->denoiser.noise_estimate +
649 avg_est) >> 2;
650 cpi->denoiser.noise_estimate_count++;
651 if (cpi->denoiser.noise_estimate_count == num_frames_estimate) {
652 // Reset counter and check noise level condition.
653 cpi->denoiser.noise_estimate_count = 0;
654 if (cpi->denoiser.noise_estimate > cpi->denoiser.thresh_noise_estimate)
655 cpi->denoiser.denoising_on = 1;
656 else
657 cpi->denoiser.denoising_on = 0;
658 }
659 }
660 }
661 copy_frame(&cpi->denoiser.last_source, cpi->Source);
662 }
663
664 #ifdef OUTPUT_YUV_DENOISED
make_grayscale(YV12_BUFFER_CONFIG * yuv)665 static void make_grayscale(YV12_BUFFER_CONFIG *yuv) {
666 int r, c;
667 uint8_t *u = yuv->u_buffer;
668 uint8_t *v = yuv->v_buffer;
669
670 for (r = 0; r < yuv->uv_height; ++r) {
671 for (c = 0; c < yuv->uv_width; ++c) {
672 u[c] = UINT8_MAX / 2;
673 v[c] = UINT8_MAX / 2;
674 }
675 u += yuv->uv_stride;
676 v += yuv->uv_stride;
677 }
678 }
679 #endif
680