1 /*
2  *  Copyright (c) 2012 The WebM project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include <assert.h>
12 #include <limits.h>
13 #include <math.h>
14 
15 #include "./vpx_dsp_rtcd.h"
16 #include "vpx_dsp/vpx_dsp_common.h"
17 #include "vpx_scale/yv12config.h"
18 #include "vpx/vpx_integer.h"
19 #include "vp9/common/vp9_reconinter.h"
20 #include "vp9/encoder/vp9_context_tree.h"
21 #include "vp9/encoder/vp9_denoiser.h"
22 #include "vp9/encoder/vp9_encoder.h"
23 
24 /* The VP9 denoiser is similar to that of the VP8 denoiser. While
25  * choosing the motion vectors / reference frames, the denoiser is run, and if
26  * it did not modify the signal to much, the denoised block is copied to the
27  * signal.
28  */
29 
30 #ifdef OUTPUT_YUV_DENOISED
31 static void make_grayscale(YV12_BUFFER_CONFIG *yuv);
32 #endif
33 
absdiff_thresh(BLOCK_SIZE bs,int increase_denoising)34 static int absdiff_thresh(BLOCK_SIZE bs, int increase_denoising) {
35   (void)bs;
36   return 3 + (increase_denoising ? 1 : 0);
37 }
38 
delta_thresh(BLOCK_SIZE bs,int increase_denoising)39 static int delta_thresh(BLOCK_SIZE bs, int increase_denoising) {
40   (void)bs;
41   (void)increase_denoising;
42   return 4;
43 }
44 
noise_motion_thresh(BLOCK_SIZE bs,int increase_denoising)45 static int noise_motion_thresh(BLOCK_SIZE bs, int increase_denoising) {
46   (void)bs;
47   (void)increase_denoising;
48   return 625;
49 }
50 
sse_thresh(BLOCK_SIZE bs,int increase_denoising)51 static unsigned int sse_thresh(BLOCK_SIZE bs, int increase_denoising) {
52   return (1 << num_pels_log2_lookup[bs]) * (increase_denoising ? 60 : 40);
53 }
54 
sse_diff_thresh(BLOCK_SIZE bs,int increase_denoising,int motion_magnitude)55 static int sse_diff_thresh(BLOCK_SIZE bs, int increase_denoising,
56                            int motion_magnitude) {
57   if (motion_magnitude >
58       noise_motion_thresh(bs, increase_denoising)) {
59     return 0;
60   } else {
61     return (1 << num_pels_log2_lookup[bs]) * 20;
62   }
63 }
64 
total_adj_weak_thresh(BLOCK_SIZE bs,int increase_denoising)65 static int total_adj_weak_thresh(BLOCK_SIZE bs, int increase_denoising) {
66   return (1 << num_pels_log2_lookup[bs]) * (increase_denoising ? 3 : 2);
67 }
68 
69 // TODO(jackychen): If increase_denoising is enabled in the future,
70 // we might need to update the code for calculating 'total_adj' in
71 // case the C code is not bit-exact with corresponding sse2 code.
vp9_denoiser_filter_c(const uint8_t * sig,int sig_stride,const uint8_t * mc_avg,int mc_avg_stride,uint8_t * avg,int avg_stride,int increase_denoising,BLOCK_SIZE bs,int motion_magnitude)72 int vp9_denoiser_filter_c(const uint8_t *sig, int sig_stride,
73                           const uint8_t *mc_avg,
74                           int mc_avg_stride,
75                           uint8_t *avg, int avg_stride,
76                           int increase_denoising,
77                           BLOCK_SIZE bs,
78                           int motion_magnitude) {
79   int r, c;
80   const uint8_t *sig_start = sig;
81   const uint8_t *mc_avg_start = mc_avg;
82   uint8_t *avg_start = avg;
83   int diff, adj, absdiff, delta;
84   int adj_val[] = {3, 4, 6};
85   int total_adj = 0;
86   int shift_inc = 1;
87 
88   // If motion_magnitude is small, making the denoiser more aggressive by
89   // increasing the adjustment for each level. Add another increment for
90   // blocks that are labeled for increase denoising.
91   if (motion_magnitude <= MOTION_MAGNITUDE_THRESHOLD) {
92     if (increase_denoising) {
93       shift_inc = 2;
94     }
95     adj_val[0] += shift_inc;
96     adj_val[1] += shift_inc;
97     adj_val[2] += shift_inc;
98   }
99 
100   // First attempt to apply a strong temporal denoising filter.
101   for (r = 0; r < (4 << b_height_log2_lookup[bs]); ++r) {
102     for (c = 0; c < (4 << b_width_log2_lookup[bs]); ++c) {
103       diff = mc_avg[c] - sig[c];
104       absdiff = abs(diff);
105 
106       if (absdiff <= absdiff_thresh(bs, increase_denoising)) {
107         avg[c] = mc_avg[c];
108         total_adj += diff;
109       } else {
110         switch (absdiff) {
111           case 4: case 5: case 6: case 7:
112             adj = adj_val[0];
113             break;
114           case 8: case 9: case 10: case 11:
115           case 12: case 13: case 14: case 15:
116             adj = adj_val[1];
117             break;
118           default:
119             adj = adj_val[2];
120         }
121         if (diff > 0) {
122           avg[c] = VPXMIN(UINT8_MAX, sig[c] + adj);
123           total_adj += adj;
124         } else {
125           avg[c] = VPXMAX(0, sig[c] - adj);
126           total_adj -= adj;
127         }
128       }
129     }
130     sig += sig_stride;
131     avg += avg_stride;
132     mc_avg += mc_avg_stride;
133   }
134 
135   // If the strong filter did not modify the signal too much, we're all set.
136   if (abs(total_adj) <= total_adj_strong_thresh(bs, increase_denoising)) {
137     return FILTER_BLOCK;
138   }
139 
140   // Otherwise, we try to dampen the filter if the delta is not too high.
141   delta = ((abs(total_adj) - total_adj_strong_thresh(bs, increase_denoising))
142            >> num_pels_log2_lookup[bs]) + 1;
143 
144   if (delta >= delta_thresh(bs, increase_denoising)) {
145     return COPY_BLOCK;
146   }
147 
148   mc_avg =  mc_avg_start;
149   avg = avg_start;
150   sig = sig_start;
151   for (r = 0; r < (4 << b_height_log2_lookup[bs]); ++r) {
152     for (c = 0; c < (4 << b_width_log2_lookup[bs]); ++c) {
153       diff = mc_avg[c] - sig[c];
154       adj = abs(diff);
155       if (adj > delta) {
156         adj = delta;
157       }
158       if (diff > 0) {
159         // Diff positive means we made positive adjustment above
160         // (in first try/attempt), so now make negative adjustment to bring
161         // denoised signal down.
162         avg[c] = VPXMAX(0, avg[c] - adj);
163         total_adj -= adj;
164       } else {
165         // Diff negative means we made negative adjustment above
166         // (in first try/attempt), so now make positive adjustment to bring
167         // denoised signal up.
168         avg[c] = VPXMIN(UINT8_MAX, avg[c] + adj);
169         total_adj += adj;
170       }
171     }
172     sig += sig_stride;
173     avg += avg_stride;
174     mc_avg += mc_avg_stride;
175   }
176 
177   // We can use the filter if it has been sufficiently dampened
178   if (abs(total_adj) <= total_adj_weak_thresh(bs, increase_denoising)) {
179     return FILTER_BLOCK;
180   }
181   return COPY_BLOCK;
182 }
183 
block_start(uint8_t * framebuf,int stride,int mi_row,int mi_col)184 static uint8_t *block_start(uint8_t *framebuf, int stride,
185                             int mi_row, int mi_col) {
186   return framebuf + (stride * mi_row * 8) + (mi_col * 8);
187 }
188 
perform_motion_compensation(VP9_DENOISER * denoiser,MACROBLOCK * mb,BLOCK_SIZE bs,int increase_denoising,int mi_row,int mi_col,PICK_MODE_CONTEXT * ctx,int * motion_magnitude,int is_skin)189 static VP9_DENOISER_DECISION perform_motion_compensation(VP9_DENOISER *denoiser,
190                                                          MACROBLOCK *mb,
191                                                          BLOCK_SIZE bs,
192                                                          int increase_denoising,
193                                                          int mi_row,
194                                                          int mi_col,
195                                                          PICK_MODE_CONTEXT *ctx,
196                                                          int *motion_magnitude,
197                                                          int is_skin) {
198   int mv_col, mv_row;
199   int sse_diff = ctx->zeromv_sse - ctx->newmv_sse;
200   MV_REFERENCE_FRAME frame;
201   MACROBLOCKD *filter_mbd = &mb->e_mbd;
202   MB_MODE_INFO *mbmi = &filter_mbd->mi[0]->mbmi;
203   MB_MODE_INFO saved_mbmi;
204   int i, j;
205   struct buf_2d saved_dst[MAX_MB_PLANE];
206   struct buf_2d saved_pre[MAX_MB_PLANE][2];  // 2 pre buffers
207 
208   mv_col = ctx->best_sse_mv.as_mv.col;
209   mv_row = ctx->best_sse_mv.as_mv.row;
210   *motion_magnitude = mv_row * mv_row + mv_col * mv_col;
211   frame = ctx->best_reference_frame;
212 
213   saved_mbmi = *mbmi;
214 
215   if (is_skin && *motion_magnitude > 16)
216     return COPY_BLOCK;
217 
218   // If the best reference frame uses inter-prediction and there is enough of a
219   // difference in sum-squared-error, use it.
220   if (frame != INTRA_FRAME &&
221       sse_diff > sse_diff_thresh(bs, increase_denoising, *motion_magnitude)) {
222     mbmi->ref_frame[0] = ctx->best_reference_frame;
223     mbmi->mode = ctx->best_sse_inter_mode;
224     mbmi->mv[0] = ctx->best_sse_mv;
225   } else {
226     // Otherwise, use the zero reference frame.
227     frame = ctx->best_zeromv_reference_frame;
228 
229     mbmi->ref_frame[0] = ctx->best_zeromv_reference_frame;
230     mbmi->mode = ZEROMV;
231     mbmi->mv[0].as_int = 0;
232 
233     ctx->best_sse_inter_mode = ZEROMV;
234     ctx->best_sse_mv.as_int = 0;
235     ctx->newmv_sse = ctx->zeromv_sse;
236   }
237 
238   if (ctx->newmv_sse > sse_thresh(bs, increase_denoising)) {
239     // Restore everything to its original state
240     *mbmi = saved_mbmi;
241     return COPY_BLOCK;
242   }
243   if (*motion_magnitude >
244      (noise_motion_thresh(bs, increase_denoising) << 3)) {
245     // Restore everything to its original state
246     *mbmi = saved_mbmi;
247     return COPY_BLOCK;
248   }
249 
250   // We will restore these after motion compensation.
251   for (i = 0; i < MAX_MB_PLANE; ++i) {
252     for (j = 0; j < 2; ++j) {
253       saved_pre[i][j] = filter_mbd->plane[i].pre[j];
254     }
255     saved_dst[i] = filter_mbd->plane[i].dst;
256   }
257 
258   // Set the pointers in the MACROBLOCKD to point to the buffers in the denoiser
259   // struct.
260   for (j = 0; j < 2; ++j) {
261     filter_mbd->plane[0].pre[j].buf =
262         block_start(denoiser->running_avg_y[frame].y_buffer,
263                     denoiser->running_avg_y[frame].y_stride,
264                     mi_row, mi_col);
265     filter_mbd->plane[0].pre[j].stride =
266         denoiser->running_avg_y[frame].y_stride;
267     filter_mbd->plane[1].pre[j].buf =
268         block_start(denoiser->running_avg_y[frame].u_buffer,
269                     denoiser->running_avg_y[frame].uv_stride,
270                     mi_row, mi_col);
271     filter_mbd->plane[1].pre[j].stride =
272         denoiser->running_avg_y[frame].uv_stride;
273     filter_mbd->plane[2].pre[j].buf =
274         block_start(denoiser->running_avg_y[frame].v_buffer,
275                     denoiser->running_avg_y[frame].uv_stride,
276                     mi_row, mi_col);
277     filter_mbd->plane[2].pre[j].stride =
278         denoiser->running_avg_y[frame].uv_stride;
279   }
280   filter_mbd->plane[0].dst.buf =
281       block_start(denoiser->mc_running_avg_y.y_buffer,
282                   denoiser->mc_running_avg_y.y_stride,
283                   mi_row, mi_col);
284   filter_mbd->plane[0].dst.stride = denoiser->mc_running_avg_y.y_stride;
285   filter_mbd->plane[1].dst.buf =
286       block_start(denoiser->mc_running_avg_y.u_buffer,
287                   denoiser->mc_running_avg_y.uv_stride,
288                   mi_row, mi_col);
289   filter_mbd->plane[1].dst.stride = denoiser->mc_running_avg_y.uv_stride;
290   filter_mbd->plane[2].dst.buf =
291       block_start(denoiser->mc_running_avg_y.v_buffer,
292                   denoiser->mc_running_avg_y.uv_stride,
293                   mi_row, mi_col);
294   filter_mbd->plane[2].dst.stride = denoiser->mc_running_avg_y.uv_stride;
295 
296   vp9_build_inter_predictors_sby(filter_mbd, mv_row, mv_col, bs);
297 
298   // Restore everything to its original state
299   *mbmi = saved_mbmi;
300   for (i = 0; i < MAX_MB_PLANE; ++i) {
301     for (j = 0; j < 2; ++j) {
302       filter_mbd->plane[i].pre[j] = saved_pre[i][j];
303     }
304     filter_mbd->plane[i].dst = saved_dst[i];
305   }
306 
307   mv_row = ctx->best_sse_mv.as_mv.row;
308   mv_col = ctx->best_sse_mv.as_mv.col;
309 
310   return FILTER_BLOCK;
311 }
312 
vp9_denoiser_denoise(VP9_DENOISER * denoiser,MACROBLOCK * mb,int mi_row,int mi_col,BLOCK_SIZE bs,PICK_MODE_CONTEXT * ctx)313 void vp9_denoiser_denoise(VP9_DENOISER *denoiser, MACROBLOCK *mb,
314                           int mi_row, int mi_col, BLOCK_SIZE bs,
315                           PICK_MODE_CONTEXT *ctx) {
316   int motion_magnitude = 0;
317   VP9_DENOISER_DECISION decision = COPY_BLOCK;
318   YV12_BUFFER_CONFIG avg = denoiser->running_avg_y[INTRA_FRAME];
319   YV12_BUFFER_CONFIG mc_avg = denoiser->mc_running_avg_y;
320   uint8_t *avg_start = block_start(avg.y_buffer, avg.y_stride, mi_row, mi_col);
321   uint8_t *mc_avg_start = block_start(mc_avg.y_buffer, mc_avg.y_stride,
322                                           mi_row, mi_col);
323   struct buf_2d src = mb->plane[0].src;
324   int is_skin = 0;
325 
326   if (bs <= BLOCK_16X16 && denoiser->denoising_on) {
327     // Take center pixel in block to determine is_skin.
328     const int y_width_shift = (4 << b_width_log2_lookup[bs]) >> 1;
329     const int y_height_shift = (4 << b_height_log2_lookup[bs]) >> 1;
330     const int uv_width_shift = y_width_shift >> 1;
331     const int uv_height_shift = y_height_shift >> 1;
332     const int stride = mb->plane[0].src.stride;
333     const int strideuv = mb->plane[1].src.stride;
334     const uint8_t ysource =
335       mb->plane[0].src.buf[y_height_shift * stride + y_width_shift];
336     const uint8_t usource =
337       mb->plane[1].src.buf[uv_height_shift * strideuv + uv_width_shift];
338     const uint8_t vsource =
339       mb->plane[2].src.buf[uv_height_shift * strideuv + uv_width_shift];
340     is_skin = vp9_skin_pixel(ysource, usource, vsource);
341   }
342 
343   if (denoiser->denoising_on)
344     decision = perform_motion_compensation(denoiser, mb, bs,
345                                            denoiser->increase_denoising,
346                                            mi_row, mi_col, ctx,
347                                            &motion_magnitude,
348                                            is_skin);
349 
350   if (decision == FILTER_BLOCK) {
351     decision = vp9_denoiser_filter(src.buf, src.stride,
352                                  mc_avg_start, mc_avg.y_stride,
353                                  avg_start, avg.y_stride,
354                                  0, bs, motion_magnitude);
355   }
356 
357   if (decision == FILTER_BLOCK) {
358     vpx_convolve_copy(avg_start, avg.y_stride, src.buf, src.stride,
359                       NULL, 0, NULL, 0,
360                       num_4x4_blocks_wide_lookup[bs] << 2,
361                       num_4x4_blocks_high_lookup[bs] << 2);
362   } else {  // COPY_BLOCK
363     vpx_convolve_copy(src.buf, src.stride, avg_start, avg.y_stride,
364                       NULL, 0, NULL, 0,
365                       num_4x4_blocks_wide_lookup[bs] << 2,
366                       num_4x4_blocks_high_lookup[bs] << 2);
367   }
368 }
369 
copy_frame(YV12_BUFFER_CONFIG * const dest,const YV12_BUFFER_CONFIG * const src)370 static void copy_frame(YV12_BUFFER_CONFIG * const dest,
371                        const YV12_BUFFER_CONFIG * const src) {
372   int r;
373   const uint8_t *srcbuf = src->y_buffer;
374   uint8_t *destbuf = dest->y_buffer;
375 
376   assert(dest->y_width == src->y_width);
377   assert(dest->y_height == src->y_height);
378 
379   for (r = 0; r < dest->y_height; ++r) {
380     memcpy(destbuf, srcbuf, dest->y_width);
381     destbuf += dest->y_stride;
382     srcbuf += src->y_stride;
383   }
384 }
385 
swap_frame_buffer(YV12_BUFFER_CONFIG * const dest,YV12_BUFFER_CONFIG * const src)386 static void swap_frame_buffer(YV12_BUFFER_CONFIG * const dest,
387                               YV12_BUFFER_CONFIG * const src) {
388   uint8_t *tmp_buf = dest->y_buffer;
389   assert(dest->y_width == src->y_width);
390   assert(dest->y_height == src->y_height);
391   dest->y_buffer = src->y_buffer;
392   src->y_buffer = tmp_buf;
393 }
394 
vp9_denoiser_update_frame_info(VP9_DENOISER * denoiser,YV12_BUFFER_CONFIG src,FRAME_TYPE frame_type,int refresh_alt_ref_frame,int refresh_golden_frame,int refresh_last_frame,int resized)395 void vp9_denoiser_update_frame_info(VP9_DENOISER *denoiser,
396                                     YV12_BUFFER_CONFIG src,
397                                     FRAME_TYPE frame_type,
398                                     int refresh_alt_ref_frame,
399                                     int refresh_golden_frame,
400                                     int refresh_last_frame,
401                                     int resized) {
402   // Copy source into denoised reference buffers on KEY_FRAME or
403   // if the just encoded frame was resized.
404   if (frame_type == KEY_FRAME || resized != 0) {
405     int i;
406     // Start at 1 so as not to overwrite the INTRA_FRAME
407     for (i = 1; i < MAX_REF_FRAMES; ++i)
408       copy_frame(&denoiser->running_avg_y[i], &src);
409     return;
410   }
411 
412   // If more than one refresh occurs, must copy frame buffer.
413   if ((refresh_alt_ref_frame + refresh_golden_frame + refresh_last_frame)
414       > 1) {
415     if (refresh_alt_ref_frame) {
416       copy_frame(&denoiser->running_avg_y[ALTREF_FRAME],
417                  &denoiser->running_avg_y[INTRA_FRAME]);
418     }
419     if (refresh_golden_frame) {
420       copy_frame(&denoiser->running_avg_y[GOLDEN_FRAME],
421                  &denoiser->running_avg_y[INTRA_FRAME]);
422     }
423     if (refresh_last_frame) {
424       copy_frame(&denoiser->running_avg_y[LAST_FRAME],
425                  &denoiser->running_avg_y[INTRA_FRAME]);
426     }
427   } else {
428     if (refresh_alt_ref_frame) {
429       swap_frame_buffer(&denoiser->running_avg_y[ALTREF_FRAME],
430                         &denoiser->running_avg_y[INTRA_FRAME]);
431     }
432     if (refresh_golden_frame) {
433       swap_frame_buffer(&denoiser->running_avg_y[GOLDEN_FRAME],
434                         &denoiser->running_avg_y[INTRA_FRAME]);
435     }
436     if (refresh_last_frame) {
437       swap_frame_buffer(&denoiser->running_avg_y[LAST_FRAME],
438                         &denoiser->running_avg_y[INTRA_FRAME]);
439     }
440   }
441 }
442 
vp9_denoiser_reset_frame_stats(PICK_MODE_CONTEXT * ctx)443 void vp9_denoiser_reset_frame_stats(PICK_MODE_CONTEXT *ctx) {
444   ctx->zeromv_sse = UINT_MAX;
445   ctx->newmv_sse = UINT_MAX;
446 }
447 
vp9_denoiser_update_frame_stats(MB_MODE_INFO * mbmi,unsigned int sse,PREDICTION_MODE mode,PICK_MODE_CONTEXT * ctx)448 void vp9_denoiser_update_frame_stats(MB_MODE_INFO *mbmi, unsigned int sse,
449                                      PREDICTION_MODE mode,
450                                      PICK_MODE_CONTEXT *ctx) {
451   // TODO(tkopp): Use both MVs if possible
452   if (mbmi->mv[0].as_int == 0 && sse < ctx->zeromv_sse) {
453     ctx->zeromv_sse = sse;
454     ctx->best_zeromv_reference_frame = mbmi->ref_frame[0];
455   }
456 
457   if (mbmi->mv[0].as_int != 0 && sse < ctx->newmv_sse) {
458     ctx->newmv_sse = sse;
459     ctx->best_sse_inter_mode = mode;
460     ctx->best_sse_mv = mbmi->mv[0];
461     ctx->best_reference_frame = mbmi->ref_frame[0];
462   }
463 }
464 
vp9_denoiser_alloc(VP9_DENOISER * denoiser,int width,int height,int ssx,int ssy,int use_highbitdepth,int border)465 int vp9_denoiser_alloc(VP9_DENOISER *denoiser, int width, int height,
466                        int ssx, int ssy,
467 #if CONFIG_VP9_HIGHBITDEPTH
468                        int use_highbitdepth,
469 #endif
470                        int border) {
471   int i, fail;
472   const int legacy_byte_alignment = 0;
473   assert(denoiser != NULL);
474 
475   for (i = 0; i < MAX_REF_FRAMES; ++i) {
476     fail = vpx_alloc_frame_buffer(&denoiser->running_avg_y[i], width, height,
477                                   ssx, ssy,
478 #if CONFIG_VP9_HIGHBITDEPTH
479                                   use_highbitdepth,
480 #endif
481                                   border, legacy_byte_alignment);
482     if (fail) {
483       vp9_denoiser_free(denoiser);
484       return 1;
485     }
486 #ifdef OUTPUT_YUV_DENOISED
487     make_grayscale(&denoiser->running_avg_y[i]);
488 #endif
489   }
490 
491   fail = vpx_alloc_frame_buffer(&denoiser->mc_running_avg_y, width, height,
492                                 ssx, ssy,
493 #if CONFIG_VP9_HIGHBITDEPTH
494                                 use_highbitdepth,
495 #endif
496                                 border, legacy_byte_alignment);
497   if (fail) {
498     vp9_denoiser_free(denoiser);
499     return 1;
500   }
501 
502   fail = vpx_alloc_frame_buffer(&denoiser->last_source, width, height,
503                                 ssx, ssy,
504 #if CONFIG_VP9_HIGHBITDEPTH
505                                 use_highbitdepth,
506 #endif
507                                 border, legacy_byte_alignment);
508   if (fail) {
509     vp9_denoiser_free(denoiser);
510     return 1;
511   }
512 #ifdef OUTPUT_YUV_DENOISED
513   make_grayscale(&denoiser->running_avg_y[i]);
514 #endif
515   denoiser->increase_denoising = 0;
516   denoiser->frame_buffer_initialized = 1;
517   vp9_denoiser_init_noise_estimate(denoiser, width, height);
518   return 0;
519 }
520 
vp9_denoiser_init_noise_estimate(VP9_DENOISER * denoiser,int width,int height)521 void vp9_denoiser_init_noise_estimate(VP9_DENOISER *denoiser,
522                                       int width,
523                                       int height) {
524   // Denoiser is off by default, i.e., no denoising is performed.
525   // Noise level is measured periodically, and if observed to be above
526   // thresh_noise_estimate, then denoising is performed, i.e., denoising_on = 1.
527   denoiser->denoising_on = 0;
528   denoiser->noise_estimate = 0;
529   denoiser->noise_estimate_count = 0;
530   denoiser->thresh_noise_estimate = 20;
531   if (width * height >= 1920 * 1080) {
532     denoiser->thresh_noise_estimate = 70;
533   } else if (width * height >= 1280 * 720) {
534     denoiser->thresh_noise_estimate = 40;
535   }
536 }
537 
vp9_denoiser_free(VP9_DENOISER * denoiser)538 void vp9_denoiser_free(VP9_DENOISER *denoiser) {
539   int i;
540   denoiser->frame_buffer_initialized = 0;
541   if (denoiser == NULL) {
542     return;
543   }
544   for (i = 0; i < MAX_REF_FRAMES; ++i) {
545     vpx_free_frame_buffer(&denoiser->running_avg_y[i]);
546   }
547   vpx_free_frame_buffer(&denoiser->mc_running_avg_y);
548   vpx_free_frame_buffer(&denoiser->last_source);
549 }
550 
vp9_denoiser_update_noise_estimate(VP9_COMP * const cpi)551 void vp9_denoiser_update_noise_estimate(VP9_COMP *const cpi) {
552   const VP9_COMMON *const cm = &cpi->common;
553   CYCLIC_REFRESH *const cr = cpi->cyclic_refresh;
554   int frame_period = 10;
555   int thresh_consec_zeromv = 8;
556   unsigned int thresh_sum_diff = 128;
557   int num_frames_estimate = 20;
558   int min_blocks_estimate = cm->mi_rows * cm->mi_cols >> 7;
559   // Estimate of noise level every frame_period frames.
560   // Estimate is between current source and last source.
561   if (cm->current_video_frame % frame_period != 0 ||
562      cpi->denoiser.last_source.y_buffer == NULL) {
563     copy_frame(&cpi->denoiser.last_source, cpi->Source);
564     return;
565   } else {
566     int num_samples = 0;
567     uint64_t avg_est = 0;
568     int bsize = BLOCK_16X16;
569     static const unsigned char const_source[16] = {
570          128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
571          128, 128};
572     // Loop over sub-sample of 16x16 blocks of frame, and for blocks that have
573     // been encoded as zero/small mv at least x consecutive frames, compute
574     // the variance to update estimate of noise in the source.
575     const uint8_t *src_y = cpi->Source->y_buffer;
576     const int src_ystride = cpi->Source->y_stride;
577     const uint8_t *last_src_y = cpi->denoiser.last_source.y_buffer;
578     const int last_src_ystride = cpi->denoiser.last_source.y_stride;
579     const uint8_t *src_u = cpi->Source->u_buffer;
580     const uint8_t *src_v = cpi->Source->v_buffer;
581     const int src_uvstride = cpi->Source->uv_stride;
582     const int y_width_shift = (4 << b_width_log2_lookup[bsize]) >> 1;
583     const int y_height_shift = (4 << b_height_log2_lookup[bsize]) >> 1;
584     const int uv_width_shift = y_width_shift >> 1;
585     const int uv_height_shift = y_height_shift >> 1;
586     int mi_row, mi_col;
587     for (mi_row = 0; mi_row < cm->mi_rows; mi_row ++) {
588       for (mi_col = 0; mi_col < cm->mi_cols; mi_col ++) {
589         // 16x16 blocks, 1/4 sample of frame.
590         if (mi_row % 4 == 0 && mi_col % 4 == 0) {
591           int bl_index = mi_row * cm->mi_cols + mi_col;
592           int bl_index1 = bl_index + 1;
593           int bl_index2 = bl_index + cm->mi_cols;
594           int bl_index3 = bl_index2 + 1;
595           // Only consider blocks that are likely steady background. i.e, have
596           // been encoded as zero/low motion x (= thresh_consec_zeromv) frames
597           // in a row. consec_zero_mv[] defined for 8x8 blocks, so consider all
598           // 4 sub-blocks for 16x16 block. Also, avoid skin blocks.
599           const uint8_t ysource =
600             src_y[y_height_shift * src_ystride + y_width_shift];
601           const uint8_t usource =
602             src_u[uv_height_shift * src_uvstride + uv_width_shift];
603           const uint8_t vsource =
604             src_v[uv_height_shift * src_uvstride + uv_width_shift];
605           int is_skin = vp9_skin_pixel(ysource, usource, vsource);
606           if (cr->consec_zero_mv[bl_index] > thresh_consec_zeromv &&
607               cr->consec_zero_mv[bl_index1] > thresh_consec_zeromv &&
608               cr->consec_zero_mv[bl_index2] > thresh_consec_zeromv &&
609               cr->consec_zero_mv[bl_index3] > thresh_consec_zeromv &&
610               !is_skin) {
611             // Compute variance.
612             unsigned int sse;
613             unsigned int variance = cpi->fn_ptr[bsize].vf(src_y,
614                                                           src_ystride,
615                                                           last_src_y,
616                                                           last_src_ystride,
617                                                           &sse);
618             // Only consider this block as valid for noise measurement if the
619             // average term (sse - variance = N * avg^{2}, N = 16X16) of the
620             // temporal residual is small (avoid effects from lighting change).
621             if ((sse - variance) < thresh_sum_diff) {
622               unsigned int sse2;
623               const unsigned int spatial_variance =
624                   cpi->fn_ptr[bsize].vf(src_y, src_ystride, const_source,
625                                         0, &sse2);
626               avg_est += variance / (10 + spatial_variance);
627               num_samples++;
628             }
629           }
630         }
631         src_y += 8;
632         last_src_y += 8;
633         src_u += 4;
634         src_v += 4;
635       }
636       src_y += (src_ystride << 3) - (cm->mi_cols << 3);
637       last_src_y += (last_src_ystride << 3) - (cm->mi_cols << 3);
638       src_u += (src_uvstride << 2) - (cm->mi_cols << 2);
639       src_v += (src_uvstride << 2) - (cm->mi_cols << 2);
640     }
641     // Update noise estimate if we have at a minimum number of block samples,
642     // and avg_est > 0 (avg_est == 0 can happen if the application inputs
643     // duplicate frames).
644     if (num_samples > min_blocks_estimate && avg_est > 0) {
645       // Normalize.
646       avg_est = (avg_est << 8) / num_samples;
647       // Update noise estimate.
648       cpi->denoiser.noise_estimate =  (3 * cpi->denoiser.noise_estimate +
649           avg_est) >> 2;
650       cpi->denoiser.noise_estimate_count++;
651       if (cpi->denoiser.noise_estimate_count == num_frames_estimate) {
652         // Reset counter and check noise level condition.
653         cpi->denoiser.noise_estimate_count = 0;
654        if (cpi->denoiser.noise_estimate > cpi->denoiser.thresh_noise_estimate)
655          cpi->denoiser.denoising_on = 1;
656        else
657          cpi->denoiser.denoising_on = 0;
658       }
659     }
660   }
661   copy_frame(&cpi->denoiser.last_source, cpi->Source);
662 }
663 
664 #ifdef OUTPUT_YUV_DENOISED
make_grayscale(YV12_BUFFER_CONFIG * yuv)665 static void make_grayscale(YV12_BUFFER_CONFIG *yuv) {
666   int r, c;
667   uint8_t *u = yuv->u_buffer;
668   uint8_t *v = yuv->v_buffer;
669 
670   for (r = 0; r < yuv->uv_height; ++r) {
671     for (c = 0; c < yuv->uv_width; ++c) {
672       u[c] = UINT8_MAX / 2;
673       v[c] = UINT8_MAX / 2;
674     }
675     u += yuv->uv_stride;
676     v += yuv->uv_stride;
677   }
678 }
679 #endif
680