1 /*
2  *  Copyright (c) 2012 The WebM project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include <assert.h>
12 #include <limits.h>
13 #include "vpx_scale/yv12config.h"
14 #include "vpx/vpx_integer.h"
15 #include "vp9/common/vp9_reconinter.h"
16 #include "vp9/encoder/vp9_context_tree.h"
17 #include "vp9/encoder/vp9_denoiser.h"
18 
19 /* The VP9 denoiser is a work-in-progress. It currently is only designed to work
20  * with speed 6, though it (inexplicably) seems to also work with speed 5 (one
21  * would need to modify the source code in vp9_pickmode.c and vp9_encoder.c to
22  * make the calls to the vp9_denoiser_* functions when in speed 5).
23  *
24  * The implementation is very similar to that of the VP8 denoiser. While
25  * choosing the motion vectors / reference frames, the denoiser is run, and if
26  * it did not modify the signal to much, the denoised block is copied to the
27  * signal.
28  */
29 
30 #ifdef OUTPUT_YUV_DENOISED
31 static void make_grayscale(YV12_BUFFER_CONFIG *yuv);
32 #endif
33 
34 static const int widths[]  = {4, 4, 8, 8,  8, 16, 16, 16, 32, 32, 32, 64, 64};
35 static const int heights[] = {4, 8, 4, 8, 16,  8, 16, 32, 16, 32, 64, 32, 64};
36 
absdiff_thresh(BLOCK_SIZE bs,int increase_denoising)37 static int absdiff_thresh(BLOCK_SIZE bs, int increase_denoising) {
38   (void)bs;
39   return 3 + (increase_denoising ? 1 : 0);
40 }
41 
delta_thresh(BLOCK_SIZE bs,int increase_denoising)42 static int delta_thresh(BLOCK_SIZE bs, int increase_denoising) {
43   (void)bs;
44   (void)increase_denoising;
45   return 4;
46 }
47 
noise_motion_thresh(BLOCK_SIZE bs,int increase_denoising)48 static int noise_motion_thresh(BLOCK_SIZE bs, int increase_denoising) {
49   (void)bs;
50   (void)increase_denoising;
51   return 25 * 25;
52 }
53 
sse_thresh(BLOCK_SIZE bs,int increase_denoising)54 static unsigned int sse_thresh(BLOCK_SIZE bs, int increase_denoising) {
55   return widths[bs] * heights[bs] * (increase_denoising ? 60 : 40);
56 }
57 
sse_diff_thresh(BLOCK_SIZE bs,int increase_denoising,int mv_row,int mv_col)58 static int sse_diff_thresh(BLOCK_SIZE bs, int increase_denoising,
59                            int mv_row, int mv_col) {
60   if (mv_row * mv_row + mv_col * mv_col >
61       noise_motion_thresh(bs, increase_denoising)) {
62     return 0;
63   } else {
64     return widths[bs] * heights[bs] * 20;
65   }
66 }
67 
total_adj_strong_thresh(BLOCK_SIZE bs,int increase_denoising)68 static int total_adj_strong_thresh(BLOCK_SIZE bs, int increase_denoising) {
69   return widths[bs] * heights[bs] * (increase_denoising ? 3 : 2);
70 }
71 
total_adj_weak_thresh(BLOCK_SIZE bs,int increase_denoising)72 static int total_adj_weak_thresh(BLOCK_SIZE bs, int increase_denoising) {
73   return widths[bs] * heights[bs] * (increase_denoising ? 3 : 2);
74 }
75 
denoiser_filter(const uint8_t * sig,int sig_stride,const uint8_t * mc_avg,int mc_avg_stride,uint8_t * avg,int avg_stride,int increase_denoising,BLOCK_SIZE bs)76 static VP9_DENOISER_DECISION denoiser_filter(const uint8_t *sig, int sig_stride,
77                                              const uint8_t *mc_avg,
78                                              int mc_avg_stride,
79                                              uint8_t *avg, int avg_stride,
80                                              int increase_denoising,
81                                              BLOCK_SIZE bs) {
82   int r, c;
83   const uint8_t *sig_start = sig;
84   const uint8_t *mc_avg_start = mc_avg;
85   uint8_t *avg_start = avg;
86   int diff, adj, absdiff, delta;
87   int adj_val[] = {3, 4, 6};
88   int total_adj = 0;
89 
90   // First attempt to apply a strong temporal denoising filter.
91   for (r = 0; r < heights[bs]; ++r) {
92     for (c = 0; c < widths[bs]; ++c) {
93       diff = mc_avg[c] - sig[c];
94       absdiff = abs(diff);
95 
96       if (absdiff <= absdiff_thresh(bs, increase_denoising)) {
97         avg[c] = mc_avg[c];
98         total_adj += diff;
99       } else {
100         switch (absdiff) {
101           case 4: case 5: case 6: case 7:
102             adj = adj_val[0];
103             break;
104           case 8: case 9: case 10: case 11:
105           case 12: case 13: case 14: case 15:
106             adj = adj_val[1];
107             break;
108           default:
109             adj = adj_val[2];
110         }
111         if (diff > 0) {
112           avg[c] = MIN(UINT8_MAX, sig[c] + adj);
113           total_adj += adj;
114         } else {
115           avg[c] = MAX(0, sig[c] - adj);
116           total_adj -= adj;
117         }
118       }
119     }
120     sig += sig_stride;
121     avg += avg_stride;
122     mc_avg += mc_avg_stride;
123   }
124 
125   // If the strong filter did not modify the signal too much, we're all set.
126   if (abs(total_adj) <= total_adj_strong_thresh(bs, increase_denoising)) {
127     return FILTER_BLOCK;
128   }
129 
130   // Otherwise, we try to dampen the filter if the delta is not too high.
131   delta = ((abs(total_adj) - total_adj_strong_thresh(bs, increase_denoising))
132            >> 8) + 1;
133   if (delta > delta_thresh(bs, increase_denoising)) {
134     return COPY_BLOCK;
135   }
136 
137   mc_avg =  mc_avg_start;
138   avg = avg_start;
139   sig = sig_start;
140   for (r = 0; r < heights[bs]; ++r) {
141     for (c = 0; c < widths[bs]; ++c) {
142       diff = mc_avg[c] - sig[c];
143       adj = abs(diff);
144       if (adj > delta) {
145         adj = delta;
146       }
147       if (diff > 0) {
148         avg[c] = MAX(0, avg[c] - adj);
149         total_adj += adj;
150       } else {
151         avg[c] = MIN(UINT8_MAX, avg[c] + adj);
152         total_adj -= adj;
153       }
154     }
155     sig += sig_stride;
156     avg += avg_stride;
157     mc_avg += mc_avg_stride;
158   }
159 
160   // We can use the filter if it has been sufficiently dampened
161   if (abs(total_adj) <= total_adj_weak_thresh(bs, increase_denoising)) {
162     return FILTER_BLOCK;
163   }
164   return COPY_BLOCK;
165 }
166 
block_start(uint8_t * framebuf,int stride,int mi_row,int mi_col)167 static uint8_t *block_start(uint8_t *framebuf, int stride,
168                             int mi_row, int mi_col) {
169   return framebuf + (stride * mi_row * 8) + (mi_col * 8);
170 }
171 
copy_block(uint8_t * dest,int dest_stride,const uint8_t * src,int src_stride,BLOCK_SIZE bs)172 static void copy_block(uint8_t *dest, int dest_stride,
173                        const uint8_t *src, int src_stride, BLOCK_SIZE bs) {
174   int r;
175   for (r = 0; r < heights[bs]; ++r) {
176     vpx_memcpy(dest, src, widths[bs]);
177     dest += dest_stride;
178     src += src_stride;
179   }
180 }
181 
perform_motion_compensation(VP9_DENOISER * denoiser,MACROBLOCK * mb,BLOCK_SIZE bs,int increase_denoising,int mi_row,int mi_col,PICK_MODE_CONTEXT * ctx)182 static VP9_DENOISER_DECISION perform_motion_compensation(VP9_DENOISER *denoiser,
183                                                          MACROBLOCK *mb,
184                                                          BLOCK_SIZE bs,
185                                                          int increase_denoising,
186                                                          int mi_row,
187                                                          int mi_col,
188                                                          PICK_MODE_CONTEXT *ctx
189                                                          ) {
190   int mv_col, mv_row;
191   int sse_diff = ctx->zeromv_sse - ctx->newmv_sse;
192   MV_REFERENCE_FRAME frame;
193   MACROBLOCKD *filter_mbd = &mb->e_mbd;
194   MB_MODE_INFO *mbmi = &filter_mbd->mi[0]->mbmi;
195 
196   MB_MODE_INFO saved_mbmi;
197   int i, j;
198   struct buf_2d saved_dst[MAX_MB_PLANE];
199   struct buf_2d saved_pre[MAX_MB_PLANE][2];  // 2 pre buffers
200 
201   // We will restore these after motion compensation.
202   saved_mbmi = *mbmi;
203   for (i = 0; i < MAX_MB_PLANE; ++i) {
204     for (j = 0; j < 2; ++j) {
205       saved_pre[i][j] = filter_mbd->plane[i].pre[j];
206     }
207     saved_dst[i] = filter_mbd->plane[i].dst;
208   }
209 
210   mv_col = ctx->best_sse_mv.as_mv.col;
211   mv_row = ctx->best_sse_mv.as_mv.row;
212 
213   frame = ctx->best_reference_frame;
214 
215   // If the best reference frame uses inter-prediction and there is enough of a
216   // difference in sum-squared-error, use it.
217   if (frame != INTRA_FRAME &&
218       sse_diff > sse_diff_thresh(bs, increase_denoising, mv_row, mv_col)) {
219     mbmi->ref_frame[0] = ctx->best_reference_frame;
220     mbmi->mode = ctx->best_sse_inter_mode;
221     mbmi->mv[0] = ctx->best_sse_mv;
222   } else {
223     // Otherwise, use the zero reference frame.
224     frame = ctx->best_zeromv_reference_frame;
225 
226     mbmi->ref_frame[0] = ctx->best_zeromv_reference_frame;
227     mbmi->mode = ZEROMV;
228     mbmi->mv[0].as_int = 0;
229 
230     ctx->best_sse_inter_mode = ZEROMV;
231     ctx->best_sse_mv.as_int = 0;
232     ctx->newmv_sse = ctx->zeromv_sse;
233   }
234 
235   // Set the pointers in the MACROBLOCKD to point to the buffers in the denoiser
236   // struct.
237   for (j = 0; j < 2; ++j) {
238     filter_mbd->plane[0].pre[j].buf =
239         block_start(denoiser->running_avg_y[frame].y_buffer,
240                     denoiser->running_avg_y[frame].y_stride,
241                     mi_row, mi_col);
242     filter_mbd->plane[0].pre[j].stride =
243         denoiser->running_avg_y[frame].y_stride;
244     filter_mbd->plane[1].pre[j].buf =
245         block_start(denoiser->running_avg_y[frame].u_buffer,
246                     denoiser->running_avg_y[frame].uv_stride,
247                     mi_row, mi_col);
248     filter_mbd->plane[1].pre[j].stride =
249         denoiser->running_avg_y[frame].uv_stride;
250     filter_mbd->plane[2].pre[j].buf =
251         block_start(denoiser->running_avg_y[frame].v_buffer,
252                     denoiser->running_avg_y[frame].uv_stride,
253                     mi_row, mi_col);
254     filter_mbd->plane[2].pre[j].stride =
255         denoiser->running_avg_y[frame].uv_stride;
256   }
257   filter_mbd->plane[0].dst.buf =
258       block_start(denoiser->mc_running_avg_y.y_buffer,
259                   denoiser->mc_running_avg_y.y_stride,
260                   mi_row, mi_col);
261   filter_mbd->plane[0].dst.stride = denoiser->mc_running_avg_y.y_stride;
262   filter_mbd->plane[1].dst.buf =
263       block_start(denoiser->mc_running_avg_y.u_buffer,
264                   denoiser->mc_running_avg_y.uv_stride,
265                   mi_row, mi_col);
266   filter_mbd->plane[1].dst.stride = denoiser->mc_running_avg_y.uv_stride;
267   filter_mbd->plane[2].dst.buf =
268       block_start(denoiser->mc_running_avg_y.v_buffer,
269                   denoiser->mc_running_avg_y.uv_stride,
270                   mi_row, mi_col);
271   filter_mbd->plane[2].dst.stride = denoiser->mc_running_avg_y.uv_stride;
272 
273   vp9_build_inter_predictors_sby(filter_mbd, mv_row, mv_col, bs);
274 
275   // Restore everything to its original state
276   *mbmi = saved_mbmi;
277   for (i = 0; i < MAX_MB_PLANE; ++i) {
278     for (j = 0; j < 2; ++j) {
279       filter_mbd->plane[i].pre[j] = saved_pre[i][j];
280     }
281     filter_mbd->plane[i].dst = saved_dst[i];
282   }
283 
284   mv_row = ctx->best_sse_mv.as_mv.row;
285   mv_col = ctx->best_sse_mv.as_mv.col;
286 
287   if (ctx->newmv_sse > sse_thresh(bs, increase_denoising)) {
288     return COPY_BLOCK;
289   }
290   if (mv_row * mv_row + mv_col * mv_col >
291       8 * noise_motion_thresh(bs, increase_denoising)) {
292     return COPY_BLOCK;
293   }
294   return FILTER_BLOCK;
295 }
296 
vp9_denoiser_denoise(VP9_DENOISER * denoiser,MACROBLOCK * mb,int mi_row,int mi_col,BLOCK_SIZE bs,PICK_MODE_CONTEXT * ctx)297 void vp9_denoiser_denoise(VP9_DENOISER *denoiser, MACROBLOCK *mb,
298                           int mi_row, int mi_col, BLOCK_SIZE bs,
299                           PICK_MODE_CONTEXT *ctx) {
300   VP9_DENOISER_DECISION decision = FILTER_BLOCK;
301   YV12_BUFFER_CONFIG avg = denoiser->running_avg_y[INTRA_FRAME];
302   YV12_BUFFER_CONFIG mc_avg = denoiser->mc_running_avg_y;
303   uint8_t *avg_start = block_start(avg.y_buffer, avg.y_stride, mi_row, mi_col);
304   uint8_t *mc_avg_start = block_start(mc_avg.y_buffer, mc_avg.y_stride,
305                                           mi_row, mi_col);
306   struct buf_2d src = mb->plane[0].src;
307 
308   decision = perform_motion_compensation(denoiser, mb, bs,
309                                          denoiser->increase_denoising,
310                                          mi_row, mi_col, ctx);
311 
312   if (decision == FILTER_BLOCK) {
313     decision = denoiser_filter(src.buf, src.stride,
314                                mc_avg_start, mc_avg.y_stride,
315                                avg_start, avg.y_stride,
316                                0, bs);
317   }
318 
319   if (decision == FILTER_BLOCK) {
320     copy_block(src.buf, src.stride, avg_start, avg.y_stride, bs);
321   } else {  // COPY_BLOCK
322     copy_block(avg_start, avg.y_stride, src.buf, src.stride, bs);
323   }
324 }
325 
copy_frame(YV12_BUFFER_CONFIG dest,const YV12_BUFFER_CONFIG src)326 static void copy_frame(YV12_BUFFER_CONFIG dest, const YV12_BUFFER_CONFIG src) {
327   int r;
328   const uint8_t *srcbuf = src.y_buffer;
329   uint8_t *destbuf = dest.y_buffer;
330   assert(dest.y_width == src.y_width);
331   assert(dest.y_height == src.y_height);
332 
333   for (r = 0; r < dest.y_height; ++r) {
334     vpx_memcpy(destbuf, srcbuf, dest.y_width);
335     destbuf += dest.y_stride;
336     srcbuf += src.y_stride;
337   }
338 }
339 
vp9_denoiser_update_frame_info(VP9_DENOISER * denoiser,YV12_BUFFER_CONFIG src,FRAME_TYPE frame_type,int refresh_alt_ref_frame,int refresh_golden_frame,int refresh_last_frame)340 void vp9_denoiser_update_frame_info(VP9_DENOISER *denoiser,
341                                     YV12_BUFFER_CONFIG src,
342                                     FRAME_TYPE frame_type,
343                                     int refresh_alt_ref_frame,
344                                     int refresh_golden_frame,
345                                     int refresh_last_frame) {
346   if (frame_type == KEY_FRAME) {
347     int i;
348     // Start at 1 so as not to overwrite the INTRA_FRAME
349     for (i = 1; i < MAX_REF_FRAMES; ++i) {
350       copy_frame(denoiser->running_avg_y[i], src);
351     }
352   } else {  /* For non key frames */
353     if (refresh_alt_ref_frame) {
354       copy_frame(denoiser->running_avg_y[ALTREF_FRAME],
355                  denoiser->running_avg_y[INTRA_FRAME]);
356     }
357     if (refresh_golden_frame) {
358       copy_frame(denoiser->running_avg_y[GOLDEN_FRAME],
359                  denoiser->running_avg_y[INTRA_FRAME]);
360     }
361     if (refresh_last_frame) {
362       copy_frame(denoiser->running_avg_y[LAST_FRAME],
363                  denoiser->running_avg_y[INTRA_FRAME]);
364     }
365   }
366 }
367 
vp9_denoiser_reset_frame_stats(PICK_MODE_CONTEXT * ctx)368 void vp9_denoiser_reset_frame_stats(PICK_MODE_CONTEXT *ctx) {
369   ctx->zeromv_sse = UINT_MAX;
370   ctx->newmv_sse = UINT_MAX;
371 }
372 
vp9_denoiser_update_frame_stats(VP9_DENOISER * denoiser,MB_MODE_INFO * mbmi,unsigned int sse,PREDICTION_MODE mode,PICK_MODE_CONTEXT * ctx)373 void vp9_denoiser_update_frame_stats(VP9_DENOISER *denoiser, MB_MODE_INFO *mbmi,
374                                      unsigned int sse, PREDICTION_MODE mode,
375                                      PICK_MODE_CONTEXT *ctx) {
376   // TODO(tkopp): Use both MVs if possible
377   if (mbmi->mv[0].as_int == 0 && sse < ctx->zeromv_sse) {
378     ctx->zeromv_sse = sse;
379     ctx->best_zeromv_reference_frame = mbmi->ref_frame[0];
380   }
381 
382   if (mode == NEWMV) {
383     ctx->newmv_sse = sse;
384     ctx->best_sse_inter_mode = mode;
385     ctx->best_sse_mv = mbmi->mv[0];
386     ctx->best_reference_frame = mbmi->ref_frame[0];
387   }
388 }
389 
vp9_denoiser_alloc(VP9_DENOISER * denoiser,int width,int height,int ssx,int ssy,int border)390 int vp9_denoiser_alloc(VP9_DENOISER *denoiser, int width, int height,
391                        int ssx, int ssy, int border) {
392   int i, fail;
393   assert(denoiser != NULL);
394 
395   for (i = 0; i < MAX_REF_FRAMES; ++i) {
396     fail = vp9_alloc_frame_buffer(&denoiser->running_avg_y[i], width, height,
397                                   ssx, ssy, border);
398     if (fail) {
399       vp9_denoiser_free(denoiser);
400       return 1;
401     }
402 #ifdef OUTPUT_YUV_DENOISED
403     make_grayscale(&denoiser->running_avg_y[i]);
404 #endif
405   }
406 
407   fail = vp9_alloc_frame_buffer(&denoiser->mc_running_avg_y, width, height,
408                                 ssx, ssy, border);
409   if (fail) {
410     vp9_denoiser_free(denoiser);
411     return 1;
412   }
413 #ifdef OUTPUT_YUV_DENOISED
414   make_grayscale(&denoiser->running_avg_y[i]);
415 #endif
416   denoiser->increase_denoising = 0;
417 
418   return 0;
419 }
420 
vp9_denoiser_free(VP9_DENOISER * denoiser)421 void vp9_denoiser_free(VP9_DENOISER *denoiser) {
422   int i;
423   if (denoiser == NULL) {
424     return;
425   }
426   for (i = 0; i < MAX_REF_FRAMES; ++i) {
427     if (&denoiser->running_avg_y[i] != NULL) {
428       vp9_free_frame_buffer(&denoiser->running_avg_y[i]);
429     }
430   }
431   if (&denoiser->mc_running_avg_y != NULL) {
432     vp9_free_frame_buffer(&denoiser->mc_running_avg_y);
433   }
434 }
435 
436 #ifdef OUTPUT_YUV_DENOISED
make_grayscale(YV12_BUFFER_CONFIG * yuv)437 static void make_grayscale(YV12_BUFFER_CONFIG *yuv) {
438   int r, c;
439   uint8_t *u = yuv->u_buffer;
440   uint8_t *v = yuv->v_buffer;
441 
442   // The '/2's are there because we have a 440 buffer, but we want to output
443   // 420.
444   for (r = 0; r < yuv->uv_height / 2; ++r) {
445     for (c = 0; c < yuv->uv_width / 2; ++c) {
446       u[c] = UINT8_MAX / 2;
447       v[c] = UINT8_MAX / 2;
448     }
449     u += yuv->uv_stride + yuv->uv_width / 2;
450     v += yuv->uv_stride + yuv->uv_width / 2;
451   }
452 }
453 #endif
454