1 /*
2 * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <arm_neon.h>
12 #include "./vp9_rtcd.h"
13
14 #include "vpx_ports/mem.h"
15 #include "vpx/vpx_integer.h"
16
17 #include "vp9/common/vp9_common.h"
18 #include "vp9/common/vp9_filter.h"
19
20 #include "vp9/encoder/vp9_variance.h"
21
22 enum { kWidth8 = 8 };
23 enum { kHeight8 = 8 };
24 enum { kHeight8PlusOne = 9 };
25 enum { kWidth16 = 16 };
26 enum { kHeight16 = 16 };
27 enum { kHeight16PlusOne = 17 };
28 enum { kWidth32 = 32 };
29 enum { kHeight32 = 32 };
30 enum { kHeight32PlusOne = 33 };
31 enum { kPixelStepOne = 1 };
32 enum { kAlign16 = 16 };
33
horizontal_add_s16x8(const int16x8_t v_16x8)34 static INLINE int horizontal_add_s16x8(const int16x8_t v_16x8) {
35 const int32x4_t a = vpaddlq_s16(v_16x8);
36 const int64x2_t b = vpaddlq_s32(a);
37 const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)),
38 vreinterpret_s32_s64(vget_high_s64(b)));
39 return vget_lane_s32(c, 0);
40 }
41
horizontal_add_s32x4(const int32x4_t v_32x4)42 static INLINE int horizontal_add_s32x4(const int32x4_t v_32x4) {
43 const int64x2_t b = vpaddlq_s32(v_32x4);
44 const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)),
45 vreinterpret_s32_s64(vget_high_s64(b)));
46 return vget_lane_s32(c, 0);
47 }
48
variance_neon_w8(const uint8_t * a,int a_stride,const uint8_t * b,int b_stride,int w,int h,unsigned int * sse,int * sum)49 static void variance_neon_w8(const uint8_t *a, int a_stride,
50 const uint8_t *b, int b_stride,
51 int w, int h, unsigned int *sse, int *sum) {
52 int i, j;
53 int16x8_t v_sum = vdupq_n_s16(0);
54 int32x4_t v_sse_lo = vdupq_n_s32(0);
55 int32x4_t v_sse_hi = vdupq_n_s32(0);
56
57 for (i = 0; i < h; ++i) {
58 for (j = 0; j < w; j += 8) {
59 const uint8x8_t v_a = vld1_u8(&a[j]);
60 const uint8x8_t v_b = vld1_u8(&b[j]);
61 const uint16x8_t v_diff = vsubl_u8(v_a, v_b);
62 const int16x8_t sv_diff = vreinterpretq_s16_u16(v_diff);
63 v_sum = vaddq_s16(v_sum, sv_diff);
64 v_sse_lo = vmlal_s16(v_sse_lo,
65 vget_low_s16(sv_diff),
66 vget_low_s16(sv_diff));
67 v_sse_hi = vmlal_s16(v_sse_hi,
68 vget_high_s16(sv_diff),
69 vget_high_s16(sv_diff));
70 }
71 a += a_stride;
72 b += b_stride;
73 }
74
75 *sum = horizontal_add_s16x8(v_sum);
76 *sse = (unsigned int)horizontal_add_s32x4(vaddq_s32(v_sse_lo, v_sse_hi));
77 }
78
vp9_get8x8var_neon(const uint8_t * src_ptr,int source_stride,const uint8_t * ref_ptr,int ref_stride,unsigned int * sse,int * sum)79 void vp9_get8x8var_neon(const uint8_t *src_ptr, int source_stride,
80 const uint8_t *ref_ptr, int ref_stride,
81 unsigned int *sse, int *sum) {
82 variance_neon_w8(src_ptr, source_stride, ref_ptr, ref_stride, kWidth8,
83 kHeight8, sse, sum);
84 }
85
vp9_variance8x8_neon(const uint8_t * a,int a_stride,const uint8_t * b,int b_stride,unsigned int * sse)86 unsigned int vp9_variance8x8_neon(const uint8_t *a, int a_stride,
87 const uint8_t *b, int b_stride,
88 unsigned int *sse) {
89 int sum;
90 variance_neon_w8(a, a_stride, b, b_stride, kWidth8, kHeight8, sse, &sum);
91 return *sse - (((int64_t)sum * sum) / (kWidth8 * kHeight8));
92 }
93
vp9_get16x16var_neon(const uint8_t * src_ptr,int source_stride,const uint8_t * ref_ptr,int ref_stride,unsigned int * sse,int * sum)94 void vp9_get16x16var_neon(const uint8_t *src_ptr, int source_stride,
95 const uint8_t *ref_ptr, int ref_stride,
96 unsigned int *sse, int *sum) {
97 variance_neon_w8(src_ptr, source_stride, ref_ptr, ref_stride, kWidth16,
98 kHeight16, sse, sum);
99 }
100
vp9_variance16x16_neon(const uint8_t * a,int a_stride,const uint8_t * b,int b_stride,unsigned int * sse)101 unsigned int vp9_variance16x16_neon(const uint8_t *a, int a_stride,
102 const uint8_t *b, int b_stride,
103 unsigned int *sse) {
104 int sum;
105 variance_neon_w8(a, a_stride, b, b_stride, kWidth16, kHeight16, sse, &sum);
106 return *sse - (((int64_t)sum * sum) / (kWidth16 * kHeight16));
107 }
108
var_filter_block2d_bil_w8(const uint8_t * src_ptr,uint8_t * output_ptr,unsigned int src_pixels_per_line,int pixel_step,unsigned int output_height,unsigned int output_width,const int16_t * vp9_filter)109 static void var_filter_block2d_bil_w8(const uint8_t *src_ptr,
110 uint8_t *output_ptr,
111 unsigned int src_pixels_per_line,
112 int pixel_step,
113 unsigned int output_height,
114 unsigned int output_width,
115 const int16_t *vp9_filter) {
116 const uint8x8_t f0 = vmov_n_u8((uint8_t)vp9_filter[0]);
117 const uint8x8_t f1 = vmov_n_u8((uint8_t)vp9_filter[1]);
118 unsigned int i;
119 for (i = 0; i < output_height; ++i) {
120 const uint8x8_t src_0 = vld1_u8(&src_ptr[0]);
121 const uint8x8_t src_1 = vld1_u8(&src_ptr[pixel_step]);
122 const uint16x8_t a = vmull_u8(src_0, f0);
123 const uint16x8_t b = vmlal_u8(a, src_1, f1);
124 const uint8x8_t out = vrshrn_n_u16(b, FILTER_BITS);
125 vst1_u8(&output_ptr[0], out);
126 // Next row...
127 src_ptr += src_pixels_per_line;
128 output_ptr += output_width;
129 }
130 }
131
var_filter_block2d_bil_w16(const uint8_t * src_ptr,uint8_t * output_ptr,unsigned int src_pixels_per_line,int pixel_step,unsigned int output_height,unsigned int output_width,const int16_t * vp9_filter)132 static void var_filter_block2d_bil_w16(const uint8_t *src_ptr,
133 uint8_t *output_ptr,
134 unsigned int src_pixels_per_line,
135 int pixel_step,
136 unsigned int output_height,
137 unsigned int output_width,
138 const int16_t *vp9_filter) {
139 const uint8x8_t f0 = vmov_n_u8((uint8_t)vp9_filter[0]);
140 const uint8x8_t f1 = vmov_n_u8((uint8_t)vp9_filter[1]);
141 unsigned int i, j;
142 for (i = 0; i < output_height; ++i) {
143 for (j = 0; j < output_width; j += 16) {
144 const uint8x16_t src_0 = vld1q_u8(&src_ptr[j]);
145 const uint8x16_t src_1 = vld1q_u8(&src_ptr[j + pixel_step]);
146 const uint16x8_t a = vmull_u8(vget_low_u8(src_0), f0);
147 const uint16x8_t b = vmlal_u8(a, vget_low_u8(src_1), f1);
148 const uint8x8_t out_lo = vrshrn_n_u16(b, FILTER_BITS);
149 const uint16x8_t c = vmull_u8(vget_high_u8(src_0), f0);
150 const uint16x8_t d = vmlal_u8(c, vget_high_u8(src_1), f1);
151 const uint8x8_t out_hi = vrshrn_n_u16(d, FILTER_BITS);
152 vst1q_u8(&output_ptr[j], vcombine_u8(out_lo, out_hi));
153 }
154 // Next row...
155 src_ptr += src_pixels_per_line;
156 output_ptr += output_width;
157 }
158 }
159
vp9_sub_pixel_variance8x8_neon(const uint8_t * src,int src_stride,int xoffset,int yoffset,const uint8_t * dst,int dst_stride,unsigned int * sse)160 unsigned int vp9_sub_pixel_variance8x8_neon(const uint8_t *src,
161 int src_stride,
162 int xoffset,
163 int yoffset,
164 const uint8_t *dst,
165 int dst_stride,
166 unsigned int *sse) {
167 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, temp2, kHeight8 * kWidth8);
168 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, fdata3, kHeight8PlusOne * kWidth8);
169
170 var_filter_block2d_bil_w8(src, fdata3, src_stride, kPixelStepOne,
171 kHeight8PlusOne, kWidth8,
172 BILINEAR_FILTERS_2TAP(xoffset));
173 var_filter_block2d_bil_w8(fdata3, temp2, kWidth8, kWidth8, kHeight8,
174 kWidth8, BILINEAR_FILTERS_2TAP(yoffset));
175 return vp9_variance8x8_neon(temp2, kWidth8, dst, dst_stride, sse);
176 }
177
vp9_sub_pixel_variance16x16_neon(const uint8_t * src,int src_stride,int xoffset,int yoffset,const uint8_t * dst,int dst_stride,unsigned int * sse)178 unsigned int vp9_sub_pixel_variance16x16_neon(const uint8_t *src,
179 int src_stride,
180 int xoffset,
181 int yoffset,
182 const uint8_t *dst,
183 int dst_stride,
184 unsigned int *sse) {
185 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, temp2, kHeight16 * kWidth16);
186 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, fdata3, kHeight16PlusOne * kWidth16);
187
188 var_filter_block2d_bil_w16(src, fdata3, src_stride, kPixelStepOne,
189 kHeight16PlusOne, kWidth16,
190 BILINEAR_FILTERS_2TAP(xoffset));
191 var_filter_block2d_bil_w16(fdata3, temp2, kWidth16, kWidth16, kHeight16,
192 kWidth16, BILINEAR_FILTERS_2TAP(yoffset));
193 return vp9_variance16x16_neon(temp2, kWidth16, dst, dst_stride, sse);
194 }
195
vp9_get32x32var_neon(const uint8_t * src_ptr,int source_stride,const uint8_t * ref_ptr,int ref_stride,unsigned int * sse,int * sum)196 void vp9_get32x32var_neon(const uint8_t *src_ptr, int source_stride,
197 const uint8_t *ref_ptr, int ref_stride,
198 unsigned int *sse, int *sum) {
199 variance_neon_w8(src_ptr, source_stride, ref_ptr, ref_stride, kWidth32,
200 kHeight32, sse, sum);
201 }
202
vp9_variance32x32_neon(const uint8_t * a,int a_stride,const uint8_t * b,int b_stride,unsigned int * sse)203 unsigned int vp9_variance32x32_neon(const uint8_t *a, int a_stride,
204 const uint8_t *b, int b_stride,
205 unsigned int *sse) {
206 int sum;
207 variance_neon_w8(a, a_stride, b, b_stride, kWidth32, kHeight32, sse, &sum);
208 return *sse - (((int64_t)sum * sum) / (kWidth32 * kHeight32));
209 }
210
vp9_sub_pixel_variance32x32_neon(const uint8_t * src,int src_stride,int xoffset,int yoffset,const uint8_t * dst,int dst_stride,unsigned int * sse)211 unsigned int vp9_sub_pixel_variance32x32_neon(const uint8_t *src,
212 int src_stride,
213 int xoffset,
214 int yoffset,
215 const uint8_t *dst,
216 int dst_stride,
217 unsigned int *sse) {
218 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, temp2, kHeight32 * kWidth32);
219 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, fdata3, kHeight32PlusOne * kWidth32);
220
221 var_filter_block2d_bil_w16(src, fdata3, src_stride, kPixelStepOne,
222 kHeight32PlusOne, kWidth32,
223 BILINEAR_FILTERS_2TAP(xoffset));
224 var_filter_block2d_bil_w16(fdata3, temp2, kWidth32, kWidth32, kHeight32,
225 kWidth32, BILINEAR_FILTERS_2TAP(yoffset));
226 return vp9_variance32x32_neon(temp2, kWidth32, dst, dst_stride, sse);
227 }
228