1 /*
2 * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <arm_neon.h>
12 #include <assert.h>
13
14 #include "./vpx_dsp_rtcd.h"
15 #include "./vpx_config.h"
16
17 #include "vpx/vpx_integer.h"
18 #include "vpx_dsp/arm/mem_neon.h"
19 #include "vpx_dsp/arm/sum_neon.h"
20 #include "vpx_ports/mem.h"
21
22 // The variance helper functions use int16_t for sum. 8 values are accumulated
23 // and then added (at which point they expand up to int32_t). To avoid overflow,
24 // there can be no more than 32767 / 255 ~= 128 values accumulated in each
25 // column. For a 32x32 buffer, this results in 32 / 8 = 4 values per row * 32
26 // rows = 128. Asserts have been added to each function to warn against reaching
27 // this limit.
28
29 // Process a block of width 4 four rows at a time.
variance_neon_w4x4(const uint8_t * src_ptr,int src_stride,const uint8_t * ref_ptr,int ref_stride,int h,uint32_t * sse,int * sum)30 static void variance_neon_w4x4(const uint8_t *src_ptr, int src_stride,
31 const uint8_t *ref_ptr, int ref_stride, int h,
32 uint32_t *sse, int *sum) {
33 int i;
34 int16x8_t sum_s16 = vdupq_n_s16(0);
35 int32x4_t sse_lo_s32 = vdupq_n_s32(0);
36 int32x4_t sse_hi_s32 = vdupq_n_s32(0);
37
38 // Since width is only 4, sum_s16 only loads a half row per loop.
39 assert(h <= 256);
40
41 for (i = 0; i < h; i += 4) {
42 const uint8x16_t a_u8 = load_unaligned_u8q(src_ptr, src_stride);
43 const uint8x16_t b_u8 = load_unaligned_u8q(ref_ptr, ref_stride);
44 const uint16x8_t diff_lo_u16 =
45 vsubl_u8(vget_low_u8(a_u8), vget_low_u8(b_u8));
46 const uint16x8_t diff_hi_u16 =
47 vsubl_u8(vget_high_u8(a_u8), vget_high_u8(b_u8));
48
49 const int16x8_t diff_lo_s16 = vreinterpretq_s16_u16(diff_lo_u16);
50 const int16x8_t diff_hi_s16 = vreinterpretq_s16_u16(diff_hi_u16);
51
52 sum_s16 = vaddq_s16(sum_s16, diff_lo_s16);
53 sum_s16 = vaddq_s16(sum_s16, diff_hi_s16);
54
55 sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_lo_s16),
56 vget_low_s16(diff_lo_s16));
57 sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_high_s16(diff_lo_s16),
58 vget_high_s16(diff_lo_s16));
59
60 sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_low_s16(diff_hi_s16),
61 vget_low_s16(diff_hi_s16));
62 sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_high_s16(diff_hi_s16),
63 vget_high_s16(diff_hi_s16));
64
65 src_ptr += 4 * src_stride;
66 ref_ptr += 4 * ref_stride;
67 }
68
69 *sum = vget_lane_s32(horizontal_add_int16x8(sum_s16), 0);
70 *sse = vget_lane_u32(horizontal_add_uint32x4(vreinterpretq_u32_s32(
71 vaddq_s32(sse_lo_s32, sse_hi_s32))),
72 0);
73 }
74
75 // Process a block of any size where the width is divisible by 16.
variance_neon_w16(const uint8_t * src_ptr,int src_stride,const uint8_t * ref_ptr,int ref_stride,int w,int h,uint32_t * sse,int * sum)76 static void variance_neon_w16(const uint8_t *src_ptr, int src_stride,
77 const uint8_t *ref_ptr, int ref_stride, int w,
78 int h, uint32_t *sse, int *sum) {
79 int i, j;
80 int16x8_t sum_s16 = vdupq_n_s16(0);
81 int32x4_t sse_lo_s32 = vdupq_n_s32(0);
82 int32x4_t sse_hi_s32 = vdupq_n_s32(0);
83
84 // The loop loads 16 values at a time but doubles them up when accumulating
85 // into sum_s16.
86 assert(w / 8 * h <= 128);
87
88 for (i = 0; i < h; ++i) {
89 for (j = 0; j < w; j += 16) {
90 const uint8x16_t a_u8 = vld1q_u8(src_ptr + j);
91 const uint8x16_t b_u8 = vld1q_u8(ref_ptr + j);
92
93 const uint16x8_t diff_lo_u16 =
94 vsubl_u8(vget_low_u8(a_u8), vget_low_u8(b_u8));
95 const uint16x8_t diff_hi_u16 =
96 vsubl_u8(vget_high_u8(a_u8), vget_high_u8(b_u8));
97
98 const int16x8_t diff_lo_s16 = vreinterpretq_s16_u16(diff_lo_u16);
99 const int16x8_t diff_hi_s16 = vreinterpretq_s16_u16(diff_hi_u16);
100
101 sum_s16 = vaddq_s16(sum_s16, diff_lo_s16);
102 sum_s16 = vaddq_s16(sum_s16, diff_hi_s16);
103
104 sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_lo_s16),
105 vget_low_s16(diff_lo_s16));
106 sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_high_s16(diff_lo_s16),
107 vget_high_s16(diff_lo_s16));
108
109 sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_low_s16(diff_hi_s16),
110 vget_low_s16(diff_hi_s16));
111 sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_high_s16(diff_hi_s16),
112 vget_high_s16(diff_hi_s16));
113 }
114 src_ptr += src_stride;
115 ref_ptr += ref_stride;
116 }
117
118 *sum = vget_lane_s32(horizontal_add_int16x8(sum_s16), 0);
119 *sse = vget_lane_u32(horizontal_add_uint32x4(vreinterpretq_u32_s32(
120 vaddq_s32(sse_lo_s32, sse_hi_s32))),
121 0);
122 }
123
124 // Process a block of width 8 two rows at a time.
variance_neon_w8x2(const uint8_t * src_ptr,int src_stride,const uint8_t * ref_ptr,int ref_stride,int h,uint32_t * sse,int * sum)125 static void variance_neon_w8x2(const uint8_t *src_ptr, int src_stride,
126 const uint8_t *ref_ptr, int ref_stride, int h,
127 uint32_t *sse, int *sum) {
128 int i = 0;
129 int16x8_t sum_s16 = vdupq_n_s16(0);
130 int32x4_t sse_lo_s32 = vdupq_n_s32(0);
131 int32x4_t sse_hi_s32 = vdupq_n_s32(0);
132
133 // Each column has it's own accumulator entry in sum_s16.
134 assert(h <= 128);
135
136 do {
137 const uint8x8_t a_0_u8 = vld1_u8(src_ptr);
138 const uint8x8_t a_1_u8 = vld1_u8(src_ptr + src_stride);
139 const uint8x8_t b_0_u8 = vld1_u8(ref_ptr);
140 const uint8x8_t b_1_u8 = vld1_u8(ref_ptr + ref_stride);
141 const uint16x8_t diff_0_u16 = vsubl_u8(a_0_u8, b_0_u8);
142 const uint16x8_t diff_1_u16 = vsubl_u8(a_1_u8, b_1_u8);
143 const int16x8_t diff_0_s16 = vreinterpretq_s16_u16(diff_0_u16);
144 const int16x8_t diff_1_s16 = vreinterpretq_s16_u16(diff_1_u16);
145 sum_s16 = vaddq_s16(sum_s16, diff_0_s16);
146 sum_s16 = vaddq_s16(sum_s16, diff_1_s16);
147 sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_0_s16),
148 vget_low_s16(diff_0_s16));
149 sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_1_s16),
150 vget_low_s16(diff_1_s16));
151 sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_high_s16(diff_0_s16),
152 vget_high_s16(diff_0_s16));
153 sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_high_s16(diff_1_s16),
154 vget_high_s16(diff_1_s16));
155 src_ptr += src_stride + src_stride;
156 ref_ptr += ref_stride + ref_stride;
157 i += 2;
158 } while (i < h);
159
160 *sum = vget_lane_s32(horizontal_add_int16x8(sum_s16), 0);
161 *sse = vget_lane_u32(horizontal_add_uint32x4(vreinterpretq_u32_s32(
162 vaddq_s32(sse_lo_s32, sse_hi_s32))),
163 0);
164 }
165
vpx_get8x8var_neon(const uint8_t * src_ptr,int src_stride,const uint8_t * ref_ptr,int ref_stride,unsigned int * sse,int * sum)166 void vpx_get8x8var_neon(const uint8_t *src_ptr, int src_stride,
167 const uint8_t *ref_ptr, int ref_stride,
168 unsigned int *sse, int *sum) {
169 variance_neon_w8x2(src_ptr, src_stride, ref_ptr, ref_stride, 8, sse, sum);
170 }
171
vpx_get16x16var_neon(const uint8_t * src_ptr,int src_stride,const uint8_t * ref_ptr,int ref_stride,unsigned int * sse,int * sum)172 void vpx_get16x16var_neon(const uint8_t *src_ptr, int src_stride,
173 const uint8_t *ref_ptr, int ref_stride,
174 unsigned int *sse, int *sum) {
175 variance_neon_w16(src_ptr, src_stride, ref_ptr, ref_stride, 16, 16, sse, sum);
176 }
177
178 #define varianceNxM(n, m, shift) \
179 unsigned int vpx_variance##n##x##m##_neon( \
180 const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \
181 int ref_stride, unsigned int *sse) { \
182 int sum; \
183 if (n == 4) \
184 variance_neon_w4x4(src_ptr, src_stride, ref_ptr, ref_stride, m, sse, \
185 &sum); \
186 else if (n == 8) \
187 variance_neon_w8x2(src_ptr, src_stride, ref_ptr, ref_stride, m, sse, \
188 &sum); \
189 else \
190 variance_neon_w16(src_ptr, src_stride, ref_ptr, ref_stride, n, m, sse, \
191 &sum); \
192 if (n * m < 16 * 16) \
193 return *sse - ((sum * sum) >> shift); \
194 else \
195 return *sse - (uint32_t)(((int64_t)sum * sum) >> shift); \
196 }
197
198 varianceNxM(4, 4, 4);
199 varianceNxM(4, 8, 5);
200 varianceNxM(8, 4, 5);
201 varianceNxM(8, 8, 6);
202 varianceNxM(8, 16, 7);
203 varianceNxM(16, 8, 7);
204 varianceNxM(16, 16, 8);
205 varianceNxM(16, 32, 9);
206 varianceNxM(32, 16, 9);
207 varianceNxM(32, 32, 10);
208
vpx_variance32x64_neon(const uint8_t * src_ptr,int src_stride,const uint8_t * ref_ptr,int ref_stride,unsigned int * sse)209 unsigned int vpx_variance32x64_neon(const uint8_t *src_ptr, int src_stride,
210 const uint8_t *ref_ptr, int ref_stride,
211 unsigned int *sse) {
212 int sum1, sum2;
213 uint32_t sse1, sse2;
214 variance_neon_w16(src_ptr, src_stride, ref_ptr, ref_stride, 32, 32, &sse1,
215 &sum1);
216 variance_neon_w16(src_ptr + (32 * src_stride), src_stride,
217 ref_ptr + (32 * ref_stride), ref_stride, 32, 32, &sse2,
218 &sum2);
219 *sse = sse1 + sse2;
220 sum1 += sum2;
221 return *sse - (unsigned int)(((int64_t)sum1 * sum1) >> 11);
222 }
223
vpx_variance64x32_neon(const uint8_t * src_ptr,int src_stride,const uint8_t * ref_ptr,int ref_stride,unsigned int * sse)224 unsigned int vpx_variance64x32_neon(const uint8_t *src_ptr, int src_stride,
225 const uint8_t *ref_ptr, int ref_stride,
226 unsigned int *sse) {
227 int sum1, sum2;
228 uint32_t sse1, sse2;
229 variance_neon_w16(src_ptr, src_stride, ref_ptr, ref_stride, 64, 16, &sse1,
230 &sum1);
231 variance_neon_w16(src_ptr + (16 * src_stride), src_stride,
232 ref_ptr + (16 * ref_stride), ref_stride, 64, 16, &sse2,
233 &sum2);
234 *sse = sse1 + sse2;
235 sum1 += sum2;
236 return *sse - (unsigned int)(((int64_t)sum1 * sum1) >> 11);
237 }
238
vpx_variance64x64_neon(const uint8_t * src_ptr,int src_stride,const uint8_t * ref_ptr,int ref_stride,unsigned int * sse)239 unsigned int vpx_variance64x64_neon(const uint8_t *src_ptr, int src_stride,
240 const uint8_t *ref_ptr, int ref_stride,
241 unsigned int *sse) {
242 int sum1, sum2;
243 uint32_t sse1, sse2;
244
245 variance_neon_w16(src_ptr, src_stride, ref_ptr, ref_stride, 64, 16, &sse1,
246 &sum1);
247 variance_neon_w16(src_ptr + (16 * src_stride), src_stride,
248 ref_ptr + (16 * ref_stride), ref_stride, 64, 16, &sse2,
249 &sum2);
250 sse1 += sse2;
251 sum1 += sum2;
252
253 variance_neon_w16(src_ptr + (16 * 2 * src_stride), src_stride,
254 ref_ptr + (16 * 2 * ref_stride), ref_stride, 64, 16, &sse2,
255 &sum2);
256 sse1 += sse2;
257 sum1 += sum2;
258
259 variance_neon_w16(src_ptr + (16 * 3 * src_stride), src_stride,
260 ref_ptr + (16 * 3 * ref_stride), ref_stride, 64, 16, &sse2,
261 &sum2);
262 *sse = sse1 + sse2;
263 sum1 += sum2;
264 return *sse - (unsigned int)(((int64_t)sum1 * sum1) >> 12);
265 }
266
vpx_mse16x16_neon(const unsigned char * src_ptr,int src_stride,const unsigned char * ref_ptr,int ref_stride,unsigned int * sse)267 unsigned int vpx_mse16x16_neon(const unsigned char *src_ptr, int src_stride,
268 const unsigned char *ref_ptr, int ref_stride,
269 unsigned int *sse) {
270 int i;
271 int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16;
272 int64x1_t d0s64;
273 uint8x16_t q0u8, q1u8, q2u8, q3u8;
274 int32x4_t q7s32, q8s32, q9s32, q10s32;
275 uint16x8_t q11u16, q12u16, q13u16, q14u16;
276 int64x2_t q1s64;
277
278 q7s32 = vdupq_n_s32(0);
279 q8s32 = vdupq_n_s32(0);
280 q9s32 = vdupq_n_s32(0);
281 q10s32 = vdupq_n_s32(0);
282
283 for (i = 0; i < 8; i++) { // mse16x16_neon_loop
284 q0u8 = vld1q_u8(src_ptr);
285 src_ptr += src_stride;
286 q1u8 = vld1q_u8(src_ptr);
287 src_ptr += src_stride;
288 q2u8 = vld1q_u8(ref_ptr);
289 ref_ptr += ref_stride;
290 q3u8 = vld1q_u8(ref_ptr);
291 ref_ptr += ref_stride;
292
293 q11u16 = vsubl_u8(vget_low_u8(q0u8), vget_low_u8(q2u8));
294 q12u16 = vsubl_u8(vget_high_u8(q0u8), vget_high_u8(q2u8));
295 q13u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q3u8));
296 q14u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q3u8));
297
298 d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
299 d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
300 q7s32 = vmlal_s16(q7s32, d22s16, d22s16);
301 q8s32 = vmlal_s16(q8s32, d23s16, d23s16);
302
303 d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
304 d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
305 q9s32 = vmlal_s16(q9s32, d24s16, d24s16);
306 q10s32 = vmlal_s16(q10s32, d25s16, d25s16);
307
308 d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16));
309 d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16));
310 q7s32 = vmlal_s16(q7s32, d26s16, d26s16);
311 q8s32 = vmlal_s16(q8s32, d27s16, d27s16);
312
313 d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16));
314 d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16));
315 q9s32 = vmlal_s16(q9s32, d28s16, d28s16);
316 q10s32 = vmlal_s16(q10s32, d29s16, d29s16);
317 }
318
319 q7s32 = vaddq_s32(q7s32, q8s32);
320 q9s32 = vaddq_s32(q9s32, q10s32);
321 q10s32 = vaddq_s32(q7s32, q9s32);
322
323 q1s64 = vpaddlq_s32(q10s32);
324 d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
325
326 vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d0s64), 0);
327 return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0);
328 }
329
vpx_get4x4sse_cs_neon(const unsigned char * src_ptr,int src_stride,const unsigned char * ref_ptr,int ref_stride)330 unsigned int vpx_get4x4sse_cs_neon(const unsigned char *src_ptr, int src_stride,
331 const unsigned char *ref_ptr,
332 int ref_stride) {
333 int16x4_t d22s16, d24s16, d26s16, d28s16;
334 int64x1_t d0s64;
335 uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8;
336 int32x4_t q7s32, q8s32, q9s32, q10s32;
337 uint16x8_t q11u16, q12u16, q13u16, q14u16;
338 int64x2_t q1s64;
339
340 d0u8 = vld1_u8(src_ptr);
341 src_ptr += src_stride;
342 d4u8 = vld1_u8(ref_ptr);
343 ref_ptr += ref_stride;
344 d1u8 = vld1_u8(src_ptr);
345 src_ptr += src_stride;
346 d5u8 = vld1_u8(ref_ptr);
347 ref_ptr += ref_stride;
348 d2u8 = vld1_u8(src_ptr);
349 src_ptr += src_stride;
350 d6u8 = vld1_u8(ref_ptr);
351 ref_ptr += ref_stride;
352 d3u8 = vld1_u8(src_ptr);
353 src_ptr += src_stride;
354 d7u8 = vld1_u8(ref_ptr);
355 ref_ptr += ref_stride;
356
357 q11u16 = vsubl_u8(d0u8, d4u8);
358 q12u16 = vsubl_u8(d1u8, d5u8);
359 q13u16 = vsubl_u8(d2u8, d6u8);
360 q14u16 = vsubl_u8(d3u8, d7u8);
361
362 d22s16 = vget_low_s16(vreinterpretq_s16_u16(q11u16));
363 d24s16 = vget_low_s16(vreinterpretq_s16_u16(q12u16));
364 d26s16 = vget_low_s16(vreinterpretq_s16_u16(q13u16));
365 d28s16 = vget_low_s16(vreinterpretq_s16_u16(q14u16));
366
367 q7s32 = vmull_s16(d22s16, d22s16);
368 q8s32 = vmull_s16(d24s16, d24s16);
369 q9s32 = vmull_s16(d26s16, d26s16);
370 q10s32 = vmull_s16(d28s16, d28s16);
371
372 q7s32 = vaddq_s32(q7s32, q8s32);
373 q9s32 = vaddq_s32(q9s32, q10s32);
374 q9s32 = vaddq_s32(q7s32, q9s32);
375
376 q1s64 = vpaddlq_s32(q9s32);
377 d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
378
379 return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0);
380 }
381