1 /*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
12
13 #include <arm_neon.h>
14
15 // NEON intrinsics version of WebRtcSpl_DownsampleFast()
16 // for ARM 32-bit/64-bit platforms.
WebRtcSpl_DownsampleFastNeon(const int16_t * data_in,size_t data_in_length,int16_t * data_out,size_t data_out_length,const int16_t * __restrict coefficients,size_t coefficients_length,int factor,size_t delay)17 int WebRtcSpl_DownsampleFastNeon(const int16_t* data_in,
18 size_t data_in_length,
19 int16_t* data_out,
20 size_t data_out_length,
21 const int16_t* __restrict coefficients,
22 size_t coefficients_length,
23 int factor,
24 size_t delay) {
25 size_t i = 0;
26 size_t j = 0;
27 int32_t out_s32 = 0;
28 size_t endpos = delay + factor * (data_out_length - 1) + 1;
29 size_t res = data_out_length & 0x7;
30 size_t endpos1 = endpos - factor * res;
31
32 // Return error if any of the running conditions doesn't meet.
33 if (data_out_length == 0 || coefficients_length == 0
34 || data_in_length < endpos) {
35 return -1;
36 }
37
38 // First part, unroll the loop 8 times, with 3 subcases
39 // (factor == 2, 4, others).
40 switch (factor) {
41 case 2: {
42 for (i = delay; i < endpos1; i += 16) {
43 // Round value, 0.5 in Q12.
44 int32x4_t out32x4_0 = vdupq_n_s32(2048);
45 int32x4_t out32x4_1 = vdupq_n_s32(2048);
46
47 #if defined(WEBRTC_ARCH_ARM64)
48 // Unroll the loop 2 times.
49 for (j = 0; j < coefficients_length - 1; j += 2) {
50 int32x2_t coeff32 = vld1_dup_s32((int32_t*)&coefficients[j]);
51 int16x4_t coeff16x4 = vreinterpret_s16_s32(coeff32);
52 int16x8x2_t in16x8x2 = vld2q_s16(&data_in[i - j - 1]);
53
54 // Mul and accumulate low 64-bit data.
55 int16x4_t in16x4_0 = vget_low_s16(in16x8x2.val[0]);
56 int16x4_t in16x4_1 = vget_low_s16(in16x8x2.val[1]);
57 out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_0, coeff16x4, 1);
58 out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_1, coeff16x4, 0);
59
60 // Mul and accumulate high 64-bit data.
61 // TODO: vget_high_s16 need extra cost on ARM64. This could be
62 // replaced by vmlal_high_lane_s16. But for the interface of
63 // vmlal_high_lane_s16, there is a bug in gcc 4.9.
64 // This issue need to be tracked in the future.
65 int16x4_t in16x4_2 = vget_high_s16(in16x8x2.val[0]);
66 int16x4_t in16x4_3 = vget_high_s16(in16x8x2.val[1]);
67 out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_2, coeff16x4, 1);
68 out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_3, coeff16x4, 0);
69 }
70
71 for (; j < coefficients_length; j++) {
72 int16x4_t coeff16x4 = vld1_dup_s16(&coefficients[j]);
73 int16x8x2_t in16x8x2 = vld2q_s16(&data_in[i - j]);
74
75 // Mul and accumulate low 64-bit data.
76 int16x4_t in16x4_0 = vget_low_s16(in16x8x2.val[0]);
77 out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_0, coeff16x4, 0);
78
79 // Mul and accumulate high 64-bit data.
80 // TODO: vget_high_s16 need extra cost on ARM64. This could be
81 // replaced by vmlal_high_lane_s16. But for the interface of
82 // vmlal_high_lane_s16, there is a bug in gcc 4.9.
83 // This issue need to be tracked in the future.
84 int16x4_t in16x4_1 = vget_high_s16(in16x8x2.val[0]);
85 out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_1, coeff16x4, 0);
86 }
87 #else
88 // On ARMv7, the loop unrolling 2 times results in performance
89 // regression.
90 for (j = 0; j < coefficients_length; j++) {
91 int16x4_t coeff16x4 = vld1_dup_s16(&coefficients[j]);
92 int16x8x2_t in16x8x2 = vld2q_s16(&data_in[i - j]);
93
94 // Mul and accumulate.
95 int16x4_t in16x4_0 = vget_low_s16(in16x8x2.val[0]);
96 int16x4_t in16x4_1 = vget_high_s16(in16x8x2.val[0]);
97 out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_0, coeff16x4, 0);
98 out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_1, coeff16x4, 0);
99 }
100 #endif
101
102 // Saturate and store the output.
103 int16x4_t out16x4_0 = vqshrn_n_s32(out32x4_0, 12);
104 int16x4_t out16x4_1 = vqshrn_n_s32(out32x4_1, 12);
105 vst1q_s16(data_out, vcombine_s16(out16x4_0, out16x4_1));
106 data_out += 8;
107 }
108 break;
109 }
110 case 4: {
111 for (i = delay; i < endpos1; i += 32) {
112 // Round value, 0.5 in Q12.
113 int32x4_t out32x4_0 = vdupq_n_s32(2048);
114 int32x4_t out32x4_1 = vdupq_n_s32(2048);
115
116 // Unroll the loop 4 times.
117 for (j = 0; j < coefficients_length - 3; j += 4) {
118 int16x4_t coeff16x4 = vld1_s16(&coefficients[j]);
119 int16x8x4_t in16x8x4 = vld4q_s16(&data_in[i - j - 3]);
120
121 // Mul and accumulate low 64-bit data.
122 int16x4_t in16x4_0 = vget_low_s16(in16x8x4.val[0]);
123 int16x4_t in16x4_2 = vget_low_s16(in16x8x4.val[1]);
124 int16x4_t in16x4_4 = vget_low_s16(in16x8x4.val[2]);
125 int16x4_t in16x4_6 = vget_low_s16(in16x8x4.val[3]);
126 out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_0, coeff16x4, 3);
127 out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_2, coeff16x4, 2);
128 out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_4, coeff16x4, 1);
129 out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_6, coeff16x4, 0);
130
131 // Mul and accumulate high 64-bit data.
132 // TODO: vget_high_s16 need extra cost on ARM64. This could be
133 // replaced by vmlal_high_lane_s16. But for the interface of
134 // vmlal_high_lane_s16, there is a bug in gcc 4.9.
135 // This issue need to be tracked in the future.
136 int16x4_t in16x4_1 = vget_high_s16(in16x8x4.val[0]);
137 int16x4_t in16x4_3 = vget_high_s16(in16x8x4.val[1]);
138 int16x4_t in16x4_5 = vget_high_s16(in16x8x4.val[2]);
139 int16x4_t in16x4_7 = vget_high_s16(in16x8x4.val[3]);
140 out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_1, coeff16x4, 3);
141 out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_3, coeff16x4, 2);
142 out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_5, coeff16x4, 1);
143 out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_7, coeff16x4, 0);
144 }
145
146 for (; j < coefficients_length; j++) {
147 int16x4_t coeff16x4 = vld1_dup_s16(&coefficients[j]);
148 int16x8x4_t in16x8x4 = vld4q_s16(&data_in[i - j]);
149
150 // Mul and accumulate low 64-bit data.
151 int16x4_t in16x4_0 = vget_low_s16(in16x8x4.val[0]);
152 out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_0, coeff16x4, 0);
153
154 // Mul and accumulate high 64-bit data.
155 // TODO: vget_high_s16 need extra cost on ARM64. This could be
156 // replaced by vmlal_high_lane_s16. But for the interface of
157 // vmlal_high_lane_s16, there is a bug in gcc 4.9.
158 // This issue need to be tracked in the future.
159 int16x4_t in16x4_1 = vget_high_s16(in16x8x4.val[0]);
160 out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_1, coeff16x4, 0);
161 }
162
163 // Saturate and store the output.
164 int16x4_t out16x4_0 = vqshrn_n_s32(out32x4_0, 12);
165 int16x4_t out16x4_1 = vqshrn_n_s32(out32x4_1, 12);
166 vst1q_s16(data_out, vcombine_s16(out16x4_0, out16x4_1));
167 data_out += 8;
168 }
169 break;
170 }
171 default: {
172 for (i = delay; i < endpos1; i += factor * 8) {
173 // Round value, 0.5 in Q12.
174 int32x4_t out32x4_0 = vdupq_n_s32(2048);
175 int32x4_t out32x4_1 = vdupq_n_s32(2048);
176
177 for (j = 0; j < coefficients_length; j++) {
178 int16x4_t coeff16x4 = vld1_dup_s16(&coefficients[j]);
179 int16x4_t in16x4_0 = vld1_dup_s16(&data_in[i - j]);
180 in16x4_0 = vld1_lane_s16(&data_in[i + factor - j], in16x4_0, 1);
181 in16x4_0 = vld1_lane_s16(&data_in[i + factor * 2 - j], in16x4_0, 2);
182 in16x4_0 = vld1_lane_s16(&data_in[i + factor * 3 - j], in16x4_0, 3);
183 int16x4_t in16x4_1 = vld1_dup_s16(&data_in[i + factor * 4 - j]);
184 in16x4_1 = vld1_lane_s16(&data_in[i + factor * 5 - j], in16x4_1, 1);
185 in16x4_1 = vld1_lane_s16(&data_in[i + factor * 6 - j], in16x4_1, 2);
186 in16x4_1 = vld1_lane_s16(&data_in[i + factor * 7 - j], in16x4_1, 3);
187
188 // Mul and accumulate.
189 out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_0, coeff16x4, 0);
190 out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_1, coeff16x4, 0);
191 }
192
193 // Saturate and store the output.
194 int16x4_t out16x4_0 = vqshrn_n_s32(out32x4_0, 12);
195 int16x4_t out16x4_1 = vqshrn_n_s32(out32x4_1, 12);
196 vst1q_s16(data_out, vcombine_s16(out16x4_0, out16x4_1));
197 data_out += 8;
198 }
199 break;
200 }
201 }
202
203 // Second part, do the rest iterations (if any).
204 for (; i < endpos; i += factor) {
205 out_s32 = 2048; // Round value, 0.5 in Q12.
206
207 for (j = 0; j < coefficients_length; j++) {
208 out_s32 = WebRtc_MulAccumW16(coefficients[j], data_in[i - j], out_s32);
209 }
210
211 // Saturate and store the output.
212 out_s32 >>= 12;
213 *data_out++ = WebRtcSpl_SatW32ToW16(out_s32);
214 }
215
216 return 0;
217 }
218