1 // Auto-generated file. Do not edit!
2 //   Template: src/qs8-gavgpool/unipass-wasmsimd.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <wasm_simd128.h>
13 
14 #include <xnnpack/gavgpool.h>
15 
16 
xnn_qs8_gavgpool_minmax_ukernel_7x__wasmsimd_c16_acc2(size_t rows,size_t channels,const int8_t * input,size_t input_stride,const int8_t * zero,int8_t * output,const union xnn_qs8_avgpool_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_qs8_gavgpool_minmax_ukernel_7x__wasmsimd_c16_acc2(
18     size_t rows,
19     size_t channels,
20     const int8_t* input,
21     size_t input_stride,
22     const int8_t* zero,
23     int8_t* output,
24     const union xnn_qs8_avgpool_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
25 {
26   assert(rows != 0);
27   assert(rows <= 7);
28   assert(channels != 0);
29 
30   const int8_t* i0 = input;
31   const int8_t* i1 = (const int8_t*) ((uintptr_t) i0 + input_stride);
32   if XNN_UNPREDICTABLE(rows < 2) {
33     i1 = zero;
34   }
35   const int8_t* i2 = (const int8_t*) ((uintptr_t) i1 + input_stride);
36   if XNN_UNPREDICTABLE(rows <= 2) {
37     i2 = zero;
38   }
39   const int8_t* i3 = (const int8_t*) ((uintptr_t) i2 + input_stride);
40   if XNN_UNPREDICTABLE(rows < 4) {
41     i3 = zero;
42   }
43   const int8_t* i4 = (const int8_t*) ((uintptr_t) i3 + input_stride);
44   if XNN_UNPREDICTABLE(rows <= 4) {
45     i4 = zero;
46   }
47   const int8_t* i5 = (const int8_t*) ((uintptr_t) i4 + input_stride);
48   if XNN_UNPREDICTABLE(rows < 6) {
49     i5 = zero;
50   }
51   const int8_t* i6 = (const int8_t*) ((uintptr_t) i5 + input_stride);
52   if XNN_UNPREDICTABLE(rows <= 6) {
53     i6 = zero;
54   }
55 
56   const v128_t vbias = wasm_v128_load(params->wasmsimd.bias);
57   const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
58   const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
59   const int32_t vshift = params->wasmsimd.shift;
60   const v128_t vzero = wasm_f64x2_splat(0.0);
61   while (channels >= 16) {
62     const v128_t vxi0x01234567 = wasm_i16x8_load_8x8(i0);
63     const v128_t vxi0x89ABCDEF = wasm_i16x8_load_8x8(i0 + 8);
64     i0 += 16;
65     const v128_t vxi1x01234567 = wasm_i16x8_load_8x8(i1);
66     const v128_t vxi1x89ABCDEF = wasm_i16x8_load_8x8(i1 + 8);
67     i1 += 16;
68     const v128_t vxi2x01234567 = wasm_i16x8_load_8x8(i2);
69     const v128_t vxi2x89ABCDEF = wasm_i16x8_load_8x8(i2 + 8);
70     i2 += 16;
71     const v128_t vxi3x01234567 = wasm_i16x8_load_8x8(i3);
72     const v128_t vxi3x89ABCDEF = wasm_i16x8_load_8x8(i3 + 8);
73     i3 += 16;
74     const v128_t vxi4x01234567 = wasm_i16x8_load_8x8(i4);
75     const v128_t vxi4x89ABCDEF = wasm_i16x8_load_8x8(i4 + 8);
76     i4 += 16;
77     const v128_t vxi5x01234567 = wasm_i16x8_load_8x8(i5);
78     const v128_t vxi5x89ABCDEF = wasm_i16x8_load_8x8(i5 + 8);
79     i5 += 16;
80     const v128_t vxi6x01234567 = wasm_i16x8_load_8x8(i6);
81     const v128_t vxi6x89ABCDEF = wasm_i16x8_load_8x8(i6 + 8);
82     i6 += 16;
83 
84     v128_t vacc0x01234567 = wasm_i16x8_add(vxi0x01234567, vxi1x01234567);
85     v128_t vacc0x89ABCDEF = wasm_i16x8_add(vxi0x89ABCDEF, vxi1x89ABCDEF);
86     v128_t vacc1x01234567 = wasm_i16x8_add(vxi2x01234567, vxi3x01234567);
87     v128_t vacc1x89ABCDEF = wasm_i16x8_add(vxi2x89ABCDEF, vxi3x89ABCDEF);
88 
89     vacc0x01234567 = wasm_i16x8_add(vacc0x01234567, vxi4x01234567);
90     vacc0x89ABCDEF = wasm_i16x8_add(vacc0x89ABCDEF, vxi4x89ABCDEF);
91     vacc1x01234567 = wasm_i16x8_add(vacc1x01234567, vxi5x01234567);
92     vacc1x89ABCDEF = wasm_i16x8_add(vacc1x89ABCDEF, vxi5x89ABCDEF);
93     vacc0x01234567 = wasm_i16x8_add(vacc0x01234567, vxi6x01234567);
94     vacc0x89ABCDEF = wasm_i16x8_add(vacc0x89ABCDEF, vxi6x89ABCDEF);
95 
96     // Add up all accumulators to vacc0x0123456789ABCDEF
97     vacc0x01234567 = wasm_i16x8_add(vacc0x01234567, vacc1x01234567);
98     vacc0x89ABCDEF = wasm_i16x8_add(vacc0x89ABCDEF, vacc1x89ABCDEF);
99 
100     const v128_t vacc0123 = wasm_i32x4_add(vbias, wasm_i32x4_widen_low_i16x8(vacc0x01234567));
101     const v128_t vacc4567 = wasm_i32x4_add(vbias, wasm_i32x4_widen_high_i16x8(vacc0x01234567));
102     const v128_t vacc89AB = wasm_i32x4_add(vbias, wasm_i32x4_widen_low_i16x8(vacc0x89ABCDEF));
103     const v128_t vaccCDEF = wasm_i32x4_add(vbias, wasm_i32x4_widen_high_i16x8(vacc0x89ABCDEF));
104 
105     const v128_t vabsacc0123 = wasm_i32x4_abs(vacc0123);
106     const v128_t vabsacc4567 = wasm_i32x4_abs(vacc4567);
107     const v128_t vabsacc89AB = wasm_i32x4_abs(vacc89AB);
108     const v128_t vabsaccCDEF = wasm_i32x4_abs(vaccCDEF);
109 
110     const v128_t vsgnacc0123 = wasm_i32x4_gt(vabsacc0123, vacc0123);
111     const v128_t vsgnacc4567 = wasm_i32x4_gt(vabsacc4567, vacc4567);
112     const v128_t vsgnacc89AB = wasm_i32x4_gt(vabsacc89AB, vacc89AB);
113     const v128_t vsgnaccCDEF = wasm_i32x4_gt(vabsaccCDEF, vaccCDEF);
114 
115     const v128_t vabsacc01 = wasm_v32x4_shuffle(vabsacc0123, vzero, 0, 4, 1, 5);
116     const v128_t vabsacc23 = wasm_v32x4_shuffle(vabsacc0123, vzero, 2, 6, 3, 7);
117     const v128_t vabsacc45 = wasm_v32x4_shuffle(vabsacc4567, vzero, 0, 4, 1, 5);
118     const v128_t vabsacc67 = wasm_v32x4_shuffle(vabsacc4567, vzero, 2, 6, 3, 7);
119     const v128_t vabsacc89 = wasm_v32x4_shuffle(vabsacc89AB, vzero, 0, 4, 1, 5);
120     const v128_t vabsaccAB = wasm_v32x4_shuffle(vabsacc89AB, vzero, 2, 6, 3, 7);
121     const v128_t vabsaccCD = wasm_v32x4_shuffle(vabsaccCDEF, vzero, 0, 4, 1, 5);
122     const v128_t vabsaccEF = wasm_v32x4_shuffle(vabsaccCDEF, vzero, 2, 6, 3, 7);
123 
124     const v128_t vabsprod01 = wasm_i64x2_mul(vabsacc01, vmultiplier);
125     const v128_t vabsprod23 = wasm_i64x2_mul(vabsacc23, vmultiplier);
126     const v128_t vabsprod45 = wasm_i64x2_mul(vabsacc45, vmultiplier);
127     const v128_t vabsprod67 = wasm_i64x2_mul(vabsacc67, vmultiplier);
128     const v128_t vabsprod89 = wasm_i64x2_mul(vabsacc89, vmultiplier);
129     const v128_t vabsprodAB = wasm_i64x2_mul(vabsaccAB, vmultiplier);
130     const v128_t vabsprodCD = wasm_i64x2_mul(vabsaccCD, vmultiplier);
131     const v128_t vabsprodEF = wasm_i64x2_mul(vabsaccEF, vmultiplier);
132 
133     const v128_t vabsout01 = wasm_u64x2_shr(wasm_i64x2_add(vabsprod01, vrounding), vshift);
134     const v128_t vabsout23 = wasm_u64x2_shr(wasm_i64x2_add(vabsprod23, vrounding), vshift);
135     const v128_t vabsout45 = wasm_u64x2_shr(wasm_i64x2_add(vabsprod45, vrounding), vshift);
136     const v128_t vabsout67 = wasm_u64x2_shr(wasm_i64x2_add(vabsprod67, vrounding), vshift);
137     const v128_t vabsout89 = wasm_u64x2_shr(wasm_i64x2_add(vabsprod89, vrounding), vshift);
138     const v128_t vabsoutAB = wasm_u64x2_shr(wasm_i64x2_add(vabsprodAB, vrounding), vshift);
139     const v128_t vabsoutCD = wasm_u64x2_shr(wasm_i64x2_add(vabsprodCD, vrounding), vshift);
140     const v128_t vabsoutEF = wasm_u64x2_shr(wasm_i64x2_add(vabsprodEF, vrounding), vshift);
141 
142     const v128_t vabsout0123 = wasm_v32x4_shuffle(vabsout01, vabsout23, 0, 2, 4, 6);
143     const v128_t vabsout4567 = wasm_v32x4_shuffle(vabsout45, vabsout67, 0, 2, 4, 6);
144     const v128_t vabsout89AB = wasm_v32x4_shuffle(vabsout89, vabsoutAB, 0, 2, 4, 6);
145     const v128_t vabsoutCDEF = wasm_v32x4_shuffle(vabsoutCD, vabsoutEF, 0, 2, 4, 6);
146 
147     const v128_t vout0123 = wasm_i32x4_sub(wasm_v128_xor(vabsout0123, vsgnacc0123), vsgnacc0123);
148     const v128_t vout4567 = wasm_i32x4_sub(wasm_v128_xor(vabsout4567, vsgnacc4567), vsgnacc4567);
149     const v128_t vout89AB = wasm_i32x4_sub(wasm_v128_xor(vabsout89AB, vsgnacc89AB), vsgnacc89AB);
150     const v128_t voutCDEF = wasm_i32x4_sub(wasm_v128_xor(vabsoutCDEF, vsgnaccCDEF), vsgnaccCDEF);
151 
152     const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
153     const v128_t vout01234567 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vout0123, vout4567), voutput_zero_point);
154     const v128_t vout89ABCDEF = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vout89AB, voutCDEF), voutput_zero_point);
155 
156     const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
157     const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
158     const v128_t vout0123456789ABCDEF = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout01234567, vout89ABCDEF), voutput_min), voutput_max);
159 
160     wasm_v128_store(output, vout0123456789ABCDEF);
161     output += 16;
162 
163     channels -= 16;
164   }
165   if XNN_UNLIKELY(channels != 0) {
166     do {
167       const v128_t vxi0x01234567 = wasm_i16x8_load_8x8(i0);
168       i0 += 8;
169       const v128_t vxi1x01234567 = wasm_i16x8_load_8x8(i1);
170       i1 += 8;
171       const v128_t vxi2x01234567 = wasm_i16x8_load_8x8(i2);
172       i2 += 8;
173       const v128_t vxi3x01234567 = wasm_i16x8_load_8x8(i3);
174       i3 += 8;
175       const v128_t vxi4x01234567 = wasm_i16x8_load_8x8(i4);
176       i4 += 8;
177       const v128_t vxi5x01234567 = wasm_i16x8_load_8x8(i5);
178       i5 += 8;
179       const v128_t vxi6x01234567 = wasm_i16x8_load_8x8(i6);
180       i6 += 8;
181 
182       v128_t vacc0x01234567 = wasm_i16x8_add(vxi0x01234567, vxi1x01234567);
183       v128_t vacc1x01234567 = wasm_i16x8_add(vxi2x01234567, vxi3x01234567);
184 
185       vacc0x01234567 = wasm_i16x8_add(vacc0x01234567, vxi4x01234567);
186       vacc1x01234567 = wasm_i16x8_add(vacc1x01234567, vxi5x01234567);
187       vacc0x01234567 = wasm_i16x8_add(vacc0x01234567, vxi6x01234567);
188 
189       // Add up all accumulators to vacc0x01234567
190       vacc0x01234567 = wasm_i16x8_add(vacc0x01234567, vacc1x01234567);
191 
192       const v128_t vacc0123 = wasm_i32x4_add(vbias, wasm_i32x4_widen_low_i16x8(vacc0x01234567));
193       const v128_t vacc4567 = wasm_i32x4_add(vbias, wasm_i32x4_widen_high_i16x8(vacc0x01234567));
194 
195       const v128_t vabsacc0123 = wasm_i32x4_abs(vacc0123);
196       const v128_t vabsacc4567 = wasm_i32x4_abs(vacc4567);
197 
198       const v128_t vsgnacc0123 = wasm_i32x4_gt(vabsacc0123, vacc0123);
199       const v128_t vsgnacc4567 = wasm_i32x4_gt(vabsacc4567, vacc4567);
200 
201       const v128_t vabsacc01 = wasm_v32x4_shuffle(vabsacc0123, vzero, 0, 4, 1, 5);
202       const v128_t vabsacc23 = wasm_v32x4_shuffle(vabsacc0123, vzero, 2, 6, 3, 7);
203       const v128_t vabsacc45 = wasm_v32x4_shuffle(vabsacc4567, vzero, 0, 4, 1, 5);
204       const v128_t vabsacc67 = wasm_v32x4_shuffle(vabsacc4567, vzero, 2, 6, 3, 7);
205 
206       const v128_t vabsprod01 = wasm_i64x2_mul(vabsacc01, vmultiplier);
207       const v128_t vabsprod23 = wasm_i64x2_mul(vabsacc23, vmultiplier);
208       const v128_t vabsprod45 = wasm_i64x2_mul(vabsacc45, vmultiplier);
209       const v128_t vabsprod67 = wasm_i64x2_mul(vabsacc67, vmultiplier);
210 
211       const v128_t vabsout01 = wasm_u64x2_shr(wasm_i64x2_add(vabsprod01, vrounding), vshift);
212       const v128_t vabsout23 = wasm_u64x2_shr(wasm_i64x2_add(vabsprod23, vrounding), vshift);
213       const v128_t vabsout45 = wasm_u64x2_shr(wasm_i64x2_add(vabsprod45, vrounding), vshift);
214       const v128_t vabsout67 = wasm_u64x2_shr(wasm_i64x2_add(vabsprod67, vrounding), vshift);
215 
216       const v128_t vabsout0123 = wasm_v32x4_shuffle(vabsout01, vabsout23, 0, 2, 4, 6);
217       const v128_t vabsout4567 = wasm_v32x4_shuffle(vabsout45, vabsout67, 0, 2, 4, 6);
218 
219       const v128_t vout0123 = wasm_i32x4_sub(wasm_v128_xor(vabsout0123, vsgnacc0123), vsgnacc0123);
220       const v128_t vout4567 = wasm_i32x4_sub(wasm_v128_xor(vabsout4567, vsgnacc4567), vsgnacc4567);
221 
222       const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
223       const v128_t vout01234567 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vout0123, vout4567), voutput_zero_point);
224 
225       const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
226       const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
227       v128_t vout0123456701234567 = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout01234567, vout01234567), voutput_min), voutput_max);
228 
229       if XNN_LIKELY(channels >= 8) {
230         *((double*) output) = wasm_f64x2_extract_lane(vout0123456701234567, 0);
231         output += 8;
232         channels -= 8;
233       } else {
234         if (channels & 4) {
235           *((float*) output) = wasm_f32x4_extract_lane(vout0123456701234567, 0);
236           vout0123456701234567 = wasm_u64x2_shr(vout0123456701234567, 32);
237           output += 4;
238         }
239         if (channels & 2) {
240           *((uint16_t*) output) = (uint16_t) wasm_i16x8_extract_lane(vout0123456701234567, 0);
241           vout0123456701234567 = wasm_u32x4_shr(vout0123456701234567, 16);
242           output += 2;
243         }
244         if (channels & 1) {
245           *output = (int8_t) wasm_i8x16_extract_lane(vout0123456701234567, 0);
246           output += 1;
247         }
248         channels = 0;
249       }
250     } while (channels != 0);
251   }
252 }
253