1 // Auto-generated file. Do not edit!
2 //   Template: src/f32-dwconv/up-avx.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <immintrin.h>
13 
14 #include <xnnpack/dwconv.h>
15 
16 
17 static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
18 
xnn_f32_dwconv_minmax_ukernel_up16x25__fma3(size_t channels,size_t output_width,const float ** input,const float * weights,float * output,size_t input_stride,size_t output_increment,size_t input_offset,const float * zero,const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])19 void xnn_f32_dwconv_minmax_ukernel_up16x25__fma3(
20     size_t channels,
21     size_t output_width,
22     const float** input,
23     const float* weights,
24     float* output,
25     size_t input_stride,
26     size_t output_increment,
27     size_t input_offset,
28     const float* zero,
29     const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
30 {
31   assert(channels != 0);
32   assert(output_width != 0);
33 
34   const __m256 vmax = _mm256_broadcast_ps((const __m128*) params->sse.max);
35   const __m256 vmin = _mm256_broadcast_ps((const __m128*) params->sse.min);
36   do {
37     const float* i0 = input[0];
38     assert(i0 != NULL);
39     if XNN_UNPREDICTABLE(i0 != zero) {
40       i0 = (const float*) ((uintptr_t) i0 + input_offset);
41     }
42     const float* i1 = input[1];
43     assert(i1 != NULL);
44     if XNN_UNPREDICTABLE(i1 != zero) {
45       i1 = (const float*) ((uintptr_t) i1 + input_offset);
46     }
47     const float* i2 = input[2];
48     assert(i2 != NULL);
49     if XNN_UNPREDICTABLE(i2 != zero) {
50       i2 = (const float*) ((uintptr_t) i2 + input_offset);
51     }
52     const float* i3 = input[3];
53     assert(i3 != NULL);
54     if XNN_UNPREDICTABLE(i3 != zero) {
55       i3 = (const float*) ((uintptr_t) i3 + input_offset);
56     }
57     const float* i4 = input[4];
58     assert(i4 != NULL);
59     if XNN_UNPREDICTABLE(i4 != zero) {
60       i4 = (const float*) ((uintptr_t) i4 + input_offset);
61     }
62     const float* i5 = input[5];
63     assert(i5 != NULL);
64     if XNN_UNPREDICTABLE(i5 != zero) {
65       i5 = (const float*) ((uintptr_t) i5 + input_offset);
66     }
67     const float* i6 = input[6];
68     assert(i6 != NULL);
69     if XNN_UNPREDICTABLE(i6 != zero) {
70       i6 = (const float*) ((uintptr_t) i6 + input_offset);
71     }
72     const float* i7 = input[7];
73     assert(i7 != NULL);
74     if XNN_UNPREDICTABLE(i7 != zero) {
75       i7 = (const float*) ((uintptr_t) i7 + input_offset);
76     }
77     const float* i8 = input[8];
78     assert(i8 != NULL);
79     if XNN_UNPREDICTABLE(i8 != zero) {
80       i8 = (const float*) ((uintptr_t) i8 + input_offset);
81     }
82     const float* i9 = input[9];
83     assert(i9 != NULL);
84     if XNN_UNPREDICTABLE(i9 != zero) {
85       i9 = (const float*) ((uintptr_t) i9 + input_offset);
86     }
87     const float* i10 = input[10];
88     assert(i10 != NULL);
89     if XNN_UNPREDICTABLE(i10 != zero) {
90       i10 = (const float*) ((uintptr_t) i10 + input_offset);
91     }
92     const float* i11 = input[11];
93     assert(i11 != NULL);
94     if XNN_UNPREDICTABLE(i11 != zero) {
95       i11 = (const float*) ((uintptr_t) i11 + input_offset);
96     }
97     const float* i12 = input[12];
98     assert(i12 != NULL);
99     if XNN_UNPREDICTABLE(i12 != zero) {
100       i12 = (const float*) ((uintptr_t) i12 + input_offset);
101     }
102     const float* i13 = input[13];
103     assert(i13 != NULL);
104     if XNN_UNPREDICTABLE(i13 != zero) {
105       i13 = (const float*) ((uintptr_t) i13 + input_offset);
106     }
107     const float* i14 = input[14];
108     assert(i14 != NULL);
109     if XNN_UNPREDICTABLE(i14 != zero) {
110       i14 = (const float*) ((uintptr_t) i14 + input_offset);
111     }
112     const float* i15 = input[15];
113     assert(i15 != NULL);
114     if XNN_UNPREDICTABLE(i15 != zero) {
115       i15 = (const float*) ((uintptr_t) i15 + input_offset);
116     }
117     const float* i16 = input[16];
118     assert(i16 != NULL);
119     if XNN_UNPREDICTABLE(i16 != zero) {
120       i16 = (const float*) ((uintptr_t) i16 + input_offset);
121     }
122     const float* i17 = input[17];
123     assert(i17 != NULL);
124     if XNN_UNPREDICTABLE(i17 != zero) {
125       i17 = (const float*) ((uintptr_t) i17 + input_offset);
126     }
127     const float* i18 = input[18];
128     assert(i18 != NULL);
129     if XNN_UNPREDICTABLE(i18 != zero) {
130       i18 = (const float*) ((uintptr_t) i18 + input_offset);
131     }
132     const float* i19 = input[19];
133     assert(i19 != NULL);
134     if XNN_UNPREDICTABLE(i19 != zero) {
135       i19 = (const float*) ((uintptr_t) i19 + input_offset);
136     }
137     const float* i20 = input[20];
138     assert(i20 != NULL);
139     if XNN_UNPREDICTABLE(i20 != zero) {
140       i20 = (const float*) ((uintptr_t) i20 + input_offset);
141     }
142     const float* i21 = input[21];
143     assert(i21 != NULL);
144     if XNN_UNPREDICTABLE(i21 != zero) {
145       i21 = (const float*) ((uintptr_t) i21 + input_offset);
146     }
147     const float* i22 = input[22];
148     assert(i22 != NULL);
149     if XNN_UNPREDICTABLE(i22 != zero) {
150       i22 = (const float*) ((uintptr_t) i22 + input_offset);
151     }
152     const float* i23 = input[23];
153     assert(i23 != NULL);
154     if XNN_UNPREDICTABLE(i23 != zero) {
155       i23 = (const float*) ((uintptr_t) i23 + input_offset);
156     }
157     const float* i24 = input[24];
158     assert(i24 != NULL);
159     if XNN_UNPREDICTABLE(i24 != zero) {
160       i24 = (const float*) ((uintptr_t) i24 + input_offset);
161     }
162     input = (const float**) ((uintptr_t) input + input_stride);
163 
164     size_t c = channels;
165     const float* w = weights;
166     for (; c >= 16; c -= 16) {
167       __m256 vacc01234567p0 = _mm256_load_ps(w);
168       __m256 vacc89ABCDEFp0 = _mm256_load_ps(w + 8);
169 
170 
171       const __m256 vi0x01234567 = _mm256_loadu_ps(i0);
172       const __m256 vi0x89ABCDEF = _mm256_loadu_ps(i0 + 8);
173       i0 += 16;
174 
175       const __m256 vk0x01234567 = _mm256_load_ps(w + 16);
176       const __m256 vk0x89ABCDEF = _mm256_load_ps(w + 24);
177       vacc01234567p0 = _mm256_fmadd_ps(vi0x01234567, vk0x01234567, vacc01234567p0);
178       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi0x89ABCDEF, vk0x89ABCDEF, vacc89ABCDEFp0);
179 
180       const __m256 vi1x01234567 = _mm256_loadu_ps(i1);
181       const __m256 vi1x89ABCDEF = _mm256_loadu_ps(i1 + 8);
182       i1 += 16;
183 
184       const __m256 vk1x01234567 = _mm256_load_ps(w + 32);
185       const __m256 vk1x89ABCDEF = _mm256_load_ps(w + 40);
186       vacc01234567p0 = _mm256_fmadd_ps(vi1x01234567, vk1x01234567, vacc01234567p0);
187       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi1x89ABCDEF, vk1x89ABCDEF, vacc89ABCDEFp0);
188 
189       const __m256 vi2x01234567 = _mm256_loadu_ps(i2);
190       const __m256 vi2x89ABCDEF = _mm256_loadu_ps(i2 + 8);
191       i2 += 16;
192 
193       const __m256 vk2x01234567 = _mm256_load_ps(w + 48);
194       const __m256 vk2x89ABCDEF = _mm256_load_ps(w + 56);
195       vacc01234567p0 = _mm256_fmadd_ps(vi2x01234567, vk2x01234567, vacc01234567p0);
196       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi2x89ABCDEF, vk2x89ABCDEF, vacc89ABCDEFp0);
197 
198       const __m256 vi3x01234567 = _mm256_loadu_ps(i3);
199       const __m256 vi3x89ABCDEF = _mm256_loadu_ps(i3 + 8);
200       i3 += 16;
201 
202       const __m256 vk3x01234567 = _mm256_load_ps(w + 64);
203       const __m256 vk3x89ABCDEF = _mm256_load_ps(w + 72);
204       vacc01234567p0 = _mm256_fmadd_ps(vi3x01234567, vk3x01234567, vacc01234567p0);
205       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi3x89ABCDEF, vk3x89ABCDEF, vacc89ABCDEFp0);
206 
207       const __m256 vi4x01234567 = _mm256_loadu_ps(i4);
208       const __m256 vi4x89ABCDEF = _mm256_loadu_ps(i4 + 8);
209       i4 += 16;
210 
211       const __m256 vk4x01234567 = _mm256_load_ps(w + 80);
212       const __m256 vk4x89ABCDEF = _mm256_load_ps(w + 88);
213       vacc01234567p0 = _mm256_fmadd_ps(vi4x01234567, vk4x01234567, vacc01234567p0);
214       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi4x89ABCDEF, vk4x89ABCDEF, vacc89ABCDEFp0);
215 
216       const __m256 vi5x01234567 = _mm256_loadu_ps(i5);
217       const __m256 vi5x89ABCDEF = _mm256_loadu_ps(i5 + 8);
218       i5 += 16;
219 
220       const __m256 vk5x01234567 = _mm256_load_ps(w + 96);
221       const __m256 vk5x89ABCDEF = _mm256_load_ps(w + 104);
222       vacc01234567p0 = _mm256_fmadd_ps(vi5x01234567, vk5x01234567, vacc01234567p0);
223       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi5x89ABCDEF, vk5x89ABCDEF, vacc89ABCDEFp0);
224 
225       const __m256 vi6x01234567 = _mm256_loadu_ps(i6);
226       const __m256 vi6x89ABCDEF = _mm256_loadu_ps(i6 + 8);
227       i6 += 16;
228 
229       const __m256 vk6x01234567 = _mm256_load_ps(w + 112);
230       const __m256 vk6x89ABCDEF = _mm256_load_ps(w + 120);
231       vacc01234567p0 = _mm256_fmadd_ps(vi6x01234567, vk6x01234567, vacc01234567p0);
232       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi6x89ABCDEF, vk6x89ABCDEF, vacc89ABCDEFp0);
233 
234       const __m256 vi7x01234567 = _mm256_loadu_ps(i7);
235       const __m256 vi7x89ABCDEF = _mm256_loadu_ps(i7 + 8);
236       i7 += 16;
237 
238       const __m256 vk7x01234567 = _mm256_load_ps(w + 128);
239       const __m256 vk7x89ABCDEF = _mm256_load_ps(w + 136);
240       vacc01234567p0 = _mm256_fmadd_ps(vi7x01234567, vk7x01234567, vacc01234567p0);
241       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi7x89ABCDEF, vk7x89ABCDEF, vacc89ABCDEFp0);
242 
243       const __m256 vi8x01234567 = _mm256_loadu_ps(i8);
244       const __m256 vi8x89ABCDEF = _mm256_loadu_ps(i8 + 8);
245       i8 += 16;
246 
247       const __m256 vk8x01234567 = _mm256_load_ps(w + 144);
248       const __m256 vk8x89ABCDEF = _mm256_load_ps(w + 152);
249       vacc01234567p0 = _mm256_fmadd_ps(vi8x01234567, vk8x01234567, vacc01234567p0);
250       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi8x89ABCDEF, vk8x89ABCDEF, vacc89ABCDEFp0);
251 
252       const __m256 vi9x01234567 = _mm256_loadu_ps(i9);
253       const __m256 vi9x89ABCDEF = _mm256_loadu_ps(i9 + 8);
254       i9 += 16;
255 
256       const __m256 vk9x01234567 = _mm256_load_ps(w + 160);
257       const __m256 vk9x89ABCDEF = _mm256_load_ps(w + 168);
258       vacc01234567p0 = _mm256_fmadd_ps(vi9x01234567, vk9x01234567, vacc01234567p0);
259       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi9x89ABCDEF, vk9x89ABCDEF, vacc89ABCDEFp0);
260 
261       const __m256 vi10x01234567 = _mm256_loadu_ps(i10);
262       const __m256 vi10x89ABCDEF = _mm256_loadu_ps(i10 + 8);
263       i10 += 16;
264 
265       const __m256 vk10x01234567 = _mm256_load_ps(w + 176);
266       const __m256 vk10x89ABCDEF = _mm256_load_ps(w + 184);
267       vacc01234567p0 = _mm256_fmadd_ps(vi10x01234567, vk10x01234567, vacc01234567p0);
268       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi10x89ABCDEF, vk10x89ABCDEF, vacc89ABCDEFp0);
269 
270       const __m256 vi11x01234567 = _mm256_loadu_ps(i11);
271       const __m256 vi11x89ABCDEF = _mm256_loadu_ps(i11 + 8);
272       i11 += 16;
273 
274       const __m256 vk11x01234567 = _mm256_load_ps(w + 192);
275       const __m256 vk11x89ABCDEF = _mm256_load_ps(w + 200);
276       vacc01234567p0 = _mm256_fmadd_ps(vi11x01234567, vk11x01234567, vacc01234567p0);
277       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi11x89ABCDEF, vk11x89ABCDEF, vacc89ABCDEFp0);
278 
279       const __m256 vi12x01234567 = _mm256_loadu_ps(i12);
280       const __m256 vi12x89ABCDEF = _mm256_loadu_ps(i12 + 8);
281       i12 += 16;
282 
283       const __m256 vk12x01234567 = _mm256_load_ps(w + 208);
284       const __m256 vk12x89ABCDEF = _mm256_load_ps(w + 216);
285       vacc01234567p0 = _mm256_fmadd_ps(vi12x01234567, vk12x01234567, vacc01234567p0);
286       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi12x89ABCDEF, vk12x89ABCDEF, vacc89ABCDEFp0);
287 
288       const __m256 vi13x01234567 = _mm256_loadu_ps(i13);
289       const __m256 vi13x89ABCDEF = _mm256_loadu_ps(i13 + 8);
290       i13 += 16;
291 
292       const __m256 vk13x01234567 = _mm256_load_ps(w + 224);
293       const __m256 vk13x89ABCDEF = _mm256_load_ps(w + 232);
294       vacc01234567p0 = _mm256_fmadd_ps(vi13x01234567, vk13x01234567, vacc01234567p0);
295       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi13x89ABCDEF, vk13x89ABCDEF, vacc89ABCDEFp0);
296 
297       const __m256 vi14x01234567 = _mm256_loadu_ps(i14);
298       const __m256 vi14x89ABCDEF = _mm256_loadu_ps(i14 + 8);
299       i14 += 16;
300 
301       const __m256 vk14x01234567 = _mm256_load_ps(w + 240);
302       const __m256 vk14x89ABCDEF = _mm256_load_ps(w + 248);
303       vacc01234567p0 = _mm256_fmadd_ps(vi14x01234567, vk14x01234567, vacc01234567p0);
304       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi14x89ABCDEF, vk14x89ABCDEF, vacc89ABCDEFp0);
305 
306       const __m256 vi15x01234567 = _mm256_loadu_ps(i15);
307       const __m256 vi15x89ABCDEF = _mm256_loadu_ps(i15 + 8);
308       i15 += 16;
309 
310       const __m256 vk15x01234567 = _mm256_load_ps(w + 256);
311       const __m256 vk15x89ABCDEF = _mm256_load_ps(w + 264);
312       vacc01234567p0 = _mm256_fmadd_ps(vi15x01234567, vk15x01234567, vacc01234567p0);
313       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi15x89ABCDEF, vk15x89ABCDEF, vacc89ABCDEFp0);
314 
315       const __m256 vi16x01234567 = _mm256_loadu_ps(i16);
316       const __m256 vi16x89ABCDEF = _mm256_loadu_ps(i16 + 8);
317       i16 += 16;
318 
319       const __m256 vk16x01234567 = _mm256_load_ps(w + 272);
320       const __m256 vk16x89ABCDEF = _mm256_load_ps(w + 280);
321       vacc01234567p0 = _mm256_fmadd_ps(vi16x01234567, vk16x01234567, vacc01234567p0);
322       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi16x89ABCDEF, vk16x89ABCDEF, vacc89ABCDEFp0);
323 
324       const __m256 vi17x01234567 = _mm256_loadu_ps(i17);
325       const __m256 vi17x89ABCDEF = _mm256_loadu_ps(i17 + 8);
326       i17 += 16;
327 
328       const __m256 vk17x01234567 = _mm256_load_ps(w + 288);
329       const __m256 vk17x89ABCDEF = _mm256_load_ps(w + 296);
330       vacc01234567p0 = _mm256_fmadd_ps(vi17x01234567, vk17x01234567, vacc01234567p0);
331       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi17x89ABCDEF, vk17x89ABCDEF, vacc89ABCDEFp0);
332 
333       const __m256 vi18x01234567 = _mm256_loadu_ps(i18);
334       const __m256 vi18x89ABCDEF = _mm256_loadu_ps(i18 + 8);
335       i18 += 16;
336 
337       const __m256 vk18x01234567 = _mm256_load_ps(w + 304);
338       const __m256 vk18x89ABCDEF = _mm256_load_ps(w + 312);
339       vacc01234567p0 = _mm256_fmadd_ps(vi18x01234567, vk18x01234567, vacc01234567p0);
340       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi18x89ABCDEF, vk18x89ABCDEF, vacc89ABCDEFp0);
341 
342       const __m256 vi19x01234567 = _mm256_loadu_ps(i19);
343       const __m256 vi19x89ABCDEF = _mm256_loadu_ps(i19 + 8);
344       i19 += 16;
345 
346       const __m256 vk19x01234567 = _mm256_load_ps(w + 320);
347       const __m256 vk19x89ABCDEF = _mm256_load_ps(w + 328);
348       vacc01234567p0 = _mm256_fmadd_ps(vi19x01234567, vk19x01234567, vacc01234567p0);
349       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi19x89ABCDEF, vk19x89ABCDEF, vacc89ABCDEFp0);
350 
351       const __m256 vi20x01234567 = _mm256_loadu_ps(i20);
352       const __m256 vi20x89ABCDEF = _mm256_loadu_ps(i20 + 8);
353       i20 += 16;
354 
355       const __m256 vk20x01234567 = _mm256_load_ps(w + 336);
356       const __m256 vk20x89ABCDEF = _mm256_load_ps(w + 344);
357       vacc01234567p0 = _mm256_fmadd_ps(vi20x01234567, vk20x01234567, vacc01234567p0);
358       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi20x89ABCDEF, vk20x89ABCDEF, vacc89ABCDEFp0);
359 
360       const __m256 vi21x01234567 = _mm256_loadu_ps(i21);
361       const __m256 vi21x89ABCDEF = _mm256_loadu_ps(i21 + 8);
362       i21 += 16;
363 
364       const __m256 vk21x01234567 = _mm256_load_ps(w + 352);
365       const __m256 vk21x89ABCDEF = _mm256_load_ps(w + 360);
366       vacc01234567p0 = _mm256_fmadd_ps(vi21x01234567, vk21x01234567, vacc01234567p0);
367       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi21x89ABCDEF, vk21x89ABCDEF, vacc89ABCDEFp0);
368 
369       const __m256 vi22x01234567 = _mm256_loadu_ps(i22);
370       const __m256 vi22x89ABCDEF = _mm256_loadu_ps(i22 + 8);
371       i22 += 16;
372 
373       const __m256 vk22x01234567 = _mm256_load_ps(w + 368);
374       const __m256 vk22x89ABCDEF = _mm256_load_ps(w + 376);
375       vacc01234567p0 = _mm256_fmadd_ps(vi22x01234567, vk22x01234567, vacc01234567p0);
376       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi22x89ABCDEF, vk22x89ABCDEF, vacc89ABCDEFp0);
377 
378       const __m256 vi23x01234567 = _mm256_loadu_ps(i23);
379       const __m256 vi23x89ABCDEF = _mm256_loadu_ps(i23 + 8);
380       i23 += 16;
381 
382       const __m256 vk23x01234567 = _mm256_load_ps(w + 384);
383       const __m256 vk23x89ABCDEF = _mm256_load_ps(w + 392);
384       vacc01234567p0 = _mm256_fmadd_ps(vi23x01234567, vk23x01234567, vacc01234567p0);
385       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi23x89ABCDEF, vk23x89ABCDEF, vacc89ABCDEFp0);
386 
387       const __m256 vi24x01234567 = _mm256_loadu_ps(i24);
388       const __m256 vi24x89ABCDEF = _mm256_loadu_ps(i24 + 8);
389       i24 += 16;
390 
391       const __m256 vk24x01234567 = _mm256_load_ps(w + 400);
392       const __m256 vk24x89ABCDEF = _mm256_load_ps(w + 408);
393       vacc01234567p0 = _mm256_fmadd_ps(vi24x01234567, vk24x01234567, vacc01234567p0);
394       vacc89ABCDEFp0 = _mm256_fmadd_ps(vi24x89ABCDEF, vk24x89ABCDEF, vacc89ABCDEFp0);
395 
396       w += 416;
397 
398 
399       __m256 vacc01234567 = _mm256_max_ps(vacc01234567p0, vmin);
400       __m256 vacc89ABCDEF = _mm256_max_ps(vacc89ABCDEFp0, vmin);
401       vacc01234567 = _mm256_min_ps(vacc01234567, vmax);
402       vacc89ABCDEF = _mm256_min_ps(vacc89ABCDEF, vmax);
403 
404       _mm256_storeu_ps(output, vacc01234567);
405       _mm256_storeu_ps(output + 8, vacc89ABCDEF);
406       output += 16;
407     }
408     for (; c >= 8; c -= 8) {
409       __m256 vacc01234567p0 = _mm256_load_ps(w);
410 
411       const __m256 vi0x01234567 = _mm256_loadu_ps(i0);
412       i0 += 8;
413 
414       const __m256 vk0x01234567 = _mm256_load_ps(w + 16);
415       vacc01234567p0 = _mm256_fmadd_ps(vi0x01234567, vk0x01234567, vacc01234567p0);
416 
417       const __m256 vi1x01234567 = _mm256_loadu_ps(i1);
418       i1 += 8;
419 
420       const __m256 vk1x01234567 = _mm256_load_ps(w + 32);
421       vacc01234567p0 = _mm256_fmadd_ps(vi1x01234567, vk1x01234567, vacc01234567p0);
422 
423       const __m256 vi2x01234567 = _mm256_loadu_ps(i2);
424       i2 += 8;
425 
426       const __m256 vk2x01234567 = _mm256_load_ps(w + 48);
427       vacc01234567p0 = _mm256_fmadd_ps(vi2x01234567, vk2x01234567, vacc01234567p0);
428 
429       const __m256 vi3x01234567 = _mm256_loadu_ps(i3);
430       i3 += 8;
431 
432       const __m256 vk3x01234567 = _mm256_load_ps(w + 64);
433       vacc01234567p0 = _mm256_fmadd_ps(vi3x01234567, vk3x01234567, vacc01234567p0);
434 
435       const __m256 vi4x01234567 = _mm256_loadu_ps(i4);
436       i4 += 8;
437 
438       const __m256 vk4x01234567 = _mm256_load_ps(w + 80);
439       vacc01234567p0 = _mm256_fmadd_ps(vi4x01234567, vk4x01234567, vacc01234567p0);
440 
441       const __m256 vi5x01234567 = _mm256_loadu_ps(i5);
442       i5 += 8;
443 
444       const __m256 vk5x01234567 = _mm256_load_ps(w + 96);
445       vacc01234567p0 = _mm256_fmadd_ps(vi5x01234567, vk5x01234567, vacc01234567p0);
446 
447       const __m256 vi6x01234567 = _mm256_loadu_ps(i6);
448       i6 += 8;
449 
450       const __m256 vk6x01234567 = _mm256_load_ps(w + 112);
451       vacc01234567p0 = _mm256_fmadd_ps(vi6x01234567, vk6x01234567, vacc01234567p0);
452 
453       const __m256 vi7x01234567 = _mm256_loadu_ps(i7);
454       i7 += 8;
455 
456       const __m256 vk7x01234567 = _mm256_load_ps(w + 128);
457       vacc01234567p0 = _mm256_fmadd_ps(vi7x01234567, vk7x01234567, vacc01234567p0);
458 
459       const __m256 vi8x01234567 = _mm256_loadu_ps(i8);
460       i8 += 8;
461 
462       const __m256 vk8x01234567 = _mm256_load_ps(w + 144);
463       vacc01234567p0 = _mm256_fmadd_ps(vi8x01234567, vk8x01234567, vacc01234567p0);
464 
465       const __m256 vi9x01234567 = _mm256_loadu_ps(i9);
466       i9 += 8;
467 
468       const __m256 vk9x01234567 = _mm256_load_ps(w + 160);
469       vacc01234567p0 = _mm256_fmadd_ps(vi9x01234567, vk9x01234567, vacc01234567p0);
470 
471       const __m256 vi10x01234567 = _mm256_loadu_ps(i10);
472       i10 += 8;
473 
474       const __m256 vk10x01234567 = _mm256_load_ps(w + 176);
475       vacc01234567p0 = _mm256_fmadd_ps(vi10x01234567, vk10x01234567, vacc01234567p0);
476 
477       const __m256 vi11x01234567 = _mm256_loadu_ps(i11);
478       i11 += 8;
479 
480       const __m256 vk11x01234567 = _mm256_load_ps(w + 192);
481       vacc01234567p0 = _mm256_fmadd_ps(vi11x01234567, vk11x01234567, vacc01234567p0);
482 
483       const __m256 vi12x01234567 = _mm256_loadu_ps(i12);
484       i12 += 8;
485 
486       const __m256 vk12x01234567 = _mm256_load_ps(w + 208);
487       vacc01234567p0 = _mm256_fmadd_ps(vi12x01234567, vk12x01234567, vacc01234567p0);
488 
489       const __m256 vi13x01234567 = _mm256_loadu_ps(i13);
490       i13 += 8;
491 
492       const __m256 vk13x01234567 = _mm256_load_ps(w + 224);
493       vacc01234567p0 = _mm256_fmadd_ps(vi13x01234567, vk13x01234567, vacc01234567p0);
494 
495       const __m256 vi14x01234567 = _mm256_loadu_ps(i14);
496       i14 += 8;
497 
498       const __m256 vk14x01234567 = _mm256_load_ps(w + 240);
499       vacc01234567p0 = _mm256_fmadd_ps(vi14x01234567, vk14x01234567, vacc01234567p0);
500 
501       const __m256 vi15x01234567 = _mm256_loadu_ps(i15);
502       i15 += 8;
503 
504       const __m256 vk15x01234567 = _mm256_load_ps(w + 256);
505       vacc01234567p0 = _mm256_fmadd_ps(vi15x01234567, vk15x01234567, vacc01234567p0);
506 
507       const __m256 vi16x01234567 = _mm256_loadu_ps(i16);
508       i16 += 8;
509 
510       const __m256 vk16x01234567 = _mm256_load_ps(w + 272);
511       vacc01234567p0 = _mm256_fmadd_ps(vi16x01234567, vk16x01234567, vacc01234567p0);
512 
513       const __m256 vi17x01234567 = _mm256_loadu_ps(i17);
514       i17 += 8;
515 
516       const __m256 vk17x01234567 = _mm256_load_ps(w + 288);
517       vacc01234567p0 = _mm256_fmadd_ps(vi17x01234567, vk17x01234567, vacc01234567p0);
518 
519       const __m256 vi18x01234567 = _mm256_loadu_ps(i18);
520       i18 += 8;
521 
522       const __m256 vk18x01234567 = _mm256_load_ps(w + 304);
523       vacc01234567p0 = _mm256_fmadd_ps(vi18x01234567, vk18x01234567, vacc01234567p0);
524 
525       const __m256 vi19x01234567 = _mm256_loadu_ps(i19);
526       i19 += 8;
527 
528       const __m256 vk19x01234567 = _mm256_load_ps(w + 320);
529       vacc01234567p0 = _mm256_fmadd_ps(vi19x01234567, vk19x01234567, vacc01234567p0);
530 
531       const __m256 vi20x01234567 = _mm256_loadu_ps(i20);
532       i20 += 8;
533 
534       const __m256 vk20x01234567 = _mm256_load_ps(w + 336);
535       vacc01234567p0 = _mm256_fmadd_ps(vi20x01234567, vk20x01234567, vacc01234567p0);
536 
537       const __m256 vi21x01234567 = _mm256_loadu_ps(i21);
538       i21 += 8;
539 
540       const __m256 vk21x01234567 = _mm256_load_ps(w + 352);
541       vacc01234567p0 = _mm256_fmadd_ps(vi21x01234567, vk21x01234567, vacc01234567p0);
542 
543       const __m256 vi22x01234567 = _mm256_loadu_ps(i22);
544       i22 += 8;
545 
546       const __m256 vk22x01234567 = _mm256_load_ps(w + 368);
547       vacc01234567p0 = _mm256_fmadd_ps(vi22x01234567, vk22x01234567, vacc01234567p0);
548 
549       const __m256 vi23x01234567 = _mm256_loadu_ps(i23);
550       i23 += 8;
551 
552       const __m256 vk23x01234567 = _mm256_load_ps(w + 384);
553       vacc01234567p0 = _mm256_fmadd_ps(vi23x01234567, vk23x01234567, vacc01234567p0);
554 
555       const __m256 vi24x01234567 = _mm256_loadu_ps(i24);
556       i24 += 8;
557 
558       const __m256 vk24x01234567 = _mm256_load_ps(w + 400);
559       vacc01234567p0 = _mm256_fmadd_ps(vi24x01234567, vk24x01234567, vacc01234567p0);
560 
561       w += 8;
562 
563 
564       __m256 vacc01234567 = _mm256_max_ps(vacc01234567p0, vmin);
565       vacc01234567 = _mm256_min_ps(vacc01234567, vmax);
566 
567       _mm256_storeu_ps(output, vacc01234567);
568       output += 8;
569     }
570     if XNN_UNLIKELY(c != 0) {
571       assert(c >= 1);
572       assert(c <= 7);
573       __m256i vmask = _mm256_loadu_si256((const __m256i*) &mask_table[7 - c]);
574 
575       __m256 vacc01234567p0 = _mm256_load_ps(w);
576 
577       const __m256 vi0x01234567 = _mm256_maskload_ps(i0, vmask);
578       const __m256 vk0x01234567 = _mm256_load_ps(w + 16);
579       vacc01234567p0 = _mm256_fmadd_ps(vi0x01234567, vk0x01234567, vacc01234567p0);
580 
581       const __m256 vi1x01234567 = _mm256_maskload_ps(i1, vmask);
582       const __m256 vk1x01234567 = _mm256_load_ps(w + 32);
583       vacc01234567p0 = _mm256_fmadd_ps(vi1x01234567, vk1x01234567, vacc01234567p0);
584 
585       const __m256 vi2x01234567 = _mm256_maskload_ps(i2, vmask);
586       const __m256 vk2x01234567 = _mm256_load_ps(w + 48);
587       vacc01234567p0 = _mm256_fmadd_ps(vi2x01234567, vk2x01234567, vacc01234567p0);
588 
589       const __m256 vi3x01234567 = _mm256_maskload_ps(i3, vmask);
590       const __m256 vk3x01234567 = _mm256_load_ps(w + 64);
591       vacc01234567p0 = _mm256_fmadd_ps(vi3x01234567, vk3x01234567, vacc01234567p0);
592 
593       const __m256 vi4x01234567 = _mm256_maskload_ps(i4, vmask);
594       const __m256 vk4x01234567 = _mm256_load_ps(w + 80);
595       vacc01234567p0 = _mm256_fmadd_ps(vi4x01234567, vk4x01234567, vacc01234567p0);
596 
597       const __m256 vi5x01234567 = _mm256_maskload_ps(i5, vmask);
598       const __m256 vk5x01234567 = _mm256_load_ps(w + 96);
599       vacc01234567p0 = _mm256_fmadd_ps(vi5x01234567, vk5x01234567, vacc01234567p0);
600 
601       const __m256 vi6x01234567 = _mm256_maskload_ps(i6, vmask);
602       const __m256 vk6x01234567 = _mm256_load_ps(w + 112);
603       vacc01234567p0 = _mm256_fmadd_ps(vi6x01234567, vk6x01234567, vacc01234567p0);
604 
605       const __m256 vi7x01234567 = _mm256_maskload_ps(i7, vmask);
606       const __m256 vk7x01234567 = _mm256_load_ps(w + 128);
607       vacc01234567p0 = _mm256_fmadd_ps(vi7x01234567, vk7x01234567, vacc01234567p0);
608 
609       const __m256 vi8x01234567 = _mm256_maskload_ps(i8, vmask);
610       const __m256 vk8x01234567 = _mm256_load_ps(w + 144);
611       vacc01234567p0 = _mm256_fmadd_ps(vi8x01234567, vk8x01234567, vacc01234567p0);
612 
613       const __m256 vi9x01234567 = _mm256_maskload_ps(i9, vmask);
614       const __m256 vk9x01234567 = _mm256_load_ps(w + 160);
615       vacc01234567p0 = _mm256_fmadd_ps(vi9x01234567, vk9x01234567, vacc01234567p0);
616 
617       const __m256 vi10x01234567 = _mm256_maskload_ps(i10, vmask);
618       const __m256 vk10x01234567 = _mm256_load_ps(w + 176);
619       vacc01234567p0 = _mm256_fmadd_ps(vi10x01234567, vk10x01234567, vacc01234567p0);
620 
621       const __m256 vi11x01234567 = _mm256_maskload_ps(i11, vmask);
622       const __m256 vk11x01234567 = _mm256_load_ps(w + 192);
623       vacc01234567p0 = _mm256_fmadd_ps(vi11x01234567, vk11x01234567, vacc01234567p0);
624 
625       const __m256 vi12x01234567 = _mm256_maskload_ps(i12, vmask);
626       const __m256 vk12x01234567 = _mm256_load_ps(w + 208);
627       vacc01234567p0 = _mm256_fmadd_ps(vi12x01234567, vk12x01234567, vacc01234567p0);
628 
629       const __m256 vi13x01234567 = _mm256_maskload_ps(i13, vmask);
630       const __m256 vk13x01234567 = _mm256_load_ps(w + 224);
631       vacc01234567p0 = _mm256_fmadd_ps(vi13x01234567, vk13x01234567, vacc01234567p0);
632 
633       const __m256 vi14x01234567 = _mm256_maskload_ps(i14, vmask);
634       const __m256 vk14x01234567 = _mm256_load_ps(w + 240);
635       vacc01234567p0 = _mm256_fmadd_ps(vi14x01234567, vk14x01234567, vacc01234567p0);
636 
637       const __m256 vi15x01234567 = _mm256_maskload_ps(i15, vmask);
638       const __m256 vk15x01234567 = _mm256_load_ps(w + 256);
639       vacc01234567p0 = _mm256_fmadd_ps(vi15x01234567, vk15x01234567, vacc01234567p0);
640 
641       const __m256 vi16x01234567 = _mm256_maskload_ps(i16, vmask);
642       const __m256 vk16x01234567 = _mm256_load_ps(w + 272);
643       vacc01234567p0 = _mm256_fmadd_ps(vi16x01234567, vk16x01234567, vacc01234567p0);
644 
645       const __m256 vi17x01234567 = _mm256_maskload_ps(i17, vmask);
646       const __m256 vk17x01234567 = _mm256_load_ps(w + 288);
647       vacc01234567p0 = _mm256_fmadd_ps(vi17x01234567, vk17x01234567, vacc01234567p0);
648 
649       const __m256 vi18x01234567 = _mm256_maskload_ps(i18, vmask);
650       const __m256 vk18x01234567 = _mm256_load_ps(w + 304);
651       vacc01234567p0 = _mm256_fmadd_ps(vi18x01234567, vk18x01234567, vacc01234567p0);
652 
653       const __m256 vi19x01234567 = _mm256_maskload_ps(i19, vmask);
654       const __m256 vk19x01234567 = _mm256_load_ps(w + 320);
655       vacc01234567p0 = _mm256_fmadd_ps(vi19x01234567, vk19x01234567, vacc01234567p0);
656 
657       const __m256 vi20x01234567 = _mm256_maskload_ps(i20, vmask);
658       const __m256 vk20x01234567 = _mm256_load_ps(w + 336);
659       vacc01234567p0 = _mm256_fmadd_ps(vi20x01234567, vk20x01234567, vacc01234567p0);
660 
661       const __m256 vi21x01234567 = _mm256_maskload_ps(i21, vmask);
662       const __m256 vk21x01234567 = _mm256_load_ps(w + 352);
663       vacc01234567p0 = _mm256_fmadd_ps(vi21x01234567, vk21x01234567, vacc01234567p0);
664 
665       const __m256 vi22x01234567 = _mm256_maskload_ps(i22, vmask);
666       const __m256 vk22x01234567 = _mm256_load_ps(w + 368);
667       vacc01234567p0 = _mm256_fmadd_ps(vi22x01234567, vk22x01234567, vacc01234567p0);
668 
669       const __m256 vi23x01234567 = _mm256_maskload_ps(i23, vmask);
670       const __m256 vk23x01234567 = _mm256_load_ps(w + 384);
671       vacc01234567p0 = _mm256_fmadd_ps(vi23x01234567, vk23x01234567, vacc01234567p0);
672 
673       const __m256 vi24x01234567 = _mm256_maskload_ps(i24, vmask);
674       const __m256 vk24x01234567 = _mm256_load_ps(w + 400);
675       vacc01234567p0 = _mm256_fmadd_ps(vi24x01234567, vk24x01234567, vacc01234567p0);
676 
677 
678       __m256 vacc01234567 = _mm256_max_ps(vacc01234567p0, vmin);
679       vacc01234567 = _mm256_min_ps(vacc01234567, vmax);
680 
681       // _mm256_maskstore_ps(output, vmask, vacc01234567); output += c; could be used here, but triggers msan failures (probably an msan bug).
682       __m128 vacc0123 = _mm256_castps256_ps128(vacc01234567);
683       if (c & 4) {
684         _mm_storeu_ps(output, vacc0123);
685         vacc0123 = _mm256_extractf128_ps(vacc01234567, 1);
686         output += 4;
687       }
688       if (c & 2) {
689         _mm_storel_pi((__m64*) output, vacc0123);
690         vacc0123 = _mm_movehl_ps(vacc0123, vacc0123);
691         output += 2;
692       }
693       if (c & 1) {
694         _mm_store_ss(output, vacc0123);
695         output += 1;
696       }
697     }
698 
699     output = (float*) ((uintptr_t) output + output_increment);
700   } while (--output_width != 0);
701 }
702