1 // Auto-generated file. Do not edit!
2 //   Template: src/f16-spmm/neonfp16arith.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <arm_neon.h>
13 
14 #include <xnnpack/spmm.h>
15 
16 
xnn_f16_spmm_minmax_ukernel_32x1__neonfp16arith_x2(size_t mc,size_t nc,const void * restrict input,const void * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,void * restrict output,size_t output_stride,const struct xnn_f16_scaleminmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f16_spmm_minmax_ukernel_32x1__neonfp16arith_x2(
18     size_t mc,
19     size_t nc,
20     const void*restrict input,
21     const void*restrict weights,
22     const int32_t*restrict widx_dmap,
23     const uint32_t*restrict nidx_nnzmap,
24     void*restrict output,
25     size_t output_stride,
26     const struct xnn_f16_scaleminmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27 {
28   assert(mc != 0);
29   assert(mc % sizeof(__fp16) == 0);
30   assert(nc != 0);
31 
32   const __fp16*restrict i = (const __fp16*) input;
33   __fp16*restrict o = (__fp16*) output;
34 
35   const float16x8_t vscale = vld1q_dup_f16((const __fp16*) &params->scale);
36   const float16x8_t vmax = vld1q_dup_f16((const __fp16*) &params->max);
37   const float16x8_t vmin = vld1q_dup_f16((const __fp16*) &params->min);
38 
39   size_t output_decrement = output_stride * nc - 32 * sizeof(__fp16);
40   while XNN_LIKELY(mc >= 32 * sizeof(__fp16)) {
41     const __fp16*restrict w = (const __fp16*) weights;
42     const int32_t* dmap = widx_dmap;
43     const uint32_t* nnzmap = nidx_nnzmap;
44     size_t n = nc;
45     do {
46       uint32_t nnz = *nnzmap++;
47       float16x8_t vacc01234567x0 = vld1q_dup_f16(w); w += 1;
48       float16x8_t vacc01234567x1 = vmovq_n_f16(0.0f);
49       float16x8_t vacc89ABCDEFx0 = vacc01234567x0;
50       float16x8_t vacc89ABCDEFx1 = vmovq_n_f16(0.0f);
51       float16x8_t vaccGHIJKLMNx0 = vacc01234567x0;
52       float16x8_t vaccGHIJKLMNx1 = vmovq_n_f16(0.0f);
53       float16x8_t vaccOPQRSTUVx0 = vacc01234567x0;
54       float16x8_t vaccOPQRSTUVx1 = vmovq_n_f16(0.0f);
55       for (; nnz >= 2; nnz -= 2) {
56         const intptr_t diff0 = dmap[0];
57         const intptr_t diff1 = dmap[1];
58         dmap += 2;
59         const float16x8_t va01234567x0 = vld1q_f16(i);
60         const float16x8_t va89ABCDEFx0 = vld1q_f16(i + 8);
61         const float16x8_t vaGHIJKLMNx0 = vld1q_f16(i + 16);
62         const float16x8_t vaOPQRSTUVx0 = vld1q_f16(i + 24);
63         i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff0);
64         const float16x8_t vb0 = vld1q_dup_f16(w); w += 1;
65         vacc01234567x0 = vfmaq_f16(vacc01234567x0, va01234567x0, vb0);
66         vacc89ABCDEFx0 = vfmaq_f16(vacc89ABCDEFx0, va89ABCDEFx0, vb0);
67         vaccGHIJKLMNx0 = vfmaq_f16(vaccGHIJKLMNx0, vaGHIJKLMNx0, vb0);
68         vaccOPQRSTUVx0 = vfmaq_f16(vaccOPQRSTUVx0, vaOPQRSTUVx0, vb0);
69         const float16x8_t va01234567x1 = vld1q_f16(i);
70         const float16x8_t va89ABCDEFx1 = vld1q_f16(i + 8);
71         const float16x8_t vaGHIJKLMNx1 = vld1q_f16(i + 16);
72         const float16x8_t vaOPQRSTUVx1 = vld1q_f16(i + 24);
73         i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff1);
74         const float16x8_t vb1 = vld1q_dup_f16(w); w += 1;
75         vacc01234567x1 = vfmaq_f16(vacc01234567x1, va01234567x1, vb1);
76         vacc89ABCDEFx1 = vfmaq_f16(vacc89ABCDEFx1, va89ABCDEFx1, vb1);
77         vaccGHIJKLMNx1 = vfmaq_f16(vaccGHIJKLMNx1, vaGHIJKLMNx1, vb1);
78         vaccOPQRSTUVx1 = vfmaq_f16(vaccOPQRSTUVx1, vaOPQRSTUVx1, vb1);
79       }
80       float16x8_t vacc01234567 = vacc01234567x0;
81       float16x8_t vacc89ABCDEF = vacc89ABCDEFx0;
82       float16x8_t vaccGHIJKLMN = vaccGHIJKLMNx0;
83       float16x8_t vaccOPQRSTUV = vaccOPQRSTUVx0;
84       vacc01234567 = vaddq_f16(vacc01234567, vacc01234567x1);
85       vacc89ABCDEF = vaddq_f16(vacc89ABCDEF, vacc89ABCDEFx1);
86       vaccGHIJKLMN = vaddq_f16(vaccGHIJKLMN, vaccGHIJKLMNx1);
87       vaccOPQRSTUV = vaddq_f16(vaccOPQRSTUV, vaccOPQRSTUVx1);
88       if XNN_LIKELY(nnz != 0) {
89         do {
90           const intptr_t diff = *dmap++;
91           const float16x8_t va01234567 = vld1q_f16(i);
92           const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
93           const float16x8_t vaGHIJKLMN = vld1q_f16(i + 16);
94           const float16x8_t vaOPQRSTUV = vld1q_f16(i + 24);
95           i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
96           const float16x8_t vb = vld1q_dup_f16(w); w += 1;
97           vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
98           vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
99           vaccGHIJKLMN = vfmaq_f16(vaccGHIJKLMN, vaGHIJKLMN, vb);
100           vaccOPQRSTUV = vfmaq_f16(vaccOPQRSTUV, vaOPQRSTUV, vb);
101         } while (--nnz != 0);
102       }
103       float16x8_t vout01234567 = vmulq_f16(vacc01234567, vscale);
104       float16x8_t vout89ABCDEF = vmulq_f16(vacc89ABCDEF, vscale);
105       float16x8_t voutGHIJKLMN = vmulq_f16(vaccGHIJKLMN, vscale);
106       float16x8_t voutOPQRSTUV = vmulq_f16(vaccOPQRSTUV, vscale);
107       vout01234567 = vminq_f16(vout01234567, vmax);
108       vout89ABCDEF = vminq_f16(vout89ABCDEF, vmax);
109       voutGHIJKLMN = vminq_f16(voutGHIJKLMN, vmax);
110       voutOPQRSTUV = vminq_f16(voutOPQRSTUV, vmax);
111       vout01234567 = vmaxq_f16(vout01234567, vmin);
112       vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
113       voutGHIJKLMN = vmaxq_f16(voutGHIJKLMN, vmin);
114       voutOPQRSTUV = vmaxq_f16(voutOPQRSTUV, vmin);
115       vst1q_f16(o, vout01234567);
116       vst1q_f16(o + 8, vout89ABCDEF);
117       vst1q_f16(o + 16, voutGHIJKLMN);
118       vst1q_f16(o + 24, voutOPQRSTUV);
119       o = (__fp16*restrict) ((uintptr_t) o + output_stride);
120     } while (--n != 0);
121     o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
122     i += 32;
123     mc -= 32 * sizeof(__fp16);
124   }
125   if XNN_UNLIKELY(mc != 0) {
126     output_decrement += 16 * sizeof(__fp16);
127     if (mc & (16 * sizeof(__fp16))) {
128       const __fp16*restrict w = (const __fp16*) weights;
129       const int32_t* dmap = widx_dmap;
130       const uint32_t* nnzmap = nidx_nnzmap;
131       size_t n = nc;
132       do {
133         uint32_t nnz = *nnzmap++;
134         float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
135         float16x8_t vacc89ABCDEF = vacc01234567;
136         if XNN_LIKELY(nnz != 0) {
137           do {
138             const intptr_t diff = *dmap++;
139             const float16x8_t va01234567 = vld1q_f16(i);
140             const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
141             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
142             const float16x8_t vb = vld1q_dup_f16(w); w += 1;
143             vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
144             vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
145           } while (--nnz != 0);
146         }
147         float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
148         float16x8_t vout89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
149         vout01234567 = vmaxq_f16(vout01234567, vmin);
150         vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
151         vst1q_f16(o, vout01234567);
152         vst1q_f16(o + 8, vout89ABCDEF);
153         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
154       } while (--n != 0);
155       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
156       i += 16;
157     }
158     output_decrement += 8 * sizeof(__fp16);
159     if (mc & (8 * sizeof(__fp16))) {
160       const __fp16*restrict w = (const __fp16*) weights;
161       const int32_t* dmap = widx_dmap;
162       const uint32_t* nnzmap = nidx_nnzmap;
163       size_t n = nc;
164       do {
165         uint32_t nnz = *nnzmap++;
166         float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
167         if XNN_LIKELY(nnz != 0) {
168           do {
169             const intptr_t diff = *dmap++;
170             const float16x8_t va01234567 = vld1q_f16(i);
171             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
172             const float16x8_t vb = vld1q_dup_f16(w); w += 1;
173             vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
174           } while (--nnz != 0);
175         }
176         float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
177         vout01234567 = vmaxq_f16(vout01234567, vmin);
178         vst1q_f16(o, vout01234567);
179         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
180       } while (--n != 0);
181       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
182       i += 8;
183     }
184     output_decrement += 4 * sizeof(__fp16);
185     if (mc & (4 * sizeof(__fp16))) {
186       const __fp16*restrict w = (const __fp16*) weights;
187       const int32_t* dmap = widx_dmap;
188       const uint32_t* nnzmap = nidx_nnzmap;
189       size_t n = nc;
190       do {
191         uint32_t nnz = *nnzmap++;
192         float16x4_t vacc0123 = vld1_dup_f16(w); w += 1;
193         if XNN_LIKELY(nnz != 0) {
194           do {
195             const intptr_t diff = *dmap++;
196             const float16x4_t va0123 = vld1_f16(i);
197             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
198             const float16x4_t vb = vld1_dup_f16(w); w += 1;
199             vacc0123 = vfma_f16(vacc0123, va0123, vb);
200           } while (--nnz != 0);
201         }
202         float16x4_t vout0123 = vmin_f16(vacc0123, vget_low_f16(vmax));
203         vout0123 = vmax_f16(vout0123, vget_low_f16(vmin));
204         vst1_f16(o, vout0123);
205         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
206       } while (--n != 0);
207       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
208       i += 4;
209     }
210     output_decrement += 2 * sizeof(__fp16);
211     if (mc & (2 * sizeof(__fp16))) {
212       const __fp16*restrict w = (const __fp16*) weights;
213       const int32_t* dmap = widx_dmap;
214       const uint32_t* nnzmap = nidx_nnzmap;
215       size_t n = nc;
216       do {
217         uint32_t nnz = *nnzmap++;
218         float16x4_t vacc01 = vld1_dup_f16(w); w += 1;
219         if XNN_LIKELY(nnz != 0) {
220           do {
221             const intptr_t diff = *dmap++;
222             const float16x4_t va01 = vreinterpret_f16_f32(vld1_dup_f32(__builtin_assume_aligned(i, 1)));
223             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
224             const float16x4_t vb = vld1_dup_f16(w); w += 1;
225             vacc01 = vfma_f16(vacc01, va01, vb);
226           } while (--nnz != 0);
227         }
228         float16x4_t vout01 = vmin_f16(vacc01, vget_low_f16(vmax));
229         vout01 = vmax_f16(vout01, vget_low_f16(vmin));
230         vst1_lane_f32(__builtin_assume_aligned(o, 1), vreinterpret_f32_f16(vout01), 0);
231         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
232       } while (--n != 0);
233       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
234       i += 2;
235     }
236     output_decrement += 1 * sizeof(__fp16);
237     if (mc & (1 * sizeof(__fp16))) {
238       const __fp16*restrict w = (const __fp16*) weights;
239       const int32_t* dmap = widx_dmap;
240       const uint32_t* nnzmap = nidx_nnzmap;
241       size_t n = nc;
242       do {
243         uint32_t nnz = *nnzmap++;
244         float16x4_t vacc0 = vld1_dup_f16(w); w += 1;
245         if XNN_LIKELY(nnz != 0) {
246           do {
247             const intptr_t diff = *dmap++;
248             const float16x4_t va0 = vld1_dup_f16(i);
249             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
250             const float16x4_t vb = vld1_dup_f16(w); w += 1;
251             vacc0 = vfma_f16(vacc0, va0, vb);
252           } while (--nnz != 0);
253         }
254         float16x4_t vout0 = vmin_f16(vacc0, vget_low_f16(vmax));
255         vout0 = vmax_f16(vout0, vget_low_f16(vmin));
256         vst1_lane_f16(o, vout0, 0);
257         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
258       } while (--n != 0);
259       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
260       i += 1;
261     }
262   }
263 }
264