1 // Auto-generated file. Do not edit!
2 //   Template: src/f16-spmm/neonfp16arith.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <arm_neon.h>
13 
14 #include <xnnpack/spmm.h>
15 
16 
xnn_f16_spmm_minmax_ukernel_32x1__neonfp16arith(size_t mc,size_t nc,const void * restrict input,const void * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,void * restrict output,size_t output_stride,const struct xnn_f16_scaleminmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f16_spmm_minmax_ukernel_32x1__neonfp16arith(
18     size_t mc,
19     size_t nc,
20     const void*restrict input,
21     const void*restrict weights,
22     const int32_t*restrict widx_dmap,
23     const uint32_t*restrict nidx_nnzmap,
24     void*restrict output,
25     size_t output_stride,
26     const struct xnn_f16_scaleminmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27 {
28   assert(mc != 0);
29   assert(mc % sizeof(__fp16) == 0);
30   assert(nc != 0);
31 
32   const __fp16*restrict i = (const __fp16*) input;
33   __fp16*restrict o = (__fp16*) output;
34 
35   const float16x8_t vscale = vld1q_dup_f16((const __fp16*) &params->scale);
36   const float16x8_t vmax = vld1q_dup_f16((const __fp16*) &params->max);
37   const float16x8_t vmin = vld1q_dup_f16((const __fp16*) &params->min);
38 
39   size_t output_decrement = output_stride * nc - 32 * sizeof(__fp16);
40   while XNN_LIKELY(mc >= 32 * sizeof(__fp16)) {
41     const __fp16*restrict w = (const __fp16*) weights;
42     const int32_t* dmap = widx_dmap;
43     const uint32_t* nnzmap = nidx_nnzmap;
44     size_t n = nc;
45     do {
46       uint32_t nnz = *nnzmap++;
47       float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
48       float16x8_t vacc89ABCDEF = vacc01234567;
49       float16x8_t vaccGHIJKLMN = vacc01234567;
50       float16x8_t vaccOPQRSTUV = vacc01234567;
51       if XNN_LIKELY(nnz != 0) {
52         do {
53           const intptr_t diff = *dmap++;
54           const float16x8_t va01234567 = vld1q_f16(i);
55           const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
56           const float16x8_t vaGHIJKLMN = vld1q_f16(i + 16);
57           const float16x8_t vaOPQRSTUV = vld1q_f16(i + 24);
58           i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
59           const float16x8_t vb = vld1q_dup_f16(w); w += 1;
60           vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
61           vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
62           vaccGHIJKLMN = vfmaq_f16(vaccGHIJKLMN, vaGHIJKLMN, vb);
63           vaccOPQRSTUV = vfmaq_f16(vaccOPQRSTUV, vaOPQRSTUV, vb);
64         } while (--nnz != 0);
65       }
66       float16x8_t vout01234567 = vmulq_f16(vacc01234567, vscale);
67       float16x8_t vout89ABCDEF = vmulq_f16(vacc89ABCDEF, vscale);
68       float16x8_t voutGHIJKLMN = vmulq_f16(vaccGHIJKLMN, vscale);
69       float16x8_t voutOPQRSTUV = vmulq_f16(vaccOPQRSTUV, vscale);
70       vout01234567 = vminq_f16(vout01234567, vmax);
71       vout89ABCDEF = vminq_f16(vout89ABCDEF, vmax);
72       voutGHIJKLMN = vminq_f16(voutGHIJKLMN, vmax);
73       voutOPQRSTUV = vminq_f16(voutOPQRSTUV, vmax);
74       vout01234567 = vmaxq_f16(vout01234567, vmin);
75       vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
76       voutGHIJKLMN = vmaxq_f16(voutGHIJKLMN, vmin);
77       voutOPQRSTUV = vmaxq_f16(voutOPQRSTUV, vmin);
78       vst1q_f16(o, vout01234567);
79       vst1q_f16(o + 8, vout89ABCDEF);
80       vst1q_f16(o + 16, voutGHIJKLMN);
81       vst1q_f16(o + 24, voutOPQRSTUV);
82       o = (__fp16*restrict) ((uintptr_t) o + output_stride);
83     } while (--n != 0);
84     o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
85     i += 32;
86     mc -= 32 * sizeof(__fp16);
87   }
88   if XNN_UNLIKELY(mc != 0) {
89     output_decrement += 16 * sizeof(__fp16);
90     if (mc & (16 * sizeof(__fp16))) {
91       const __fp16*restrict w = (const __fp16*) weights;
92       const int32_t* dmap = widx_dmap;
93       const uint32_t* nnzmap = nidx_nnzmap;
94       size_t n = nc;
95       do {
96         uint32_t nnz = *nnzmap++;
97         float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
98         float16x8_t vacc89ABCDEF = vacc01234567;
99         if XNN_LIKELY(nnz != 0) {
100           do {
101             const intptr_t diff = *dmap++;
102             const float16x8_t va01234567 = vld1q_f16(i);
103             const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
104             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
105             const float16x8_t vb = vld1q_dup_f16(w); w += 1;
106             vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
107             vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
108           } while (--nnz != 0);
109         }
110         float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
111         float16x8_t vout89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
112         vout01234567 = vmaxq_f16(vout01234567, vmin);
113         vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
114         vst1q_f16(o, vout01234567);
115         vst1q_f16(o + 8, vout89ABCDEF);
116         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
117       } while (--n != 0);
118       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
119       i += 16;
120     }
121     output_decrement += 8 * sizeof(__fp16);
122     if (mc & (8 * sizeof(__fp16))) {
123       const __fp16*restrict w = (const __fp16*) weights;
124       const int32_t* dmap = widx_dmap;
125       const uint32_t* nnzmap = nidx_nnzmap;
126       size_t n = nc;
127       do {
128         uint32_t nnz = *nnzmap++;
129         float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
130         if XNN_LIKELY(nnz != 0) {
131           do {
132             const intptr_t diff = *dmap++;
133             const float16x8_t va01234567 = vld1q_f16(i);
134             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
135             const float16x8_t vb = vld1q_dup_f16(w); w += 1;
136             vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
137           } while (--nnz != 0);
138         }
139         float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
140         vout01234567 = vmaxq_f16(vout01234567, vmin);
141         vst1q_f16(o, vout01234567);
142         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
143       } while (--n != 0);
144       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
145       i += 8;
146     }
147     output_decrement += 4 * sizeof(__fp16);
148     if (mc & (4 * sizeof(__fp16))) {
149       const __fp16*restrict w = (const __fp16*) weights;
150       const int32_t* dmap = widx_dmap;
151       const uint32_t* nnzmap = nidx_nnzmap;
152       size_t n = nc;
153       do {
154         uint32_t nnz = *nnzmap++;
155         float16x4_t vacc0123 = vld1_dup_f16(w); w += 1;
156         if XNN_LIKELY(nnz != 0) {
157           do {
158             const intptr_t diff = *dmap++;
159             const float16x4_t va0123 = vld1_f16(i);
160             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
161             const float16x4_t vb = vld1_dup_f16(w); w += 1;
162             vacc0123 = vfma_f16(vacc0123, va0123, vb);
163           } while (--nnz != 0);
164         }
165         float16x4_t vout0123 = vmin_f16(vacc0123, vget_low_f16(vmax));
166         vout0123 = vmax_f16(vout0123, vget_low_f16(vmin));
167         vst1_f16(o, vout0123);
168         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
169       } while (--n != 0);
170       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
171       i += 4;
172     }
173     output_decrement += 2 * sizeof(__fp16);
174     if (mc & (2 * sizeof(__fp16))) {
175       const __fp16*restrict w = (const __fp16*) weights;
176       const int32_t* dmap = widx_dmap;
177       const uint32_t* nnzmap = nidx_nnzmap;
178       size_t n = nc;
179       do {
180         uint32_t nnz = *nnzmap++;
181         float16x4_t vacc01 = vld1_dup_f16(w); w += 1;
182         if XNN_LIKELY(nnz != 0) {
183           do {
184             const intptr_t diff = *dmap++;
185             const float16x4_t va01 = vreinterpret_f16_f32(vld1_dup_f32(__builtin_assume_aligned(i, 1)));
186             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
187             const float16x4_t vb = vld1_dup_f16(w); w += 1;
188             vacc01 = vfma_f16(vacc01, va01, vb);
189           } while (--nnz != 0);
190         }
191         float16x4_t vout01 = vmin_f16(vacc01, vget_low_f16(vmax));
192         vout01 = vmax_f16(vout01, vget_low_f16(vmin));
193         vst1_lane_f32(__builtin_assume_aligned(o, 1), vreinterpret_f32_f16(vout01), 0);
194         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
195       } while (--n != 0);
196       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
197       i += 2;
198     }
199     output_decrement += 1 * sizeof(__fp16);
200     if (mc & (1 * sizeof(__fp16))) {
201       const __fp16*restrict w = (const __fp16*) weights;
202       const int32_t* dmap = widx_dmap;
203       const uint32_t* nnzmap = nidx_nnzmap;
204       size_t n = nc;
205       do {
206         uint32_t nnz = *nnzmap++;
207         float16x4_t vacc0 = vld1_dup_f16(w); w += 1;
208         if XNN_LIKELY(nnz != 0) {
209           do {
210             const intptr_t diff = *dmap++;
211             const float16x4_t va0 = vld1_dup_f16(i);
212             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
213             const float16x4_t vb = vld1_dup_f16(w); w += 1;
214             vacc0 = vfma_f16(vacc0, va0, vb);
215           } while (--nnz != 0);
216         }
217         float16x4_t vout0 = vmin_f16(vacc0, vget_low_f16(vmax));
218         vout0 = vmax_f16(vout0, vget_low_f16(vmin));
219         vst1_lane_f16(o, vout0, 0);
220         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
221       } while (--n != 0);
222       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
223       i += 1;
224     }
225   }
226 }
227