1 // Auto-generated file. Do not edit!
2 //   Template: src/f16-spmm/neonfp16arith.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <arm_neon.h>
13 
14 #include <xnnpack/spmm.h>
15 
16 
xnn_f16_spmm_minmax_ukernel_8x1__neonfp16arith_x2(size_t mc,size_t nc,const void * restrict input,const void * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,void * restrict output,size_t output_stride,const struct xnn_f16_scaleminmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f16_spmm_minmax_ukernel_8x1__neonfp16arith_x2(
18     size_t mc,
19     size_t nc,
20     const void*restrict input,
21     const void*restrict weights,
22     const int32_t*restrict widx_dmap,
23     const uint32_t*restrict nidx_nnzmap,
24     void*restrict output,
25     size_t output_stride,
26     const struct xnn_f16_scaleminmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27 {
28   assert(mc != 0);
29   assert(mc % sizeof(__fp16) == 0);
30   assert(nc != 0);
31 
32   const __fp16*restrict i = (const __fp16*) input;
33   __fp16*restrict o = (__fp16*) output;
34 
35   const float16x8_t vscale = vld1q_dup_f16((const __fp16*) &params->scale);
36   const float16x8_t vmax = vld1q_dup_f16((const __fp16*) &params->max);
37   const float16x8_t vmin = vld1q_dup_f16((const __fp16*) &params->min);
38 
39   size_t output_decrement = output_stride * nc - 8 * sizeof(__fp16);
40   while XNN_LIKELY(mc >= 8 * sizeof(__fp16)) {
41     const __fp16*restrict w = (const __fp16*) weights;
42     const int32_t* dmap = widx_dmap;
43     const uint32_t* nnzmap = nidx_nnzmap;
44     size_t n = nc;
45     do {
46       uint32_t nnz = *nnzmap++;
47       float16x8_t vacc01234567x0 = vld1q_dup_f16(w); w += 1;
48       float16x8_t vacc01234567x1 = vmovq_n_f16(0.0f);
49       for (; nnz >= 2; nnz -= 2) {
50         const intptr_t diff0 = dmap[0];
51         const intptr_t diff1 = dmap[1];
52         dmap += 2;
53         const float16x8_t va01234567x0 = vld1q_f16(i);
54         i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff0);
55         const float16x8_t vb0 = vld1q_dup_f16(w); w += 1;
56         vacc01234567x0 = vfmaq_f16(vacc01234567x0, va01234567x0, vb0);
57         const float16x8_t va01234567x1 = vld1q_f16(i);
58         i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff1);
59         const float16x8_t vb1 = vld1q_dup_f16(w); w += 1;
60         vacc01234567x1 = vfmaq_f16(vacc01234567x1, va01234567x1, vb1);
61       }
62       float16x8_t vacc01234567 = vacc01234567x0;
63       vacc01234567 = vaddq_f16(vacc01234567, vacc01234567x1);
64       if XNN_LIKELY(nnz != 0) {
65         do {
66           const intptr_t diff = *dmap++;
67           const float16x8_t va01234567 = vld1q_f16(i);
68           i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
69           const float16x8_t vb = vld1q_dup_f16(w); w += 1;
70           vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
71         } while (--nnz != 0);
72       }
73       float16x8_t vout01234567 = vmulq_f16(vacc01234567, vscale);
74       vout01234567 = vminq_f16(vout01234567, vmax);
75       vout01234567 = vmaxq_f16(vout01234567, vmin);
76       vst1q_f16(o, vout01234567);
77       o = (__fp16*restrict) ((uintptr_t) o + output_stride);
78     } while (--n != 0);
79     o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
80     i += 8;
81     mc -= 8 * sizeof(__fp16);
82   }
83   if XNN_UNLIKELY(mc != 0) {
84     output_decrement += 4 * sizeof(__fp16);
85     if (mc & (4 * sizeof(__fp16))) {
86       const __fp16*restrict w = (const __fp16*) weights;
87       const int32_t* dmap = widx_dmap;
88       const uint32_t* nnzmap = nidx_nnzmap;
89       size_t n = nc;
90       do {
91         uint32_t nnz = *nnzmap++;
92         float16x4_t vacc0123 = vld1_dup_f16(w); w += 1;
93         if XNN_LIKELY(nnz != 0) {
94           do {
95             const intptr_t diff = *dmap++;
96             const float16x4_t va0123 = vld1_f16(i);
97             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
98             const float16x4_t vb = vld1_dup_f16(w); w += 1;
99             vacc0123 = vfma_f16(vacc0123, va0123, vb);
100           } while (--nnz != 0);
101         }
102         float16x4_t vout0123 = vmin_f16(vacc0123, vget_low_f16(vmax));
103         vout0123 = vmax_f16(vout0123, vget_low_f16(vmin));
104         vst1_f16(o, vout0123);
105         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
106       } while (--n != 0);
107       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
108       i += 4;
109     }
110     output_decrement += 2 * sizeof(__fp16);
111     if (mc & (2 * sizeof(__fp16))) {
112       const __fp16*restrict w = (const __fp16*) weights;
113       const int32_t* dmap = widx_dmap;
114       const uint32_t* nnzmap = nidx_nnzmap;
115       size_t n = nc;
116       do {
117         uint32_t nnz = *nnzmap++;
118         float16x4_t vacc01 = vld1_dup_f16(w); w += 1;
119         if XNN_LIKELY(nnz != 0) {
120           do {
121             const intptr_t diff = *dmap++;
122             const float16x4_t va01 = vreinterpret_f16_f32(vld1_dup_f32(__builtin_assume_aligned(i, 1)));
123             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
124             const float16x4_t vb = vld1_dup_f16(w); w += 1;
125             vacc01 = vfma_f16(vacc01, va01, vb);
126           } while (--nnz != 0);
127         }
128         float16x4_t vout01 = vmin_f16(vacc01, vget_low_f16(vmax));
129         vout01 = vmax_f16(vout01, vget_low_f16(vmin));
130         vst1_lane_f32(__builtin_assume_aligned(o, 1), vreinterpret_f32_f16(vout01), 0);
131         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
132       } while (--n != 0);
133       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
134       i += 2;
135     }
136     output_decrement += 1 * sizeof(__fp16);
137     if (mc & (1 * sizeof(__fp16))) {
138       const __fp16*restrict w = (const __fp16*) weights;
139       const int32_t* dmap = widx_dmap;
140       const uint32_t* nnzmap = nidx_nnzmap;
141       size_t n = nc;
142       do {
143         uint32_t nnz = *nnzmap++;
144         float16x4_t vacc0 = vld1_dup_f16(w); w += 1;
145         if XNN_LIKELY(nnz != 0) {
146           do {
147             const intptr_t diff = *dmap++;
148             const float16x4_t va0 = vld1_dup_f16(i);
149             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
150             const float16x4_t vb = vld1_dup_f16(w); w += 1;
151             vacc0 = vfma_f16(vacc0, va0, vb);
152           } while (--nnz != 0);
153         }
154         float16x4_t vout0 = vmin_f16(vacc0, vget_low_f16(vmax));
155         vout0 = vmax_f16(vout0, vget_low_f16(vmin));
156         vst1_lane_f16(o, vout0, 0);
157         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
158       } while (--n != 0);
159       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
160       i += 1;
161     }
162   }
163 }
164