1 // Auto-generated file. Do not edit!
2 //   Template: src/f32-spmm/neon.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <arm_neon.h>
13 
14 #include <xnnpack/spmm.h>
15 
16 
xnn_f32_spmm_minmax_ukernel_12x1__neonfma(size_t mc,size_t nc,const float * restrict input,const float * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,float * restrict output,size_t output_stride,const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f32_spmm_minmax_ukernel_12x1__neonfma(
18     size_t mc,
19     size_t nc,
20     const float*restrict input,
21     const float*restrict weights,
22     const int32_t*restrict widx_dmap,
23     const uint32_t*restrict nidx_nnzmap,
24     float*restrict output,
25     size_t output_stride,
26     const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27 {
28   assert(mc != 0);
29   assert(mc % sizeof(float) == 0);
30   assert(nc != 0);
31 
32   const float32x4_t vmin = vld1q_dup_f32(&params->scalar.min);
33   const float32x4_t vmax = vld1q_dup_f32(&params->scalar.max);
34   size_t output_decrement = output_stride * nc - 12 * sizeof(float);
35   while XNN_LIKELY(mc >= 12 * sizeof(float)) {
36     const float*restrict w = weights;
37     const int32_t* dmap = widx_dmap;
38     const uint32_t* nnzmap = nidx_nnzmap;
39     size_t n = nc;
40     do {
41       uint32_t nnz = *nnzmap++;
42       float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
43       float32x4_t vacc4567 = vacc0123;
44       float32x4_t vacc89AB = vacc0123;
45       if XNN_LIKELY(nnz != 0) {
46         do {
47           const intptr_t diff = *dmap++;
48           const float32x4_t vi0123 = vld1q_f32(input);
49           const float32x4_t vi4567 = vld1q_f32(input + 4);
50           const float32x4_t vi89AB = vld1q_f32(input + 8);
51           input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
52           __builtin_prefetch(input + 16);
53           const float32x4_t vw = vld1q_dup_f32(w); w += 1;
54           __builtin_prefetch(w + 32);
55           vacc0123 = vfmaq_f32(vacc0123, vi0123, vw);
56           vacc4567 = vfmaq_f32(vacc4567, vi4567, vw);
57           vacc89AB = vfmaq_f32(vacc89AB, vi89AB, vw);
58         } while (--nnz != 0);
59       }
60       float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
61       float32x4_t vout4567 = vminq_f32(vacc4567, vmax);
62       float32x4_t vout89AB = vminq_f32(vacc89AB, vmax);
63       vout0123 = vmaxq_f32(vout0123, vmin);
64       vout4567 = vmaxq_f32(vout4567, vmin);
65       vout89AB = vmaxq_f32(vout89AB, vmin);
66       vst1q_f32(output, vout0123);
67       vst1q_f32(output + 4, vout4567);
68       vst1q_f32(output + 8, vout89AB);
69       output = (float*restrict) ((uintptr_t) output + output_stride);
70     } while (--n != 0);
71     output = (float*restrict) ((uintptr_t) output - output_decrement);
72     input += 12;
73     mc -= 12 * sizeof(float);
74   }
75   if XNN_UNLIKELY(mc != 0) {
76     output_decrement += 4 * sizeof(float);
77     if (mc & (8 * sizeof(float))) {
78       const float*restrict w = weights;
79       const int32_t* dmap = widx_dmap;
80       const uint32_t* nnzmap = nidx_nnzmap;
81       size_t n = nc;
82       do {
83         uint32_t nnz = *nnzmap++;
84         float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
85         float32x4_t vacc4567 = vacc0123;
86         if XNN_LIKELY(nnz != 0) {
87           do {
88             const intptr_t diff = *dmap++;
89             const float32x4_t vi0123 = vld1q_f32(input);
90             const float32x4_t vi4567 = vld1q_f32(input + 4);
91             input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
92             const float32x4_t vw = vld1q_dup_f32(w); w += 1;
93             vacc0123 = vfmaq_f32(vacc0123, vi0123, vw);
94             vacc4567 = vfmaq_f32(vacc4567, vi4567, vw);
95           } while (--nnz != 0);
96         }
97         float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
98         float32x4_t vout4567 = vminq_f32(vacc4567, vmax);
99         vout0123 = vmaxq_f32(vout0123, vmin);
100         vout4567 = vmaxq_f32(vout4567, vmin);
101         vst1q_f32(output, vout0123);
102         vst1q_f32(output + 4, vout4567);
103         output = (float*restrict) ((uintptr_t) output + output_stride);
104       } while (--n != 0);
105       output = (float*restrict) ((uintptr_t) output - output_decrement);
106       input += 8;
107     }
108     output_decrement += 4 * sizeof(float);
109     if (mc & (4 * sizeof(float))) {
110       const float*restrict w = weights;
111       const int32_t* dmap = widx_dmap;
112       const uint32_t* nnzmap = nidx_nnzmap;
113       size_t n = nc;
114       do {
115         uint32_t nnz = *nnzmap++;
116         float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
117         if XNN_LIKELY(nnz != 0) {
118           do {
119             const intptr_t diff = *dmap++;
120             const float32x4_t vi0123 = vld1q_f32(input);
121             input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
122             const float32x4_t vw = vld1q_dup_f32(w); w += 1;
123             vacc0123 = vfmaq_f32(vacc0123, vi0123, vw);
124           } while (--nnz != 0);
125         }
126         float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
127         vout0123 = vmaxq_f32(vout0123, vmin);
128         vst1q_f32(output, vout0123);
129         output = (float*restrict) ((uintptr_t) output + output_stride);
130       } while (--n != 0);
131       output = (float*restrict) ((uintptr_t) output - output_decrement);
132       input += 4;
133     }
134     output_decrement += 2 * sizeof(float);
135     if (mc & (2 * sizeof(float))) {
136       const float*restrict w = weights;
137       const int32_t* dmap = widx_dmap;
138       const uint32_t* nnzmap = nidx_nnzmap;
139       size_t n = nc;
140       do {
141         uint32_t nnz = *nnzmap++;
142         float32x2_t vacc01 = vld1_dup_f32(w); w += 1;
143         if XNN_LIKELY(nnz != 0) {
144           do {
145             const intptr_t diff = *dmap++;
146             const float32x2_t vi01 = vld1_f32(input);
147             input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
148             const float32x2_t vw = vld1_dup_f32(w); w += 1;
149             vacc01 = vfma_f32(vacc01, vi01, vw);
150           } while (--nnz != 0);
151         }
152         float32x2_t vout01 = vmin_f32(vacc01, vget_low_f32(vmax));
153         vout01 = vmax_f32(vout01, vget_low_f32(vmin));
154         vst1_f32(output, vout01);
155         output = (float*restrict) ((uintptr_t) output + output_stride);
156       } while (--n != 0);
157       output = (float*restrict) ((uintptr_t) output - output_decrement);
158       input += 2;
159     }
160     output_decrement += 1 * sizeof(float);
161     if (mc & (1 * sizeof(float))) {
162       const float*restrict w = weights;
163       const int32_t* dmap = widx_dmap;
164       const uint32_t* nnzmap = nidx_nnzmap;
165       size_t n = nc;
166       do {
167         uint32_t nnz = *nnzmap++;
168         float32x2_t vacc0 = vld1_dup_f32(w); w += 1;
169         if XNN_LIKELY(nnz != 0) {
170           do {
171             const intptr_t diff = *dmap++;
172             const float32x2_t vi0 = vld1_dup_f32(input);
173             input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
174             const float32x2_t vw = vld1_dup_f32(w); w += 1;
175             vacc0 = vfma_f32(vacc0, vi0, vw);
176           } while (--nnz != 0);
177         }
178         float32x2_t vout0 = vmin_f32(vacc0, vget_low_f32(vmax));
179         vout0 = vmax_f32(vout0, vget_low_f32(vmin));
180         vst1_lane_f32(output, vout0, 0);
181         output = (float*restrict) ((uintptr_t) output + output_stride);
182       } while (--n != 0);
183       output = (float*restrict) ((uintptr_t) output - output_decrement);
184       input += 1;
185     }
186   }
187 }
188