1 // Auto-generated file. Do not edit!
2 //   Template: src/f32-spmm/scalar-pipelined.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <xnnpack/math.h>
13 #include <xnnpack/spmm.h>
14 
15 
xnn_f32_spmm_minmax_ukernel_8x1__scalar_pipelined(size_t mc,size_t nc,const float * restrict input,const float * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,float * restrict output,size_t output_stride,const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])16 void xnn_f32_spmm_minmax_ukernel_8x1__scalar_pipelined(
17     size_t mc,
18     size_t nc,
19     const float*restrict input,
20     const float*restrict weights,
21     const int32_t*restrict widx_dmap,
22     const uint32_t*restrict nidx_nnzmap,
23     float*restrict output,
24     size_t output_stride,
25     const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
26 {
27   assert(mc != 0);
28   assert(mc % sizeof(float) == 0);
29   assert(nc != 0);
30 
31   const float vmin = params->scalar.min;
32   const float vmax = params->scalar.max;
33   size_t output_decrement = output_stride * nc - 8 * sizeof(float);
34   while XNN_LIKELY(mc >= 8 * sizeof(float)) {
35     const float*restrict w = weights;
36     const int32_t* dmap = widx_dmap;
37     const uint32_t* nnzmap = nidx_nnzmap;
38     float vw = *w++;
39     intptr_t diff = *dmap++;
40     float vi0 = input[0];
41     float vi1 = input[1];
42     float vi2 = input[2];
43     float vi3 = input[3];
44     float vi4 = input[4];
45     float vi5 = input[5];
46     float vi6 = input[6];
47     float vi7 = input[7];
48     size_t n = nc;
49     do {
50       uint32_t nnz = *nnzmap++;
51       float vacc0 = vw;
52       float vacc1 = vw;
53       float vacc2 = vw;
54       float vacc3 = vw;
55       float vacc4 = vw;
56       float vacc5 = vw;
57       float vacc6 = vw;
58       float vacc7 = vw;
59       vw = *w++;
60       if XNN_LIKELY(nnz != 0) {
61         do {
62           vacc0 += vi0 * vw;
63           vacc1 += vi1 * vw;
64           vacc2 += vi2 * vw;
65           vacc3 += vi3 * vw;
66           vacc4 += vi4 * vw;
67           vacc5 += vi5 * vw;
68           vacc6 += vi6 * vw;
69           vacc7 += vi7 * vw;
70           input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
71 
72           diff = *dmap++;
73           vw = *w++;
74           vi0 = input[0];
75           vi1 = input[1];
76           vi2 = input[2];
77           vi3 = input[3];
78           vi4 = input[4];
79           vi5 = input[5];
80           vi6 = input[6];
81           vi7 = input[7];
82         } while (--nnz != 0);
83       }
84       float vout0 = math_min_f32(vacc0, vmax);
85       float vout1 = math_min_f32(vacc1, vmax);
86       float vout2 = math_min_f32(vacc2, vmax);
87       float vout3 = math_min_f32(vacc3, vmax);
88       float vout4 = math_min_f32(vacc4, vmax);
89       float vout5 = math_min_f32(vacc5, vmax);
90       float vout6 = math_min_f32(vacc6, vmax);
91       float vout7 = math_min_f32(vacc7, vmax);
92       vout0 = math_max_f32(vout0, vmin);
93       vout1 = math_max_f32(vout1, vmin);
94       vout2 = math_max_f32(vout2, vmin);
95       vout3 = math_max_f32(vout3, vmin);
96       vout4 = math_max_f32(vout4, vmin);
97       vout5 = math_max_f32(vout5, vmin);
98       vout6 = math_max_f32(vout6, vmin);
99       vout7 = math_max_f32(vout7, vmin);
100       output[0] = vout0;
101       output[1] = vout1;
102       output[2] = vout2;
103       output[3] = vout3;
104       output[4] = vout4;
105       output[5] = vout5;
106       output[6] = vout6;
107       output[7] = vout7;
108       output = (float*restrict) ((uintptr_t) output + output_stride);
109     } while (--n != 0);
110     output = (float*restrict) ((uintptr_t) output - output_decrement);
111     input += 8;
112     mc -= 8 * sizeof(float);
113   }
114   if XNN_UNLIKELY(mc != 0) {
115     output_decrement += 4 * sizeof(float);
116     if (mc & (4 * sizeof(float))) {
117       const float*restrict w = weights;
118       const int32_t* dmap = widx_dmap;
119       const uint32_t* nnzmap = nidx_nnzmap;
120       float vw = *w++;
121       intptr_t diff = *dmap++;
122       float vi0 = input[0];
123       float vi1 = input[1];
124       float vi2 = input[2];
125       float vi3 = input[3];
126       size_t n = nc;
127       do {
128         uint32_t nnz = *nnzmap++;
129         float vacc0 = vw;
130         float vacc1 = vw;
131         float vacc2 = vw;
132         float vacc3 = vw;
133         vw = *w++;
134         if XNN_LIKELY(nnz != 0) {
135           do {
136             vacc0 += vi0 * vw;
137             vacc1 += vi1 * vw;
138             vacc2 += vi2 * vw;
139             vacc3 += vi3 * vw;
140             input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
141 
142             diff = *dmap++;
143             vw = *w++;
144             vi0 = input[0];
145             vi1 = input[1];
146             vi2 = input[2];
147             vi3 = input[3];
148           } while (--nnz != 0);
149         }
150         float vout0 = math_min_f32(vacc0, vmax);
151         float vout1 = math_min_f32(vacc1, vmax);
152         float vout2 = math_min_f32(vacc2, vmax);
153         float vout3 = math_min_f32(vacc3, vmax);
154         vout0 = math_max_f32(vout0, vmin);
155         vout1 = math_max_f32(vout1, vmin);
156         vout2 = math_max_f32(vout2, vmin);
157         vout3 = math_max_f32(vout3, vmin);
158         output[0] = vout0;
159         output[1] = vout1;
160         output[2] = vout2;
161         output[3] = vout3;
162         output = (float*restrict) ((uintptr_t) output + output_stride);
163       } while (--n != 0);
164       output = (float*restrict) ((uintptr_t) output - output_decrement);
165       input += 4;
166     }
167     output_decrement += 2 * sizeof(float);
168     if (mc & (2 * sizeof(float))) {
169       const float*restrict w = weights;
170       const int32_t* dmap = widx_dmap;
171       const uint32_t* nnzmap = nidx_nnzmap;
172       float vw = *w++;
173       intptr_t diff = *dmap++;
174       float vi0 = input[0];
175       float vi1 = input[1];
176       size_t n = nc;
177       do {
178         uint32_t nnz = *nnzmap++;
179         float vacc0 = vw;
180         float vacc1 = vw;
181         vw = *w++;
182         if XNN_LIKELY(nnz != 0) {
183           do {
184             vacc0 += vi0 * vw;
185             vacc1 += vi1 * vw;
186             input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
187 
188             diff = *dmap++;
189             vw = *w++;
190             vi0 = input[0];
191             vi1 = input[1];
192           } while (--nnz != 0);
193         }
194         float vout0 = math_min_f32(vacc0, vmax);
195         float vout1 = math_min_f32(vacc1, vmax);
196         vout0 = math_max_f32(vout0, vmin);
197         vout1 = math_max_f32(vout1, vmin);
198         output[0] = vout0;
199         output[1] = vout1;
200         output = (float*restrict) ((uintptr_t) output + output_stride);
201       } while (--n != 0);
202       output = (float*restrict) ((uintptr_t) output - output_decrement);
203       input += 2;
204     }
205     output_decrement += 1 * sizeof(float);
206     if (mc & (1 * sizeof(float))) {
207       const float*restrict w = weights;
208       const int32_t* dmap = widx_dmap;
209       const uint32_t* nnzmap = nidx_nnzmap;
210       float vw = *w++;
211       intptr_t diff = *dmap++;
212       float vi0 = input[0];
213       size_t n = nc;
214       do {
215         uint32_t nnz = *nnzmap++;
216         float vacc0 = vw;
217         vw = *w++;
218         if XNN_LIKELY(nnz != 0) {
219           do {
220             vacc0 += vi0 * vw;
221             input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
222 
223             diff = *dmap++;
224             vw = *w++;
225             vi0 = input[0];
226           } while (--nnz != 0);
227         }
228         float vout0 = math_min_f32(vacc0, vmax);
229         vout0 = math_max_f32(vout0, vmin);
230         output[0] = vout0;
231         output = (float*restrict) ((uintptr_t) output + output_stride);
232       } while (--n != 0);
233       output = (float*restrict) ((uintptr_t) output - output_decrement);
234       input += 1;
235     }
236   }
237 }
238