1 // Auto-generated file. Do not edit!
2 //   Template: src/f32-spmm/wasmsimd-pipelined.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <wasm_simd128.h>
13 
14 #include <xnnpack/spmm.h>
15 
16 
xnn_f32_spmm_minmax_ukernel_8x1__wasmsimd_x86_pipelined(size_t mc,size_t nc,const float * restrict input,const float * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,float * restrict output,size_t output_stride,const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f32_spmm_minmax_ukernel_8x1__wasmsimd_x86_pipelined(
18     size_t mc,
19     size_t nc,
20     const float*restrict input,
21     const float*restrict weights,
22     const int32_t*restrict widx_dmap,
23     const uint32_t*restrict nidx_nnzmap,
24     float*restrict output,
25     size_t output_stride,
26     const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27 {
28   assert(mc != 0);
29   assert(mc % sizeof(float) == 0);
30   assert(nc != 0);
31 
32   const v128_t vmin = wasm_v32x4_load_splat(&params->scalar.min);
33   const v128_t vmax = wasm_v32x4_load_splat(&params->scalar.max);
34   size_t output_decrement = output_stride * nc - 8 * sizeof(float);
35   while XNN_LIKELY(mc >= 8 * sizeof(float)) {
36     const float*restrict w = weights;
37     const int32_t* dmap = widx_dmap;
38     const uint32_t* nnzmap = nidx_nnzmap;
39     v128_t vw = wasm_v32x4_load_splat(w); w += 1;
40     intptr_t diff = *dmap++;
41     v128_t vi0123 = wasm_v128_load(input + 0);
42     v128_t vi4567 = wasm_v128_load(input + 4);
43     size_t n = nc;
44     do {
45       uint32_t nnz = *nnzmap++;
46        v128_t vacc0123 = vw;
47        v128_t vacc4567 = vw;
48       vw = wasm_v32x4_load_splat(w); w += 1;
49 
50 
51       if XNN_LIKELY(nnz != 0) {
52         do {
53           vacc0123 = wasm_f32x4_add(vacc0123, wasm_f32x4_mul(vi0123, vw));
54           vacc4567 = wasm_f32x4_add(vacc4567, wasm_f32x4_mul(vi4567, vw));
55           input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
56 
57           diff = *dmap++;
58           vw = wasm_v32x4_load_splat(w); w += 1;
59           vi0123 = wasm_v128_load(input + 0);
60           vi4567 = wasm_v128_load(input + 4);
61         } while (--nnz != 0);
62       }
63       v128_t vout0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax));
64       v128_t vout4567 = wasm_v128_bitselect(vacc4567, vmax, wasm_f32x4_le(vacc4567, vmax));
65       vout0123 = wasm_v128_bitselect(vmin, vout0123, wasm_f32x4_lt(vout0123, vmin));
66       vout4567 = wasm_v128_bitselect(vmin, vout4567, wasm_f32x4_lt(vout4567, vmin));
67       wasm_v128_store(output, vout0123);
68       wasm_v128_store(output + 4, vout4567);
69       output = (float*restrict) ((uintptr_t) output + output_stride);
70     } while (--n != 0);
71     output = (float*restrict) ((uintptr_t) output - output_decrement);
72     input += 8;
73     mc -= 8 * sizeof(float);
74   }
75   if XNN_UNLIKELY(mc != 0) {
76     output_decrement += 4 * sizeof(float);
77     if (mc & (4 * sizeof(float))) {
78       const float*restrict w = weights;
79       const int32_t* dmap = widx_dmap;
80       const uint32_t* nnzmap = nidx_nnzmap;
81       size_t n = nc;
82       do {
83         uint32_t nnz = *nnzmap++;
84         v128_t vacc0123 = wasm_v32x4_load_splat(w); w += 1;
85         if XNN_LIKELY(nnz != 0) {
86           do {
87             const intptr_t diff = *dmap++;
88             const v128_t vi0123 = wasm_v128_load(input);
89             input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
90             const v128_t vw = wasm_v32x4_load_splat(w); w += 1;
91             vacc0123 = wasm_f32x4_add(vacc0123, wasm_f32x4_mul(vi0123, vw));
92           } while (--nnz != 0);
93         }
94         v128_t vout0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax));
95         vout0123 = wasm_v128_bitselect(vmin, vout0123, wasm_f32x4_lt(vout0123, vmin));
96         wasm_v128_store(output, vout0123);
97 
98         output = (float*restrict) ((uintptr_t) output + output_stride);
99       } while (--n != 0);
100       output = (float*restrict) ((uintptr_t) output - output_decrement);
101       input += 4;
102     }
103     output_decrement += 2 * sizeof(float);
104     if (mc & (2 * sizeof(float))) {
105       const float*restrict w = weights;
106       const int32_t* dmap = widx_dmap;
107       const uint32_t* nnzmap = nidx_nnzmap;
108       size_t n = nc;
109       do {
110         uint32_t nnz = *nnzmap++;
111         v128_t vacc01 = wasm_v32x4_load_splat(w); w += 1;
112         if XNN_LIKELY(nnz != 0) {
113           do {
114             const intptr_t diff = *dmap++;
115             const v128_t vi01 = wasm_v64x2_load_splat(input);
116             input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
117             const v128_t vw = wasm_v32x4_load_splat(w); w += 1;
118             vacc01 = wasm_f32x4_add(vacc01, wasm_f32x4_mul(vi01, vw));
119           } while (--nnz != 0);
120         }
121         v128_t vout01 = wasm_v128_bitselect(vacc01, vmax, wasm_f32x4_le(vacc01, vmax));
122         vout01 = wasm_v128_bitselect(vmin, vout01, wasm_f32x4_lt(vout01, vmin));
123         *((double*) output) = wasm_f64x2_extract_lane(vout01, 0);
124 
125         output = (float*restrict) ((uintptr_t) output + output_stride);
126       } while (--n != 0);
127       output = (float*restrict) ((uintptr_t) output - output_decrement);
128       input += 2;
129     }
130     output_decrement += 1 * sizeof(float);
131     if (mc & (1 * sizeof(float))) {
132       const float*restrict w = weights;
133       const int32_t* dmap = widx_dmap;
134       const uint32_t* nnzmap = nidx_nnzmap;
135       size_t n = nc;
136       do {
137         uint32_t nnz = *nnzmap++;
138         v128_t vacc0 = wasm_v32x4_load_splat(w); w += 1;
139         if XNN_LIKELY(nnz != 0) {
140           do {
141             const intptr_t diff = *dmap++;
142             const v128_t vi0 = wasm_v32x4_load_splat(input);
143             input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
144             const v128_t vw = wasm_v32x4_load_splat(w); w += 1;
145             vacc0 = wasm_f32x4_add(vacc0, wasm_f32x4_mul(vi0, vw));
146           } while (--nnz != 0);
147         }
148         v128_t vout0 = wasm_v128_bitselect(vacc0, vmax, wasm_f32x4_le(vacc0, vmax));
149         vout0 = wasm_v128_bitselect(vmin, vout0, wasm_f32x4_lt(vout0, vmin));
150         *output = wasm_f32x4_extract_lane(vout0, 0);
151 
152         output = (float*restrict) ((uintptr_t) output + output_stride);
153       } while (--n != 0);
154       output = (float*restrict) ((uintptr_t) output - output_decrement);
155       input += 1;
156     }
157   }
158 }
159