1 // Auto-generated file. Do not edit!
2 //   Template: src/f32-gemm/wasmsimd-loadsplat.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <wasm_simd128.h>
13 
14 #include <xnnpack/gemm.h>
15 
16 
xnn_f32_gemminc_minmax_ukernel_3x8__wasmsimd_arm_loadsplat(size_t mr,size_t nc,size_t kc,const float * restrict a,size_t a_stride,const float * restrict w,float * restrict c,size_t cm_stride,size_t cn_stride,const float * restrict acc,const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f32_gemminc_minmax_ukernel_3x8__wasmsimd_arm_loadsplat(
18     size_t mr,
19     size_t nc,
20     size_t kc,
21     const float*restrict a,
22     size_t a_stride,
23     const float*restrict w,
24     float*restrict c,
25     size_t cm_stride,
26     size_t cn_stride,
27     const float*restrict acc,
28     const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
29 {
30   assert(mr != 0);
31   assert(mr <= 3);
32   assert(nc != 0);
33   assert(kc != 0);
34   assert(kc % sizeof(float) == 0);
35   assert(a != NULL);
36   assert(w != NULL);
37   assert(c != NULL);
38   assert(acc != NULL);
39 
40   const float* a0 = a;
41   float* c0 = c;
42   const float* a1 = (const float*) ((uintptr_t) a0 + a_stride);
43   float* c1 = (float*) ((uintptr_t) c0 + cm_stride);
44   if XNN_UNPREDICTABLE(mr < 2) {
45     a1 = a0;
46     c1 = c0;
47   }
48   const float* a2 = (const float*) ((uintptr_t) a1 + a_stride);
49   float* c2 = (float*) ((uintptr_t) c1 + cm_stride);
50   if XNN_UNPREDICTABLE(mr <= 2) {
51     a2 = a1;
52     c2 = c1;
53   }
54 
55   const v128_t vmin = wasm_v32x4_load_splat(&params->scalar.min);
56   const v128_t vmax = wasm_v32x4_load_splat(&params->scalar.max);
57   do {
58     v128_t vacc0x0123 = wasm_v128_load(acc + 0);
59     v128_t vacc0x4567 = wasm_v128_load(acc + 4);
60     v128_t vacc1x0123 = wasm_v128_load(acc + 8);
61     v128_t vacc1x4567 = wasm_v128_load(acc + 12);
62     v128_t vacc2x0123 = wasm_v128_load(acc + 16);
63     v128_t vacc2x4567 = wasm_v128_load(acc + 20);
64     acc += 24;
65 
66     size_t k = kc;
67     do {
68       const v128_t va0 = wasm_v32x4_load_splat(a0);
69       a0 += 1;
70       const v128_t va1 = wasm_v32x4_load_splat(a1);
71       a1 += 1;
72       const v128_t va2 = wasm_v32x4_load_splat(a2);
73       a2 += 1;
74 
75       const v128_t vb0123 = wasm_v128_load(w);
76       const v128_t vb4567 = wasm_v128_load(w + 4);
77       w += 8;
78 
79       vacc0x0123 = wasm_f32x4_add(vacc0x0123, wasm_f32x4_mul(va0, vb0123));
80       vacc1x0123 = wasm_f32x4_add(vacc1x0123, wasm_f32x4_mul(va1, vb0123));
81       vacc2x0123 = wasm_f32x4_add(vacc2x0123, wasm_f32x4_mul(va2, vb0123));
82       vacc0x4567 = wasm_f32x4_add(vacc0x4567, wasm_f32x4_mul(va0, vb4567));
83       vacc1x4567 = wasm_f32x4_add(vacc1x4567, wasm_f32x4_mul(va1, vb4567));
84       vacc2x4567 = wasm_f32x4_add(vacc2x4567, wasm_f32x4_mul(va2, vb4567));
85 
86       k -= sizeof(float);
87     } while (k != 0);
88 
89     vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin);
90     vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin);
91     vacc2x0123 = wasm_f32x4_max(vacc2x0123, vmin);
92     vacc0x4567 = wasm_f32x4_max(vacc0x4567, vmin);
93     vacc1x4567 = wasm_f32x4_max(vacc1x4567, vmin);
94     vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin);
95 
96     vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax);
97     vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax);
98     vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax);
99     vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax);
100     vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax);
101     vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax);
102 
103     if XNN_LIKELY(nc >= 8) {
104       wasm_v128_store(c2, vacc2x0123);
105       wasm_v128_store(c2 + 4, vacc2x4567);
106       c2 = (float*) ((uintptr_t) c2 + cn_stride);
107       wasm_v128_store(c1, vacc1x0123);
108       wasm_v128_store(c1 + 4, vacc1x4567);
109       c1 = (float*) ((uintptr_t) c1 + cn_stride);
110       wasm_v128_store(c0, vacc0x0123);
111       wasm_v128_store(c0 + 4, vacc0x4567);
112       c0 = (float*) ((uintptr_t) c0 + cn_stride);
113 
114       a2 = (const float*) ((uintptr_t) a2 - kc);
115       a1 = (const float*) ((uintptr_t) a1 - kc);
116       a0 = (const float*) ((uintptr_t) a0 - kc);
117 
118       nc -= 8;
119     } else {
120       if (nc & 4) {
121         wasm_v128_store(c2, vacc2x0123);
122         wasm_v128_store(c1, vacc1x0123);
123         wasm_v128_store(c0, vacc0x0123);
124 
125         vacc2x0123 = vacc2x4567;
126         vacc1x0123 = vacc1x4567;
127         vacc0x0123 = vacc0x4567;
128 
129         c2 += 4;
130         c1 += 4;
131         c0 += 4;
132       }
133       if (nc & 2) {
134         *((double*) c2) = wasm_f64x2_extract_lane(vacc2x0123, 0);
135         *((double*) c1) = wasm_f64x2_extract_lane(vacc1x0123, 0);
136         *((double*) c0) = wasm_f64x2_extract_lane(vacc0x0123, 0);
137 
138         vacc2x0123 = wasm_v32x4_shuffle(vacc2x0123, vacc2x0123, 2, 3, 2, 3);
139         vacc1x0123 = wasm_v32x4_shuffle(vacc1x0123, vacc1x0123, 2, 3, 2, 3);
140         vacc0x0123 = wasm_v32x4_shuffle(vacc0x0123, vacc0x0123, 2, 3, 2, 3);
141 
142         c2 += 2;
143         c1 += 2;
144         c0 += 2;
145       }
146       if (nc & 1) {
147         *c2 = wasm_f32x4_extract_lane(vacc2x0123, 0);
148         *c1 = wasm_f32x4_extract_lane(vacc1x0123, 0);
149         *c0 = wasm_f32x4_extract_lane(vacc0x0123, 0);
150       }
151 
152       nc = 0;
153     }
154   } while (nc != 0);
155 }
156