1 // Auto-generated file. Do not edit!
2 //   Template: src/qs8-gemm/neon-mlal-lane.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <arm_neon.h>
13 
14 #include <xnnpack/common.h>
15 #include <xnnpack/gemm.h>
16 
17 
xnn_qs8_gemm_minmax_ukernel_1x8__neon_mlal_lane(size_t mr,size_t nc,size_t kc,const int8_t * restrict a,size_t a_stride,const void * restrict w,int8_t * restrict c,size_t cm_stride,size_t cn_stride,const union xnn_qs8_gemm_params params[restrict XNN_MIN_ELEMENTS (1)])18 void xnn_qs8_gemm_minmax_ukernel_1x8__neon_mlal_lane(
19     size_t mr,
20     size_t nc,
21     size_t kc,
22     const int8_t* restrict a,
23     size_t a_stride,
24     const void* restrict w,
25     int8_t* restrict c,
26     size_t cm_stride,
27     size_t cn_stride,
28     const union xnn_qs8_gemm_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
29 {
30   assert(mr != 0);
31   assert(mr <= 1);
32   assert(nc != 0);
33   assert(kc != 0);
34   assert(kc % sizeof(int8_t) == 0);
35   assert(a != NULL);
36   assert(w != NULL);
37   assert(c != NULL);
38 
39   const int8_t* a0 = a;
40   int8_t* c0 = c;
41 
42   do {
43     int32x4_t vacc0x0123 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
44     int32x4_t vacc0x4567 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
45 
46     size_t k = kc;
47     while (k >= 8 * sizeof(int8_t)) {
48       const int8x8_t va0 = vld1_s8(a0); a0 += 8;
49       const int16x8_t vxa0 = vmovl_s8(va0);
50 
51       const int8x8_t vb01234567c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
52       const int16x8_t vxb01234567c0 = vmovl_s8(vb01234567c0);
53 
54       vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c0), vget_low_s16(vxa0), 0);
55       vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c0), vget_low_s16(vxa0), 0);
56       const int8x8_t vb01234567c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
57       const int16x8_t vxb01234567c1 = vmovl_s8(vb01234567c1);
58 
59       vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c1), vget_low_s16(vxa0), 1);
60       vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c1), vget_low_s16(vxa0), 1);
61       const int8x8_t vb01234567c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
62       const int16x8_t vxb01234567c2 = vmovl_s8(vb01234567c2);
63 
64       vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c2), vget_low_s16(vxa0), 2);
65       vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c2), vget_low_s16(vxa0), 2);
66       const int8x8_t vb01234567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
67       const int16x8_t vxb01234567c3 = vmovl_s8(vb01234567c3);
68 
69       vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c3), vget_low_s16(vxa0), 3);
70       vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c3), vget_low_s16(vxa0), 3);
71 
72       const int8x8_t vb01234567c4 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
73       const int16x8_t vxb01234567c4 = vmovl_s8(vb01234567c4);
74 
75       vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c4), vget_high_s16(vxa0), 0);
76       vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c4), vget_high_s16(vxa0), 0);
77       const int8x8_t vb01234567c5 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
78       const int16x8_t vxb01234567c5 = vmovl_s8(vb01234567c5);
79 
80       vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c5), vget_high_s16(vxa0), 1);
81       vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c5), vget_high_s16(vxa0), 1);
82       const int8x8_t vb01234567c6 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
83       const int16x8_t vxb01234567c6 = vmovl_s8(vb01234567c6);
84 
85       vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c6), vget_high_s16(vxa0), 2);
86       vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c6), vget_high_s16(vxa0), 2);
87       const int8x8_t vb01234567c7 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
88       const int16x8_t vxb01234567c7 = vmovl_s8(vb01234567c7);
89 
90       vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c7), vget_high_s16(vxa0), 3);
91       vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c7), vget_high_s16(vxa0), 3);
92 
93       k -= 8 * sizeof(int8_t);
94     }
95     if XNN_UNLIKELY(k != 0) {
96       const int8x8_t va0 = vld1_s8(a0); a0 = (const int8_t*) ((uintptr_t) a0 + k);
97       const int16x8_t vxa0 = vmovl_s8(va0);
98 
99       const int8x8_t vb01234567c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
100       const int16x8_t vxb01234567c0 = vmovl_s8(vb01234567c0);
101 
102       vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c0), vget_low_s16(vxa0), 0);
103       vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c0), vget_low_s16(vxa0), 0);
104 
105       if (k >= 2 * sizeof(int8_t)) {
106         const int8x8_t vb01234567c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
107         const int16x8_t vxb01234567c1 = vmovl_s8(vb01234567c1);
108 
109         vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c1), vget_low_s16(vxa0), 1);
110         vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c1), vget_low_s16(vxa0), 1);
111 
112         if (k > 2 * sizeof(int8_t)) {
113           const int8x8_t vb01234567c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
114           const int16x8_t vxb01234567c2 = vmovl_s8(vb01234567c2);
115 
116           vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c2), vget_low_s16(vxa0), 2);
117           vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c2), vget_low_s16(vxa0), 2);
118 
119           if (k >= 4 * sizeof(int8_t)) {
120             const int8x8_t vb01234567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
121             const int16x8_t vxb01234567c3 = vmovl_s8(vb01234567c3);
122 
123             vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c3), vget_low_s16(vxa0), 3);
124             vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c3), vget_low_s16(vxa0), 3);
125 
126             if (k > 4 * sizeof(int8_t)) {
127               const int8x8_t vb01234567c4 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
128               const int16x8_t vxb01234567c4 = vmovl_s8(vb01234567c4);
129 
130               vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c4), vget_high_s16(vxa0), 0);
131               vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c4), vget_high_s16(vxa0), 0);
132 
133               if (k >= 6 * sizeof(int8_t)) {
134                 const int8x8_t vb01234567c5 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
135                 const int16x8_t vxb01234567c5 = vmovl_s8(vb01234567c5);
136 
137                 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c5), vget_high_s16(vxa0), 1);
138                 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c5), vget_high_s16(vxa0), 1);
139 
140                 if (k > 6 * sizeof(int8_t)) {
141                   const int8x8_t vb01234567c6 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
142                   const int16x8_t vxb01234567c6 = vmovl_s8(vb01234567c6);
143 
144                   vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c6), vget_high_s16(vxa0), 2);
145                   vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c6), vget_high_s16(vxa0), 2);
146                 }
147               }
148             }
149           }
150         }
151       }
152     }
153 
154     const int32x4_t vmultiplier = vld1q_dup_s32(&params->neon.multiplier);
155     vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
156     vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
157 
158     const int32x4_t vright_shift = vld1q_dup_s32(&params->neon.right_shift);
159     const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
160     vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
161     vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
162 
163     vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
164     vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
165 
166     const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->neon.output_zero_point);
167 #if XNN_ARCH_ARM64
168     const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
169 
170     int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
171 #else
172     const int16x8_t vacc0x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0x0123), vqmovn_s32(vacc0x4567)), voutput_zero_point);
173 
174     int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
175 #endif
176     const int8x8_t voutput_min = vld1_dup_s8(&params->neon.output_min);
177     const int8x8_t voutput_max = vld1_dup_s8(&params->neon.output_max);
178 
179     vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
180 
181     vout0x01234567 = vmin_s8(vout0x01234567, voutput_max);
182 
183     if (nc >= 8) {
184       vst1_s8(c0 + 0, vout0x01234567);
185 
186       c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
187 
188       a0 = (const int8_t*) ((uintptr_t) a0 - kc);
189 
190       nc -= 8;
191     } else {
192       if (nc & 4) {
193         vst1_lane_u32(__builtin_assume_aligned(c0, 1), vreinterpret_u32_s8(vout0x01234567), 0); c0 += 4;
194         vout0x01234567 = vext_s8(vout0x01234567, vout0x01234567, 4);
195       }
196       if (nc & 2) {
197         vst1_lane_u16(__builtin_assume_aligned(c0, 1), vreinterpret_u16_s8(vout0x01234567), 0); c0 += 2;
198         vout0x01234567 = vext_s8(vout0x01234567, vout0x01234567, 2);
199       }
200       if (nc & 1) {
201         vst1_lane_s8(c0, vout0x01234567, 0);
202       }
203 
204       nc = 0;
205     }
206   } while (nc != 0);
207 }
208