1 // Auto-generated file. Do not edit!
2 // Template: src/f16-spmm/neonfp16arith.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/spmm.h>
15
16
xnn_f16_spmm_minmax_ukernel_8x1__neonfp16arith(size_t mc,size_t nc,const void * restrict input,const void * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,void * restrict output,size_t output_stride,const struct xnn_f16_scaleminmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f16_spmm_minmax_ukernel_8x1__neonfp16arith(
18 size_t mc,
19 size_t nc,
20 const void*restrict input,
21 const void*restrict weights,
22 const int32_t*restrict widx_dmap,
23 const uint32_t*restrict nidx_nnzmap,
24 void*restrict output,
25 size_t output_stride,
26 const struct xnn_f16_scaleminmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27 {
28 assert(mc != 0);
29 assert(mc % sizeof(__fp16) == 0);
30 assert(nc != 0);
31
32 const __fp16*restrict i = (const __fp16*) input;
33 __fp16*restrict o = (__fp16*) output;
34
35 const float16x8_t vscale = vld1q_dup_f16((const __fp16*) ¶ms->scale);
36 const float16x8_t vmax = vld1q_dup_f16((const __fp16*) ¶ms->max);
37 const float16x8_t vmin = vld1q_dup_f16((const __fp16*) ¶ms->min);
38
39 size_t output_decrement = output_stride * nc - 8 * sizeof(__fp16);
40 while XNN_LIKELY(mc >= 8 * sizeof(__fp16)) {
41 const __fp16*restrict w = (const __fp16*) weights;
42 const int32_t* dmap = widx_dmap;
43 const uint32_t* nnzmap = nidx_nnzmap;
44 size_t n = nc;
45 do {
46 uint32_t nnz = *nnzmap++;
47 float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
48 if XNN_LIKELY(nnz != 0) {
49 do {
50 const intptr_t diff = *dmap++;
51 const float16x8_t va01234567 = vld1q_f16(i);
52 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
53 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
54 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
55 } while (--nnz != 0);
56 }
57 float16x8_t vout01234567 = vmulq_f16(vacc01234567, vscale);
58 vout01234567 = vminq_f16(vout01234567, vmax);
59 vout01234567 = vmaxq_f16(vout01234567, vmin);
60 vst1q_f16(o, vout01234567);
61 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
62 } while (--n != 0);
63 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
64 i += 8;
65 mc -= 8 * sizeof(__fp16);
66 }
67 if XNN_UNLIKELY(mc != 0) {
68 output_decrement += 4 * sizeof(__fp16);
69 if (mc & (4 * sizeof(__fp16))) {
70 const __fp16*restrict w = (const __fp16*) weights;
71 const int32_t* dmap = widx_dmap;
72 const uint32_t* nnzmap = nidx_nnzmap;
73 size_t n = nc;
74 do {
75 uint32_t nnz = *nnzmap++;
76 float16x4_t vacc0123 = vld1_dup_f16(w); w += 1;
77 if XNN_LIKELY(nnz != 0) {
78 do {
79 const intptr_t diff = *dmap++;
80 const float16x4_t va0123 = vld1_f16(i);
81 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
82 const float16x4_t vb = vld1_dup_f16(w); w += 1;
83 vacc0123 = vfma_f16(vacc0123, va0123, vb);
84 } while (--nnz != 0);
85 }
86 float16x4_t vout0123 = vmin_f16(vacc0123, vget_low_f16(vmax));
87 vout0123 = vmax_f16(vout0123, vget_low_f16(vmin));
88 vst1_f16(o, vout0123);
89 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
90 } while (--n != 0);
91 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
92 i += 4;
93 }
94 output_decrement += 2 * sizeof(__fp16);
95 if (mc & (2 * sizeof(__fp16))) {
96 const __fp16*restrict w = (const __fp16*) weights;
97 const int32_t* dmap = widx_dmap;
98 const uint32_t* nnzmap = nidx_nnzmap;
99 size_t n = nc;
100 do {
101 uint32_t nnz = *nnzmap++;
102 float16x4_t vacc01 = vld1_dup_f16(w); w += 1;
103 if XNN_LIKELY(nnz != 0) {
104 do {
105 const intptr_t diff = *dmap++;
106 const float16x4_t va01 = vreinterpret_f16_f32(vld1_dup_f32(__builtin_assume_aligned(i, 1)));
107 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
108 const float16x4_t vb = vld1_dup_f16(w); w += 1;
109 vacc01 = vfma_f16(vacc01, va01, vb);
110 } while (--nnz != 0);
111 }
112 float16x4_t vout01 = vmin_f16(vacc01, vget_low_f16(vmax));
113 vout01 = vmax_f16(vout01, vget_low_f16(vmin));
114 vst1_lane_f32(__builtin_assume_aligned(o, 1), vreinterpret_f32_f16(vout01), 0);
115 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
116 } while (--n != 0);
117 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
118 i += 2;
119 }
120 output_decrement += 1 * sizeof(__fp16);
121 if (mc & (1 * sizeof(__fp16))) {
122 const __fp16*restrict w = (const __fp16*) weights;
123 const int32_t* dmap = widx_dmap;
124 const uint32_t* nnzmap = nidx_nnzmap;
125 size_t n = nc;
126 do {
127 uint32_t nnz = *nnzmap++;
128 float16x4_t vacc0 = vld1_dup_f16(w); w += 1;
129 if XNN_LIKELY(nnz != 0) {
130 do {
131 const intptr_t diff = *dmap++;
132 const float16x4_t va0 = vld1_dup_f16(i);
133 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
134 const float16x4_t vb = vld1_dup_f16(w); w += 1;
135 vacc0 = vfma_f16(vacc0, va0, vb);
136 } while (--nnz != 0);
137 }
138 float16x4_t vout0 = vmin_f16(vacc0, vget_low_f16(vmax));
139 vout0 = vmax_f16(vout0, vget_low_f16(vmin));
140 vst1_lane_f16(o, vout0, 0);
141 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
142 } while (--n != 0);
143 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
144 i += 1;
145 }
146 }
147 }
148