1// Copyright 2019 Google LLC
2//
3// This source code is licensed under the BSD-style license found in the
4// LICENSE file in the root directory of this source tree.
5
6$assert NR % 4 == 0
7$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
8$VMULADDQ_F32 = "vfmaq_f32" if FMA else "vmlaq_f32"
9
10#include <assert.h>
11
12#include <arm_neon.h>
13
14#include <xnnpack/gemm.h>
15
16
17void xnn_f32_gemm${"inc" if INC else ""}_minmax_ukernel_${MR}x${NR}s4__${"neonfma" if FMA else "neon"}(
18    size_t mr,
19    size_t nc,
20    size_t kc,
21    const float* restrict a,
22    size_t a_stride,
23    const float* restrict w,
24    float* restrict c,
25    size_t cm_stride,
26    size_t cn_stride,
27    $if INC:
28      const float*restrict acc,
29    const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
30{
31  assert(mr != 0);
32  assert(mr <= ${MR});
33  assert(nc != 0);
34  assert(kc != 0);
35  assert(kc % sizeof(float) == 0);
36  assert(a != NULL);
37  assert(w != NULL);
38  assert(c != NULL);
39  $if INC:
40    assert(acc != NULL);
41
42  const float* a0 = a;
43  float* c0 = c;
44  $for M in range(1, MR):
45    const float* a${M} = (const float*) ((uintptr_t) a${M-1} + a_stride);
46    float* c${M} = (float*) ((uintptr_t) c${M-1} + cm_stride);
47    $if M % 2 == 0:
48      if XNN_UNPREDICTABLE(mr <= ${M}) {
49        a${M} = a${M-1};
50        c${M} = c${M-1};
51      }
52    $elif M + 1 == MR:
53      if XNN_UNPREDICTABLE(mr != ${M+1}) {
54        a${M} = a${M-1};
55        c${M} = c${M-1};
56      }
57    $else:
58      if XNN_UNPREDICTABLE(mr < ${M+1}) {
59        a${M} = a${M-1};
60        c${M} = c${M-1};
61      }
62
63  do {
64    $if INC:
65      $for M in range(MR):
66        $for N in range(0, NR, 4):
67          float32x4_t vacc${M}x${ABC[N:N+4]} = vld1q_f32(acc); acc += 4;
68    $else:
69      $for N in range(0, NR, 4):
70        float32x4_t vacc0x${ABC[N:N+4]} = vld1q_f32(w); w += 4;
71      $for M in range(1, MR):
72        $for N in range(0, NR, 4):
73          float32x4_t vacc${M}x${ABC[N:N+4]} = vacc0x${ABC[N:N+4]};
74
75    size_t k = kc;
76    while (k >= 4 * sizeof(float)) {
77      $for M in range(MR):
78        float32x4_t va${M} = vld1q_f32(a${M}); a${M} += 4;
79
80      $for L in range(4):
81
82        $for N in range(0, NR, 4):
83          const float32x4_t vb${ABC[N:N+4]}c${L} = vld1q_f32(w + ${L * NR + N});
84
85        $for N in range(0, NR, 4):
86          $for M in range(MR):
87            vacc${M}x${ABC[N:N+4]} = ${VMULADDQ_F32}(vacc${M}x${ABC[N:N+4]}, va${M}, vb${ABC[N:N+4]}c${L});
88
89        $if L + 1 != 4:
90          $for M in range(MR):
91            va${M} = vextq_f32(va${M}, va${M}, 1);
92
93      w += ${4 * NR};
94      k -= 4 * sizeof(float);
95    }
96    if XNN_UNLIKELY(k != 0) {
97      do {
98        $for M in range(MR):
99          const float32x4_t va${M} = vld1q_dup_f32(a${M}); a${M} += 1;
100
101        $for N in range(0, NR, 4):
102          const float32x4_t vb${ABC[N:N+4]} = vld1q_f32(w); w += 4;
103
104        $for N in range(0, NR, 4):
105          $for M in range(MR):
106            vacc${M}x${ABC[N:N+4]} = ${VMULADDQ_F32}(vacc${M}x${ABC[N:N+4]}, va${M}, vb${ABC[N:N+4]});
107
108        k -= sizeof(float);
109      } while (k != 0);
110    }
111    const float32x4_t vmax = vld1q_dup_f32(&params->scalar.max);
112    $for N in range(0, NR, 4):
113      $for M in range(MR):
114        vacc${M}x${ABC[N:N+4]} = vminq_f32(vacc${M}x${ABC[N:N+4]}, vmax);
115
116    const float32x4_t vmin = vld1q_dup_f32(&params->scalar.min);
117    $for N in range(0, NR, 4):
118      $for M in range(MR):
119        vacc${M}x${ABC[N:N+4]} = vmaxq_f32(vacc${M}x${ABC[N:N+4]}, vmin);
120
121    if XNN_LIKELY(nc >= ${NR}) {
122      $for M in reversed(range(MR)):
123        vst1q_f32(c${M}, vacc${M}x${ABC[0:4]});
124        $for N in range(4, NR, 4):
125          vst1q_f32(c${M} + ${N}, vacc${M}x${ABC[N:N+4]});
126        c${M} = (float*) ((uintptr_t) c${M} + cn_stride);
127
128      $for M in reversed(range(MR)):
129        a${M} = (const float*) ((uintptr_t) a${M} - kc);
130
131      nc -= ${NR};
132
133    } else {
134      $for LOG2N in reversed(range(NR.bit_length())):
135        $if NR != 1 << LOG2N:
136          if (nc & ${1 << LOG2N}) {
137            $if LOG2N >= 2:
138              $for N in range(0, 1 << LOG2N, 4):
139                $for M in reversed(range(MR)):
140                  vst1q_f32(c${M}, vacc${M}x${ABC[N:N+4]}); c${M} += 4;
141
142              $for M in reversed(range(MR)):
143                $for N in range(0, 1 << (LOG2N - 1), 4):
144                  vacc${M}x${ABC[N:N+4]} = vacc${M}x${ABC[N + (1 << LOG2N):N + (1 <<   LOG2N)+4]};
145            $elif LOG2N == 1:
146              $for M in reversed(range(MR)):
147                vst1_f32(c${M}, vacc${M}x${ABC[0:2]}); c${M} += 2;
148
149              $for M in reversed(range(MR)):
150                vacc${M}x${ABC[0:2]} = vget_high_f32(vacc${M}x${ABC[0:4]});
151            $elif LOG2N == 0:
152              $for M in reversed(range(MR)):
153                vst1_lane_f32(c${M}, vacc${M}x${ABC[0:2]}, 0);
154          }
155          $if LOG2N == 2:
156            $for M in reversed(range(MR)):
157              float32x2_t vacc${M}x${ABC[0:2]} = vget_low_f32(vacc${M}x${ABC[0:4]});
158
159      nc = 0;
160    }
161  } while (nc != 0);
162}