1// Copyright 2019 Google LLC
2//
3// This source code is licensed under the BSD-style license found in the
4// LICENSE file in the root directory of this source tree.
5
6$assert NR % 4 == 0
7$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
8$VMULADDQ_F32 = "vfmaq_f32" if FMA else "vmlaq_f32"
9$VMULADDQ_LANE_F32 = "vfmaq_lane_f32" if FMA else "vmlaq_lane_f32"
10
11#include <assert.h>
12
13#include <arm_neon.h>
14
15#include <xnnpack/gemm.h>
16
17
18void xnn_f32_gemm${"inc" if INC else ""}_minmax_ukernel_${MR}x${NR}__${"neonfma" if FMA else "neon"}_${"dup" if DUP else "lane"}_ld128(
19    size_t mr,
20    size_t nc,
21    size_t kc,
22    const float* restrict a,
23    size_t a_stride,
24    const float* restrict w,
25    float* restrict c,
26    size_t cm_stride,
27    size_t cn_stride,
28    $if INC:
29      const float*restrict acc,
30    const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
31{
32  assert(mr != 0);
33  assert(mr <= ${MR});
34  assert(nc != 0);
35  assert(kc != 0);
36  assert(kc % sizeof(float) == 0);
37  assert(a != NULL);
38  assert(w != NULL);
39  assert(c != NULL);
40  $if INC:
41    assert(acc != NULL);
42
43  const float* a0 = a;
44  float* c0 = c;
45  $for M in range(1, MR):
46    const float* a${M} = (const float*) ((uintptr_t) a${M-1} + a_stride);
47    float* c${M} = (float*) ((uintptr_t) c${M-1} + cm_stride);
48    $if M % 2 == 0:
49      if XNN_UNPREDICTABLE(mr <= ${M}) {
50        a${M} = a${M-1};
51        c${M} = c${M-1};
52      }
53    $elif M + 1 == MR:
54      if XNN_UNPREDICTABLE(mr != ${M+1}) {
55        a${M} = a${M-1};
56        c${M} = c${M-1};
57      }
58    $else:
59      if XNN_UNPREDICTABLE(mr < ${M+1}) {
60        a${M} = a${M-1};
61        c${M} = c${M-1};
62      }
63
64  do {
65    $if INC:
66      $for M in range(MR):
67        $for N in range(0, NR, 4):
68          float32x4_t vacc${M}x${ABC[N:N+4]} = vld1q_f32(acc); acc += 4;
69    $else:
70      $for N in range(0, NR, 4):
71        float32x4_t vacc0x${ABC[N:N+4]} = vld1q_f32(w); w += 4;
72      $for M in range(1, MR):
73        $for N in range(0, NR, 4):
74          float32x4_t vacc${M}x${ABC[N:N+4]} = vacc0x${ABC[N:N+4]};
75
76    size_t k = kc;
77    for (; k >= 4 * sizeof(float); k -= 4 * sizeof(float)) {
78      $for M in range(MR):
79        const float32x4_t va${M} = vld1q_f32(a${M}); a${M} += 4;
80
81      $for L in range(4):
82        $VGET_PART_F32 = "vget_low_f32" if L < 2 else "vget_high_f32"
83
84        $for N in range(0, NR, 4):
85          const float32x4_t vb${ABC[N:N+4]}c${L} = vld1q_f32(w); w += 4;
86
87        $if DUP:
88          $for M in range(MR):
89            const float32x4_t va${M}c${L} = vdupq_lane_f32(${VGET_PART_F32}(va${M}), ${L % 2});
90          $for N in range(0, NR, 4):
91            $for M in range(MR):
92              vacc${M}x${ABC[N:N+4]} = ${VMULADDQ_F32}(vacc${M}x${ABC[N:N+4]}, va${M}c${L}, vb${ABC[N:N+4]}c${L});
93        $else:
94          $for N in range(0, NR, 4):
95            $for M in range(MR):
96              vacc${M}x${ABC[N:N+4]} = ${VMULADDQ_LANE_F32}(vacc${M}x${ABC[N:N+4]}, vb${ABC[N:N+4]}c${L}, ${VGET_PART_F32}(va${M}), ${L % 2});
97    }
98    if XNN_UNLIKELY(k != 0) {
99      do {
100        $for M in range(MR):
101          const float32x4_t va${M} = vld1q_dup_f32(a${M}); a${M} += 1;
102
103        $for N in range(0, NR, 4):
104          const float32x4_t vb${ABC[N:N+4]} = vld1q_f32(w); w += 4;
105
106        $for N in range(0, NR, 4):
107          $for M in range(MR):
108            vacc${M}x${ABC[N:N+4]} = ${VMULADDQ_F32}(vacc${M}x${ABC[N:N+4]}, va${M}, vb${ABC[N:N+4]});
109
110        k -= sizeof(float);
111      } while (k != 0);
112    }
113    const float32x4_t vmax = vld1q_dup_f32(&params->scalar.max);
114    $for N in range(0, NR, 4):
115      $for M in range(MR):
116        vacc${M}x${ABC[N:N+4]} = vminq_f32(vacc${M}x${ABC[N:N+4]}, vmax);
117
118    const float32x4_t vmin = vld1q_dup_f32(&params->scalar.min);
119    $for N in range(0, NR, 4):
120      $for M in range(MR):
121        vacc${M}x${ABC[N:N+4]} = vmaxq_f32(vacc${M}x${ABC[N:N+4]}, vmin);
122
123    if XNN_LIKELY(nc >= ${NR}) {
124      $for M in reversed(range(MR)):
125        vst1q_f32(c${M}, vacc${M}x${ABC[0:4]});
126        $for N in range(4, NR, 4):
127          vst1q_f32(c${M} + ${N}, vacc${M}x${ABC[N:N+4]});
128        c${M} = (float*) ((uintptr_t) c${M} + cn_stride);
129
130      $for M in reversed(range(MR)):
131        a${M} = (const float*) ((uintptr_t) a${M} - kc);
132
133      nc -= ${NR};
134
135    } else {
136      $for LOG2N in reversed(range(NR.bit_length())):
137        $if NR != 1 << LOG2N:
138          if (nc & ${1 << LOG2N}) {
139            $if LOG2N >= 2:
140              $for N in range(0, 1 << LOG2N, 4):
141                $for M in reversed(range(MR)):
142                  vst1q_f32(c${M}, vacc${M}x${ABC[N:N+4]}); c${M} += 4;
143
144              $for M in reversed(range(MR)):
145                $for N in range(0, 1 << (LOG2N - 1), 4):
146                  vacc${M}x${ABC[N:N+4]} = vacc${M}x${ABC[N + (1 << LOG2N):N + (1 <<   LOG2N)+4]};
147            $elif LOG2N == 1:
148              $for M in reversed(range(MR)):
149                vst1_f32(c${M}, vacc${M}x${ABC[0:2]}); c${M} += 2;
150
151              $for M in reversed(range(MR)):
152                vacc${M}x${ABC[0:2]} = vget_high_f32(vacc${M}x${ABC[0:4]});
153            $elif LOG2N == 0:
154              $for M in reversed(range(MR)):
155                vst1_lane_f32(c${M}, vacc${M}x${ABC[0:2]}, 0);
156          }
157          $if LOG2N == 2:
158            $for M in reversed(range(MR)):
159              float32x2_t vacc${M}x${ABC[0:2]} = vget_low_f32(vacc${M}x${ABC[0:4]});
160
161      nc = 0;
162    }
163  } while (nc != 0);
164}