1// Copyright 2019 Google LLC
2//
3// This source code is licensed under the BSD-style license found in the
4// LICENSE file in the root directory of this source tree.
5
6$assert NR % 4 == 0
7$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
8$VMULADDQ_F32 = "vfmaq_f32" if FMA else "vmlaq_f32"
9$VMULADDQ_LANE_F32 = "vfmaq_lane_f32" if FMA else "vmlaq_lane_f32"
10
11#include <assert.h>
12
13#include <arm_neon.h>
14
15#include <xnnpack/gemm.h>
16
17
18void xnn_f32_gemm${"inc" if INC else ""}_minmax_ukernel_${MR}x${NR}__${"neonfma" if FMA else "neon"}_${"dup" if DUP else "lane"}_ld64(
19    size_t mr,
20    size_t nc,
21    size_t kc,
22    const float* restrict a,
23    size_t a_stride,
24    const float* restrict w,
25    float* restrict c,
26    size_t cm_stride,
27    size_t cn_stride,
28    $if INC:
29      const float*restrict acc,
30    const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
31{
32  assert(mr != 0);
33  assert(mr <= ${MR});
34  assert(nc != 0);
35  assert(kc != 0);
36  assert(kc % sizeof(float) == 0);
37  assert(a != NULL);
38  assert(w != NULL);
39  assert(c != NULL);
40  $if INC:
41    assert(acc != NULL);
42
43  const float* a0 = a;
44  float* c0 = c;
45  $for M in range(1, MR):
46    const float* a${M} = (const float*) ((uintptr_t) a${M-1} + a_stride);
47    float* c${M} = (float*) ((uintptr_t) c${M-1} + cm_stride);
48    $if M % 2 == 0:
49      if XNN_UNPREDICTABLE(mr <= ${M}) {
50        a${M} = a${M-1};
51        c${M} = c${M-1};
52      }
53    $elif M + 1 == MR:
54      if XNN_UNPREDICTABLE(mr != ${M+1}) {
55        a${M} = a${M-1};
56        c${M} = c${M-1};
57      }
58    $else:
59      if XNN_UNPREDICTABLE(mr < ${M+1}) {
60        a${M} = a${M-1};
61        c${M} = c${M-1};
62      }
63
64  do {
65    $if INC:
66      $for M in range(MR):
67        $for N in range(0, NR, 4):
68          float32x4_t vacc${M}x${ABC[N:N+4]} = vld1q_f32(acc); acc += 4;
69    $else:
70      $for N in range(0, NR, 4):
71        float32x4_t vacc0x${ABC[N:N+4]} = vld1q_f32(w); w += 4;
72      $for M in range(1, MR):
73        $for N in range(0, NR, 4):
74          float32x4_t vacc${M}x${ABC[N:N+4]} = vacc0x${ABC[N:N+4]};
75
76    size_t k = kc;
77    for (; k >= 2 * sizeof(float); k -= 2 * sizeof(float)) {
78      $for M in range(MR):
79        const float32x2_t va${M} = vld1_f32(a${M}); a${M} += 2;
80
81      $for L in range(2):
82        $for N in range(0, NR, 4):
83          const float32x4_t vb${ABC[N:N+4]}c${L} = vld1q_f32(w); w += 4;
84
85        $if DUP:
86          $for M in range(MR):
87            const float32x4_t va${M}c${L} = vdupq_lane_f32(va${M}, ${L});
88          $for N in range(0, NR, 4):
89            $for M in range(MR):
90              vacc${M}x${ABC[N:N+4]} = ${VMULADDQ_F32}(vacc${M}x${ABC[N:N+4]}, va${M}c${L}, vb${ABC[N:N+4]}c${L});
91        $else:
92           $for N in range(0, NR, 4):
93             $for M in range(MR):
94               vacc${M}x${ABC[N:N+4]} = ${VMULADDQ_LANE_F32}(vacc${M}x${ABC[N:N+4]}, vb${ABC[N:N+4]}c${L}, va${M}, ${L});
95    }
96    if XNN_UNLIKELY(k != 0) {
97      $for M in range(MR):
98        const float32x4_t va${M} = vld1q_dup_f32(a${M}); a${M} += 1;
99
100      $for N in range(0, NR, 4):
101        const float32x4_t vb${ABC[N:N+4]} = vld1q_f32(w); w += 4;
102
103      $for N in range(0, NR, 4):
104        $for M in range(MR):
105          vacc${M}x${ABC[N:N+4]} = ${VMULADDQ_F32}(vacc${M}x${ABC[N:N+4]}, va${M}, vb${ABC[N:N+4]});
106    }
107    const float32x4_t vmax = vld1q_dup_f32(&params->scalar.max);
108    $for N in range(0, NR, 4):
109      $for M in range(MR):
110        vacc${M}x${ABC[N:N+4]} = vminq_f32(vacc${M}x${ABC[N:N+4]}, vmax);
111
112    const float32x4_t vmin = vld1q_dup_f32(&params->scalar.min);
113    $for N in range(0, NR, 4):
114      $for M in range(MR):
115        vacc${M}x${ABC[N:N+4]} = vmaxq_f32(vacc${M}x${ABC[N:N+4]}, vmin);
116
117    if XNN_LIKELY(nc >= ${NR}) {
118      $for M in reversed(range(MR)):
119        vst1q_f32(c${M}, vacc${M}x${ABC[0:4]});
120        $for N in range(4, NR, 4):
121          vst1q_f32(c${M} + ${N}, vacc${M}x${ABC[N:N+4]});
122        c${M} = (float*) ((uintptr_t) c${M} + cn_stride);
123
124      $for M in reversed(range(MR)):
125        a${M} = (const float*) ((uintptr_t) a${M} - kc);
126
127      nc -= ${NR};
128
129    } else {
130      $for LOG2N in reversed(range(NR.bit_length())):
131        $if NR != 1 << LOG2N:
132          if (nc & ${1 << LOG2N}) {
133            $if LOG2N >= 2:
134              $for N in range(0, 1 << LOG2N, 4):
135                $for M in reversed(range(MR)):
136                  vst1q_f32(c${M}, vacc${M}x${ABC[N:N+4]}); c${M} += 4;
137
138              $for M in reversed(range(MR)):
139                $for N in range(0, 1 << (LOG2N - 1), 4):
140                  vacc${M}x${ABC[N:N+4]} = vacc${M}x${ABC[N + (1 << LOG2N):N + (1 <<   LOG2N)+4]};
141            $elif LOG2N == 1:
142              $for M in reversed(range(MR)):
143                vst1_f32(c${M}, vacc${M}x${ABC[0:2]}); c${M} += 2;
144
145              $for M in reversed(range(MR)):
146                vacc${M}x${ABC[0:2]} = vget_high_f32(vacc${M}x${ABC[0:4]});
147            $elif LOG2N == 0:
148              $for M in reversed(range(MR)):
149                vst1_lane_f32(c${M}, vacc${M}x${ABC[0:2]}, 0);
150          }
151          $if LOG2N == 2:
152            $for M in reversed(range(MR)):
153              float32x2_t vacc${M}x${ABC[0:2]} = vget_low_f32(vacc${M}x${ABC[0:4]});
154
155      nc = 0;
156    }
157  } while (nc != 0);
158}
159