1// Copyright 2019 Google LLC
2//
3// This source code is licensed under the BSD-style license found in the
4// LICENSE file in the root directory of this source tree.
5
6$assert NR % 4 == 0
7$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
8$SSE_HEADER = {1: "xmmintrin.h", 2: "emmintrin.h"}[SSE]
9#include <assert.h>
10
11#include <${SSE_HEADER}>
12
13#include <xnnpack/gemm.h>
14
15
16$ISA = {1: "sse", 2: "sse2"}[SSE]
17void xnn_f32_gemm${"inc" if INC else ""}_minmax_ukernel_${MR}x${NR}__${ISA}_dup(
18    size_t mr,
19    size_t nc,
20    size_t kc,
21    const float*restrict a,
22    size_t a_stride,
23    const float*restrict w,
24    float*restrict c,
25    size_t cm_stride,
26    size_t cn_stride,
27    $if INC:
28      const float*restrict acc,
29    const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
30{
31  assert(mr != 0);
32  assert(mr <= ${MR});
33  assert(nc != 0);
34  assert(kc != 0);
35  assert(kc % sizeof(float) == 0);
36  assert(a != NULL);
37  assert(w != NULL);
38  assert(c != NULL);
39  $if INC:
40    assert(acc != NULL);
41
42  const float* a0 = a;
43  float* c0 = c;
44  $for M in range(1, MR):
45    const float* a${M} = (const float*) ((uintptr_t) a${M-1} + a_stride);
46    float* c${M} = (float*) ((uintptr_t) c${M-1} + cm_stride);
47    $if M % 2 == 0:
48      if XNN_UNPREDICTABLE(mr <= ${M}) {
49        a${M} = a${M-1};
50        c${M} = c${M-1};
51      }
52    $elif M + 1 == MR:
53      if XNN_UNPREDICTABLE(mr != ${M+1}) {
54        a${M} = a${M-1};
55        c${M} = c${M-1};
56      }
57    $else:
58      if XNN_UNPREDICTABLE(mr < ${M+1}) {
59        a${M} = a${M-1};
60        c${M} = c${M-1};
61      }
62
63  do {
64    $if INC:
65      $for M in range(MR):
66        $for N in range(0, NR, 4):
67          __m128 vacc${M}x${ABC[N:N+4]} = _mm_load_ps(acc + ${M*NR+N});
68      acc += ${MR*NR};
69    $else:
70      $for N in range(0, NR, 4):
71        __m128 vacc0x${ABC[N:N+4]} = _mm_load_ps(w + ${N});
72      $for M in range(1, MR):
73        $for N in range(0, NR, 4):
74          __m128 vacc${M}x${ABC[N:N+4]} = vacc0x${ABC[N:N+4]};
75      w += ${NR};
76
77    size_t k = kc;
78    while (k >= 4 * sizeof(float)) {
79      $for M in range(MR):
80        const __m128 va${M} = _mm_loadu_ps(a${M});
81        a${M} += 4;
82
83      $for L in range(4):
84        $LLLL = str(L) * 4
85
86        $for M in range(MR):
87          $if SSE >= 2 and L < 3:
88            const __m128 va${M}c${LLLL} = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(va${M}), _MM_SHUFFLE(${L}, ${L}, ${L}, ${L})));
89          $else:
90            const __m128 va${M}c${LLLL} = _mm_shuffle_ps(va${M}, va${M}, _MM_SHUFFLE(${L}, ${L}, ${L}, ${L}));
91
92        $for N in range(0, NR, 4):
93          const __m128 vb${ABC[N:N+4]}c${L} = _mm_load_ps(w + ${L * NR + N});
94
95        $for N in range(0, NR, 4):
96          $for M in range(MR):
97            vacc${M}x${ABC[N:N+4]} = _mm_add_ps(vacc${M}x${ABC[N:N+4]}, _mm_mul_ps(va${M}c${LLLL}, vb${ABC[N:N+4]}c${L}));
98
99      w += ${4 * NR};
100      k -= 4 * sizeof(float);
101    }
102    if XNN_UNLIKELY(k != 0) {
103      do {
104        $for M in range(MR):
105          const __m128 va${M} = _mm_load1_ps(a${M});
106          a${M} += 1;
107
108        const __m128 vb${ABC[0:4]} = _mm_load_ps(w);
109        $for N in range(4, NR, 4):
110          const __m128 vb${ABC[N:N+4]} = _mm_load_ps(w + ${N});
111        w += ${NR};
112
113        $for N in range(0, NR, 4):
114          $for M in range(MR):
115            vacc${M}x${ABC[N:N+4]} = _mm_add_ps(vacc${M}x${ABC[N:N+4]}, _mm_mul_ps(va${M}, vb${ABC[N:N+4]}));
116
117        k -= sizeof(float);
118      } while (k != 0);
119    }
120
121    const __m128 vmax = _mm_load_ps(params->sse.max);
122    $for N in range(0, NR, 4):
123      $for M in range(MR):
124        vacc${M}x${ABC[N:N+4]} = _mm_min_ps(vacc${M}x${ABC[N:N+4]}, vmax);
125
126    const __m128 vmin = _mm_load_ps(params->sse.min);
127    $for N in range(0, NR, 4):
128      $for M in range(MR):
129        vacc${M}x${ABC[N:N+4]} = _mm_max_ps(vacc${M}x${ABC[N:N+4]}, vmin);
130
131    if XNN_LIKELY(nc >= ${NR}) {
132      $for M in reversed(range(MR)):
133        _mm_storeu_ps(c${M}, vacc${M}x${ABC[0:4]});
134        $for N in range(4, NR, 4):
135          _mm_storeu_ps(c${M} + ${N}, vacc${M}x${ABC[N:N+4]});
136        c${M} = (float*) ((uintptr_t) c${M} + cn_stride);
137
138      $for M in reversed(range(MR)):
139        a${M} = (const float*) ((uintptr_t) a${M} - kc);
140
141      nc -= ${NR};
142    } else {
143      $for LOG2N in reversed(range(NR.bit_length())):
144        $if NR != 1 << LOG2N:
145          if (nc & ${1 << LOG2N}) {
146            $if LOG2N >= 2:
147              $for M in reversed(range(MR)):
148                _mm_storeu_ps(c${M}, vacc${M}x${ABC[0:4]});
149                $for N in range(4, 1 << LOG2N, 4):
150                  _mm_storeu_ps(c${M} + ${N}, vacc${M}x${ABC[N:N+4]});
151
152              $for M in reversed(range(MR)):
153                $for N in range(0, 1 << (LOG2N - 1), 4):
154                  vacc${M}x${ABC[N:N+4]} = vacc${M}x${ABC[N + (1 << LOG2N):N + (1 << LOG2N)+4]};
155
156              $for M in reversed(range(MR)):
157                c${M} += ${1 << LOG2N};
158            $elif LOG2N == 1:
159              $for M in reversed(range(MR)):
160                _mm_storel_pi((__m64*) c${M}, vacc${M}x${ABC[0:4]});
161
162              $for M in reversed(range(MR)):
163                vacc${M}x${ABC[0:4]} = _mm_movehl_ps(vacc${M}x${ABC[0:4]}, vacc${M}x${ABC[0:4]});
164
165              $for M in reversed(range(MR)):
166                c${M} += 2;
167            $elif LOG2N == 0:
168              $for M in reversed(range(MR)):
169                _mm_store_ss(c${M}, vacc${M}x${ABC[0:4]});
170          }
171
172      nc = 0;
173    }
174  } while (nc != 0);
175}
176