1// Copyright 2020 Google LLC
2//
3// This source code is licensed under the BSD-style license found in the
4// LICENSE file in the root directory of this source tree.
5
6$assert BATCH_TILE % 8 == 0
7$assert BATCH_TILE >= 8
8$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
9#include <assert.h>
10
11#include <arm_neon.h>
12
13#include <xnnpack/vadd.h>
14
15#include <stdio.h>
16#include <inttypes.h>
17
18
19void xnn_qs8_vaddc_minmax_ukernel__neon_ld64_x${BATCH_TILE}(
20    size_t n,
21    const int8_t* input_x,
22    const int8_t* input_y,
23    int8_t* output,
24    const union xnn_qs8_add_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
25{
26  const int8x8_t vx_zero_point = vld1_dup_s8(&params->neon.x_zero_point);
27  const int32x4_t vx_multiplier = vld1q_dup_s32(&params->neon.x_multiplier);
28  const int32x4_t vright_shift = vld1q_dup_s32(&params->neon.right_shift);
29  const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
30  const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->neon.output_zero_point);
31  const int8x16_t voutput_min = vld1q_dup_s8(&params->neon.output_min);
32  const int8x16_t voutput_max = vld1q_dup_s8(&params->neon.output_max);
33
34  const int32_t vey = (int32_t) *input_y - (int32_t) params->neon.y_zero_point;
35  const int32_t vy_multiplier = params->neon.y_multiplier;
36  const int32x4_t vy_bias = vdupq_n_s32(vey * vy_multiplier);
37
38  for (; n >= ${BATCH_TILE} * sizeof(int8_t); n -= ${BATCH_TILE} * sizeof(int8_t)) {
39    $for N in range(0, BATCH_TILE, 8):
40      const int8x8_t vx${ABC[N:N+8]} = vld1_s8(input_x); input_x += 8;
41
42    $for N in range(0, BATCH_TILE, 8):
43      const int16x8_t vex${ABC[N:N+8]} = vsubl_s8(vx${ABC[N:N+8]}, vx_zero_point);
44
45    $for N in range(0, BATCH_TILE, 8):
46      int32x4_t vacc${ABC[N:N+4]} = vmlaq_s32(vy_bias, vmovl_s16(vget_low_s16(vex${ABC[N:N+8]})), vx_multiplier);
47      int32x4_t vacc${ABC[N+4:N+8]} = vmlaq_s32(vy_bias, vmovl_s16(vget_high_s16(vex${ABC[N:N+8]})), vx_multiplier);
48
49    $for N in range(0, BATCH_TILE, 4):
50      vacc${ABC[N:N+4]} = vsraq_n_s32(vacc${ABC[N:N+4]}, vbicq_s32(vacc${ABC[N:N+4]}, vzero_shift_mask), 31);
51
52    $for N in range(0, BATCH_TILE, 4):
53      vacc${ABC[N:N+4]} = vrshlq_s32(vacc${ABC[N:N+4]}, vright_shift);
54
55    $for N in range(0, BATCH_TILE, 8):
56      const int16x8_t vacc${ABC[N:N+8]} = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc${ABC[N:N+4]}), vqmovn_s32(vacc${ABC[N+4:N+8]})), voutput_zero_point);
57
58    $for N in range(0, BATCH_TILE, 16):
59      $if N + 8 < BATCH_TILE:
60        int8x16_t vout${ABC[N:N+16]} = vcombine_s8(vqmovn_s16(vacc${ABC[N:N+8]}), vqmovn_s16(vacc${ABC[N+8:N+16]}));
61      $else:
62        int8x8_t vout${ABC[N:N+8]} = vqmovn_s16(vacc${ABC[N:N+8]});
63
64    $for N in range(0, BATCH_TILE, 16):
65      $if N + 8 < BATCH_TILE:
66        vout${ABC[N:N+16]} = vmaxq_s8(vout${ABC[N:N+16]}, voutput_min);
67      $else:
68        vout${ABC[N:N+8]} = vmax_s8(vout${ABC[N:N+8]}, vget_low_s8(voutput_min));
69
70    $for N in range(0, BATCH_TILE, 16):
71      $if N + 8 < BATCH_TILE:
72        vout${ABC[N:N+16]} = vminq_s8(vout${ABC[N:N+16]}, voutput_max);
73      $else:
74        vout${ABC[N:N+8]} = vmin_s8(vout${ABC[N:N+8]}, vget_low_s8(voutput_max));
75
76    $for N in range(0, BATCH_TILE, 16):
77      $if N + 8 < BATCH_TILE:
78        vst1q_s8(output, vout${ABC[N:N+16]}); output += 16;
79      $else:
80        vst1_s8(output, vout${ABC[N:N+8]}); output += 8;
81  }
82  if XNN_UNLIKELY(n != 0) {
83    ${"do " if BATCH_TILE > 8 else ""}{
84      $if BATCH_TILE > 8:
85        const int8x8_t vx${ABC[0:8]} = vld1_s8(input_x); input_x += 8;
86      $else:
87        const int8x8_t vx${ABC[0:8]} = vld1_s8(input_x);
88
89      const int16x8_t vex${ABC[0:8]} = vsubl_s8(vx${ABC[0:8]}, vx_zero_point);
90
91      int32x4_t vacc${ABC[0:4]} = vmlaq_s32(vy_bias, vmovl_s16(vget_low_s16(vex${ABC[0:8]})), vx_multiplier);
92      int32x4_t vacc${ABC[4:8]} = vmlaq_s32(vy_bias, vmovl_s16(vget_high_s16(vex${ABC[0:8]})), vx_multiplier);
93
94      vacc${ABC[0:4]} = vsraq_n_s32(vacc${ABC[0:4]}, vbicq_s32(vacc${ABC[0:4]}, vzero_shift_mask), 31);
95      vacc${ABC[4:8]} = vsraq_n_s32(vacc${ABC[4:8]}, vbicq_s32(vacc${ABC[4:8]}, vzero_shift_mask), 31);
96
97      vacc${ABC[0:4]} = vrshlq_s32(vacc${ABC[0:4]}, vright_shift);
98      vacc${ABC[4:8]} = vrshlq_s32(vacc${ABC[4:8]}, vright_shift);
99
100      const int16x8_t vacc${ABC[0:8]} = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc${ABC[0:4]}), vqmovn_s32(vacc${ABC[4:8]})), voutput_zero_point);
101
102      int8x8_t vout${ABC[0:8]} = vqmovn_s16(vacc${ABC[0:8]});
103      vout${ABC[0:8]} = vmax_s8(vout${ABC[0:8]}, vget_low_s8(voutput_min));
104      vout${ABC[0:8]} = vmin_s8(vout${ABC[0:8]}, vget_low_s8(voutput_max));
105
106      $if BATCH_TILE > 8:
107        if XNN_LIKELY(n >= (8 * sizeof(int8_t))) {
108          vst1_s8(output, vout${ABC[0:8]}); output += 8;
109          n -= 8 * sizeof(int8_t);
110        } else {
111          if (n & (4 * sizeof(int8_t))) {
112            vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout${ABC[0:8]}), 0); output += 4;
113            vout${ABC[0:8]} = vext_s8(vout${ABC[0:8]}, vout${ABC[0:8]}, 4);
114          }
115          if (n & (2 * sizeof(int8_t))) {
116            vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout${ABC[0:8]}), 0); output += 2;
117            vout${ABC[0:8]} = vext_s8(vout${ABC[0:8]}, vout${ABC[0:8]}, 2);
118          }
119          if (n & (1 * sizeof(int8_t))) {
120            vst1_lane_s8(output, vout${ABC[0:8]}, 0);
121          }
122          n = 0;
123        }
124      $else:
125        if (n & (4 * sizeof(int8_t))) {
126          vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout${ABC[0:8]}), 0); output += 4;
127          vout${ABC[0:8]} = vext_s8(vout${ABC[0:8]}, vout${ABC[0:8]}, 4);
128        }
129        if (n & (2 * sizeof(int8_t))) {
130          vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout${ABC[0:8]}), 0); output += 2;
131          vout${ABC[0:8]} = vext_s8(vout${ABC[0:8]}, vout${ABC[0:8]}, 2);
132        }
133        if (n & (1 * sizeof(int8_t))) {
134          vst1_lane_s8(output, vout${ABC[0:8]}, 0);
135        }
136    }${" while (n != 0);" if BATCH_TILE > 8 else ""}
137  }
138}
139