1// Copyright 2020 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert BATCH_TILE % 8 == 0 7$assert BATCH_TILE >= 8 8$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 9#include <assert.h> 10 11#include <arm_neon.h> 12 13#include <xnnpack/vadd.h> 14 15#include <stdio.h> 16#include <inttypes.h> 17 18 19void xnn_qs8_vaddc_minmax_ukernel__neon_ld64_x${BATCH_TILE}( 20 size_t n, 21 const int8_t* input_x, 22 const int8_t* input_y, 23 int8_t* output, 24 const union xnn_qs8_add_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN 25{ 26 const int8x8_t vx_zero_point = vld1_dup_s8(¶ms->neon.x_zero_point); 27 const int32x4_t vx_multiplier = vld1q_dup_s32(¶ms->neon.x_multiplier); 28 const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift); 29 const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0))); 30 const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point); 31 const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min); 32 const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max); 33 34 const int32_t vey = (int32_t) *input_y - (int32_t) params->neon.y_zero_point; 35 const int32_t vy_multiplier = params->neon.y_multiplier; 36 const int32x4_t vy_bias = vdupq_n_s32(vey * vy_multiplier); 37 38 for (; n >= ${BATCH_TILE} * sizeof(int8_t); n -= ${BATCH_TILE} * sizeof(int8_t)) { 39 $for N in range(0, BATCH_TILE, 8): 40 const int8x8_t vx${ABC[N:N+8]} = vld1_s8(input_x); input_x += 8; 41 42 $for N in range(0, BATCH_TILE, 8): 43 const int16x8_t vex${ABC[N:N+8]} = vsubl_s8(vx${ABC[N:N+8]}, vx_zero_point); 44 45 $for N in range(0, BATCH_TILE, 8): 46 int32x4_t vacc${ABC[N:N+4]} = vmlaq_s32(vy_bias, vmovl_s16(vget_low_s16(vex${ABC[N:N+8]})), vx_multiplier); 47 int32x4_t vacc${ABC[N+4:N+8]} = vmlaq_s32(vy_bias, vmovl_s16(vget_high_s16(vex${ABC[N:N+8]})), vx_multiplier); 48 49 $for N in range(0, BATCH_TILE, 4): 50 vacc${ABC[N:N+4]} = vsraq_n_s32(vacc${ABC[N:N+4]}, vbicq_s32(vacc${ABC[N:N+4]}, vzero_shift_mask), 31); 51 52 $for N in range(0, BATCH_TILE, 4): 53 vacc${ABC[N:N+4]} = vrshlq_s32(vacc${ABC[N:N+4]}, vright_shift); 54 55 $for N in range(0, BATCH_TILE, 8): 56 const int16x8_t vacc${ABC[N:N+8]} = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc${ABC[N:N+4]}), vqmovn_s32(vacc${ABC[N+4:N+8]})), voutput_zero_point); 57 58 $for N in range(0, BATCH_TILE, 16): 59 $if N + 8 < BATCH_TILE: 60 int8x16_t vout${ABC[N:N+16]} = vcombine_s8(vqmovn_s16(vacc${ABC[N:N+8]}), vqmovn_s16(vacc${ABC[N+8:N+16]})); 61 $else: 62 int8x8_t vout${ABC[N:N+8]} = vqmovn_s16(vacc${ABC[N:N+8]}); 63 64 $for N in range(0, BATCH_TILE, 16): 65 $if N + 8 < BATCH_TILE: 66 vout${ABC[N:N+16]} = vmaxq_s8(vout${ABC[N:N+16]}, voutput_min); 67 $else: 68 vout${ABC[N:N+8]} = vmax_s8(vout${ABC[N:N+8]}, vget_low_s8(voutput_min)); 69 70 $for N in range(0, BATCH_TILE, 16): 71 $if N + 8 < BATCH_TILE: 72 vout${ABC[N:N+16]} = vminq_s8(vout${ABC[N:N+16]}, voutput_max); 73 $else: 74 vout${ABC[N:N+8]} = vmin_s8(vout${ABC[N:N+8]}, vget_low_s8(voutput_max)); 75 76 $for N in range(0, BATCH_TILE, 16): 77 $if N + 8 < BATCH_TILE: 78 vst1q_s8(output, vout${ABC[N:N+16]}); output += 16; 79 $else: 80 vst1_s8(output, vout${ABC[N:N+8]}); output += 8; 81 } 82 if XNN_UNLIKELY(n != 0) { 83 ${"do " if BATCH_TILE > 8 else ""}{ 84 $if BATCH_TILE > 8: 85 const int8x8_t vx${ABC[0:8]} = vld1_s8(input_x); input_x += 8; 86 $else: 87 const int8x8_t vx${ABC[0:8]} = vld1_s8(input_x); 88 89 const int16x8_t vex${ABC[0:8]} = vsubl_s8(vx${ABC[0:8]}, vx_zero_point); 90 91 int32x4_t vacc${ABC[0:4]} = vmlaq_s32(vy_bias, vmovl_s16(vget_low_s16(vex${ABC[0:8]})), vx_multiplier); 92 int32x4_t vacc${ABC[4:8]} = vmlaq_s32(vy_bias, vmovl_s16(vget_high_s16(vex${ABC[0:8]})), vx_multiplier); 93 94 vacc${ABC[0:4]} = vsraq_n_s32(vacc${ABC[0:4]}, vbicq_s32(vacc${ABC[0:4]}, vzero_shift_mask), 31); 95 vacc${ABC[4:8]} = vsraq_n_s32(vacc${ABC[4:8]}, vbicq_s32(vacc${ABC[4:8]}, vzero_shift_mask), 31); 96 97 vacc${ABC[0:4]} = vrshlq_s32(vacc${ABC[0:4]}, vright_shift); 98 vacc${ABC[4:8]} = vrshlq_s32(vacc${ABC[4:8]}, vright_shift); 99 100 const int16x8_t vacc${ABC[0:8]} = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc${ABC[0:4]}), vqmovn_s32(vacc${ABC[4:8]})), voutput_zero_point); 101 102 int8x8_t vout${ABC[0:8]} = vqmovn_s16(vacc${ABC[0:8]}); 103 vout${ABC[0:8]} = vmax_s8(vout${ABC[0:8]}, vget_low_s8(voutput_min)); 104 vout${ABC[0:8]} = vmin_s8(vout${ABC[0:8]}, vget_low_s8(voutput_max)); 105 106 $if BATCH_TILE > 8: 107 if XNN_LIKELY(n >= (8 * sizeof(int8_t))) { 108 vst1_s8(output, vout${ABC[0:8]}); output += 8; 109 n -= 8 * sizeof(int8_t); 110 } else { 111 if (n & (4 * sizeof(int8_t))) { 112 vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout${ABC[0:8]}), 0); output += 4; 113 vout${ABC[0:8]} = vext_s8(vout${ABC[0:8]}, vout${ABC[0:8]}, 4); 114 } 115 if (n & (2 * sizeof(int8_t))) { 116 vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout${ABC[0:8]}), 0); output += 2; 117 vout${ABC[0:8]} = vext_s8(vout${ABC[0:8]}, vout${ABC[0:8]}, 2); 118 } 119 if (n & (1 * sizeof(int8_t))) { 120 vst1_lane_s8(output, vout${ABC[0:8]}, 0); 121 } 122 n = 0; 123 } 124 $else: 125 if (n & (4 * sizeof(int8_t))) { 126 vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout${ABC[0:8]}), 0); output += 4; 127 vout${ABC[0:8]} = vext_s8(vout${ABC[0:8]}, vout${ABC[0:8]}, 4); 128 } 129 if (n & (2 * sizeof(int8_t))) { 130 vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout${ABC[0:8]}), 0); output += 2; 131 vout${ABC[0:8]} = vext_s8(vout${ABC[0:8]}, vout${ABC[0:8]}, 2); 132 } 133 if (n & (1 * sizeof(int8_t))) { 134 vst1_lane_s8(output, vout${ABC[0:8]}, 0); 135 } 136 }${" while (n != 0);" if BATCH_TILE > 8 else ""} 137 } 138} 139