1// Copyright 2020 Google LLC
2//
3// This source code is licensed under the BSD-style license found in the
4// LICENSE file in the root directory of this source tree.
5
6$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
7$assert CHANNEL_TILE % 8 == 0
8$assert CHANNEL_TILE >= 8
9$assert KERNEL_TILE >= 2
10#include <assert.h>
11
12#include <wasm_simd128.h>
13
14#include <xnnpack/dwconv.h>
15
16
17void xnn_qs8_dwconv_minmax_ukernel_up${CHANNEL_TILE}x${KERNEL_TILE}__wasmsimd_mul16(
18    size_t channels,
19    size_t output_width,
20    const int8_t** input,
21    const void* weights,
22    int8_t* output,
23    size_t input_stride,
24    size_t output_increment,
25    size_t input_offset,
26    const int8_t* zero,
27    const union xnn_qs8_gemm_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
28{
29  assert(channels != 0);
30  assert(output_width != 0);
31
32  do {
33    $for K in range(KERNEL_TILE):
34      const int8_t* i${K} = input[${K}];
35      assert(i${K} != NULL);
36      if XNN_UNPREDICTABLE(i${K} != zero) {
37        i${K} = (const int8_t*) ((uintptr_t) i${K} + input_offset);
38      }
39    input = (const int8_t**) ((uintptr_t) input + input_stride);
40
41    size_t c = channels;
42    const void* w = weights;
43    for (; c >= ${CHANNEL_TILE}; c -= ${CHANNEL_TILE}) {
44      v128_t vacc${ABC[0:4]} = wasm_v128_load(w);
45      $for C in range(4, CHANNEL_TILE, 4):
46        v128_t vacc${ABC[C:C+4]} = wasm_v128_load((const void*) ((uintptr_t) w + ${C} * sizeof(int32_t)));
47
48      $for K in range(KERNEL_TILE):
49
50        $for C in range(0, CHANNEL_TILE, 8):
51          $if C == 0:
52            const v128_t vi${K}x${ABC[0:8]} = wasm_i16x8_load_8x8(i${K});
53          $else:
54            const v128_t vi${K}x${ABC[C:C+8]} = wasm_i16x8_load_8x8(i${K} + ${C});
55          const v128_t vk${K}x${ABC[C:C+8]} = wasm_i16x8_load_8x8((const void*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(int32_t) + ${K * CHANNEL_TILE + C} * sizeof(int8_t)));
56        i${K} += ${CHANNEL_TILE};
57
58        $for C in range(0, CHANNEL_TILE, 8):
59          const v128_t vprod${K}x${ABC[C:C+8]} = wasm_i16x8_mul(vi${K}x${ABC[C:C+8]}, vk${K}x${ABC[C:C+8]});
60
61        $for C in range(0, CHANNEL_TILE, 8):
62          vacc${ABC[C:C+4]} = wasm_i32x4_add(vacc${ABC[C:C+4]}, wasm_i32x4_widen_low_i16x8(vprod${K}x${ABC[C:C+8]}));
63          vacc${ABC[C+4:C+8]} = wasm_i32x4_add(vacc${ABC[C+4:C+8]}, wasm_i32x4_widen_high_i16x8(vprod${K}x${ABC[C:C+8]}));
64
65      w = (const void*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(int32_t) + ${KERNEL_TILE * CHANNEL_TILE} * sizeof(int8_t));
66
67      $for C in range(0, CHANNEL_TILE, 4):
68        const v128_t vsign${ABC[C:C+4]} = wasm_i32x4_shr(vacc${ABC[C:C+4]}, 31);
69
70      $for C in range(0, CHANNEL_TILE, 4):
71        const v128_t vacc${ABC[C:C+2]} = wasm_v32x4_shuffle(vacc${ABC[C:C+4]}, vsign${ABC[C:C+4]}, 0, 4, 1, 5);
72        const v128_t vacc${ABC[C+2:C+4]} = wasm_v32x4_shuffle(vacc${ABC[C:C+4]}, vsign${ABC[C:C+4]}, 2, 6, 3, 7);
73
74      const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
75      const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
76      $for C in range(0, CHANNEL_TILE, 2):
77        const v128_t vprod${ABC[C:C+2]} = wasm_i64x2_add(wasm_i64x2_mul(vacc${ABC[C:C+2]}, vmultiplier), vrounding);
78
79      $for C in range(0, CHANNEL_TILE, 4):
80        const v128_t vq31prod${ABC[C:C+4]} = wasm_v32x4_shuffle(vprod${ABC[C:C+2]}, vprod${ABC[C+2:C+4]}, 1, 3, 5, 7);
81
82      const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
83      $for C in range(0, CHANNEL_TILE, 4):
84        const v128_t vrem${ABC[C:C+4]} = wasm_i32x4_add(wasm_v128_and(vq31prod${ABC[C:C+4]}, vremainder_mask), wasm_i32x4_shr(vq31prod${ABC[C:C+4]}, 31));
85
86      const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
87      const int32_t vshift = params->wasmsimd.shift;
88      $for C in range(0, CHANNEL_TILE, 4):
89        vacc${ABC[C:C+4]} = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod${ABC[C:C+4]}, vshift), wasm_i32x4_gt(vrem${ABC[C:C+4]}, vthreshold));
90
91      const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
92      $for C in range(0, CHANNEL_TILE, 8):
93        v128_t vout${ABC[C:C+8]} = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc${ABC[C:C+4]}, vacc${ABC[C+4:C+8]}), voutput_zero_point);
94
95      const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
96      const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
97      $for C in range(0, CHANNEL_TILE, 16):
98        $if C + 8 < CHANNEL_TILE:
99          v128_t vout${ABC[C:C+16]} = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout${ABC[C:C+8]}, vout${ABC[C+8:C+16]}), voutput_min), voutput_max);
100        $else:
101          v128_t vout${ABC[C:C+8]}${ABC[C:C+8]} = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout${ABC[C:C+8]}, vout${ABC[C:C+8]}), voutput_min), voutput_max);
102
103      $if CHANNEL_TILE > 8:
104        wasm_v128_store(output, vout${ABC[0:16]});
105      $else:
106        *((double*) output) = wasm_f64x2_extract_lane(vout${ABC[0:8]}${ABC[0:8]}, 0);
107      $for C in range(16, CHANNEL_TILE, 16):
108        $if C + 8 < CHANNEL_TILE:
109          wasm_v128_store(output + ${C}, vout${ABC[C:C+16]});
110        $else:
111          *((double*) (output + ${C})) = wasm_f64x2_extract_lane(vout${ABC[C:C+8]}${ABC[C:C+8]}, 0);
112      output += ${CHANNEL_TILE};
113    }
114    if XNN_UNLIKELY(c != 0) {
115      $if CHANNEL_TILE > 8:
116        const int8_t* k = (const int8_t*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(int32_t));
117      ${"do " if CHANNEL_TILE > 8 else ""}{
118        v128_t vacc${ABC[0:4]} = wasm_v128_load(w);
119        v128_t vacc${ABC[4:8]} = wasm_v128_load((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
120
121        $for K in range(KERNEL_TILE):
122
123          const v128_t vi${K}x${ABC[0:8]} = wasm_i16x8_load_8x8(i${K});
124          $if CHANNEL_TILE > 8:
125            $if K == 0:
126              const v128_t vk${K}x${ABC[0:8]} = wasm_i16x8_load_8x8(k);
127            $else:
128              const v128_t vk${K}x${ABC[0:8]} = wasm_i16x8_load_8x8((const void*) (k + ${K * CHANNEL_TILE}));
129          $else:
130            const v128_t vk${K}x${ABC[0:8]} = wasm_i16x8_load_8x8((const void*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(int32_t) + ${K * CHANNEL_TILE} * sizeof(int8_t)));
131          $if CHANNEL_TILE > 8:
132            i${K} += 8;
133
134          const v128_t vprod${K}x${ABC[0:8]} = wasm_i16x8_mul(vi${K}x${ABC[0:8]}, vk${K}x${ABC[0:8]});
135
136          vacc${ABC[0:4]} = wasm_i32x4_add(vacc${ABC[0:4]}, wasm_i32x4_widen_low_i16x8(vprod${K}x${ABC[0:8]}));
137          vacc${ABC[4:8]} = wasm_i32x4_add(vacc${ABC[4:8]}, wasm_i32x4_widen_high_i16x8(vprod${K}x${ABC[0:8]}));
138
139        $if CHANNEL_TILE > 8:
140          w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
141          k += 8;
142
143      const v128_t vsign${ABC[0:4]} = wasm_i32x4_shr(vacc${ABC[0:4]}, 31);
144      const v128_t vsign${ABC[4:8]} = wasm_i32x4_shr(vacc${ABC[4:8]}, 31);
145
146      const v128_t vacc${ABC[0:2]} = wasm_v32x4_shuffle(vacc${ABC[0:4]}, vsign${ABC[0:4]}, 0, 4, 1, 5);
147      const v128_t vacc${ABC[2:4]} = wasm_v32x4_shuffle(vacc${ABC[0:4]}, vsign${ABC[0:4]}, 2, 6, 3, 7);
148      const v128_t vacc${ABC[4:6]} = wasm_v32x4_shuffle(vacc${ABC[4:8]}, vsign${ABC[4:8]}, 0, 4, 1, 5);
149      const v128_t vacc${ABC[6:8]} = wasm_v32x4_shuffle(vacc${ABC[4:8]}, vsign${ABC[4:8]}, 2, 6, 3, 7);
150
151      const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
152      const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
153      const v128_t vprod${ABC[0:2]} = wasm_i64x2_add(wasm_i64x2_mul(vacc${ABC[0:2]}, vmultiplier), vrounding);
154      const v128_t vprod${ABC[2:4]} = wasm_i64x2_add(wasm_i64x2_mul(vacc${ABC[2:4]}, vmultiplier), vrounding);
155      const v128_t vprod${ABC[4:6]} = wasm_i64x2_add(wasm_i64x2_mul(vacc${ABC[4:6]}, vmultiplier), vrounding);
156      const v128_t vprod${ABC[6:8]} = wasm_i64x2_add(wasm_i64x2_mul(vacc${ABC[6:8]}, vmultiplier), vrounding);
157
158      const v128_t vq31prod${ABC[0:4]} = wasm_v32x4_shuffle(vprod${ABC[0:2]}, vprod${ABC[2:4]}, 1, 3, 5, 7);
159      const v128_t vq31prod${ABC[4:8]} = wasm_v32x4_shuffle(vprod${ABC[4:6]}, vprod${ABC[6:8]}, 1, 3, 5, 7);
160
161      const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
162      const v128_t vrem${ABC[0:4]} = wasm_i32x4_add(wasm_v128_and(vq31prod${ABC[0:4]}, vremainder_mask), wasm_i32x4_shr(vq31prod${ABC[0:4]}, 31));
163      const v128_t vrem${ABC[4:8]} = wasm_i32x4_add(wasm_v128_and(vq31prod${ABC[4:8]}, vremainder_mask), wasm_i32x4_shr(vq31prod${ABC[4:8]}, 31));
164
165      const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
166      const int32_t vshift = params->wasmsimd.shift;
167      vacc${ABC[0:4]} = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod${ABC[0:4]}, vshift), wasm_i32x4_gt(vrem${ABC[0:4]}, vthreshold));
168      vacc${ABC[4:8]} = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod${ABC[4:8]}, vshift), wasm_i32x4_gt(vrem${ABC[4:8]}, vthreshold));
169
170      const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
171      v128_t vout${ABC[0:8]} = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc${ABC[0:4]}, vacc${ABC[4:8]}), voutput_zero_point);
172
173      const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
174      const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
175      v128_t vout${ABC[0:8]}${ABC[0:8]} = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout${ABC[0:8]}, vout${ABC[0:8]}), voutput_min), voutput_max);
176
177      $if CHANNEL_TILE > 8:
178        if XNN_LIKELY(c >= 8) {
179          *((double*) output) = wasm_f64x2_extract_lane(vout${ABC[0:8]}${ABC[0:8]}, 0);
180          output += 8;
181          c -= 8;
182        } else {
183          if (c & 4) {
184            *((float*) output) = wasm_f32x4_extract_lane(vout${ABC[0:8]}${ABC[0:8]}, 0);
185            vout${ABC[0:8]}${ABC[0:8]} = wasm_u64x2_shr(vout${ABC[0:8]}${ABC[0:8]}, 32);
186            output += 4;
187          }
188          if (c & 2) {
189            *((uint16_t*) output) = (uint16_t) wasm_i16x8_extract_lane(vout${ABC[0:8]}${ABC[0:8]}, 0);
190            vout${ABC[0:8]}${ABC[0:8]} = wasm_u32x4_shr(vout${ABC[0:8]}${ABC[0:8]}, 16);
191            output += 2;
192          }
193          if (c & 1) {
194            *output = (int8_t) wasm_i8x16_extract_lane(vout${ABC[0:8]}${ABC[0:8]}, 0);
195            output += 1;
196          }
197          c = 0;
198        }
199      $else:
200        if (c & 4) {
201          *((float*) output) = wasm_f32x4_extract_lane(vout${ABC[0:8]}${ABC[0:8]}, 0);
202          vout${ABC[0:8]}${ABC[0:8]} = wasm_u64x2_shr(vout${ABC[0:8]}${ABC[0:8]}, 32);
203          output += 4;
204        }
205        if (c & 2) {
206          *((uint16_t*) output) = (uint16_t) wasm_i16x8_extract_lane(vout${ABC[0:8]}${ABC[0:8]}, 0);
207          vout${ABC[0:8]}${ABC[0:8]} = wasm_u32x4_shr(vout${ABC[0:8]}${ABC[0:8]}, 16);
208          output += 2;
209        }
210        if (c & 1) {
211          *output = (int8_t) wasm_i8x16_extract_lane(vout${ABC[0:8]}${ABC[0:8]}, 0);
212          output += 1;
213        }
214      }${" while (c != 0);" if CHANNEL_TILE > 8 else ""}
215    }
216
217    output = (int8_t*) ((uintptr_t) output + output_increment);
218  } while (--output_width != 0);
219}
220