1 // Copyright 2020 Google Inc. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 // fixedpoint_wasmsimd.h: optimized WAsm SIMD specializations of the templates
16 // in fixedpoint.h.
17
18 #ifndef GEMMLOWP_INTERNAL_FIXEDPOINT_WASMSIMD_H_
19 #define GEMMLOWP_INTERNAL_FIXEDPOINT_WASMSIMD_H_
20
21 #include <wasm_simd128.h>
22
23 namespace gemmlowp {
24
25 // WAsm SIMD intrinsics are not typed: there is a single v128_t vector
26 // type that does not distinguish between "int32x4" and "int16x8" use
27 // cases, unlike the NEON equivalents. Because we had initially focused
28 // on int32x4, we did not pay attention and specialized these fixedpoint
29 // templates directly for v128_t hardcoding the int32x4 semantics,
30 // not leaving room for int16x8 semantics. Amending that by adding a separate
31 // data type, int16x8_v128_t, that wraps v128_t while being a separate
32 // type.
33 struct int16x8_v128_t {
34 v128_t v;
35 };
36
37 // Keep int16x8_v128_t trivially constructible/destructible and provide
38 // easily optimized helper function.
to_int16x8_v128_t(v128_t w)39 inline int16x8_v128_t to_int16x8_v128_t(v128_t w) {
40 int16x8_v128_t r;
41 r.v = w;
42 return r;
43 }
44
45 template <>
46 struct FixedPointRawTypeTraits<v128_t> {
47 typedef std::int32_t ScalarRawType;
48 static constexpr int kLanes = 4;
49 };
50
51 template <>
52 struct FixedPointRawTypeTraits<int16x8_v128_t> {
53 typedef std::int16_t ScalarRawType;
54 static constexpr int kLanes = 8;
55 };
56
57 template <>
58 inline v128_t BitAnd(v128_t a, v128_t b) {
59 return wasm_v128_and(a, b);
60 }
61
62 template <>
63 inline int16x8_v128_t BitAnd(int16x8_v128_t a, int16x8_v128_t b) {
64 return to_int16x8_v128_t(wasm_v128_and(a.v, b.v));
65 }
66
67 template <>
68 inline v128_t BitOr(v128_t a, v128_t b) {
69 return wasm_v128_or(a, b);
70 }
71
72 template <>
73 inline int16x8_v128_t BitOr(int16x8_v128_t a, int16x8_v128_t b) {
74 return to_int16x8_v128_t(wasm_v128_or(a.v, b.v));
75 }
76
77 template <>
78 inline v128_t BitXor(v128_t a, v128_t b) {
79 return wasm_v128_xor(a, b);
80 }
81
82 template <>
83 inline int16x8_v128_t BitXor(int16x8_v128_t a, int16x8_v128_t b) {
84 return to_int16x8_v128_t(wasm_v128_xor(a.v, b.v));
85 }
86
87 template <>
88 inline v128_t BitNot(v128_t a) {
89 return wasm_v128_not(a);
90 }
91
92 template <>
93 inline int16x8_v128_t BitNot(int16x8_v128_t a) {
94 return to_int16x8_v128_t(wasm_v128_not(a.v));
95 }
96
97 template <>
98 inline v128_t Add(v128_t a, v128_t b) {
99 return wasm_i32x4_add(a, b);
100 }
101
102 template <>
103 inline int16x8_v128_t Add(int16x8_v128_t a, int16x8_v128_t b) {
104 return to_int16x8_v128_t(wasm_i16x8_add(a.v, b.v));
105 }
106
107 template <>
108 inline v128_t Mul(v128_t a, v128_t b) {
109 return wasm_i32x4_mul(a, b);
110 }
111
112 template <>
113 inline int16x8_v128_t Mul(int16x8_v128_t a, int16x8_v128_t b) {
114 return to_int16x8_v128_t(wasm_i16x8_mul(a.v, b.v));
115 }
116
117 template <>
118 inline v128_t Sub(v128_t a, v128_t b) {
119 return wasm_i32x4_sub(a, b);
120 }
121
122 template <>
123 inline int16x8_v128_t Sub(int16x8_v128_t a, int16x8_v128_t b) {
124 return to_int16x8_v128_t(wasm_i16x8_sub(a.v, b.v));
125 }
126
127 template <>
128 inline v128_t Neg(v128_t a) {
129 return wasm_i32x4_neg(a);
130 }
131
132 template <>
133 inline int16x8_v128_t Neg(int16x8_v128_t a) {
134 return to_int16x8_v128_t(wasm_i16x8_neg(a.v));
135 }
136
137 template <>
138 inline v128_t ShiftLeft(v128_t a, int offset) {
139 return wasm_i32x4_shl(a, offset);
140 }
141
142 template <>
143 inline int16x8_v128_t ShiftLeft(int16x8_v128_t a, int offset) {
144 return to_int16x8_v128_t(wasm_i16x8_shl(a.v, offset));
145 }
146
147 template <>
148 inline v128_t ShiftRight(v128_t a, int offset) {
149 return wasm_i32x4_shr(a, offset);
150 }
151
152 template <>
153 inline int16x8_v128_t ShiftRight(int16x8_v128_t a, int offset) {
154 return to_int16x8_v128_t(wasm_i16x8_shr(a.v, offset));
155 }
156
157 template <>
158 inline v128_t SelectUsingMask(v128_t if_mask, v128_t then_val,
159 v128_t else_val) {
160 return wasm_v128_bitselect(then_val, else_val, if_mask);
161 }
162
163 template <>
164 inline int16x8_v128_t SelectUsingMask(int16x8_v128_t if_mask,
165 int16x8_v128_t then_val,
166 int16x8_v128_t else_val) {
167 return to_int16x8_v128_t(
168 wasm_v128_bitselect(then_val.v, else_val.v, if_mask.v));
169 }
170
171 template <>
172 inline v128_t MaskIfEqual(v128_t a, v128_t b) {
173 return wasm_i32x4_eq(a, b);
174 }
175
176 template <>
177 inline int16x8_v128_t MaskIfEqual(int16x8_v128_t a, int16x8_v128_t b) {
178 return to_int16x8_v128_t(wasm_i16x8_eq(a.v, b.v));
179 }
180
181 template <>
182 inline v128_t MaskIfNotEqual(v128_t a, v128_t b) {
183 return wasm_i32x4_ne(a, b);
184 }
185
186 template <>
187 inline int16x8_v128_t MaskIfNotEqual(int16x8_v128_t a, int16x8_v128_t b) {
188 return to_int16x8_v128_t(wasm_i16x8_ne(a.v, b.v));
189 }
190
191 template <>
192 inline v128_t MaskIfZero(v128_t a) {
193 return MaskIfEqual(a, wasm_i32x4_const(0, 0, 0, 0));
194 }
195
196 template <>
197 inline int16x8_v128_t MaskIfZero(int16x8_v128_t a) {
198 return MaskIfEqual(
199 a, to_int16x8_v128_t(wasm_i16x8_const(0, 0, 0, 0, 0, 0, 0, 0)));
200 }
201
202 template <>
203 inline v128_t MaskIfNonZero(v128_t a) {
204 return MaskIfNotEqual(a, wasm_i32x4_const(0, 0, 0, 0));
205 }
206
207 template <>
208 inline int16x8_v128_t MaskIfNonZero(int16x8_v128_t a) {
209 return MaskIfNotEqual(
210 a, to_int16x8_v128_t(wasm_i16x8_const(0, 0, 0, 0, 0, 0, 0, 0)));
211 }
212
213 template <>
214 inline v128_t MaskIfGreaterThan(v128_t a, v128_t b) {
215 return wasm_i32x4_gt(a, b);
216 }
217
218 template <>
219 inline int16x8_v128_t MaskIfGreaterThan(int16x8_v128_t a, int16x8_v128_t b) {
220 return to_int16x8_v128_t(wasm_i16x8_gt(a.v, b.v));
221 }
222
223 template <>
224 inline v128_t MaskIfLessThan(v128_t a, v128_t b) {
225 return wasm_i32x4_lt(a, b);
226 }
227
228 template <>
229 inline int16x8_v128_t MaskIfLessThan(int16x8_v128_t a, int16x8_v128_t b) {
230 return to_int16x8_v128_t(wasm_i16x8_lt(a.v, b.v));
231 }
232
233 template <>
234 inline v128_t MaskIfGreaterThanOrEqual(v128_t a, v128_t b) {
235 return wasm_i32x4_ge(a, b);
236 }
237
238 template <>
239 inline int16x8_v128_t MaskIfGreaterThanOrEqual(int16x8_v128_t a,
240 int16x8_v128_t b) {
241 return to_int16x8_v128_t(wasm_i16x8_ge(a.v, b.v));
242 }
243
244 template <>
245 inline v128_t MaskIfLessThanOrEqual(v128_t a, v128_t b) {
246 return wasm_i32x4_le(a, b);
247 }
248
249 template <>
250 inline int16x8_v128_t MaskIfLessThanOrEqual(int16x8_v128_t a,
251 int16x8_v128_t b) {
252 return to_int16x8_v128_t(wasm_i16x8_le(a.v, b.v));
253 }
254
255 /* Assumptions:
256 - All and Any are used on masks.
257 - masks are all_ones for true lanes, all_zeroes otherwise.
258 Hence, All means all 128bits set, and Any means any bit set.
259 */
260
261 template <>
262 inline bool All(v128_t a) {
263 return wasm_i32x4_all_true(a);
264 }
265
266 template <>
267 inline bool All(int16x8_v128_t a) {
268 return wasm_i16x8_all_true(a.v);
269 }
270
271 template <>
272 inline bool Any(v128_t a) {
273 return wasm_i32x4_any_true(a);
274 }
275
276 template <>
277 inline bool Any(int16x8_v128_t a) {
278 return wasm_i16x8_any_true(a.v);
279 }
280
281 template <>
282 inline v128_t RoundingHalfSum(v128_t a, v128_t b) {
283 // We divide the inputs before the add to avoid the overflow and costly test.
284 const v128_t one = wasm_i32x4_const(1, 1, 1, 1);
285 const v128_t sign_bit_mask =
286 wasm_i32x4_const(0x80000000, 0x80000000, 0x80000000, 0x80000000);
287 const v128_t sum = Add(a, b);
288 const v128_t rounded_half_sum = ShiftRight(Add(sum, one), 1);
289 const v128_t overflow =
290 BitAnd(BitAnd(BitXor(a, rounded_half_sum), BitXor(b, rounded_half_sum)),
291 sign_bit_mask);
292 const v128_t result = BitXor(rounded_half_sum, overflow);
293 return result;
294 }
295
296 template <>
297 inline int16x8_v128_t RoundingHalfSum(int16x8_v128_t a, int16x8_v128_t b) {
298 // Idea: go to unsigned to use wasm_u16x8_avgr,
299 // borrowed from Intel's arm_neon_sse.h header.
300 const v128_t constant_neg_32768 = wasm_i16x8_const(
301 -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768);
302 const v128_t a_unsigned = wasm_v128_xor(a.v, constant_neg_32768);
303 const v128_t b_unsigned = wasm_v128_xor(b.v, constant_neg_32768);
304 const v128_t avg_unsigned = wasm_u16x8_avgr(a_unsigned, b_unsigned);
305 const v128_t avg = wasm_v128_xor(avg_unsigned, constant_neg_32768);
306 return to_int16x8_v128_t(avg);
307 }
308
309 template <>
310 inline v128_t SaturatingRoundingDoublingHighMul(v128_t a, v128_t b) {
311 // TODO: switch to extended multiplication once implemented in the toolchain
312 const v128_t a_sign = wasm_i32x4_shr(a, 31);
313 const v128_t b_sign = wasm_i32x4_shr(b, 31);
314
315 const v128_t a_ext_lo = wasm_v32x4_shuffle(a, a_sign, 0, 4, 1, 5);
316 const v128_t a_ext_hi = wasm_v32x4_shuffle(a, a_sign, 2, 6, 3, 7);
317 const v128_t b_ext_lo = wasm_v32x4_shuffle(b, b_sign, 0, 4, 1, 5);
318 const v128_t b_ext_hi = wasm_v32x4_shuffle(b, b_sign, 2, 6, 3, 7);
319
320 const v128_t ab_lo = wasm_i64x2_mul(a_ext_lo, b_ext_lo);
321 const v128_t ab_hi = wasm_i64x2_mul(a_ext_hi, b_ext_hi);
322
323 const v128_t nudge_2x =
324 wasm_i64x2_const(INT64_C(0x80000000), INT64_C(0x80000000));
325 const v128_t ab_lo_2x = wasm_i64x2_add(ab_lo, ab_lo);
326 const v128_t ab_hi_2x = wasm_i64x2_add(ab_hi, ab_hi);
327
328 const v128_t ab_lo_rounded_2x = wasm_i64x2_add(ab_lo_2x, nudge_2x);
329 const v128_t ab_hi_rounded_2x = wasm_i64x2_add(ab_hi_2x, nudge_2x);
330
331 const v128_t prod =
332 wasm_v32x4_shuffle(ab_lo_rounded_2x, ab_hi_rounded_2x, 1, 3, 5, 7);
333
334 // Saturation only happen if a == b == INT_MIN, and this is the only case
335 // where prod == INT_MIN (0x80000000) instead of INT_MAX (0x7FFFFFFF).
336 const v128_t min = wasm_i32x4_const(INT32_C(0x80000000), INT32_C(0x80000000),
337 INT32_C(0x80000000), INT32_C(0x80000000));
338
339 return wasm_v128_xor(prod, wasm_i32x4_eq(prod, min));
340 }
341
342 template <>
343 inline int16x8_v128_t SaturatingRoundingDoublingHighMul(int16x8_v128_t a,
344 int16x8_v128_t b) {
345 #if 0
346 // TODO: enable if https://github.com/WebAssembly/simd/pull/365 is accepted
347 return to_int16x8_v128_t(__builtin_wasm_q15mulr_saturate_s_i16x8(a.v, b.v));
348 #else
349 // TODO: switch to extended multiplication once implemented in the toolchain
350 v128_t lo = wasm_i32x4_mul(wasm_i32x4_widen_low_i16x8(a.v),
351 wasm_i32x4_widen_low_i16x8(b.v));
352 v128_t hi = wasm_i32x4_mul(wasm_i32x4_widen_high_i16x8(a.v),
353 wasm_i32x4_widen_high_i16x8(b.v));
354 const v128_t inc = wasm_i32x4_const(0x4000, 0x4000, 0x4000, 0x4000);
355 lo = wasm_i32x4_add(lo, inc);
356 hi = wasm_i32x4_add(hi, inc);
357 lo = wasm_i32x4_shr(lo, 15);
358 hi = wasm_i32x4_shr(hi, 15);
359 return to_int16x8_v128_t(wasm_i16x8_narrow_i32x4(lo, hi));
360 #endif
361 }
362
363 template <>
364 inline v128_t Dup<v128_t>(std::int32_t x) {
365 return wasm_i32x4_splat(x);
366 }
367
368 template <>
369 inline int16x8_v128_t Dup<int16x8_v128_t>(std::int16_t x) {
370 return to_int16x8_v128_t(wasm_i16x8_splat(x));
371 }
372
373 // So far this is only needed for int16.
374 template <>
375 inline int16x8_v128_t SaturatingAdd(int16x8_v128_t a, int16x8_v128_t b) {
376 return to_int16x8_v128_t(wasm_i16x8_add_saturate(a.v, b.v));
377 }
378
379 } // end namespace gemmlowp
380
381 #endif // GEMMLOWP_INTERNAL_FIXEDPOINT_WASMSIMD_H_
382