1 /*===------------- avx512vlvnniintrin.h - VNNI intrinsics ------------------===
2 *
3 *
4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * See https://llvm.org/LICENSE.txt for license information.
6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 *
8 *===-----------------------------------------------------------------------===
9 */
10 #ifndef __IMMINTRIN_H
11 #error "Never use <avx512vlvnniintrin.h> directly; include <immintrin.h> instead."
12 #endif
13
14 #ifndef __AVX512VLVNNIINTRIN_H
15 #define __AVX512VLVNNIINTRIN_H
16
17 /* Define the default attributes for the functions in this file. */
18 #define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avx512vl,avx512vnni"), __min_vector_width__(128)))
19 #define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("avx512vl,avx512vnni"), __min_vector_width__(256)))
20
21 /// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a A with
22 /// corresponding signed 8-bit integers in \a B, producing 4 intermediate signed
23 /// 16-bit results. Sum these 4 results with the corresponding 32-bit integer
24 /// in \a S, and store the packed 32-bit results in DST.
25 ///
26 /// This intrinsic corresponds to the <c> VPDPBUSD </c> instructions.
27 ///
28 /// \operation
29 /// FOR j := 0 to 7
30 /// tmp1.word := Signed(ZeroExtend16(A.byte[4*j]) * SignExtend16(B.byte[4*j]))
31 /// tmp2.word := Signed(ZeroExtend16(A.byte[4*j+1]) * SignExtend16(B.byte[4*j+1]))
32 /// tmp3.word := Signed(ZeroExtend16(A.byte[4*j+2]) * SignExtend16(B.byte[4*j+2]))
33 /// tmp4.word := Signed(ZeroExtend16(A.byte[4*j+3]) * SignExtend16(B.byte[4*j+3]))
34 /// DST.dword[j] := S.dword[j] + tmp1 + tmp2 + tmp3 + tmp4
35 /// ENDFOR
36 /// DST[MAX:256] := 0
37 /// \endoperation
38 #define _mm256_dpbusd_epi32(S, A, B) \
39 (__m256i)__builtin_ia32_vpdpbusd256((__v8si)(S), (__v8si)(A), (__v8si)(B))
40
41 /// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a A with
42 /// corresponding signed 8-bit integers in \a B, producing 4 intermediate signed
43 /// 16-bit results. Sum these 4 results with the corresponding 32-bit integer
44 /// in \a S using signed saturation, and store the packed 32-bit results in DST.
45 ///
46 /// This intrinsic corresponds to the <c> VPDPBUSDS </c> instructions.
47 ///
48 /// \operation
49 /// FOR j := 0 to 7
50 /// tmp1.word := Signed(ZeroExtend16(A.byte[4*j]) * SignExtend16(B.byte[4*j]))
51 /// tmp2.word := Signed(ZeroExtend16(A.byte[4*j+1]) * SignExtend16(B.byte[4*j+1]))
52 /// tmp3.word := Signed(ZeroExtend16(A.byte[4*j+2]) * SignExtend16(B.byte[4*j+2]))
53 /// tmp4.word := Signed(ZeroExtend16(A.byte[4*j+3]) * SignExtend16(B.byte[4*j+3]))
54 /// DST.dword[j] := Saturate32(S.dword[j] + tmp1 + tmp2 + tmp3 + tmp4)
55 /// ENDFOR
56 /// DST[MAX:256] := 0
57 /// \endoperation
58 #define _mm256_dpbusds_epi32(S, A, B) \
59 (__m256i)__builtin_ia32_vpdpbusds256((__v8si)(S), (__v8si)(A), (__v8si)(B))
60
61 /// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a A with
62 /// corresponding 16-bit integers in \a B, producing 2 intermediate signed 32-bit
63 /// results. Sum these 2 results with the corresponding 32-bit integer in \a S,
64 /// and store the packed 32-bit results in DST.
65 ///
66 /// This intrinsic corresponds to the <c> VPDPWSSD </c> instructions.
67 ///
68 /// \operation
69 /// FOR j := 0 to 7
70 /// tmp1.dword := SignExtend32(A.word[2*j]) * SignExtend32(B.word[2*j])
71 /// tmp2.dword := SignExtend32(A.word[2*j+1]) * SignExtend32(B.word[2*j+1])
72 /// DST.dword[j] := S.dword[j] + tmp1 + tmp2
73 /// ENDFOR
74 /// DST[MAX:256] := 0
75 /// \endoperation
76 #define _mm256_dpwssd_epi32(S, A, B) \
77 (__m256i)__builtin_ia32_vpdpwssd256((__v8si)(S), (__v8si)(A), (__v8si)(B))
78
79 /// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a A with
80 /// corresponding 16-bit integers in \a B, producing 2 intermediate signed 32-bit
81 /// results. Sum these 2 results with the corresponding 32-bit integer in \a S
82 /// using signed saturation, and store the packed 32-bit results in DST.
83 ///
84 /// This intrinsic corresponds to the <c> VPDPWSSDS </c> instructions.
85 ///
86 /// \operation
87 /// FOR j := 0 to 7
88 /// tmp1.dword := SignExtend32(A.word[2*j]) * SignExtend32(B.word[2*j])
89 /// tmp2.dword := SignExtend32(A.word[2*j+1]) * SignExtend32(B.word[2*j+1])
90 /// DST.dword[j] := Saturate32(S.dword[j] + tmp1 + tmp2)
91 /// ENDFOR
92 /// DST[MAX:256] := 0
93 /// \endoperation
94 #define _mm256_dpwssds_epi32(S, A, B) \
95 (__m256i)__builtin_ia32_vpdpwssds256((__v8si)(S), (__v8si)(A), (__v8si)(B))
96
97 /// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a A with
98 /// corresponding signed 8-bit integers in \a B, producing 4 intermediate signed
99 /// 16-bit results. Sum these 4 results with the corresponding 32-bit integer
100 /// in \a S, and store the packed 32-bit results in DST.
101 ///
102 /// This intrinsic corresponds to the <c> VPDPBUSD </c> instructions.
103 ///
104 /// \operation
105 /// FOR j := 0 to 3
106 /// tmp1.word := Signed(ZeroExtend16(A.byte[4*j]) * SignExtend16(B.byte[4*j]))
107 /// tmp2.word := Signed(ZeroExtend16(A.byte[4*j+1]) * SignExtend16(B.byte[4*j+1]))
108 /// tmp3.word := Signed(ZeroExtend16(A.byte[4*j+2]) * SignExtend16(B.byte[4*j+2]))
109 /// tmp4.word := Signed(ZeroExtend16(A.byte[4*j+3]) * SignExtend16(B.byte[4*j+3]))
110 /// DST.dword[j] := S.dword[j] + tmp1 + tmp2 + tmp3 + tmp4
111 /// ENDFOR
112 /// DST[MAX:128] := 0
113 /// \endoperation
114 #define _mm_dpbusd_epi32(S, A, B) \
115 (__m128i)__builtin_ia32_vpdpbusd128((__v4si)(S), (__v4si)(A), (__v4si)(B))
116
117 /// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a A with
118 /// corresponding signed 8-bit integers in \a B, producing 4 intermediate signed
119 /// 16-bit results. Sum these 4 results with the corresponding 32-bit integer
120 /// in \a S using signed saturation, and store the packed 32-bit results in DST.
121 ///
122 /// This intrinsic corresponds to the <c> VPDPBUSDS </c> instructions.
123 ///
124 /// \operation
125 /// FOR j := 0 to 3
126 /// tmp1.word := Signed(ZeroExtend16(A.byte[4*j]) * SignExtend16(B.byte[4*j]))
127 /// tmp2.word := Signed(ZeroExtend16(A.byte[4*j+1]) * SignExtend16(B.byte[4*j+1]))
128 /// tmp3.word := Signed(ZeroExtend16(A.byte[4*j+2]) * SignExtend16(B.byte[4*j+2]))
129 /// tmp4.word := Signed(ZeroExtend16(A.byte[4*j+3]) * SignExtend16(B.byte[4*j+3]))
130 /// DST.dword[j] := Saturate32(S.dword[j] + tmp1 + tmp2 + tmp3 + tmp4)
131 /// ENDFOR
132 /// DST[MAX:128] := 0
133 /// \endoperation
134 #define _mm_dpbusds_epi32(S, A, B) \
135 (__m128i)__builtin_ia32_vpdpbusds128((__v4si)(S), (__v4si)(A), (__v4si)(B))
136
137 /// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a A with
138 /// corresponding 16-bit integers in \a B, producing 2 intermediate signed 32-bit
139 /// results. Sum these 2 results with the corresponding 32-bit integer in \a S,
140 /// and store the packed 32-bit results in DST.
141 ///
142 /// This intrinsic corresponds to the <c> VPDPWSSD </c> instructions.
143 ///
144 /// \operation
145 /// FOR j := 0 to 3
146 /// tmp1.dword := SignExtend32(A.word[2*j]) * SignExtend32(B.word[2*j])
147 /// tmp2.dword := SignExtend32(A.word[2*j+1]) * SignExtend32(B.word[2*j+1])
148 /// DST.dword[j] := S.dword[j] + tmp1 + tmp2
149 /// ENDFOR
150 /// DST[MAX:128] := 0
151 /// \endoperation
152 #define _mm_dpwssd_epi32(S, A, B) \
153 (__m128i)__builtin_ia32_vpdpwssd128((__v4si)(S), (__v4si)(A), (__v4si)(B))
154
155 /// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a A with
156 /// corresponding 16-bit integers in \a B, producing 2 intermediate signed 32-bit
157 /// results. Sum these 2 results with the corresponding 32-bit integer in \a S
158 /// using signed saturation, and store the packed 32-bit results in DST.
159 ///
160 /// This intrinsic corresponds to the <c> VPDPWSSDS </c> instructions.
161 ///
162 /// \operation
163 /// FOR j := 0 to 3
164 /// tmp1.dword := SignExtend32(A.word[2*j]) * SignExtend32(B.word[2*j])
165 /// tmp2.dword := SignExtend32(A.word[2*j+1]) * SignExtend32(B.word[2*j+1])
166 /// DST.dword[j] := Saturate32(S.dword[j] + tmp1 + tmp2)
167 /// ENDFOR
168 /// DST[MAX:128] := 0
169 /// \endoperation
170 #define _mm_dpwssds_epi32(S, A, B) \
171 (__m128i)__builtin_ia32_vpdpwssds128((__v4si)(S), (__v4si)(A), (__v4si)(B))
172
173 static __inline__ __m256i __DEFAULT_FN_ATTRS256
_mm256_mask_dpbusd_epi32(__m256i __S,__mmask8 __U,__m256i __A,__m256i __B)174 _mm256_mask_dpbusd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)
175 {
176 return (__m256i)__builtin_ia32_selectd_256(__U,
177 (__v8si)_mm256_dpbusd_epi32(__S, __A, __B),
178 (__v8si)__S);
179 }
180
181 static __inline__ __m256i __DEFAULT_FN_ATTRS256
_mm256_maskz_dpbusd_epi32(__mmask8 __U,__m256i __S,__m256i __A,__m256i __B)182 _mm256_maskz_dpbusd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)
183 {
184 return (__m256i)__builtin_ia32_selectd_256(__U,
185 (__v8si)_mm256_dpbusd_epi32(__S, __A, __B),
186 (__v8si)_mm256_setzero_si256());
187 }
188
189 static __inline__ __m256i __DEFAULT_FN_ATTRS256
_mm256_mask_dpbusds_epi32(__m256i __S,__mmask8 __U,__m256i __A,__m256i __B)190 _mm256_mask_dpbusds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)
191 {
192 return (__m256i)__builtin_ia32_selectd_256(__U,
193 (__v8si)_mm256_dpbusds_epi32(__S, __A, __B),
194 (__v8si)__S);
195 }
196
197 static __inline__ __m256i __DEFAULT_FN_ATTRS256
_mm256_maskz_dpbusds_epi32(__mmask8 __U,__m256i __S,__m256i __A,__m256i __B)198 _mm256_maskz_dpbusds_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)
199 {
200 return (__m256i)__builtin_ia32_selectd_256(__U,
201 (__v8si)_mm256_dpbusds_epi32(__S, __A, __B),
202 (__v8si)_mm256_setzero_si256());
203 }
204
205 static __inline__ __m256i __DEFAULT_FN_ATTRS256
_mm256_mask_dpwssd_epi32(__m256i __S,__mmask8 __U,__m256i __A,__m256i __B)206 _mm256_mask_dpwssd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)
207 {
208 return (__m256i)__builtin_ia32_selectd_256(__U,
209 (__v8si)_mm256_dpwssd_epi32(__S, __A, __B),
210 (__v8si)__S);
211 }
212
213 static __inline__ __m256i __DEFAULT_FN_ATTRS256
_mm256_maskz_dpwssd_epi32(__mmask8 __U,__m256i __S,__m256i __A,__m256i __B)214 _mm256_maskz_dpwssd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)
215 {
216 return (__m256i)__builtin_ia32_selectd_256(__U,
217 (__v8si)_mm256_dpwssd_epi32(__S, __A, __B),
218 (__v8si)_mm256_setzero_si256());
219 }
220
221 static __inline__ __m256i __DEFAULT_FN_ATTRS256
_mm256_mask_dpwssds_epi32(__m256i __S,__mmask8 __U,__m256i __A,__m256i __B)222 _mm256_mask_dpwssds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)
223 {
224 return (__m256i)__builtin_ia32_selectd_256(__U,
225 (__v8si)_mm256_dpwssds_epi32(__S, __A, __B),
226 (__v8si)__S);
227 }
228
229 static __inline__ __m256i __DEFAULT_FN_ATTRS256
_mm256_maskz_dpwssds_epi32(__mmask8 __U,__m256i __S,__m256i __A,__m256i __B)230 _mm256_maskz_dpwssds_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)
231 {
232 return (__m256i)__builtin_ia32_selectd_256(__U,
233 (__v8si)_mm256_dpwssds_epi32(__S, __A, __B),
234 (__v8si)_mm256_setzero_si256());
235 }
236
237 static __inline__ __m128i __DEFAULT_FN_ATTRS128
_mm_mask_dpbusd_epi32(__m128i __S,__mmask8 __U,__m128i __A,__m128i __B)238 _mm_mask_dpbusd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)
239 {
240 return (__m128i)__builtin_ia32_selectd_128(__U,
241 (__v4si)_mm_dpbusd_epi32(__S, __A, __B),
242 (__v4si)__S);
243 }
244
245 static __inline__ __m128i __DEFAULT_FN_ATTRS128
_mm_maskz_dpbusd_epi32(__mmask8 __U,__m128i __S,__m128i __A,__m128i __B)246 _mm_maskz_dpbusd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B)
247 {
248 return (__m128i)__builtin_ia32_selectd_128(__U,
249 (__v4si)_mm_dpbusd_epi32(__S, __A, __B),
250 (__v4si)_mm_setzero_si128());
251 }
252
253 static __inline__ __m128i __DEFAULT_FN_ATTRS128
_mm_mask_dpbusds_epi32(__m128i __S,__mmask8 __U,__m128i __A,__m128i __B)254 _mm_mask_dpbusds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)
255 {
256 return (__m128i)__builtin_ia32_selectd_128(__U,
257 (__v4si)_mm_dpbusds_epi32(__S, __A, __B),
258 (__v4si)__S);
259 }
260
261 static __inline__ __m128i __DEFAULT_FN_ATTRS128
_mm_maskz_dpbusds_epi32(__mmask8 __U,__m128i __S,__m128i __A,__m128i __B)262 _mm_maskz_dpbusds_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B)
263 {
264 return (__m128i)__builtin_ia32_selectd_128(__U,
265 (__v4si)_mm_dpbusds_epi32(__S, __A, __B),
266 (__v4si)_mm_setzero_si128());
267 }
268
269 static __inline__ __m128i __DEFAULT_FN_ATTRS128
_mm_mask_dpwssd_epi32(__m128i __S,__mmask8 __U,__m128i __A,__m128i __B)270 _mm_mask_dpwssd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)
271 {
272 return (__m128i)__builtin_ia32_selectd_128(__U,
273 (__v4si)_mm_dpwssd_epi32(__S, __A, __B),
274 (__v4si)__S);
275 }
276
277 static __inline__ __m128i __DEFAULT_FN_ATTRS128
_mm_maskz_dpwssd_epi32(__mmask8 __U,__m128i __S,__m128i __A,__m128i __B)278 _mm_maskz_dpwssd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B)
279 {
280 return (__m128i)__builtin_ia32_selectd_128(__U,
281 (__v4si)_mm_dpwssd_epi32(__S, __A, __B),
282 (__v4si)_mm_setzero_si128());
283 }
284
285 static __inline__ __m128i __DEFAULT_FN_ATTRS128
_mm_mask_dpwssds_epi32(__m128i __S,__mmask8 __U,__m128i __A,__m128i __B)286 _mm_mask_dpwssds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)
287 {
288 return (__m128i)__builtin_ia32_selectd_128(__U,
289 (__v4si)_mm_dpwssds_epi32(__S, __A, __B),
290 (__v4si)__S);
291 }
292
293 static __inline__ __m128i __DEFAULT_FN_ATTRS128
_mm_maskz_dpwssds_epi32(__mmask8 __U,__m128i __S,__m128i __A,__m128i __B)294 _mm_maskz_dpwssds_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B)
295 {
296 return (__m128i)__builtin_ia32_selectd_128(__U,
297 (__v4si)_mm_dpwssds_epi32(__S, __A, __B),
298 (__v4si)_mm_setzero_si128());
299 }
300
301 #undef __DEFAULT_FN_ATTRS128
302 #undef __DEFAULT_FN_ATTRS256
303
304 #endif
305