1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef DEFINE_MACRO_ASSEMBLER_GENERIC_FUNCTIONS
18 #error This file is supposed to be included from berberis/intrinsics/macro_assembler-inl.h
19 #else
20 #undef DEFINE_MACRO_ASSEMBLER_GENERIC_FUNCTIONS
21 #endif
22
23 using Condition = typename Assembler::Condition;
24 using Label = typename Assembler::Label;
25 using Operand = typename Assembler::Operand;
26 using Register = typename Assembler::Register;
27 using ScaleFactor = typename Assembler::ScaleFactor;
28 using XMMRegister = typename Assembler::XMMRegister;
29
30 using Float32 = intrinsics::Float32;
31 using Float64 = intrinsics::Float64;
32
33 template <typename IntType>
34 using ImmFormat =
35 std::conditional_t<sizeof(IntType) <= sizeof(int32_t), std::make_signed_t<IntType>, int32_t>;
36
37 template <typename format, typename... allowed_formats>
38 static constexpr bool kFormatIs = (std::is_same_v<format, allowed_formats> || ...);
39
40 template <typename IntType>
41 static constexpr bool kIntType =
42 kFormatIs<IntType, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t>;
43
44 template <typename IntType>
45 static constexpr bool kIntTypeBW = kFormatIs<IntType, int8_t, uint8_t, int16_t, uint16_t>;
46
47 template <typename IntType>
48 static constexpr bool kIntTypeBWL =
49 kFormatIs<IntType, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t>;
50
51 template <typename IntType>
52 static constexpr bool kIntTypeWL = kFormatIs<IntType, int16_t, uint16_t, int32_t, uint32_t>;
53
54 template <typename IntType>
55 static constexpr bool kIntTypeWLQ =
56 kFormatIs<IntType, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t>;
57
58 template <typename IntType>
59 static constexpr bool kIntTypeLQ = kFormatIs<IntType, int32_t, uint32_t, int64_t, uint64_t>;
60
61 // Psr is the only asymmetric instruction where unsigned 64bit variant was supported in MMX in CPUs
62 // released last century while signed 64bit variant is only added in AVX10 which is not yet even
63 // available for purchase!
64 template <typename IntType>
65 static constexpr bool kIntTypePsr =
66 kFormatIs<IntType, int16_t, uint16_t, int32_t, uint32_t, uint64_t>;
67
68 template <typename IntType>
69 static constexpr bool kSignedIntType = kFormatIs<IntType, int8_t, int16_t, int32_t, int64_t>;
70
71 template <typename IntType>
72 static constexpr bool kUnsignedIntType = kFormatIs<IntType, uint8_t, uint16_t, uint32_t, uint64_t>;
73
74 template <typename FloatType>
75 static constexpr bool kFloatType = kFormatIs<FloatType, Float32, Float64>;
76
77 #define DEFINE_EXPAND_INSTRUCTION(Declare_dest, Declare_src) \
78 template <typename format_out, typename format_in> \
79 std::enable_if_t<kIntType<format_out> && kIntType<format_in> && \
80 sizeof(format_in) <= sizeof(format_out)> \
81 Expand(Declare_dest, Declare_src) { \
82 if constexpr (std::is_same_v<decltype(dest), decltype(src)> && \
83 sizeof(format_out) == sizeof(format_in)) { \
84 if (dest == src) { \
85 return; \
86 } \
87 } \
88 if constexpr (kFormatIs<format_out, int8_t, uint8_t> && \
89 kFormatIs<format_in, int8_t, uint8_t>) { \
90 Assembler::Movb(dest, src); \
91 } else if constexpr (kFormatIs<format_out, int16_t, uint16_t> && \
92 kFormatIs<format_in, int8_t>) { \
93 if constexpr (std::is_same_v<decltype(dest), decltype(src)>) { \
94 if (dest == Assembler::gpr_a && src == Assembler::gpr_a) { \
95 Assembler::Cbw(); \
96 return; \
97 } \
98 } \
99 Assembler::Movsxbw(dest, src); \
100 } else if constexpr (kFormatIs<format_out, int16_t, uint16_t> && \
101 kFormatIs<format_in, uint8_t>) { \
102 Assembler::Movzxbw(dest, src); \
103 } else if constexpr (kFormatIs<format_out, int16_t, uint16_t> && \
104 kFormatIs<format_in, int16_t, uint16_t>) { \
105 Assembler::Movw(dest, src); \
106 } else if constexpr (kFormatIs<format_out, int32_t, uint32_t> && \
107 kFormatIs<format_in, int8_t>) { \
108 Assembler::Movsxbl(dest, src); \
109 } else if constexpr (kFormatIs<format_out, int32_t, uint32_t> && \
110 kFormatIs<format_in, uint8_t>) { \
111 Assembler::Movzxbl(dest, src); \
112 } else if constexpr (kFormatIs<format_out, int32_t, uint32_t> && \
113 kFormatIs<format_in, int16_t>) { \
114 if constexpr (std::is_same_v<decltype(dest), decltype(src)>) { \
115 if (dest == Assembler::gpr_a && src == Assembler::gpr_a) { \
116 Assembler::Cwde(); \
117 return; \
118 } \
119 } \
120 Assembler::Movsxwl(dest, src); \
121 } else if constexpr (kFormatIs<format_out, int32_t, uint32_t> && \
122 kFormatIs<format_in, uint16_t>) { \
123 Assembler::Movzxwl(dest, src); \
124 } else if constexpr (kFormatIs<format_out, int32_t, uint32_t> && \
125 kFormatIs<format_in, int32_t, uint32_t>) { \
126 Assembler::Movl(dest, src); \
127 } else if constexpr (kFormatIs<format_out, int64_t, uint64_t> && \
128 kFormatIs<format_in, int8_t>) { \
129 Assembler::Movsxbq(dest, src); \
130 } else if constexpr (kFormatIs<format_out, int64_t, uint64_t> && \
131 kFormatIs<format_in, uint8_t>) { \
132 Assembler::Movzxbl(dest, src); \
133 } else if constexpr (kFormatIs<format_out, int64_t, uint64_t> && \
134 kFormatIs<format_in, int16_t>) { \
135 Assembler::Movsxwq(dest, src); \
136 } else if constexpr (kFormatIs<format_out, int64_t, uint64_t> && \
137 kFormatIs<format_in, uint16_t>) { \
138 Assembler::Movzxwl(dest, src); \
139 } else if constexpr (kFormatIs<format_out, int64_t, uint64_t> && \
140 kFormatIs<format_in, int32_t>) { \
141 if constexpr (std::is_same_v<decltype(dest), decltype(src)>) { \
142 if (dest == Assembler::gpr_a && src == Assembler::gpr_a) { \
143 Assembler::Cdqe(); \
144 return; \
145 } \
146 } \
147 Assembler::Movsxlq(dest, src); \
148 } else if constexpr (kFormatIs<format_out, int64_t, uint64_t> && \
149 kFormatIs<format_in, uint32_t>) { \
150 Assembler::Movl(dest, src); \
151 } else { \
152 Assembler::Movq(dest, src); \
153 } \
154 }
DEFINE_EXPAND_INSTRUCTION(Register dest,Operand src)155 DEFINE_EXPAND_INSTRUCTION(Register dest, Operand src)
156 DEFINE_EXPAND_INSTRUCTION(Register dest, Register src)
157 #undef DEFINE_EXPAND_INSTRUCTION
158
159 #define DEFINE_INT_INSTRUCTION( \
160 insn_name, asm_name, insn_siffix, type_check, parameters, arguments) \
161 template <typename format> \
162 std::enable_if_t<type_check<format>> insn_name##insn_siffix parameters { \
163 if constexpr (kFormatIs<format, int8_t, uint8_t>) { \
164 Assembler::asm_name##b##insn_siffix arguments; \
165 } else if constexpr (kFormatIs<format, int16_t, uint16_t>) { \
166 Assembler::asm_name##w##insn_siffix arguments; \
167 } else if constexpr (kFormatIs<format, int32_t, uint32_t>) { \
168 Assembler::asm_name##l##insn_siffix arguments; \
169 } else { \
170 Assembler::asm_name##q##insn_siffix arguments; \
171 } \
172 }
173 DEFINE_INT_INSTRUCTION(CmpXchg, CmpXchg, , kIntType, (Operand dest, Register src), (dest, src))
174 DEFINE_INT_INSTRUCTION(CmpXchg, CmpXchg, , kIntType, (Register dest, Register src), (dest, src))
175 DEFINE_INT_INSTRUCTION(LockCmpXchg,
176 LockCmpXchg,
177 ,
178 kIntType,
179 (Operand dest, Register src),
180 (dest, src))
181 DEFINE_INT_INSTRUCTION(Mov, Mov, , kIntType, (Operand dest, ImmFormat<format> imm), (dest, imm))
182 DEFINE_INT_INSTRUCTION(Mov, Mov, , kIntType, (Operand dest, Register src), (dest, src))
183 DEFINE_INT_INSTRUCTION(Mov,
184 Mov,
185 ,
186 kIntType,
187 (Register dest, std::make_signed_t<format> imm),
188 (dest, imm))
189 DEFINE_INT_INSTRUCTION(Mov, Mov, , kIntType, (Register dest, Operand src), (dest, src))
190 DEFINE_INT_INSTRUCTION(Mul, Imul, , kIntTypeWLQ, (Operand dest, Register src), (dest, src))
191 DEFINE_INT_INSTRUCTION(Mul, Imul, , kIntTypeWLQ, (Operand dest, Operand src), (dest, src))
192 DEFINE_INT_INSTRUCTION(Mul,
193 Imul,
194 ,
195 kIntTypeWLQ,
196 (Operand dest, Register src, ImmFormat<format> imm),
197 (dest, src, imm))
198 DEFINE_INT_INSTRUCTION(Mul,
199 Imul,
200 ,
201 kIntTypeWLQ,
202 (Operand dest, Operand src, ImmFormat<format> imm),
203 (dest, src, imm))
204 DEFINE_INT_INSTRUCTION(Test, Test, , kIntType, (Operand dest, ImmFormat<format> imm), (dest, imm))
205 DEFINE_INT_INSTRUCTION(Test, Test, , kIntType, (Operand dest, Register src), (dest, src))
206 DEFINE_INT_INSTRUCTION(Test, Test, , kIntType, (Register dest, ImmFormat<format> imm), (dest, imm))
207 DEFINE_INT_INSTRUCTION(Test, Test, , kIntType, (Register dest, Register src), (dest, src))
208 #define DEFINE_ARITH_INSTRUCTION(insn_name, asm_name, type_check) \
209 DEFINE_INT_INSTRUCTION(insn_name, asm_name, , type_check, (Operand arg), (arg)) \
210 DEFINE_INT_INSTRUCTION(insn_name, asm_name, , type_check, (Register arg), (arg))
211 DEFINE_ARITH_INSTRUCTION(Dec, Dec, kIntType)
212 DEFINE_ARITH_INSTRUCTION(Div, Div, kUnsignedIntType)
213 DEFINE_ARITH_INSTRUCTION(Div, Idiv, kSignedIntType)
214 DEFINE_ARITH_INSTRUCTION(Inc, Inc, kIntType)
215 DEFINE_ARITH_INSTRUCTION(Mul, Imul, kSignedIntType)
216 DEFINE_ARITH_INSTRUCTION(Mul, Mul, kUnsignedIntType)
217 DEFINE_ARITH_INSTRUCTION(Neg, Neg, kIntType)
218 DEFINE_ARITH_INSTRUCTION(Not, Not, kIntType)
219 #undef DEFINE_ARITH_INSTRUCTION
220 #define DEFINE_ARITH_INSTRUCTION(insn_name) \
221 DEFINE_INT_INSTRUCTION( \
222 insn_name, insn_name, , kIntType, (Operand dest, ImmFormat<format> imm), (dest, imm)) \
223 DEFINE_INT_INSTRUCTION( \
224 insn_name, insn_name, , kIntType, (Operand dest, Register src), (dest, src)) \
225 DEFINE_INT_INSTRUCTION( \
226 insn_name, insn_name, , kIntType, (Register dest, ImmFormat<format> imm), (dest, imm)) \
227 DEFINE_INT_INSTRUCTION( \
228 insn_name, insn_name, , kIntType, (Register dest, Operand src), (dest, src)) \
229 DEFINE_INT_INSTRUCTION( \
230 insn_name, insn_name, , kIntType, (Register dest, Register src), (dest, src))
231 DEFINE_ARITH_INSTRUCTION(Adc)
232 DEFINE_ARITH_INSTRUCTION(Add)
233 DEFINE_ARITH_INSTRUCTION(And)
234 DEFINE_ARITH_INSTRUCTION(Cmp)
235 DEFINE_ARITH_INSTRUCTION(Or)
236 DEFINE_ARITH_INSTRUCTION(Sbb)
237 DEFINE_ARITH_INSTRUCTION(Sub)
238 DEFINE_ARITH_INSTRUCTION(Xor)
239 #undef DEFINE_ARITH_INSTRUCTION
240 #define DEFINE_SHIFT_INSTRUCTION(insn_name) \
241 DEFINE_INT_INSTRUCTION( \
242 insn_name, insn_name, , kIntType, (Operand dest, int8_t imm), (dest, imm)) \
243 DEFINE_INT_INSTRUCTION(insn_name, insn_name, ByCl, kIntType, (Operand dest), (dest)) \
244 DEFINE_INT_INSTRUCTION( \
245 insn_name, insn_name, , kIntType, (Register dest, int8_t imm), (dest, imm)) \
246 DEFINE_INT_INSTRUCTION(insn_name, insn_name, ByCl, kIntType, (Register dest), (dest))
247 DEFINE_SHIFT_INSTRUCTION(Rcl)
248 DEFINE_SHIFT_INSTRUCTION(Rcr)
249 DEFINE_SHIFT_INSTRUCTION(Rol)
250 DEFINE_SHIFT_INSTRUCTION(Ror)
251 DEFINE_SHIFT_INSTRUCTION(Sar)
252 DEFINE_SHIFT_INSTRUCTION(Shl)
253 DEFINE_SHIFT_INSTRUCTION(Shr)
254 #undef DEFINE_INT_INSTRUCTION
255 #undef DEFINE_SHIFT_INSTRUCTION
256
257 #define DEFINE_INT_INSTRUCTION(insn_name, parameters, arguments) \
258 template <typename format> \
259 std::enable_if_t<kIntTypeWLQ<format>> insn_name parameters { \
260 if constexpr (kFormatIs<format, int16_t, uint16_t>) { \
261 Assembler::insn_name##w arguments; \
262 } else if constexpr (kFormatIs<format, int32_t, uint32_t>) { \
263 Assembler::insn_name##l arguments; \
264 } else { \
265 Assembler::insn_name##q arguments; \
266 } \
267 }
268 DEFINE_INT_INSTRUCTION(Cmov, (Condition cond, Register dest, Operand src), (cond, dest, src))
269 DEFINE_INT_INSTRUCTION(Cmov, (Condition cond, Register dest, Register src), (cond, dest, src))
270 #define DEFINE_BIT_INSTRUCTION(insn_name) \
271 DEFINE_INT_INSTRUCTION(insn_name, (Operand dest, ImmFormat<format> imm), (dest, imm)) \
272 DEFINE_INT_INSTRUCTION(insn_name, (Operand dest, Register src), (dest, src)) \
273 DEFINE_INT_INSTRUCTION(insn_name, (Register dest, ImmFormat<format> imm), (dest, imm)) \
274 DEFINE_INT_INSTRUCTION(insn_name, (Register dest, Register src), (dest, src))
275 DEFINE_BIT_INSTRUCTION(Bt)
276 DEFINE_BIT_INSTRUCTION(Btc)
277 DEFINE_BIT_INSTRUCTION(Btr)
278 DEFINE_BIT_INSTRUCTION(Bts)
279 #undef DEFINE_BIT_INSTRUCTION
280 #define DEFINE_BIT_INSTRUCTION(insn_name) \
281 DEFINE_INT_INSTRUCTION(insn_name, (Register dest, Operand src), (dest, src)) \
282 DEFINE_INT_INSTRUCTION(insn_name, (Register dest, Register src), (dest, src))
283 DEFINE_BIT_INSTRUCTION(Bsf)
284 DEFINE_BIT_INSTRUCTION(Bsr)
285 DEFINE_BIT_INSTRUCTION(Lzcnt)
286 DEFINE_BIT_INSTRUCTION(Tzcnt)
287 #undef DEFINE_BIT_INSTRUCTION
288 #undef DEFINE_INT_INSTRUCTION
289
290 // Note: Mov<int32_t> from one register to that same register doesn't zero-out top 32bits,
291 // like real Movq would! If you want that effect then use Expand<tnt32_t, int32_t> instead!
292 template <typename format>
293 std::enable_if_t<kIntType<format>> Mov(Register dest, Register src) {
294 if (dest == src) {
295 return;
296 }
297 if constexpr (kFormatIs<format, int8_t, uint8_t>) {
298 Assembler::Movb(dest, src);
299 } else if constexpr (kFormatIs<format, int16_t, uint16_t>) {
300 Assembler::Movw(dest, src);
301 } else if constexpr (kFormatIs<format, int32_t, uint32_t>) {
302 Assembler::Movl(dest, src);
303 } else {
304 Assembler::Movq(dest, src);
305 }
306 }
307
308 #define DEFINE_INT_INSTRUCTION(insn_name, parameters, arguments) \
309 template <typename target_format> \
310 std::enable_if_t<kIntTypeBW<target_format>> insn_name parameters { \
311 if constexpr (kFormatIs<target_format, int8_t>) { \
312 Assembler::insn_name##sswb arguments; \
313 } else if constexpr (kFormatIs<target_format, uint8_t>) { \
314 Assembler::insn_name##uswb arguments; \
315 } else if constexpr (kFormatIs<target_format, int16_t>) { \
316 Assembler::insn_name##ssdw arguments; \
317 } else { \
318 Assembler::insn_name##usdw arguments; \
319 } \
320 }
321 #define DEFINE_XMM_INT_INSTRUCTIONS_GROUP(insn_name) \
322 DEFINE_INT_INSTRUCTION(P##insn_name, (XMMRegister dest, XMMRegister src), (dest, src)) \
323 DEFINE_INT_INSTRUCTION(P##insn_name, (XMMRegister dest, Operand src), (dest, src)) \
324 DEFINE_INT_INSTRUCTION( \
325 Vp##insn_name, (XMMRegister dest, XMMRegister src1, XMMRegister src2), (dest, src1, src2)) \
326 DEFINE_INT_INSTRUCTION( \
327 Vp##insn_name, (XMMRegister dest, XMMRegister src1, Operand src2), (dest, src1, src2))
328 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(ack)
329 #undef DEFINE_INT_INSTRUCTION
330 #define DEFINE_INT_INSTRUCTION(insn_name, parameters, arguments) \
331 template <typename format> \
332 std::enable_if_t<kUnsignedIntType<format>> insn_name parameters { \
333 if constexpr (kFormatIs<format, uint8_t>) { \
334 Assembler::insn_name##bw arguments; \
335 } else if constexpr (kFormatIs<format, uint16_t>) { \
336 Assembler::insn_name##wd arguments; \
337 } else if constexpr (kFormatIs<format, uint32_t>) { \
338 Assembler::insn_name##dq arguments; \
339 } else { \
340 Assembler::insn_name##qdq arguments; \
341 } \
342 }
343 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(unpckh)
344 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(unpckl)
345 #undef DEFINE_XMM_INT_INSTRUCTIONS_GROUP
346 #undef DEFINE_INT_INSTRUCTION
347
348 #define DEFINE_XMM_INT_INSTRUCTION( \
349 insn_name, asm_name, type_check, parameters, arguments, signed, unsigned) \
350 template <typename format> \
351 std::enable_if_t<type_check<format>> insn_name parameters { \
352 if constexpr (kFormatIs<format, int8_t>) { \
353 Assembler::asm_name##signed##b arguments; \
354 } else if constexpr (kFormatIs<format, uint8_t>) { \
355 Assembler::asm_name##unsigned##b arguments; \
356 } else if constexpr (kFormatIs<format, int16_t>) { \
357 Assembler::asm_name##signed##w arguments; \
358 } else if constexpr (kFormatIs<format, uint16_t>) { \
359 Assembler::asm_name##unsigned##w arguments; \
360 } else if constexpr (kFormatIs<format, int32_t>) { \
361 Assembler::asm_name##signed##d arguments; \
362 } else if constexpr (kFormatIs<format, uint32_t>) { \
363 Assembler::asm_name##unsigned##d arguments; \
364 } else if constexpr (kFormatIs<format, int64_t>) { \
365 Assembler::asm_name##signed##q arguments; \
366 } else { \
367 Assembler::asm_name##unsigned##q arguments; \
368 } \
369 }
370 #define DEFINE_XMM_INT_INSTRUCTIONS_GROUP(insn_name, asm_name, type_check, signed, unsigned) \
371 DEFINE_XMM_INT_INSTRUCTION(P##insn_name, \
372 P##asm_name, \
373 type_check, \
374 (XMMRegister dest, Operand src), \
375 (dest, src), \
376 signed, \
377 unsigned) \
378 DEFINE_XMM_INT_INSTRUCTION(P##insn_name, \
379 P##asm_name, \
380 type_check, \
381 (XMMRegister dest, XMMRegister src), \
382 (dest, src), \
383 signed, \
384 unsigned) \
385 DEFINE_XMM_INT_INSTRUCTION(Vp##insn_name, \
386 Vp##asm_name, \
387 type_check, \
388 (XMMRegister dest, XMMRegister src1, Operand src2), \
389 (dest, src1, src2), \
390 signed, \
391 unsigned) \
392 DEFINE_XMM_INT_INSTRUCTION(Vp##insn_name, \
393 Vp##asm_name, \
394 type_check, \
395 (XMMRegister dest, XMMRegister src1, XMMRegister src2), \
396 (dest, src1, src2), \
397 signed, \
398 unsigned)
399 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(add, add, kIntType, , )
400 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(adds, add, kIntTypeBW, s, us)
401 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(cmpeq, cmpeq, kIntType, , )
402 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(cmpgt, cmpgt, kSignedIntType, , )
403 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(max, max, kIntTypeBWL, s, u)
404 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(min, min, kIntTypeBWL, s, u)
405 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(mull, mull, kIntTypeWL, , )
406 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(sl, sl, kIntTypeWLQ, l, l)
407 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(sr, sr, kIntTypePsr, a, l)
408 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(subs, sub, kIntTypeBW, s, us)
409 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(sub, sub, kIntType, , )
410 #undef DEFINE_XMM_INT_INSTRUCTIONS_GROUP
411 // Shifts have immediate form in addition to register-to-register form.
412 #define DEFINE_XMM_INT_INSTRUCTIONS_GROUP(insn_name, asm_name, type_check, signed, unsigned) \
413 DEFINE_XMM_INT_INSTRUCTION(P##insn_name, \
414 P##asm_name, \
415 type_check, \
416 (XMMRegister dest, int8_t imm), \
417 (dest, imm), \
418 signed, \
419 unsigned) \
420 DEFINE_XMM_INT_INSTRUCTION(Vp##insn_name, \
421 Vp##asm_name, \
422 type_check, \
423 (XMMRegister dest, XMMRegister src, int8_t imm), \
424 (dest, src, imm), \
425 signed, \
426 unsigned)
427 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(sl, sl, kIntTypeWLQ, l, l)
428 DEFINE_XMM_INT_INSTRUCTIONS_GROUP(sr, sr, kIntTypePsr, a, l)
429 #undef DEFINE_XMM_INT_INSTRUCTIONS_GROUP
430 #undef DEFINE_XMM_INT_INSTRUCTION
431
432 #define DEFINE_MOVS_INSTRUCTION(insn_name, opt_check, parameters, arguments) \
433 template <typename format> \
434 std::enable_if_t<kFloatType<format>> insn_name parameters { \
435 if constexpr (kFormatIs<format, Float32>) { \
436 opt_check; \
437 Assembler::insn_name##s arguments; \
438 } else { \
439 opt_check; \
440 Assembler::insn_name##d arguments; \
441 } \
442 }
443 DEFINE_MOVS_INSTRUCTION(Movs, , (XMMRegister dest, Operand src), (dest, src))
444 DEFINE_MOVS_INSTRUCTION(Movs, , (Operand dest, XMMRegister src), (dest, src))
445 DEFINE_MOVS_INSTRUCTION(Movs,
446 if (dest == src) return,
447 (XMMRegister dest, XMMRegister src),
448 (dest, src))
449 DEFINE_MOVS_INSTRUCTION(Vmovs, , (XMMRegister dest, Operand src), (dest, src))
450 DEFINE_MOVS_INSTRUCTION(Vmovs, , (Operand dest, XMMRegister src), (dest, src))
451 DEFINE_MOVS_INSTRUCTION(Vmovs,
452 if ((dest == src1) && (dest == src2)) return,
453 (XMMRegister dest, XMMRegister src1, XMMRegister src2),
454 (dest, src1, src2))
455 #undef DEFINE_MOVS_INSTRUCTION
456
457 #define DEFINE_XMM_MOV_INSTRUCTION(insn_name, parameters, arguments) \
458 template <typename format> \
459 std::enable_if_t<kFloatType<format>> insn_name parameters { \
460 if constexpr (kFormatIs<format, Float32>) { \
461 Assembler::insn_name##d arguments; \
462 } else { \
463 Assembler::insn_name##q arguments; \
464 } \
465 }
466 DEFINE_XMM_MOV_INSTRUCTION(Mov, (XMMRegister dest, Operand src), (dest, src))
467 DEFINE_XMM_MOV_INSTRUCTION(Mov, (Operand dest, XMMRegister src), (dest, src))
468 DEFINE_XMM_MOV_INSTRUCTION(Mov, (XMMRegister dest, Register src), (dest, src))
469 DEFINE_XMM_MOV_INSTRUCTION(Mov, (Register dest, XMMRegister src), (dest, src))
470 DEFINE_XMM_MOV_INSTRUCTION(Vmov, (XMMRegister dest, Operand src), (dest, src))
471 DEFINE_XMM_MOV_INSTRUCTION(Vmov, (Operand dest, XMMRegister src), (dest, src))
472 DEFINE_XMM_MOV_INSTRUCTION(Vmov, (XMMRegister dest, Register src), (dest, src))
473 DEFINE_XMM_MOV_INSTRUCTION(Vmov, (Register dest, XMMRegister src), (dest, src))
474 #undef DEFINE_XMM_MOV_INSTRUCTION
475
476 #define DEFINE_XMM_CVT_INSTRUCTION(insn_name, parameters, arguments) \
477 template <typename FormatFrom, typename FormatTo> \
478 std::enable_if_t<kFloatType<FormatFrom> && kSignedIntType<FormatTo> && kIntTypeLQ<FormatTo>> \
479 insn_name parameters { \
480 if constexpr (kFormatIs<FormatFrom, Float32> && kFormatIs<FormatTo, int32_t>) { \
481 Assembler::insn_name##ss2sil arguments; \
482 } else if constexpr (kFormatIs<FormatFrom, Float32> && kFormatIs<FormatTo, int64_t>) { \
483 Assembler::insn_name##ss2siq(dest, src); \
484 } else if constexpr (kFormatIs<FormatFrom, Float64> && kFormatIs<FormatTo, int32_t>) { \
485 Assembler::insn_name##sd2sil(dest, src); \
486 } else { \
487 Assembler::insn_name##sd2siq(dest, src); \
488 } \
489 }
490 DEFINE_XMM_CVT_INSTRUCTION(Cvt, (Register dest, XMMRegister src), (dest, src))
491 DEFINE_XMM_CVT_INSTRUCTION(Cvt, (Register dest, Operand src), (dest, src))
492 DEFINE_XMM_CVT_INSTRUCTION(Cvtt, (Register dest, XMMRegister src), (dest, src))
493 DEFINE_XMM_CVT_INSTRUCTION(Cvtt, (Register dest, Operand src), (dest, src))
494 DEFINE_XMM_CVT_INSTRUCTION(Vcvt, (Register dest, XMMRegister src), (dest, src))
495 DEFINE_XMM_CVT_INSTRUCTION(Vcvt, (Register dest, Operand src), (dest, src))
496 DEFINE_XMM_CVT_INSTRUCTION(Vcvtt, (Register dest, XMMRegister src), (dest, src))
497 DEFINE_XMM_CVT_INSTRUCTION(Vcvtt, (Register dest, Operand src), (dest, src))
498 #undef DEFINE_XMM_CVT_INSTRUCTION
499
500 #define DEFINE_XMM_CVT_INSTRUCTION(insn_name, parameters, arguments) \
501 template <typename FormatFrom, typename FormatTo> \
502 std::enable_if_t<kSignedIntType<FormatFrom> && kIntTypeWL<FormatFrom> && kFloatType<FormatTo>> \
503 insn_name parameters { \
504 if constexpr (kFormatIs<FormatFrom, Float32> && kFormatIs<FormatTo, int32_t>) { \
505 Assembler::insn_name##sil2ss arguments; \
506 } else if constexpr (kFormatIs<FormatFrom, Float32> && kFormatIs<FormatTo, int64_t>) { \
507 Assembler::insn_name##siq2ss(dest, src); \
508 } else if constexpr (kFormatIs<FormatFrom, Float64> && kFormatIs<FormatTo, int32_t>) { \
509 Assembler::insn_name##sil2sd(dest, src); \
510 } else { \
511 Assembler::insn_name##siq2sd(dest, src); \
512 } \
513 }
514 DEFINE_XMM_CVT_INSTRUCTION(Cvt, (XMMRegister dest, Register src), (dest, src))
515 DEFINE_XMM_CVT_INSTRUCTION(Cvt, (XMMRegister dest, Operand src), (dest, src))
516 DEFINE_XMM_CVT_INSTRUCTION(Vcvt, (XMMRegister dest, Register src), (dest, src))
517 DEFINE_XMM_CVT_INSTRUCTION(Vcvt, (XMMRegister dest, Operand src), (dest, src))
518 #undef DEFINE_XMM_CVT_INSTRUCTION
519
520 #define DEFINE_XMM_CVT_INSTRUCTION(insn_name, insn_suffix, parameters, arguments) \
521 template <typename FormatFrom, typename FormatTo> \
522 std::enable_if_t<kFloatType<FormatFrom> && kFloatType<FormatTo> && \
523 sizeof(FormatFrom) != sizeof(FormatTo)> \
524 insn_name##insn_suffix parameters { \
525 if constexpr (kFormatIs<FormatFrom, Float32> && kFormatIs<FormatTo, Float64>) { \
526 Assembler::insn_name##insn_suffix##s2##insn_suffix##d arguments; \
527 } else { \
528 Assembler::insn_name##insn_suffix##d2##insn_suffix##s arguments; \
529 } \
530 }
531 DEFINE_XMM_CVT_INSTRUCTION(Cvt, p, (XMMRegister dest, XMMRegister src), (dest, src))
532 DEFINE_XMM_CVT_INSTRUCTION(Cvt, p, (XMMRegister dest, Operand src), (dest, src))
533 DEFINE_XMM_CVT_INSTRUCTION(Cvt, s, (XMMRegister dest, XMMRegister src), (dest, src))
534 DEFINE_XMM_CVT_INSTRUCTION(Cvt, s, (XMMRegister dest, Operand src), (dest, src))
535 DEFINE_XMM_CVT_INSTRUCTION(Vcvt, p, (XMMRegister dest, XMMRegister src), (dest, src))
536 DEFINE_XMM_CVT_INSTRUCTION(Vcvt, p, (XMMRegister dest, Operand src), (dest, src))
537 DEFINE_XMM_CVT_INSTRUCTION(Vcvt,
538 s,
539 (XMMRegister dest, XMMRegister src1, XMMRegister src2),
540 (dest, src1, src2))
541 DEFINE_XMM_CVT_INSTRUCTION(Vcvt,
542 s,
543 (XMMRegister dest, XMMRegister src1, Operand src2),
544 (dest, src1, src2))
545 #undef DEFINE_XMM_CVT_INSTRUCTION
546
547 #define DEFINE_XMM_FLOAT_INSTRUCTION(insn_name, parameters, arguments) \
548 template <typename format> \
549 std::enable_if_t<kFloatType<format>> insn_name parameters { \
550 if constexpr (kFormatIs<format, Float32>) { \
551 Assembler::insn_name##s arguments; \
552 } else { \
553 Assembler::insn_name##d arguments; \
554 } \
555 }
556 DEFINE_XMM_FLOAT_INSTRUCTION(Comis, (XMMRegister dest, Operand src), (dest, src))
557 DEFINE_XMM_FLOAT_INSTRUCTION(Comis, (XMMRegister dest, XMMRegister src), (dest, src))
558 DEFINE_XMM_FLOAT_INSTRUCTION(Ucomis, (XMMRegister dest, Operand src), (dest, src))
559 DEFINE_XMM_FLOAT_INSTRUCTION(Ucomis, (XMMRegister dest, XMMRegister src), (dest, src))
560 DEFINE_XMM_FLOAT_INSTRUCTION(Vcomis, (XMMRegister dest, Operand src), (dest, src))
561 DEFINE_XMM_FLOAT_INSTRUCTION(Vcomis, (XMMRegister dest, XMMRegister src), (dest, src))
562 DEFINE_XMM_FLOAT_INSTRUCTION(Vucomis, (XMMRegister dest, Operand src), (dest, src))
563 DEFINE_XMM_FLOAT_INSTRUCTION(Vucomis, (XMMRegister dest, XMMRegister src), (dest, src))
564 #define DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_SCALAR(insn_name, vinsn_name) \
565 DEFINE_XMM_FLOAT_INSTRUCTION(insn_name##s, (XMMRegister dest, Operand src), (dest, src)) \
566 DEFINE_XMM_FLOAT_INSTRUCTION(insn_name##s, (XMMRegister dest, XMMRegister src), (dest, src)) \
567 DEFINE_XMM_FLOAT_INSTRUCTION( \
568 vinsn_name##s, (XMMRegister dest, XMMRegister src1, Operand src2), (dest, src1, src2)) \
569 DEFINE_XMM_FLOAT_INSTRUCTION( \
570 vinsn_name##s, (XMMRegister dest, XMMRegister src1, XMMRegister src2), (dest, src1, src2))
571 #define DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_PACKED(insn_name, vinsn_name) \
572 DEFINE_XMM_FLOAT_INSTRUCTION(insn_name##p, (XMMRegister dest, Operand src), (dest, src)) \
573 DEFINE_XMM_FLOAT_INSTRUCTION(insn_name##p, (XMMRegister dest, XMMRegister src), (dest, src)) \
574 DEFINE_XMM_FLOAT_INSTRUCTION( \
575 vinsn_name##p, (XMMRegister dest, XMMRegister src1, Operand src2), (dest, src1, src2)) \
576 DEFINE_XMM_FLOAT_INSTRUCTION( \
577 vinsn_name##p, (XMMRegister dest, XMMRegister src1, XMMRegister src2), (dest, src1, src2))
578 #define DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(insn_name, vinsn_name) \
579 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_SCALAR(insn_name, vinsn_name) \
580 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_PACKED(insn_name, vinsn_name)
581 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Add, Vadd)
582 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Cmpeq, Vcmpeq)
583 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Cmple, Vcmple)
584 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Cmplt, Vcmplt)
585 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Cmpord, Vcmpord)
586 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Cmpneq, Vcmpneq)
587 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Cmpnle, Vcmpnle)
588 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Cmpnlt, Vcmpnlt)
589 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Cmpunord, Vcmpunord)
590 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Hadd, Vhadd)
591 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Max, Vmax)
592 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Min, Vmin)
593 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Mul, Vmul)
594 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Sub, Vsub)
595 // Note: logical operations ������������ don't have scalar versions!
596 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_PACKED(And, Vand)
597 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_PACKED(Or, Vor)
598 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_PACKED(Xor, Vxor)
599 #undef DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_PACKED
600 #undef DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_SCALAR
601 #undef DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP
602 #define DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_SCALAR(insn_name) \
603 DEFINE_XMM_FLOAT_INSTRUCTION( \
604 insn_name##s, (XMMRegister dest, XMMRegister src1, Operand src2), (dest, src1, src2)) \
605 DEFINE_XMM_FLOAT_INSTRUCTION( \
606 insn_name##s, (XMMRegister dest, XMMRegister src1, XMMRegister src2), (dest, src1, src2))
607 #define DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_PACKED(insn_name) \
608 DEFINE_XMM_FLOAT_INSTRUCTION( \
609 insn_name##p, (XMMRegister dest, XMMRegister src1, Operand src2), (dest, src1, src2)) \
610 DEFINE_XMM_FLOAT_INSTRUCTION( \
611 insn_name##p, (XMMRegister dest, XMMRegister src1, XMMRegister src2), (dest, src1, src2))
612 #define DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(insn_name) \
613 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_SCALAR(insn_name) \
614 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_PACKED(insn_name)
615 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmadd132)
616 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmadd213)
617 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmadd231)
618 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmaddsub132)
619 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmaddsub213)
620 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmaddsub231)
621 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmsub132)
622 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmsub213)
623 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmsub231)
624 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmsubadd132)
625 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmsubadd213)
626 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmsubadd231)
627 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfnmadd132)
628 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfnmadd213)
629 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfnmadd231)
630 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfnmsub132)
631 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfnmsub213)
632 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfnmsub231)
633 #undef DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_PACKED
634 #undef DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_SCALAR
635 #define DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_SCALAR(insn_name) \
636 DEFINE_XMM_FLOAT_INSTRUCTION( \
637 insn_name##s, \
638 (XMMRegister dest, XMMRegister src1, XMMRegister src2, Operand src3), \
639 (dest, src1, src2, src3)) \
640 DEFINE_XMM_FLOAT_INSTRUCTION( \
641 insn_name##s, \
642 (XMMRegister dest, XMMRegister src1, Operand src2, XMMRegister src3), \
643 (dest, src1, src2, src3)) \
644 DEFINE_XMM_FLOAT_INSTRUCTION( \
645 insn_name##s, \
646 (XMMRegister dest, XMMRegister src1, XMMRegister src2, XMMRegister src3), \
647 (dest, src1, src2, src3))
648 #define DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_PACKED(insn_name) \
649 DEFINE_XMM_FLOAT_INSTRUCTION( \
650 insn_name##p, \
651 (XMMRegister dest, XMMRegister src1, XMMRegister src2, Operand src3), \
652 (dest, src1, src2, src3)) \
653 DEFINE_XMM_FLOAT_INSTRUCTION( \
654 insn_name##p, \
655 (XMMRegister dest, XMMRegister src1, Operand src2, XMMRegister src3), \
656 (dest, src1, src2, src3)) \
657 DEFINE_XMM_FLOAT_INSTRUCTION( \
658 insn_name##p, \
659 (XMMRegister dest, XMMRegister src1, XMMRegister src2, XMMRegister src3), \
660 (dest, src1, src2, src3))
661 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmadd)
662 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmaddsub)
663 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmsubadd)
664 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfmsub)
665 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfnmadd)
666 DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP(Vfnmsub)
667 #undef DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_PACKED
668 #undef DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP_SCALAR
669 #undef DEFINE_XMM_FLOAT_INSTRUCTIONS_GROUP
670 // Special, unique, instructions
671 // Movmskps/Movmskpd doesn't support memoru operand, Round[sp][sd] two arguments and immediate.
672 DEFINE_XMM_FLOAT_INSTRUCTION(Movmskp, (Register dest, XMMRegister src), (dest, src))
673 DEFINE_XMM_FLOAT_INSTRUCTION(Vmovmskp, (Register dest, XMMRegister src), (dest, src))
674 DEFINE_XMM_FLOAT_INSTRUCTION(Roundp,
675 (XMMRegister dest, XMMRegister src, uint8_t imm8),
676 (dest, src, imm8))
677 DEFINE_XMM_FLOAT_INSTRUCTION(Rounds,
678 (XMMRegister dest, XMMRegister src, uint8_t imm8),
679 (dest, src, imm8))
680 DEFINE_XMM_FLOAT_INSTRUCTION(Roundp,
681 (XMMRegister dest, Operand src, uint8_t imm8),
682 (dest, src, imm8))
683 DEFINE_XMM_FLOAT_INSTRUCTION(Rounds,
684 (XMMRegister dest, Operand src, uint8_t imm8),
685 (dest, src, imm8))
686 DEFINE_XMM_FLOAT_INSTRUCTION(Vroundp,
687 (XMMRegister dest, XMMRegister src, uint8_t imm8),
688 (dest, src, imm8))
689 DEFINE_XMM_FLOAT_INSTRUCTION(Vrounds,
690 (XMMRegister dest, XMMRegister src, uint8_t imm8),
691 (dest, src, imm8))
692 DEFINE_XMM_FLOAT_INSTRUCTION(Vroundp,
693 (XMMRegister dest, Operand src, uint8_t imm8),
694 (dest, src, imm8))
695 DEFINE_XMM_FLOAT_INSTRUCTION(Vrounds,
696 (XMMRegister dest, Operand src, uint8_t imm8),
697 (dest, src, imm8))
698 #undef DEFINE_XMM_FLOAT_INSTRUCTION
699