1 // Copyright 2019, VIXL authors
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
6 //
7 // * Redistributions of source code must retain the above copyright notice,
8 // this list of conditions and the following disclaimer.
9 // * Redistributions in binary form must reproduce the above copyright notice,
10 // this list of conditions and the following disclaimer in the documentation
11 // and/or other materials provided with the distribution.
12 // * Neither the name of ARM Limited nor the names of its contributors may be
13 // used to endorse or promote products derived from this software without
14 // specific prior written permission.
15 //
16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
17 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27 #include "assembler-aarch64.h"
28
29 namespace vixl {
30 namespace aarch64 {
31
ResolveSVEImm8Shift(int * imm8,int * shift)32 void Assembler::ResolveSVEImm8Shift(int* imm8, int* shift) {
33 if (*shift < 0) {
34 VIXL_ASSERT(*shift == -1);
35 // Derive the shift amount from the immediate.
36 if (IsInt8(*imm8)) {
37 *shift = 0;
38 } else if ((*imm8 % 256) == 0) {
39 *imm8 /= 256;
40 *shift = 8;
41 }
42 }
43
44 VIXL_ASSERT(IsInt8(*imm8));
45 VIXL_ASSERT((*shift == 0) || (*shift == 8));
46 }
47
48 // SVEAddressGeneration.
49
adr(const ZRegister & zd,const SVEMemOperand & addr)50 void Assembler::adr(const ZRegister& zd, const SVEMemOperand& addr) {
51 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
52 VIXL_ASSERT(addr.IsVectorPlusVector());
53 VIXL_ASSERT(
54 AreSameLaneSize(zd, addr.GetVectorBase(), addr.GetVectorOffset()));
55
56 int lane_size = zd.GetLaneSizeInBits();
57 VIXL_ASSERT((lane_size == kSRegSize) || (lane_size == kDRegSize));
58
59 int shift_amount = addr.GetShiftAmount();
60 VIXL_ASSERT((shift_amount >= 0) && (shift_amount <= 3));
61
62 Instr op = 0xffffffff;
63 Instr msz = shift_amount << 10;
64 SVEOffsetModifier mod = addr.GetOffsetModifier();
65 switch (mod) {
66 case SVE_UXTW:
67 VIXL_ASSERT(lane_size == kDRegSize);
68 op = ADR_z_az_d_u32_scaled;
69 break;
70 case SVE_SXTW:
71 VIXL_ASSERT(lane_size == kDRegSize);
72 op = ADR_z_az_d_s32_scaled;
73 break;
74 case SVE_LSL:
75 case NO_SVE_OFFSET_MODIFIER:
76 op = (lane_size == kSRegSize) ? ADR_z_az_s_same_scaled
77 : ADR_z_az_d_same_scaled;
78 break;
79 default:
80 VIXL_UNIMPLEMENTED();
81 }
82 Emit(op | msz | Rd(zd) | Rn(addr.GetVectorBase()) |
83 Rm(addr.GetVectorOffset()));
84 }
85
SVELogicalImmediate(const ZRegister & zdn,uint64_t imm,Instr op)86 void Assembler::SVELogicalImmediate(const ZRegister& zdn,
87 uint64_t imm,
88 Instr op) {
89 unsigned bit_n, imm_s, imm_r;
90 unsigned lane_size = zdn.GetLaneSizeInBits();
91 // Check that the immediate can be encoded in the instruction.
92 if (IsImmLogical(imm, lane_size, &bit_n, &imm_s, &imm_r)) {
93 Emit(op | Rd(zdn) | SVEBitN(bit_n) | SVEImmRotate(imm_r, lane_size) |
94 SVEImmSetBits(imm_s, lane_size));
95 } else {
96 VIXL_UNREACHABLE();
97 }
98 }
99
and_(const ZRegister & zd,const ZRegister & zn,uint64_t imm)100 void Assembler::and_(const ZRegister& zd, const ZRegister& zn, uint64_t imm) {
101 USE(zn);
102 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
103 VIXL_ASSERT(zd.Is(zn));
104 SVELogicalImmediate(zd, imm, AND_z_zi);
105 }
106
dupm(const ZRegister & zd,uint64_t imm)107 void Assembler::dupm(const ZRegister& zd, uint64_t imm) {
108 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
109 // DUPM_z_i is an SVEBroadcastBitmaskImmOp, but its encoding and constraints
110 // are similar enough to SVEBitwiseLogicalWithImm_UnpredicatedOp, that we can
111 // use the logical immediate encoder to get the correct behaviour.
112 SVELogicalImmediate(zd, imm, DUPM_z_i);
113 }
114
eor(const ZRegister & zd,const ZRegister & zn,uint64_t imm)115 void Assembler::eor(const ZRegister& zd, const ZRegister& zn, uint64_t imm) {
116 USE(zn);
117 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
118 VIXL_ASSERT(zd.Is(zn));
119 SVELogicalImmediate(zd, imm, EOR_z_zi);
120 }
121
orr(const ZRegister & zd,const ZRegister & zn,uint64_t imm)122 void Assembler::orr(const ZRegister& zd, const ZRegister& zn, uint64_t imm) {
123 USE(zn);
124 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
125 VIXL_ASSERT(zd.Is(zn));
126 SVELogicalImmediate(zd, imm, ORR_z_zi);
127 }
128
129 // SVEBitwiseLogicalUnpredicated.
and_(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)130 void Assembler::and_(const ZRegister& zd,
131 const ZRegister& zn,
132 const ZRegister& zm) {
133 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
134 VIXL_ASSERT(zd.IsLaneSizeD());
135 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
136 Emit(AND_z_zz | Rd(zd) | Rn(zn) | Rm(zm));
137 }
138
bic(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)139 void Assembler::bic(const ZRegister& zd,
140 const ZRegister& zn,
141 const ZRegister& zm) {
142 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
143 VIXL_ASSERT(zd.IsLaneSizeD());
144 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
145 Emit(BIC_z_zz | Rd(zd) | Rn(zn) | Rm(zm));
146 }
147
eor(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)148 void Assembler::eor(const ZRegister& zd,
149 const ZRegister& zn,
150 const ZRegister& zm) {
151 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
152 VIXL_ASSERT(zd.IsLaneSizeD());
153 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
154 Emit(EOR_z_zz | Rd(zd) | Rn(zn) | Rm(zm));
155 }
156
orr(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)157 void Assembler::orr(const ZRegister& zd,
158 const ZRegister& zn,
159 const ZRegister& zm) {
160 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
161 VIXL_ASSERT(zd.IsLaneSizeD());
162 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
163 Emit(ORR_z_zz | Rd(zd) | Rn(zn) | Rm(zm));
164 }
165
166 // SVEBitwiseShiftPredicated.
167
SVEBitwiseShiftImmediatePred(const ZRegister & zdn,const PRegisterM & pg,Instr encoded_imm_and_tsz,SVEBitwiseShiftByImm_PredicatedOp op)168 void Assembler::SVEBitwiseShiftImmediatePred(
169 const ZRegister& zdn,
170 const PRegisterM& pg,
171 Instr encoded_imm_and_tsz,
172 SVEBitwiseShiftByImm_PredicatedOp op) {
173 Instr tszl_and_imm = ExtractUnsignedBitfield32(4, 0, encoded_imm_and_tsz)
174 << 5;
175 Instr tszh = ExtractUnsignedBitfield32(6, 5, encoded_imm_and_tsz) << 22;
176 Emit(op | tszh | tszl_and_imm | PgLow8(pg) | Rd(zdn));
177 }
178
asr(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,int shift)179 void Assembler::asr(const ZRegister& zd,
180 const PRegisterM& pg,
181 const ZRegister& zn,
182 int shift) {
183 // ASR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const>
184 // 0000 0100 ..00 0000 100. .... .... ....
185 // tszh<23:22> | opc<19:18> = 00 | L<17> = 0 | U<16> = 0 | Pg<12:10> |
186 // tszl<9:8> | imm3<7:5> | Zdn<4:0>
187
188 USE(zn);
189 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
190 VIXL_ASSERT(zd.Is(zn));
191 Instr encoded_imm =
192 EncodeSVEShiftImmediate(ASR, shift, zd.GetLaneSizeInBits());
193 SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, ASR_z_p_zi);
194 }
195
asr(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)196 void Assembler::asr(const ZRegister& zd,
197 const PRegisterM& pg,
198 const ZRegister& zn,
199 const ZRegister& zm) {
200 // ASR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.D
201 // 0000 0100 ..01 1000 100. .... .... ....
202 // size<23:22> | R<18> = 0 | L<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> |
203 // Zdn<4:0>
204
205 USE(zn);
206 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
207 VIXL_ASSERT(zd.Is(zn));
208 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm) ||
209 ((zm.GetLaneSizeInBytes() == kDRegSizeInBytes) &&
210 (zd.GetLaneSizeInBytes() != kDRegSizeInBytes)));
211 Instr op = ASR_z_p_zw;
212 if (AreSameLaneSize(zd, zn, zm)) {
213 op = ASR_z_p_zz;
214 }
215 Emit(op | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
216 }
217
asrd(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,int shift)218 void Assembler::asrd(const ZRegister& zd,
219 const PRegisterM& pg,
220 const ZRegister& zn,
221 int shift) {
222 // ASRD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const>
223 // 0000 0100 ..00 0100 100. .... .... ....
224 // tszh<23:22> | opc<19:18> = 01 | L<17> = 0 | U<16> = 0 | Pg<12:10> |
225 // tszl<9:8> | imm3<7:5> | Zdn<4:0>
226
227 USE(zn);
228 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
229 VIXL_ASSERT(zd.Is(zn));
230
231 Instr encoded_imm =
232 EncodeSVEShiftImmediate(ASR, shift, zd.GetLaneSizeInBits());
233 SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, ASRD_z_p_zi);
234 }
235
asrr(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)236 void Assembler::asrr(const ZRegister& zd,
237 const PRegisterM& pg,
238 const ZRegister& zn,
239 const ZRegister& zm) {
240 // ASRR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
241 // 0000 0100 ..01 0100 100. .... .... ....
242 // size<23:22> | R<18> = 1 | L<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> |
243 // Zdn<4:0>
244
245 USE(zn);
246 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
247 VIXL_ASSERT(zd.Is(zn));
248 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
249
250 Emit(ASRR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
251 }
252
lsl(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,int shift)253 void Assembler::lsl(const ZRegister& zd,
254 const PRegisterM& pg,
255 const ZRegister& zn,
256 int shift) {
257 // LSL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const>
258 // 0000 0100 ..00 0011 100. .... .... ....
259 // tszh<23:22> | opc<19:18> = 00 | L<17> = 1 | U<16> = 1 | Pg<12:10> |
260 // tszl<9:8> | imm3<7:5> | Zdn<4:0>
261
262 USE(zn);
263 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
264 VIXL_ASSERT(zd.Is(zn));
265
266 Instr encoded_imm =
267 EncodeSVEShiftImmediate(LSL, shift, zd.GetLaneSizeInBits());
268 SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, LSL_z_p_zi);
269 }
270
lsl(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)271 void Assembler::lsl(const ZRegister& zd,
272 const PRegisterM& pg,
273 const ZRegister& zn,
274 const ZRegister& zm) {
275 // LSL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.D
276 // 0000 0100 ..01 1011 100. .... .... ....
277 // size<23:22> | R<18> = 0 | L<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> |
278 // Zdn<4:0>
279
280 USE(zn);
281 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
282 VIXL_ASSERT(zd.Is(zn));
283 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm) ||
284 ((zm.GetLaneSizeInBytes() == kDRegSizeInBytes) &&
285 (zd.GetLaneSizeInBytes() != kDRegSizeInBytes)));
286 Instr op = LSL_z_p_zw;
287 if (AreSameLaneSize(zd, zn, zm)) {
288 op = LSL_z_p_zz;
289 }
290 Emit(op | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
291 }
292
lslr(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)293 void Assembler::lslr(const ZRegister& zd,
294 const PRegisterM& pg,
295 const ZRegister& zn,
296 const ZRegister& zm) {
297 // LSLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
298 // 0000 0100 ..01 0111 100. .... .... ....
299 // size<23:22> | R<18> = 1 | L<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> |
300 // Zdn<4:0>
301
302 USE(zn);
303 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
304 VIXL_ASSERT(zd.Is(zn));
305 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
306
307 Emit(LSLR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
308 }
309
lsr(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,int shift)310 void Assembler::lsr(const ZRegister& zd,
311 const PRegisterM& pg,
312 const ZRegister& zn,
313 int shift) {
314 // LSR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const>
315 // 0000 0100 ..00 0001 100. .... .... ....
316 // tszh<23:22> | opc<19:18> = 00 | L<17> = 0 | U<16> = 1 | Pg<12:10> |
317 // tszl<9:8> | imm3<7:5> | Zdn<4:0>
318
319 USE(zn);
320 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
321 VIXL_ASSERT(zd.Is(zn));
322
323 Instr encoded_imm =
324 EncodeSVEShiftImmediate(LSR, shift, zd.GetLaneSizeInBits());
325 SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, LSR_z_p_zi);
326 }
327
lsr(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)328 void Assembler::lsr(const ZRegister& zd,
329 const PRegisterM& pg,
330 const ZRegister& zn,
331 const ZRegister& zm) {
332 // LSR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.D
333 // 0000 0100 ..01 1001 100. .... .... ....
334 // size<23:22> | R<18> = 0 | L<17> = 0 | U<16> = 1 | Pg<12:10> | Zm<9:5> |
335 // Zdn<4:0>
336
337 USE(zn);
338 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
339 VIXL_ASSERT(zd.Is(zn));
340 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm) ||
341 ((zm.GetLaneSizeInBytes() == kDRegSizeInBytes) &&
342 (zd.GetLaneSizeInBytes() != kDRegSizeInBytes)));
343 Instr op = LSR_z_p_zw;
344 if (AreSameLaneSize(zd, zn, zm)) {
345 op = LSR_z_p_zz;
346 }
347 Emit(op | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
348 }
349
lsrr(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)350 void Assembler::lsrr(const ZRegister& zd,
351 const PRegisterM& pg,
352 const ZRegister& zn,
353 const ZRegister& zm) {
354 // LSRR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
355 // 0000 0100 ..01 0101 100. .... .... ....
356 // size<23:22> | R<18> = 1 | L<17> = 0 | U<16> = 1 | Pg<12:10> | Zm<9:5> |
357 // Zdn<4:0>
358
359 USE(zn);
360 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
361 VIXL_ASSERT(zd.Is(zn));
362 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
363
364 Emit(LSRR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
365 }
366
367 // SVEBitwiseShiftUnpredicated.
368
EncodeSVEShiftImmediate(Shift shift_op,int shift,int lane_size_in_bits)369 Instr Assembler::EncodeSVEShiftImmediate(Shift shift_op,
370 int shift,
371 int lane_size_in_bits) {
372 if (shift_op == LSL) {
373 VIXL_ASSERT((shift >= 0) && (shift < lane_size_in_bits));
374 return lane_size_in_bits + shift;
375 }
376
377 VIXL_ASSERT((shift_op == ASR) || (shift_op == LSR));
378 VIXL_ASSERT((shift > 0) && (shift <= lane_size_in_bits));
379 return (2 * lane_size_in_bits) - shift;
380 }
381
SVEBitwiseShiftImmediate(const ZRegister & zd,const ZRegister & zn,Instr encoded_imm_and_tsz,SVEBitwiseShiftUnpredicatedOp op)382 void Assembler::SVEBitwiseShiftImmediate(const ZRegister& zd,
383 const ZRegister& zn,
384 Instr encoded_imm_and_tsz,
385 SVEBitwiseShiftUnpredicatedOp op) {
386 Instr tszl_and_imm = ExtractUnsignedBitfield32(4, 0, encoded_imm_and_tsz)
387 << 16;
388 Instr tszh = ExtractUnsignedBitfield32(6, 5, encoded_imm_and_tsz) << 22;
389 Emit(op | tszh | tszl_and_imm | Rd(zd) | Rn(zn));
390 }
391
asr(const ZRegister & zd,const ZRegister & zn,int shift)392 void Assembler::asr(const ZRegister& zd, const ZRegister& zn, int shift) {
393 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
394 VIXL_ASSERT(AreSameLaneSize(zd, zn));
395 Instr encoded_imm =
396 EncodeSVEShiftImmediate(ASR, shift, zd.GetLaneSizeInBits());
397 SVEBitwiseShiftImmediate(zd, zn, encoded_imm, ASR_z_zi);
398 }
399
asr(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)400 void Assembler::asr(const ZRegister& zd,
401 const ZRegister& zn,
402 const ZRegister& zm) {
403 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
404 VIXL_ASSERT(AreSameLaneSize(zd, zn));
405 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kDRegSizeInBytes);
406
407 Emit(ASR_z_zw | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
408 }
409
lsl(const ZRegister & zd,const ZRegister & zn,int shift)410 void Assembler::lsl(const ZRegister& zd, const ZRegister& zn, int shift) {
411 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
412 Instr encoded_imm =
413 EncodeSVEShiftImmediate(LSL, shift, zd.GetLaneSizeInBits());
414 SVEBitwiseShiftImmediate(zd, zn, encoded_imm, LSL_z_zi);
415 }
416
lsl(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)417 void Assembler::lsl(const ZRegister& zd,
418 const ZRegister& zn,
419 const ZRegister& zm) {
420 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
421 VIXL_ASSERT(AreSameLaneSize(zd, zn));
422 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kDRegSizeInBytes);
423
424 Emit(LSL_z_zw | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
425 }
426
lsr(const ZRegister & zd,const ZRegister & zn,int shift)427 void Assembler::lsr(const ZRegister& zd, const ZRegister& zn, int shift) {
428 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
429 Instr encoded_imm =
430 EncodeSVEShiftImmediate(LSR, shift, zd.GetLaneSizeInBits());
431 SVEBitwiseShiftImmediate(zd, zn, encoded_imm, LSR_z_zi);
432 }
433
lsr(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)434 void Assembler::lsr(const ZRegister& zd,
435 const ZRegister& zn,
436 const ZRegister& zm) {
437 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
438 VIXL_ASSERT(AreSameLaneSize(zd, zn));
439 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kDRegSizeInBytes);
440
441 Emit(LSR_z_zw | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
442 }
443
444 // SVEElementCount.
445
446 #define VIXL_SVE_INC_DEC_LIST(V) \
447 V(cntb, CNTB_r_s) \
448 V(cnth, CNTH_r_s) \
449 V(cntw, CNTW_r_s) \
450 V(cntd, CNTD_r_s) \
451 V(decb, DECB_r_rs) \
452 V(dech, DECH_r_rs) \
453 V(decw, DECW_r_rs) \
454 V(decd, DECD_r_rs) \
455 V(incb, INCB_r_rs) \
456 V(inch, INCH_r_rs) \
457 V(incw, INCW_r_rs) \
458 V(incd, INCD_r_rs) \
459 V(sqdecb, SQDECB_r_rs_x) \
460 V(sqdech, SQDECH_r_rs_x) \
461 V(sqdecw, SQDECW_r_rs_x) \
462 V(sqdecd, SQDECD_r_rs_x) \
463 V(sqincb, SQINCB_r_rs_x) \
464 V(sqinch, SQINCH_r_rs_x) \
465 V(sqincw, SQINCW_r_rs_x) \
466 V(sqincd, SQINCD_r_rs_x)
467
468 #define VIXL_DEFINE_ASM_FUNC(FN, OP) \
469 void Assembler::FN(const Register& rdn, int pattern, int multiplier) { \
470 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \
471 VIXL_ASSERT(rdn.IsX()); \
472 Emit(OP | Rd(rdn) | ImmSVEPredicateConstraint(pattern) | \
473 ImmUnsignedField<19, 16>(multiplier - 1)); \
474 }
475 VIXL_SVE_INC_DEC_LIST(VIXL_DEFINE_ASM_FUNC)
476 #undef VIXL_DEFINE_ASM_FUNC
477
478 #define VIXL_SVE_UQINC_UQDEC_LIST(V) \
479 V(uqdecb, (rdn.IsX() ? UQDECB_r_rs_x : UQDECB_r_rs_uw)) \
480 V(uqdech, (rdn.IsX() ? UQDECH_r_rs_x : UQDECH_r_rs_uw)) \
481 V(uqdecw, (rdn.IsX() ? UQDECW_r_rs_x : UQDECW_r_rs_uw)) \
482 V(uqdecd, (rdn.IsX() ? UQDECD_r_rs_x : UQDECD_r_rs_uw)) \
483 V(uqincb, (rdn.IsX() ? UQINCB_r_rs_x : UQINCB_r_rs_uw)) \
484 V(uqinch, (rdn.IsX() ? UQINCH_r_rs_x : UQINCH_r_rs_uw)) \
485 V(uqincw, (rdn.IsX() ? UQINCW_r_rs_x : UQINCW_r_rs_uw)) \
486 V(uqincd, (rdn.IsX() ? UQINCD_r_rs_x : UQINCD_r_rs_uw))
487
488 #define VIXL_DEFINE_ASM_FUNC(FN, OP) \
489 void Assembler::FN(const Register& rdn, int pattern, int multiplier) { \
490 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \
491 Emit(OP | Rd(rdn) | ImmSVEPredicateConstraint(pattern) | \
492 ImmUnsignedField<19, 16>(multiplier - 1)); \
493 }
VIXL_SVE_UQINC_UQDEC_LIST(VIXL_DEFINE_ASM_FUNC)494 VIXL_SVE_UQINC_UQDEC_LIST(VIXL_DEFINE_ASM_FUNC)
495 #undef VIXL_DEFINE_ASM_FUNC
496
497 #define VIXL_SVE_SQX_INC_DEC_LIST(V) \
498 V(sqdecb, SQDECB) \
499 V(sqdech, SQDECH) \
500 V(sqdecw, SQDECW) \
501 V(sqdecd, SQDECD) \
502 V(sqincb, SQINCB) \
503 V(sqinch, SQINCH) \
504 V(sqincw, SQINCW) \
505 V(sqincd, SQINCD)
506
507 #define VIXL_DEFINE_ASM_FUNC(FN, OP) \
508 void Assembler::FN(const Register& xd, \
509 const Register& wn, \
510 int pattern, \
511 int multiplier) { \
512 USE(wn); \
513 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \
514 VIXL_ASSERT(wn.IsW() && xd.Is(wn.X())); \
515 Emit(OP##_r_rs_sx | Rd(xd) | ImmSVEPredicateConstraint(pattern) | \
516 ImmUnsignedField<19, 16>(multiplier - 1)); \
517 }
518 VIXL_SVE_SQX_INC_DEC_LIST(VIXL_DEFINE_ASM_FUNC)
519 #undef VIXL_DEFINE_ASM_FUNC
520
521 #define VIXL_SVE_INC_DEC_VEC_LIST(V) \
522 V(dech, DEC, H) \
523 V(decw, DEC, W) \
524 V(decd, DEC, D) \
525 V(inch, INC, H) \
526 V(incw, INC, W) \
527 V(incd, INC, D) \
528 V(sqdech, SQDEC, H) \
529 V(sqdecw, SQDEC, W) \
530 V(sqdecd, SQDEC, D) \
531 V(sqinch, SQINC, H) \
532 V(sqincw, SQINC, W) \
533 V(sqincd, SQINC, D) \
534 V(uqdech, UQDEC, H) \
535 V(uqdecw, UQDEC, W) \
536 V(uqdecd, UQDEC, D) \
537 V(uqinch, UQINC, H) \
538 V(uqincw, UQINC, W) \
539 V(uqincd, UQINC, D)
540
541 #define VIXL_DEFINE_ASM_FUNC(FN, OP, T) \
542 void Assembler::FN(const ZRegister& zdn, int pattern, int multiplier) { \
543 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \
544 VIXL_ASSERT(zdn.GetLaneSizeInBytes() == k##T##RegSizeInBytes); \
545 Emit(OP##T##_z_zs | Rd(zdn) | ImmSVEPredicateConstraint(pattern) | \
546 ImmUnsignedField<19, 16>(multiplier - 1)); \
547 }
548 VIXL_SVE_INC_DEC_VEC_LIST(VIXL_DEFINE_ASM_FUNC)
549 #undef VIXL_DEFINE_ASM_FUNC
550
551 // SVEFPAccumulatingReduction.
552
553 void Assembler::fadda(const VRegister& vd,
554 const PRegister& pg,
555 const VRegister& vn,
556 const ZRegister& zm) {
557 // FADDA <V><dn>, <Pg>, <V><dn>, <Zm>.<T>
558 // 0110 0101 ..01 1000 001. .... .... ....
559 // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zm<9:5> | Vdn<4:0>
560
561 USE(vn);
562 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
563 VIXL_ASSERT(vd.Is(vn));
564 VIXL_ASSERT(vd.IsScalar());
565 VIXL_ASSERT(zm.GetLaneSizeInBytes() != kBRegSizeInBytes);
566 VIXL_ASSERT(AreSameLaneSize(zm, vd));
567
568 Emit(FADDA_v_p_z | SVESize(zm) | Rd(vd) | PgLow8(pg) | Rn(zm));
569 }
570
571 // SVEFPArithmetic_Predicated.
572
fabd(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)573 void Assembler::fabd(const ZRegister& zd,
574 const PRegisterM& pg,
575 const ZRegister& zn,
576 const ZRegister& zm) {
577 // FABD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
578 // 0110 0101 ..00 1000 100. .... .... ....
579 // size<23:22> | opc<19:16> = 1000 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
580
581 USE(zn);
582 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
583 VIXL_ASSERT(zd.Is(zn));
584 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
585 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
586
587 Emit(FABD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
588 }
589
fadd(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,double imm)590 void Assembler::fadd(const ZRegister& zd,
591 const PRegisterM& pg,
592 const ZRegister& zn,
593 double imm) {
594 // FADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
595 // 0110 0101 ..01 1000 100. ..00 00.. ....
596 // size<23:22> | opc<18:16> = 000 | Pg<12:10> | i1<5> | Zdn<4:0>
597
598 USE(zn);
599 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
600 VIXL_ASSERT(zd.Is(zn));
601 VIXL_ASSERT(AreSameLaneSize(zd, zn));
602 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
603 VIXL_ASSERT((imm == 0.5) || (imm == 1.0));
604
605 Instr i1 = (imm == 1.0) ? (1 << 5) : 0;
606 Emit(FADD_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
607 }
608
fadd(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)609 void Assembler::fadd(const ZRegister& zd,
610 const PRegisterM& pg,
611 const ZRegister& zn,
612 const ZRegister& zm) {
613 // FADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
614 // 0110 0101 ..00 0000 100. .... .... ....
615 // size<23:22> | opc<19:16> = 0000 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
616
617 USE(zn);
618 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
619 VIXL_ASSERT(zd.Is(zn));
620 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
621 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
622
623 Emit(FADD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
624 }
625
fdiv(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)626 void Assembler::fdiv(const ZRegister& zd,
627 const PRegisterM& pg,
628 const ZRegister& zn,
629 const ZRegister& zm) {
630 // FDIV <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
631 // 0110 0101 ..00 1101 100. .... .... ....
632 // size<23:22> | opc<19:16> = 1101 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
633
634 USE(zn);
635 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
636 VIXL_ASSERT(zd.Is(zn));
637 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
638 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
639
640 Emit(FDIV_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
641 }
642
fdivr(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)643 void Assembler::fdivr(const ZRegister& zd,
644 const PRegisterM& pg,
645 const ZRegister& zn,
646 const ZRegister& zm) {
647 // FDIVR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
648 // 0110 0101 ..00 1100 100. .... .... ....
649 // size<23:22> | opc<19:16> = 1100 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
650
651 USE(zn);
652 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
653 VIXL_ASSERT(zd.Is(zn));
654 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
655 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
656
657 Emit(FDIVR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
658 }
659
fmax(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,double imm)660 void Assembler::fmax(const ZRegister& zd,
661 const PRegisterM& pg,
662 const ZRegister& zn,
663 double imm) {
664 // FMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
665 // 0110 0101 ..01 1110 100. ..00 00.. ....
666 // size<23:22> | opc<18:16> = 110 | Pg<12:10> | i1<5> | Zdn<4:0>
667
668 USE(zn);
669 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
670 VIXL_ASSERT(zd.Is(zn));
671 VIXL_ASSERT(AreSameLaneSize(zd, zn));
672 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
673 VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0));
674
675 Instr i1 = (imm == 1.0) ? (1 << 5) : 0;
676 Emit(FMAX_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
677 }
678
fmax(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)679 void Assembler::fmax(const ZRegister& zd,
680 const PRegisterM& pg,
681 const ZRegister& zn,
682 const ZRegister& zm) {
683 // FMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
684 // 0110 0101 ..00 0110 100. .... .... ....
685 // size<23:22> | opc<19:16> = 0110 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
686
687 USE(zn);
688 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
689 VIXL_ASSERT(zd.Is(zn));
690 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
691 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
692
693 Emit(FMAX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
694 }
695
fmaxnm(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,double imm)696 void Assembler::fmaxnm(const ZRegister& zd,
697 const PRegisterM& pg,
698 const ZRegister& zn,
699 double imm) {
700 // FMAXNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
701 // 0110 0101 ..01 1100 100. ..00 00.. ....
702 // size<23:22> | opc<18:16> = 100 | Pg<12:10> | i1<5> | Zdn<4:0>
703
704 USE(zn);
705 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
706 VIXL_ASSERT(zd.Is(zn));
707 VIXL_ASSERT(AreSameLaneSize(zd, zn));
708 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
709 VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0));
710
711 Instr i1 = (imm == 1.0) ? (1 << 5) : 0;
712 Emit(FMAXNM_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
713 }
714
fmaxnm(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)715 void Assembler::fmaxnm(const ZRegister& zd,
716 const PRegisterM& pg,
717 const ZRegister& zn,
718 const ZRegister& zm) {
719 // FMAXNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
720 // 0110 0101 ..00 0100 100. .... .... ....
721 // size<23:22> | opc<19:16> = 0100 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
722
723 USE(zn);
724 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
725 VIXL_ASSERT(zd.Is(zn));
726 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
727 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
728
729 Emit(FMAXNM_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
730 }
731
fmin(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,double imm)732 void Assembler::fmin(const ZRegister& zd,
733 const PRegisterM& pg,
734 const ZRegister& zn,
735 double imm) {
736 // FMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
737 // 0110 0101 ..01 1111 100. ..00 00.. ....
738 // size<23:22> | opc<18:16> = 111 | Pg<12:10> | i1<5> | Zdn<4:0>
739
740 USE(zn);
741 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
742 VIXL_ASSERT(zd.Is(zn));
743 VIXL_ASSERT(AreSameLaneSize(zd, zn));
744 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
745 VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0));
746
747 Instr i1 = (imm == 1.0) ? (1 << 5) : 0;
748 Emit(FMIN_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
749 }
750
fmin(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)751 void Assembler::fmin(const ZRegister& zd,
752 const PRegisterM& pg,
753 const ZRegister& zn,
754 const ZRegister& zm) {
755 // FMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
756 // 0110 0101 ..00 0111 100. .... .... ....
757 // size<23:22> | opc<19:16> = 0111 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
758
759 USE(zn);
760 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
761 VIXL_ASSERT(zd.Is(zn));
762 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
763 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
764
765 Emit(FMIN_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
766 }
767
fminnm(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,double imm)768 void Assembler::fminnm(const ZRegister& zd,
769 const PRegisterM& pg,
770 const ZRegister& zn,
771 double imm) {
772 // FMINNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
773 // 0110 0101 ..01 1101 100. ..00 00.. ....
774 // size<23:22> | opc<18:16> = 101 | Pg<12:10> | i1<5> | Zdn<4:0>
775
776 USE(zn);
777 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
778 VIXL_ASSERT(zd.Is(zn));
779 VIXL_ASSERT(AreSameLaneSize(zd, zn));
780 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
781 VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0));
782
783 Instr i1 = (imm == 1.0) ? (1 << 5) : 0;
784 Emit(FMINNM_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
785 }
786
fminnm(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)787 void Assembler::fminnm(const ZRegister& zd,
788 const PRegisterM& pg,
789 const ZRegister& zn,
790 const ZRegister& zm) {
791 // FMINNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
792 // 0110 0101 ..00 0101 100. .... .... ....
793 // size<23:22> | opc<19:16> = 0101 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
794
795 USE(zn);
796 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
797 VIXL_ASSERT(zd.Is(zn));
798 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
799 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
800
801 Emit(FMINNM_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
802 }
803
fmul(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,double imm)804 void Assembler::fmul(const ZRegister& zd,
805 const PRegisterM& pg,
806 const ZRegister& zn,
807 double imm) {
808 // FMUL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
809 // 0110 0101 ..01 1010 100. ..00 00.. ....
810 // size<23:22> | opc<18:16> = 010 | Pg<12:10> | i1<5> | Zdn<4:0>
811
812 USE(zn);
813 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
814 VIXL_ASSERT(zd.Is(zn));
815 VIXL_ASSERT(AreSameLaneSize(zd, zn));
816 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
817 VIXL_ASSERT((imm == 0.5) || (imm == 2.0));
818
819 Instr i1 = (imm == 2.0) ? (1 << 5) : 0;
820 Emit(FMUL_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
821 }
822
fmul(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)823 void Assembler::fmul(const ZRegister& zd,
824 const PRegisterM& pg,
825 const ZRegister& zn,
826 const ZRegister& zm) {
827 // FMUL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
828 // 0110 0101 ..00 0010 100. .... .... ....
829 // size<23:22> | opc<19:16> = 0010 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
830
831 USE(zn);
832 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
833 VIXL_ASSERT(zd.Is(zn));
834 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
835 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
836
837 Emit(FMUL_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
838 }
839
fmulx(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)840 void Assembler::fmulx(const ZRegister& zd,
841 const PRegisterM& pg,
842 const ZRegister& zn,
843 const ZRegister& zm) {
844 // FMULX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
845 // 0110 0101 ..00 1010 100. .... .... ....
846 // size<23:22> | opc<19:16> = 1010 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
847
848 USE(zn);
849 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
850 VIXL_ASSERT(zd.Is(zn));
851 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
852 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
853
854 Emit(FMULX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
855 }
856
fscale(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)857 void Assembler::fscale(const ZRegister& zd,
858 const PRegisterM& pg,
859 const ZRegister& zn,
860 const ZRegister& zm) {
861 // FSCALE <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
862 // 0110 0101 ..00 1001 100. .... .... ....
863 // size<23:22> | opc<19:16> = 1001 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
864
865 USE(zn);
866 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
867 VIXL_ASSERT(zd.Is(zn));
868 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
869 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
870
871 Emit(FSCALE_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
872 }
873
fsub(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,double imm)874 void Assembler::fsub(const ZRegister& zd,
875 const PRegisterM& pg,
876 const ZRegister& zn,
877 double imm) {
878 // FSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
879 // 0110 0101 ..01 1001 100. ..00 00.. ....
880 // size<23:22> | opc<18:16> = 001 | Pg<12:10> | i1<5> | Zdn<4:0>
881
882 USE(zn);
883 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
884 VIXL_ASSERT(zd.Is(zn));
885 VIXL_ASSERT(AreSameLaneSize(zd, zn));
886 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
887 VIXL_ASSERT((imm == 0.5) || (imm == 1.0));
888
889 Instr i1 = (imm == 1.0) ? (1 << 5) : 0;
890 Emit(FSUB_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
891 }
892
fsub(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)893 void Assembler::fsub(const ZRegister& zd,
894 const PRegisterM& pg,
895 const ZRegister& zn,
896 const ZRegister& zm) {
897 // FSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
898 // 0110 0101 ..00 0001 100. .... .... ....
899 // size<23:22> | opc<19:16> = 0001 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
900
901 USE(zn);
902 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
903 VIXL_ASSERT(zd.Is(zn));
904 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
905 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
906
907 Emit(FSUB_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
908 }
909
fsubr(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,double imm)910 void Assembler::fsubr(const ZRegister& zd,
911 const PRegisterM& pg,
912 const ZRegister& zn,
913 double imm) {
914 // FSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
915 // 0110 0101 ..01 1011 100. ..00 00.. ....
916 // size<23:22> | opc<18:16> = 011 | Pg<12:10> | i1<5> | Zdn<4:0>
917
918 USE(zn);
919 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
920 VIXL_ASSERT(zd.Is(zn));
921 VIXL_ASSERT(AreSameLaneSize(zd, zn));
922 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
923 VIXL_ASSERT((imm == 0.5) || (imm == 1.0));
924
925 Instr i1 = (imm == 1.0) ? (1 << 5) : 0;
926 Emit(FSUBR_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
927 }
928
fsubr(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)929 void Assembler::fsubr(const ZRegister& zd,
930 const PRegisterM& pg,
931 const ZRegister& zn,
932 const ZRegister& zm) {
933 // FSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
934 // 0110 0101 ..00 0011 100. .... .... ....
935 // size<23:22> | opc<19:16> = 0011 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
936
937 USE(zn);
938 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
939 VIXL_ASSERT(zd.Is(zn));
940 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
941 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
942
943 Emit(FSUBR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
944 }
945
ftmad(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm,int imm3)946 void Assembler::ftmad(const ZRegister& zd,
947 const ZRegister& zn,
948 const ZRegister& zm,
949 int imm3) {
950 // FTMAD <Zdn>.<T>, <Zdn>.<T>, <Zm>.<T>, #<imm>
951 // 0110 0101 ..01 0... 1000 00.. .... ....
952 // size<23:22> | imm3<18:16> | Zm<9:5> | Zdn<4:0>
953
954 USE(zn);
955 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
956 VIXL_ASSERT(zd.Is(zn));
957 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
958 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
959
960 Emit(FTMAD_z_zzi | SVESize(zd) | Rd(zd) | Rn(zm) |
961 ImmUnsignedField<18, 16>(imm3));
962 }
963
964 // SVEFPArithmeticUnpredicated.
965
fadd(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)966 void Assembler::fadd(const ZRegister& zd,
967 const ZRegister& zn,
968 const ZRegister& zm) {
969 // FADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
970 // 0110 0101 ..0. .... 0000 00.. .... ....
971 // size<23:22> | Zm<20:16> | opc<12:10> = 000 | Zn<9:5> | Zd<4:0>
972
973 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
974 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
975 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
976
977 Emit(FADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
978 }
979
fmul(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)980 void Assembler::fmul(const ZRegister& zd,
981 const ZRegister& zn,
982 const ZRegister& zm) {
983 // FMUL <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
984 // 0110 0101 ..0. .... 0000 10.. .... ....
985 // size<23:22> | Zm<20:16> | opc<12:10> = 010 | Zn<9:5> | Zd<4:0>
986
987 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
988 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
989 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
990
991 Emit(FMUL_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
992 }
993
frecps(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)994 void Assembler::frecps(const ZRegister& zd,
995 const ZRegister& zn,
996 const ZRegister& zm) {
997 // FRECPS <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
998 // 0110 0101 ..0. .... 0001 10.. .... ....
999 // size<23:22> | Zm<20:16> | opc<12:10> = 110 | Zn<9:5> | Zd<4:0>
1000
1001 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1002 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
1003 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1004
1005 Emit(FRECPS_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
1006 }
1007
frsqrts(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)1008 void Assembler::frsqrts(const ZRegister& zd,
1009 const ZRegister& zn,
1010 const ZRegister& zm) {
1011 // FRSQRTS <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
1012 // 0110 0101 ..0. .... 0001 11.. .... ....
1013 // size<23:22> | Zm<20:16> | opc<12:10> = 111 | Zn<9:5> | Zd<4:0>
1014
1015 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1016 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
1017 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1018
1019 Emit(FRSQRTS_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
1020 }
1021
fsub(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)1022 void Assembler::fsub(const ZRegister& zd,
1023 const ZRegister& zn,
1024 const ZRegister& zm) {
1025 // FSUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
1026 // 0110 0101 ..0. .... 0000 01.. .... ....
1027 // size<23:22> | Zm<20:16> | opc<12:10> = 001 | Zn<9:5> | Zd<4:0>
1028
1029 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1030 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
1031 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1032
1033 Emit(FSUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
1034 }
1035
ftsmul(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)1036 void Assembler::ftsmul(const ZRegister& zd,
1037 const ZRegister& zn,
1038 const ZRegister& zm) {
1039 // FTSMUL <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
1040 // 0110 0101 ..0. .... 0000 11.. .... ....
1041 // size<23:22> | Zm<20:16> | opc<12:10> = 011 | Zn<9:5> | Zd<4:0>
1042
1043 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1044 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
1045 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1046
1047 Emit(FTSMUL_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
1048 }
1049
1050 // SVEFPCompareVectors.
1051
facge(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)1052 void Assembler::facge(const PRegisterWithLaneSize& pd,
1053 const PRegisterZ& pg,
1054 const ZRegister& zn,
1055 const ZRegister& zm) {
1056 // FACGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
1057 // 0110 0101 ..0. .... 110. .... ...1 ....
1058 // size<23:22> | Zm<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5> |
1059 // o3<4> = 1 | Pd<3:0>
1060
1061 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1062 VIXL_ASSERT(AreSameLaneSize(zn, zm));
1063 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1064
1065 Emit(FACGE_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
1066 }
1067
facgt(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)1068 void Assembler::facgt(const PRegisterWithLaneSize& pd,
1069 const PRegisterZ& pg,
1070 const ZRegister& zn,
1071 const ZRegister& zm) {
1072 // FACGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
1073 // 0110 0101 ..0. .... 111. .... ...1 ....
1074 // size<23:22> | Zm<20:16> | op<15> = 1 | o2<13> = 1 | Pg<12:10> | Zn<9:5> |
1075 // o3<4> = 1 | Pd<3:0>
1076
1077 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1078 VIXL_ASSERT(AreSameLaneSize(zn, zm));
1079 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1080
1081 Emit(FACGT_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
1082 }
1083
fcmeq(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)1084 void Assembler::fcmeq(const PRegisterWithLaneSize& pd,
1085 const PRegisterZ& pg,
1086 const ZRegister& zn,
1087 const ZRegister& zm) {
1088 // FCMEQ <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
1089 // 0110 0101 ..0. .... 011. .... ...0 ....
1090 // size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5> |
1091 // o3<4> = 0 | Pd<3:0>
1092
1093 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1094 VIXL_ASSERT(AreSameLaneSize(zn, zm));
1095 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1096
1097 Emit(FCMEQ_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
1098 }
1099
fcmge(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)1100 void Assembler::fcmge(const PRegisterWithLaneSize& pd,
1101 const PRegisterZ& pg,
1102 const ZRegister& zn,
1103 const ZRegister& zm) {
1104 // FCMGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
1105 // 0110 0101 ..0. .... 010. .... ...0 ....
1106 // size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5> |
1107 // o3<4> = 0 | Pd<3:0>
1108
1109 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1110 VIXL_ASSERT(AreSameLaneSize(zn, zm));
1111 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1112
1113 Emit(FCMGE_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
1114 }
1115
fcmgt(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)1116 void Assembler::fcmgt(const PRegisterWithLaneSize& pd,
1117 const PRegisterZ& pg,
1118 const ZRegister& zn,
1119 const ZRegister& zm) {
1120 // FCMGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
1121 // 0110 0101 ..0. .... 010. .... ...1 ....
1122 // size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5> |
1123 // o3<4> = 1 | Pd<3:0>
1124
1125 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1126 VIXL_ASSERT(AreSameLaneSize(zn, zm));
1127 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1128
1129 Emit(FCMGT_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
1130 }
1131
fcmne(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)1132 void Assembler::fcmne(const PRegisterWithLaneSize& pd,
1133 const PRegisterZ& pg,
1134 const ZRegister& zn,
1135 const ZRegister& zm) {
1136 // FCMNE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
1137 // 0110 0101 ..0. .... 011. .... ...1 ....
1138 // size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5> |
1139 // o3<4> = 1 | Pd<3:0>
1140
1141 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1142 VIXL_ASSERT(AreSameLaneSize(zn, zm));
1143 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1144
1145 Emit(FCMNE_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
1146 }
1147
fcmuo(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)1148 void Assembler::fcmuo(const PRegisterWithLaneSize& pd,
1149 const PRegisterZ& pg,
1150 const ZRegister& zn,
1151 const ZRegister& zm) {
1152 // FCMUO <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
1153 // 0110 0101 ..0. .... 110. .... ...0 ....
1154 // size<23:22> | Zm<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5> |
1155 // o3<4> = 0 | Pd<3:0>
1156
1157 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1158 VIXL_ASSERT(AreSameLaneSize(zn, zm));
1159 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1160
1161 Emit(FCMUO_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
1162 }
1163
1164 // SVEFPCompareWithZero.
1165
fcmeq(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,double zero)1166 void Assembler::fcmeq(const PRegisterWithLaneSize& pd,
1167 const PRegisterZ& pg,
1168 const ZRegister& zn,
1169 double zero) {
1170 // FCMEQ <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0
1171 // 0110 0101 ..01 0010 001. .... ...0 ....
1172 // size<23:22> | eq<17> = 1 | lt<16> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 0 |
1173 // Pd<3:0>
1174
1175 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1176 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1177 VIXL_ASSERT(zero == 0.0);
1178 USE(zero);
1179
1180 Emit(FCMEQ_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn));
1181 }
1182
fcmge(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,double zero)1183 void Assembler::fcmge(const PRegisterWithLaneSize& pd,
1184 const PRegisterZ& pg,
1185 const ZRegister& zn,
1186 double zero) {
1187 // FCMGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0
1188 // 0110 0101 ..01 0000 001. .... ...0 ....
1189 // size<23:22> | eq<17> = 0 | lt<16> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 0 |
1190 // Pd<3:0>
1191
1192 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1193 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1194 VIXL_ASSERT(zero == 0.0);
1195 USE(zero);
1196
1197 Emit(FCMGE_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn));
1198 }
1199
fcmgt(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,double zero)1200 void Assembler::fcmgt(const PRegisterWithLaneSize& pd,
1201 const PRegisterZ& pg,
1202 const ZRegister& zn,
1203 double zero) {
1204 // FCMGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0
1205 // 0110 0101 ..01 0000 001. .... ...1 ....
1206 // size<23:22> | eq<17> = 0 | lt<16> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 1 |
1207 // Pd<3:0>
1208
1209 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1210 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1211 VIXL_ASSERT(zero == 0.0);
1212 USE(zero);
1213
1214 Emit(FCMGT_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn));
1215 }
1216
fcmle(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,double zero)1217 void Assembler::fcmle(const PRegisterWithLaneSize& pd,
1218 const PRegisterZ& pg,
1219 const ZRegister& zn,
1220 double zero) {
1221 // FCMLE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0
1222 // 0110 0101 ..01 0001 001. .... ...1 ....
1223 // size<23:22> | eq<17> = 0 | lt<16> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 1 |
1224 // Pd<3:0>
1225
1226 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1227 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1228 VIXL_ASSERT(zero == 0.0);
1229 USE(zero);
1230
1231 Emit(FCMLE_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn));
1232 }
1233
fcmlt(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,double zero)1234 void Assembler::fcmlt(const PRegisterWithLaneSize& pd,
1235 const PRegisterZ& pg,
1236 const ZRegister& zn,
1237 double zero) {
1238 // FCMLT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0
1239 // 0110 0101 ..01 0001 001. .... ...0 ....
1240 // size<23:22> | eq<17> = 0 | lt<16> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 0 |
1241 // Pd<3:0>
1242
1243 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1244 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1245 VIXL_ASSERT(zero == 0.0);
1246 USE(zero);
1247
1248 Emit(FCMLT_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn));
1249 }
1250
fcmne(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,double zero)1251 void Assembler::fcmne(const PRegisterWithLaneSize& pd,
1252 const PRegisterZ& pg,
1253 const ZRegister& zn,
1254 double zero) {
1255 // FCMNE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0
1256 // 0110 0101 ..01 0011 001. .... ...0 ....
1257 // size<23:22> | eq<17> = 1 | lt<16> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 0 |
1258 // Pd<3:0>
1259
1260 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1261 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1262 VIXL_ASSERT(zero == 0.0);
1263 USE(zero);
1264
1265 Emit(FCMNE_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn));
1266 }
1267
1268 // SVEFPComplexAddition.
1269
fcadd(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm,int rot)1270 void Assembler::fcadd(const ZRegister& zd,
1271 const PRegisterM& pg,
1272 const ZRegister& zn,
1273 const ZRegister& zm,
1274 int rot) {
1275 // FCADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>, <const>
1276 // 0110 0100 ..00 000. 100. .... .... ....
1277 // size<23:22> | rot<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
1278
1279 USE(zn);
1280 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1281 VIXL_ASSERT(zd.Is(zn));
1282 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
1283 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1284 VIXL_ASSERT((rot == 90) || (rot == 270));
1285
1286 Instr rotate_bit = (rot == 90) ? 0 : (1 << 16);
1287 Emit(FCADD_z_p_zz | rotate_bit | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
1288 }
1289
1290 // SVEFPComplexMulAdd.
1291
fcmla(const ZRegister & zda,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm,int rot)1292 void Assembler::fcmla(const ZRegister& zda,
1293 const PRegisterM& pg,
1294 const ZRegister& zn,
1295 const ZRegister& zm,
1296 int rot) {
1297 // FCMLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>, <const>
1298 // 0110 0100 ..0. .... 0... .... .... ....
1299 // size<23:22> | Zm<20:16> | rot<14:13> | Pg<12:10> | Zn<9:5> | Zda<4:0>
1300
1301 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1302 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1303 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
1304 VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270));
1305
1306 Instr rotate_bit = (rot / 90) << 13;
1307 Emit(FCMLA_z_p_zzz | rotate_bit | SVESize(zda) | Rd(zda) | PgLow8(pg) |
1308 Rn(zn) | Rm(zm));
1309 }
1310
1311 // SVEFPComplexMulAddIndex.
1312
fcmla(const ZRegister & zda,const ZRegister & zn,const ZRegister & zm,int index,int rot)1313 void Assembler::fcmla(const ZRegister& zda,
1314 const ZRegister& zn,
1315 const ZRegister& zm,
1316 int index,
1317 int rot) {
1318 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1319 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1320 VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270));
1321 VIXL_ASSERT(index >= 0);
1322
1323 int lane_size = zda.GetLaneSizeInBytes();
1324
1325 Instr zm_and_idx = 0;
1326 Instr op = FCMLA_z_zzzi_h;
1327 if (lane_size == kHRegSizeInBytes) {
1328 // Zm<18:16> | i2<20:19>
1329 VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 3));
1330 zm_and_idx = (index << 19) | Rx<18, 16>(zm);
1331 } else {
1332 // Zm<19:16> | i1<20>
1333 VIXL_ASSERT(lane_size == kSRegSizeInBytes);
1334 VIXL_ASSERT((zm.GetCode() <= 15) && (index <= 1));
1335 zm_and_idx = (index << 20) | Rx<19, 16>(zm);
1336 op = FCMLA_z_zzzi_s;
1337 }
1338
1339 Instr rotate_bit = (rot / 90) << 10;
1340 Emit(op | zm_and_idx | rotate_bit | Rd(zda) | Rn(zn));
1341 }
1342
1343 // SVEFPFastReduction.
1344
faddv(const VRegister & vd,const PRegister & pg,const ZRegister & zn)1345 void Assembler::faddv(const VRegister& vd,
1346 const PRegister& pg,
1347 const ZRegister& zn) {
1348 // FADDV <V><d>, <Pg>, <Zn>.<T>
1349 // 0110 0101 ..00 0000 001. .... .... ....
1350 // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zn<9:5> | Vd<4:0>
1351
1352 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1353 VIXL_ASSERT(vd.IsScalar());
1354 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1355 VIXL_ASSERT(AreSameLaneSize(zn, vd));
1356
1357 Emit(FADDV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
1358 }
1359
fmaxnmv(const VRegister & vd,const PRegister & pg,const ZRegister & zn)1360 void Assembler::fmaxnmv(const VRegister& vd,
1361 const PRegister& pg,
1362 const ZRegister& zn) {
1363 // FMAXNMV <V><d>, <Pg>, <Zn>.<T>
1364 // 0110 0101 ..00 0100 001. .... .... ....
1365 // size<23:22> | opc<18:16> = 100 | Pg<12:10> | Zn<9:5> | Vd<4:0>
1366
1367 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1368 VIXL_ASSERT(vd.IsScalar());
1369 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1370 VIXL_ASSERT(AreSameLaneSize(zn, vd));
1371
1372 Emit(FMAXNMV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
1373 }
1374
fmaxv(const VRegister & vd,const PRegister & pg,const ZRegister & zn)1375 void Assembler::fmaxv(const VRegister& vd,
1376 const PRegister& pg,
1377 const ZRegister& zn) {
1378 // FMAXV <V><d>, <Pg>, <Zn>.<T>
1379 // 0110 0101 ..00 0110 001. .... .... ....
1380 // size<23:22> | opc<18:16> = 110 | Pg<12:10> | Zn<9:5> | Vd<4:0>
1381
1382 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1383 VIXL_ASSERT(vd.IsScalar());
1384 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1385 VIXL_ASSERT(AreSameLaneSize(zn, vd));
1386
1387 Emit(FMAXV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
1388 }
1389
fminnmv(const VRegister & vd,const PRegister & pg,const ZRegister & zn)1390 void Assembler::fminnmv(const VRegister& vd,
1391 const PRegister& pg,
1392 const ZRegister& zn) {
1393 // FMINNMV <V><d>, <Pg>, <Zn>.<T>
1394 // 0110 0101 ..00 0101 001. .... .... ....
1395 // size<23:22> | opc<18:16> = 101 | Pg<12:10> | Zn<9:5> | Vd<4:0>
1396
1397 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1398 VIXL_ASSERT(vd.IsScalar());
1399 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1400 VIXL_ASSERT(AreSameLaneSize(zn, vd));
1401
1402 Emit(FMINNMV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
1403 }
1404
fminv(const VRegister & vd,const PRegister & pg,const ZRegister & zn)1405 void Assembler::fminv(const VRegister& vd,
1406 const PRegister& pg,
1407 const ZRegister& zn) {
1408 // FMINV <V><d>, <Pg>, <Zn>.<T>
1409 // 0110 0101 ..00 0111 001. .... .... ....
1410 // size<23:22> | opc<18:16> = 111 | Pg<12:10> | Zn<9:5> | Vd<4:0>
1411
1412 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1413 VIXL_ASSERT(vd.IsScalar());
1414 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1415 VIXL_ASSERT(AreSameLaneSize(zn, vd));
1416
1417 Emit(FMINV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
1418 }
1419
1420 // SVEFPMulAdd.
1421
fmad(const ZRegister & zdn,const PRegisterM & pg,const ZRegister & zm,const ZRegister & za)1422 void Assembler::fmad(const ZRegister& zdn,
1423 const PRegisterM& pg,
1424 const ZRegister& zm,
1425 const ZRegister& za) {
1426 // FMAD <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T>
1427 // 0110 0101 ..1. .... 100. .... .... ....
1428 // size<23:22> | Za<20:16> | opc<14:13> = 00 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
1429
1430 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1431 VIXL_ASSERT(AreSameLaneSize(zdn, zm, za));
1432 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1433
1434 Emit(FMAD_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za));
1435 }
1436
fmla(const ZRegister & zda,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)1437 void Assembler::fmla(const ZRegister& zda,
1438 const PRegisterM& pg,
1439 const ZRegister& zn,
1440 const ZRegister& zm) {
1441 // FMLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>
1442 // 0110 0101 ..1. .... 000. .... .... ....
1443 // size<23:22> | Zm<20:16> | opc<14:13> = 00 | Pg<12:10> | Zn<9:5> | Zda<4:0>
1444
1445 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1446 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1447 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
1448
1449 Emit(FMLA_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm));
1450 }
1451
fmls(const ZRegister & zda,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)1452 void Assembler::fmls(const ZRegister& zda,
1453 const PRegisterM& pg,
1454 const ZRegister& zn,
1455 const ZRegister& zm) {
1456 // FMLS <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>
1457 // 0110 0101 ..1. .... 001. .... .... ....
1458 // size<23:22> | Zm<20:16> | opc<14:13> = 01 | Pg<12:10> | Zn<9:5> | Zda<4:0>
1459
1460 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1461 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1462 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
1463
1464 Emit(FMLS_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm));
1465 }
1466
fmsb(const ZRegister & zdn,const PRegisterM & pg,const ZRegister & zm,const ZRegister & za)1467 void Assembler::fmsb(const ZRegister& zdn,
1468 const PRegisterM& pg,
1469 const ZRegister& zm,
1470 const ZRegister& za) {
1471 // FMSB <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T>
1472 // 0110 0101 ..1. .... 101. .... .... ....
1473 // size<23:22> | Za<20:16> | opc<14:13> = 01 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
1474
1475 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1476 VIXL_ASSERT(AreSameLaneSize(zdn, zm, za));
1477 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1478
1479 Emit(FMSB_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za));
1480 }
1481
fnmad(const ZRegister & zdn,const PRegisterM & pg,const ZRegister & zm,const ZRegister & za)1482 void Assembler::fnmad(const ZRegister& zdn,
1483 const PRegisterM& pg,
1484 const ZRegister& zm,
1485 const ZRegister& za) {
1486 // FNMAD <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T>
1487 // 0110 0101 ..1. .... 110. .... .... ....
1488 // size<23:22> | Za<20:16> | opc<14:13> = 10 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
1489
1490 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1491 VIXL_ASSERT(AreSameLaneSize(zdn, zm, za));
1492 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1493
1494 Emit(FNMAD_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za));
1495 }
1496
fnmla(const ZRegister & zda,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)1497 void Assembler::fnmla(const ZRegister& zda,
1498 const PRegisterM& pg,
1499 const ZRegister& zn,
1500 const ZRegister& zm) {
1501 // FNMLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>
1502 // 0110 0101 ..1. .... 010. .... .... ....
1503 // size<23:22> | Zm<20:16> | opc<14:13> = 10 | Pg<12:10> | Zn<9:5> | Zda<4:0>
1504
1505 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1506 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1507 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
1508
1509 Emit(FNMLA_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm));
1510 }
1511
fnmls(const ZRegister & zda,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)1512 void Assembler::fnmls(const ZRegister& zda,
1513 const PRegisterM& pg,
1514 const ZRegister& zn,
1515 const ZRegister& zm) {
1516 // FNMLS <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>
1517 // 0110 0101 ..1. .... 011. .... .... ....
1518 // size<23:22> | Zm<20:16> | opc<14:13> = 11 | Pg<12:10> | Zn<9:5> | Zda<4:0>
1519
1520 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1521 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1522 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
1523
1524 Emit(FNMLS_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm));
1525 }
1526
fnmsb(const ZRegister & zdn,const PRegisterM & pg,const ZRegister & zm,const ZRegister & za)1527 void Assembler::fnmsb(const ZRegister& zdn,
1528 const PRegisterM& pg,
1529 const ZRegister& zm,
1530 const ZRegister& za) {
1531 // FNMSB <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T>
1532 // 0110 0101 ..1. .... 111. .... .... ....
1533 // size<23:22> | Za<20:16> | opc<14:13> = 11 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
1534
1535 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1536 VIXL_ASSERT(AreSameLaneSize(zdn, zm, za));
1537 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1538
1539 Emit(FNMSB_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za));
1540 }
1541
SVEFPMulIndexHelper(unsigned lane_size_in_bytes_log2,const ZRegister & zm,int index,Instr op_h,Instr op_s,Instr op_d)1542 Instr Assembler::SVEFPMulIndexHelper(unsigned lane_size_in_bytes_log2,
1543 const ZRegister& zm,
1544 int index,
1545 Instr op_h,
1546 Instr op_s,
1547 Instr op_d) {
1548 Instr size = lane_size_in_bytes_log2 << SVESize_offset;
1549 Instr zm_with_index = Rm(zm);
1550 Instr op = 0xffffffff;
1551 // Allowable register number and lane index depends on the lane size.
1552 switch (lane_size_in_bytes_log2) {
1553 case kHRegSizeInBytesLog2:
1554 VIXL_ASSERT(zm.GetCode() <= 7);
1555 VIXL_ASSERT(IsUint3(index));
1556 // For H-sized lanes, size is encoded as 0b0x, where x is used as the top
1557 // bit of the index. So, if index is less than four, the top bit of index
1558 // is zero, and therefore size is 0b00. Otherwise, it's 0b01, the usual
1559 // encoding for H-sized lanes.
1560 if (index < 4) size = 0;
1561 // Top two bits of "zm" encode the index.
1562 zm_with_index |= (index & 3) << (Rm_offset + 3);
1563 op = op_h;
1564 break;
1565 case kSRegSizeInBytesLog2:
1566 VIXL_ASSERT(zm.GetCode() <= 7);
1567 VIXL_ASSERT(IsUint2(index));
1568 // Top two bits of "zm" encode the index.
1569 zm_with_index |= (index & 3) << (Rm_offset + 3);
1570 op = op_s;
1571 break;
1572 case kDRegSizeInBytesLog2:
1573 VIXL_ASSERT(zm.GetCode() <= 15);
1574 VIXL_ASSERT(IsUint1(index));
1575 // Top bit of "zm" encodes the index.
1576 zm_with_index |= (index & 1) << (Rm_offset + 4);
1577 op = op_d;
1578 break;
1579 default:
1580 VIXL_UNIMPLEMENTED();
1581 }
1582 return op | zm_with_index | size;
1583 }
1584
1585 // SVEFPMulAddIndex.
1586
fmla(const ZRegister & zda,const ZRegister & zn,const ZRegister & zm,int index)1587 void Assembler::fmla(const ZRegister& zda,
1588 const ZRegister& zn,
1589 const ZRegister& zm,
1590 int index) {
1591 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1592 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1593
1594 // The encoding of opcode, index, Zm, and size are synthesized in this
1595 // variable.
1596 Instr synthesized_op = SVEFPMulIndexHelper(zda.GetLaneSizeInBytesLog2(),
1597 zm,
1598 index,
1599 FMLA_z_zzzi_h,
1600 FMLA_z_zzzi_s,
1601 FMLA_z_zzzi_d);
1602
1603 Emit(synthesized_op | Rd(zda) | Rn(zn));
1604 }
1605
fmls(const ZRegister & zda,const ZRegister & zn,const ZRegister & zm,int index)1606 void Assembler::fmls(const ZRegister& zda,
1607 const ZRegister& zn,
1608 const ZRegister& zm,
1609 int index) {
1610 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1611 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1612
1613 // The encoding of opcode, index, Zm, and size are synthesized in this
1614 // variable.
1615 Instr synthesized_op = SVEFPMulIndexHelper(zda.GetLaneSizeInBytesLog2(),
1616 zm,
1617 index,
1618 FMLS_z_zzzi_h,
1619 FMLS_z_zzzi_s,
1620 FMLS_z_zzzi_d);
1621
1622 Emit(synthesized_op | Rd(zda) | Rn(zn));
1623 }
1624
1625 // SVEFPMulIndex.
1626
1627 // This prototype maps to 3 instruction encodings:
fmul(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm,unsigned index)1628 void Assembler::fmul(const ZRegister& zd,
1629 const ZRegister& zn,
1630 const ZRegister& zm,
1631 unsigned index) {
1632 // FMUL <Zd>.<T>, <Zn>.<T>, <Zm>.<T>[<imm>]
1633 // 0110 0100 ..1. .... 0010 00.. .... ....
1634 // size<23:22> | opc<20:16> | Zn<9:5> | Zd<4:0>
1635
1636 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1637 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
1638
1639 // The encoding of opcode, index, Zm, and size are synthesized in this
1640 // variable.
1641 Instr synthesized_op = SVEFPMulIndexHelper(zd.GetLaneSizeInBytesLog2(),
1642 zm,
1643 index,
1644 FMUL_z_zzi_h,
1645 FMUL_z_zzi_s,
1646 FMUL_z_zzi_d);
1647
1648 Emit(synthesized_op | Rd(zd) | Rn(zn));
1649 }
1650
1651 // SVEFPUnaryOpPredicated.
1652
fcvt(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)1653 void Assembler::fcvt(const ZRegister& zd,
1654 const PRegisterM& pg,
1655 const ZRegister& zn) {
1656 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1657
1658 Instr op = 0xffffffff;
1659 switch (zn.GetLaneSizeInBytes()) {
1660 case kHRegSizeInBytes:
1661 switch (zd.GetLaneSizeInBytes()) {
1662 case kSRegSizeInBytes:
1663 op = FCVT_z_p_z_h2s;
1664 break;
1665 case kDRegSizeInBytes:
1666 op = FCVT_z_p_z_h2d;
1667 break;
1668 }
1669 break;
1670 case kSRegSizeInBytes:
1671 switch (zd.GetLaneSizeInBytes()) {
1672 case kHRegSizeInBytes:
1673 op = FCVT_z_p_z_s2h;
1674 break;
1675 case kDRegSizeInBytes:
1676 op = FCVT_z_p_z_s2d;
1677 break;
1678 }
1679 break;
1680 case kDRegSizeInBytes:
1681 switch (zd.GetLaneSizeInBytes()) {
1682 case kHRegSizeInBytes:
1683 op = FCVT_z_p_z_d2h;
1684 break;
1685 case kSRegSizeInBytes:
1686 op = FCVT_z_p_z_d2s;
1687 break;
1688 }
1689 break;
1690 }
1691 VIXL_ASSERT(op != 0xffffffff);
1692
1693 Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn));
1694 }
1695
fcvtzs(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)1696 void Assembler::fcvtzs(const ZRegister& zd,
1697 const PRegisterM& pg,
1698 const ZRegister& zn) {
1699 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1700 Instr op = 0xffffffff;
1701 switch (zn.GetLaneSizeInBytes()) {
1702 case kHRegSizeInBytes:
1703 switch (zd.GetLaneSizeInBytes()) {
1704 case kHRegSizeInBytes:
1705 op = FCVTZS_z_p_z_fp162h;
1706 break;
1707 case kSRegSizeInBytes:
1708 op = FCVTZS_z_p_z_fp162w;
1709 break;
1710 case kDRegSizeInBytes:
1711 op = FCVTZS_z_p_z_fp162x;
1712 break;
1713 }
1714 break;
1715 case kSRegSizeInBytes:
1716 switch (zd.GetLaneSizeInBytes()) {
1717 case kSRegSizeInBytes:
1718 op = FCVTZS_z_p_z_s2w;
1719 break;
1720 case kDRegSizeInBytes:
1721 op = FCVTZS_z_p_z_s2x;
1722 break;
1723 }
1724 break;
1725 case kDRegSizeInBytes:
1726 switch (zd.GetLaneSizeInBytes()) {
1727 case kSRegSizeInBytes:
1728 op = FCVTZS_z_p_z_d2w;
1729 break;
1730 case kDRegSizeInBytes:
1731 op = FCVTZS_z_p_z_d2x;
1732 break;
1733 }
1734 break;
1735 }
1736 VIXL_ASSERT(op != 0xffffffff);
1737
1738 Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn));
1739 }
1740
fcvtzu(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)1741 void Assembler::fcvtzu(const ZRegister& zd,
1742 const PRegisterM& pg,
1743 const ZRegister& zn) {
1744 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1745 Instr op = 0xffffffff;
1746 switch (zn.GetLaneSizeInBytes()) {
1747 case kHRegSizeInBytes:
1748 switch (zd.GetLaneSizeInBytes()) {
1749 case kHRegSizeInBytes:
1750 op = FCVTZU_z_p_z_fp162h;
1751 break;
1752 case kSRegSizeInBytes:
1753 op = FCVTZU_z_p_z_fp162w;
1754 break;
1755 case kDRegSizeInBytes:
1756 op = FCVTZU_z_p_z_fp162x;
1757 break;
1758 }
1759 break;
1760 case kSRegSizeInBytes:
1761 switch (zd.GetLaneSizeInBytes()) {
1762 case kSRegSizeInBytes:
1763 op = FCVTZU_z_p_z_s2w;
1764 break;
1765 case kDRegSizeInBytes:
1766 op = FCVTZU_z_p_z_s2x;
1767 break;
1768 }
1769 break;
1770 case kDRegSizeInBytes:
1771 switch (zd.GetLaneSizeInBytes()) {
1772 case kSRegSizeInBytes:
1773 op = FCVTZU_z_p_z_d2w;
1774 break;
1775 case kDRegSizeInBytes:
1776 op = FCVTZU_z_p_z_d2x;
1777 break;
1778 }
1779 break;
1780 }
1781 VIXL_ASSERT(op != 0xffffffff);
1782
1783 Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn));
1784 }
1785
frecpx(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)1786 void Assembler::frecpx(const ZRegister& zd,
1787 const PRegisterM& pg,
1788 const ZRegister& zn) {
1789 // FRECPX <Zd>.<T>, <Pg>/M, <Zn>.<T>
1790 // 0110 0101 ..00 1100 101. .... .... ....
1791 // size<23:22> | opc<17:16> = 00 | Pg<12:10> | Zn<9:5> | Zd<4:0>
1792
1793 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1794 VIXL_ASSERT(AreSameLaneSize(zd, zn));
1795 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1796
1797 Emit(FRECPX_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1798 }
1799
frinta(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)1800 void Assembler::frinta(const ZRegister& zd,
1801 const PRegisterM& pg,
1802 const ZRegister& zn) {
1803 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1804 VIXL_ASSERT(AreSameLaneSize(zd, zn));
1805 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1806
1807 Emit(FRINTA_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1808 }
1809
frinti(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)1810 void Assembler::frinti(const ZRegister& zd,
1811 const PRegisterM& pg,
1812 const ZRegister& zn) {
1813 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1814 VIXL_ASSERT(AreSameLaneSize(zd, zn));
1815 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1816
1817 Emit(FRINTI_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1818 }
1819
frintm(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)1820 void Assembler::frintm(const ZRegister& zd,
1821 const PRegisterM& pg,
1822 const ZRegister& zn) {
1823 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1824 VIXL_ASSERT(AreSameLaneSize(zd, zn));
1825 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1826
1827 Emit(FRINTM_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1828 }
1829
frintn(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)1830 void Assembler::frintn(const ZRegister& zd,
1831 const PRegisterM& pg,
1832 const ZRegister& zn) {
1833 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1834 VIXL_ASSERT(AreSameLaneSize(zd, zn));
1835 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1836
1837 Emit(FRINTN_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1838 }
1839
frintp(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)1840 void Assembler::frintp(const ZRegister& zd,
1841 const PRegisterM& pg,
1842 const ZRegister& zn) {
1843 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1844 VIXL_ASSERT(AreSameLaneSize(zd, zn));
1845 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1846
1847 Emit(FRINTP_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1848 }
1849
frintx(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)1850 void Assembler::frintx(const ZRegister& zd,
1851 const PRegisterM& pg,
1852 const ZRegister& zn) {
1853 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1854 VIXL_ASSERT(AreSameLaneSize(zd, zn));
1855 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1856
1857 Emit(FRINTX_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1858 }
1859
frintz(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)1860 void Assembler::frintz(const ZRegister& zd,
1861 const PRegisterM& pg,
1862 const ZRegister& zn) {
1863 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1864 VIXL_ASSERT(AreSameLaneSize(zd, zn));
1865 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1866
1867 Emit(FRINTZ_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1868 }
1869
fsqrt(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)1870 void Assembler::fsqrt(const ZRegister& zd,
1871 const PRegisterM& pg,
1872 const ZRegister& zn) {
1873 // FSQRT <Zd>.<T>, <Pg>/M, <Zn>.<T>
1874 // 0110 0101 ..00 1101 101. .... .... ....
1875 // size<23:22> | opc<17:16> = 01 | Pg<12:10> | Zn<9:5> | Zd<4:0>
1876
1877 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1878 VIXL_ASSERT(AreSameLaneSize(zd, zn));
1879 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1880
1881 Emit(FSQRT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1882 }
1883
scvtf(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)1884 void Assembler::scvtf(const ZRegister& zd,
1885 const PRegisterM& pg,
1886 const ZRegister& zn) {
1887 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1888 Instr op = 0xffffffff;
1889 switch (zn.GetLaneSizeInBytes()) {
1890 case kHRegSizeInBytes:
1891 switch (zd.GetLaneSizeInBytes()) {
1892 case kHRegSizeInBytes:
1893 op = SCVTF_z_p_z_h2fp16;
1894 break;
1895 }
1896 break;
1897 case kSRegSizeInBytes:
1898 switch (zd.GetLaneSizeInBytes()) {
1899 case kHRegSizeInBytes:
1900 op = SCVTF_z_p_z_w2fp16;
1901 break;
1902 case kSRegSizeInBytes:
1903 op = SCVTF_z_p_z_w2s;
1904 break;
1905 case kDRegSizeInBytes:
1906 op = SCVTF_z_p_z_w2d;
1907 break;
1908 }
1909 break;
1910 case kDRegSizeInBytes:
1911 switch (zd.GetLaneSizeInBytes()) {
1912 case kHRegSizeInBytes:
1913 op = SCVTF_z_p_z_x2fp16;
1914 break;
1915 case kSRegSizeInBytes:
1916 op = SCVTF_z_p_z_x2s;
1917 break;
1918 case kDRegSizeInBytes:
1919 op = SCVTF_z_p_z_x2d;
1920 break;
1921 }
1922 break;
1923 }
1924 VIXL_ASSERT(op != 0xffffffff);
1925
1926 Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn));
1927 }
1928
ucvtf(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)1929 void Assembler::ucvtf(const ZRegister& zd,
1930 const PRegisterM& pg,
1931 const ZRegister& zn) {
1932 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1933 Instr op = 0xffffffff;
1934 switch (zn.GetLaneSizeInBytes()) {
1935 case kHRegSizeInBytes:
1936 switch (zd.GetLaneSizeInBytes()) {
1937 case kHRegSizeInBytes:
1938 op = UCVTF_z_p_z_h2fp16;
1939 break;
1940 }
1941 break;
1942 case kSRegSizeInBytes:
1943 switch (zd.GetLaneSizeInBytes()) {
1944 case kHRegSizeInBytes:
1945 op = UCVTF_z_p_z_w2fp16;
1946 break;
1947 case kSRegSizeInBytes:
1948 op = UCVTF_z_p_z_w2s;
1949 break;
1950 case kDRegSizeInBytes:
1951 op = UCVTF_z_p_z_w2d;
1952 break;
1953 }
1954 break;
1955 case kDRegSizeInBytes:
1956 switch (zd.GetLaneSizeInBytes()) {
1957 case kHRegSizeInBytes:
1958 op = UCVTF_z_p_z_x2fp16;
1959 break;
1960 case kSRegSizeInBytes:
1961 op = UCVTF_z_p_z_x2s;
1962 break;
1963 case kDRegSizeInBytes:
1964 op = UCVTF_z_p_z_x2d;
1965 break;
1966 }
1967 break;
1968 }
1969 VIXL_ASSERT(op != 0xffffffff);
1970
1971 Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn));
1972 }
1973
1974 // SVEFPUnaryOpUnpredicated.
1975
frecpe(const ZRegister & zd,const ZRegister & zn)1976 void Assembler::frecpe(const ZRegister& zd, const ZRegister& zn) {
1977 // FRECPE <Zd>.<T>, <Zn>.<T>
1978 // 0110 0101 ..00 1110 0011 00.. .... ....
1979 // size<23:22> | opc<18:16> = 110 | Zn<9:5> | Zd<4:0>
1980
1981 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1982 VIXL_ASSERT(AreSameLaneSize(zd, zn));
1983 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1984
1985 Emit(FRECPE_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
1986 }
1987
frsqrte(const ZRegister & zd,const ZRegister & zn)1988 void Assembler::frsqrte(const ZRegister& zd, const ZRegister& zn) {
1989 // FRSQRTE <Zd>.<T>, <Zn>.<T>
1990 // 0110 0101 ..00 1111 0011 00.. .... ....
1991 // size<23:22> | opc<18:16> = 111 | Zn<9:5> | Zd<4:0>
1992
1993 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1994 VIXL_ASSERT(AreSameLaneSize(zd, zn));
1995 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1996
1997 Emit(FRSQRTE_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
1998 }
1999
2000 // SVEIncDecByPredicateCount.
2001
decp(const Register & rdn,const PRegisterWithLaneSize & pg)2002 void Assembler::decp(const Register& rdn, const PRegisterWithLaneSize& pg) {
2003 // DECP <Xdn>, <Pg>.<T>
2004 // 0010 0101 ..10 1101 1000 100. .... ....
2005 // size<23:22> | op<17> = 0 | D<16> = 1 | opc2<10:9> = 00 | Pg<8:5> |
2006 // Rdn<4:0>
2007
2008 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2009 VIXL_ASSERT(rdn.IsX());
2010
2011 Emit(DECP_r_p_r | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg));
2012 }
2013
decp(const ZRegister & zdn,const PRegister & pg)2014 void Assembler::decp(const ZRegister& zdn, const PRegister& pg) {
2015 // DECP <Zdn>.<T>, <Pg>
2016 // 0010 0101 ..10 1101 1000 000. .... ....
2017 // size<23:22> | op<17> = 0 | D<16> = 1 | opc2<10:9> = 00 | Pg<8:5> |
2018 // Zdn<4:0>
2019
2020 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2021 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
2022 VIXL_ASSERT(pg.IsUnqualified());
2023
2024 Emit(DECP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg));
2025 }
2026
incp(const Register & rdn,const PRegisterWithLaneSize & pg)2027 void Assembler::incp(const Register& rdn, const PRegisterWithLaneSize& pg) {
2028 // INCP <Xdn>, <Pg>.<T>
2029 // 0010 0101 ..10 1100 1000 100. .... ....
2030 // size<23:22> | op<17> = 0 | D<16> = 0 | opc2<10:9> = 00 | Pg<8:5> |
2031 // Rdn<4:0>
2032
2033 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2034 VIXL_ASSERT(rdn.IsX());
2035
2036 Emit(INCP_r_p_r | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg));
2037 }
2038
incp(const ZRegister & zdn,const PRegister & pg)2039 void Assembler::incp(const ZRegister& zdn, const PRegister& pg) {
2040 // INCP <Zdn>.<T>, <Pg>
2041 // 0010 0101 ..10 1100 1000 000. .... ....
2042 // size<23:22> | op<17> = 0 | D<16> = 0 | opc2<10:9> = 00 | Pg<8:5> |
2043 // Zdn<4:0>
2044
2045 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2046 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
2047 VIXL_ASSERT(pg.IsUnqualified());
2048
2049 Emit(INCP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg));
2050 }
2051
sqdecp(const Register & xd,const PRegisterWithLaneSize & pg,const Register & wn)2052 void Assembler::sqdecp(const Register& xd,
2053 const PRegisterWithLaneSize& pg,
2054 const Register& wn) {
2055 // SQDECP <Xdn>, <Pg>.<T>, <Wdn>
2056 // 0010 0101 ..10 1010 1000 100. .... ....
2057 // size<23:22> | D<17> = 1 | U<16> = 0 | sf<10> = 0 | op<9> = 0 | Pg<8:5> |
2058 // Rdn<4:0>
2059
2060 USE(wn);
2061 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2062 VIXL_ASSERT(xd.IsX() && wn.IsW() && xd.Aliases(wn));
2063
2064 Emit(SQDECP_r_p_r_sx | SVESize(pg) | Rd(xd) | Rx<8, 5>(pg));
2065 }
2066
sqdecp(const Register & xdn,const PRegisterWithLaneSize & pg)2067 void Assembler::sqdecp(const Register& xdn, const PRegisterWithLaneSize& pg) {
2068 // SQDECP <Xdn>, <Pg>.<T>
2069 // 0010 0101 ..10 1010 1000 110. .... ....
2070 // size<23:22> | D<17> = 1 | U<16> = 0 | sf<10> = 1 | op<9> = 0 | Pg<8:5> |
2071 // Rdn<4:0>
2072
2073 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2074 VIXL_ASSERT(xdn.IsX());
2075
2076 Emit(SQDECP_r_p_r_x | SVESize(pg) | Rd(xdn) | Rx<8, 5>(pg));
2077 }
2078
sqdecp(const ZRegister & zdn,const PRegister & pg)2079 void Assembler::sqdecp(const ZRegister& zdn, const PRegister& pg) {
2080 // SQDECP <Zdn>.<T>, <Pg>
2081 // 0010 0101 ..10 1010 1000 000. .... ....
2082 // size<23:22> | D<17> = 1 | U<16> = 0 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0>
2083
2084 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2085 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
2086 VIXL_ASSERT(pg.IsUnqualified());
2087
2088 Emit(SQDECP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg));
2089 }
2090
sqincp(const Register & xd,const PRegisterWithLaneSize & pg,const Register & wn)2091 void Assembler::sqincp(const Register& xd,
2092 const PRegisterWithLaneSize& pg,
2093 const Register& wn) {
2094 // SQINCP <Xdn>, <Pg>.<T>, <Wdn>
2095 // 0010 0101 ..10 1000 1000 100. .... ....
2096 // size<23:22> | D<17> = 0 | U<16> = 0 | sf<10> = 0 | op<9> = 0 | Pg<8:5> |
2097 // Rdn<4:0>
2098
2099 USE(wn);
2100 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2101 VIXL_ASSERT(xd.IsX() && wn.IsW() && xd.Aliases(wn));
2102
2103 Emit(SQINCP_r_p_r_sx | SVESize(pg) | Rd(xd) | Rx<8, 5>(pg));
2104 }
2105
sqincp(const Register & xdn,const PRegisterWithLaneSize & pg)2106 void Assembler::sqincp(const Register& xdn, const PRegisterWithLaneSize& pg) {
2107 // SQINCP <Xdn>, <Pg>.<T>
2108 // 0010 0101 ..10 1000 1000 110. .... ....
2109 // size<23:22> | D<17> = 0 | U<16> = 0 | sf<10> = 1 | op<9> = 0 | Pg<8:5> |
2110 // Rdn<4:0>
2111
2112 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2113 VIXL_ASSERT(xdn.IsX());
2114
2115 Emit(SQINCP_r_p_r_x | SVESize(pg) | Rd(xdn) | Rx<8, 5>(pg));
2116 }
2117
sqincp(const ZRegister & zdn,const PRegister & pg)2118 void Assembler::sqincp(const ZRegister& zdn, const PRegister& pg) {
2119 // SQINCP <Zdn>.<T>, <Pg>
2120 // 0010 0101 ..10 1000 1000 000. .... ....
2121 // size<23:22> | D<17> = 0 | U<16> = 0 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0>
2122
2123 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2124 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
2125 VIXL_ASSERT(pg.IsUnqualified());
2126
2127 Emit(SQINCP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg));
2128 }
2129
uqdecp(const Register & rdn,const PRegisterWithLaneSize & pg)2130 void Assembler::uqdecp(const Register& rdn, const PRegisterWithLaneSize& pg) {
2131 // UQDECP <Wdn>, <Pg>.<T>
2132 // UQDECP <Xdn>, <Pg>.<T>
2133 // 0010 0101 ..10 1011 1000 10.. .... ....
2134 // size<23:22> | D<17> = 1 | U<16> = 1 | sf<10> | op<9> = 0 | Pg<8:5> |
2135 // Rdn<4:0>
2136
2137 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2138
2139 Instr op = rdn.IsX() ? UQDECP_r_p_r_x : UQDECP_r_p_r_uw;
2140 Emit(op | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg));
2141 }
2142
uqdecp(const ZRegister & zdn,const PRegister & pg)2143 void Assembler::uqdecp(const ZRegister& zdn, const PRegister& pg) {
2144 // UQDECP <Zdn>.<T>, <Pg>
2145 // 0010 0101 ..10 1011 1000 000. .... ....
2146 // size<23:22> | D<17> = 1 | U<16> = 1 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0>
2147
2148 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2149 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
2150 VIXL_ASSERT(pg.IsUnqualified());
2151
2152 Emit(UQDECP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg));
2153 }
2154
uqincp(const Register & rdn,const PRegisterWithLaneSize & pg)2155 void Assembler::uqincp(const Register& rdn, const PRegisterWithLaneSize& pg) {
2156 // UQINCP <Wdn>, <Pg>.<T>
2157 // 0010 0101 ..10 1001 1000 100. .... ....
2158 // size<23:22> | D<17> = 0 | U<16> = 1 | sf<10> = 0 | op<9> = 0 | Pg<8:5> |
2159 // Rdn<4:0>
2160
2161 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2162
2163 Instr op = rdn.IsX() ? UQINCP_r_p_r_x : UQINCP_r_p_r_uw;
2164 Emit(op | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg));
2165 }
2166
uqincp(const ZRegister & zdn,const PRegister & pg)2167 void Assembler::uqincp(const ZRegister& zdn, const PRegister& pg) {
2168 // UQINCP <Zdn>.<T>, <Pg>
2169 // 0010 0101 ..10 1001 1000 000. .... ....
2170 // size<23:22> | D<17> = 0 | U<16> = 1 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0>
2171
2172 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2173 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
2174 VIXL_ASSERT(pg.IsUnqualified());
2175
2176 Emit(UQINCP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg));
2177 }
2178
2179 // SVEIndexGeneration.
2180
index(const ZRegister & zd,int start,int step)2181 void Assembler::index(const ZRegister& zd, int start, int step) {
2182 // INDEX <Zd>.<T>, #<imm1>, #<imm2>
2183 // 0000 0100 ..1. .... 0100 00.. .... ....
2184 // size<23:22> | step<20:16> | start<9:5> | Zd<4:0>
2185
2186 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2187
2188 Emit(INDEX_z_ii | SVESize(zd) | ImmField<20, 16>(step) |
2189 ImmField<9, 5>(start) | Rd(zd));
2190 }
2191
index(const ZRegister & zd,const Register & rn,const Register & rm)2192 void Assembler::index(const ZRegister& zd,
2193 const Register& rn,
2194 const Register& rm) {
2195 // INDEX <Zd>.<T>, <R><n>, <R><m>
2196 // 0000 0100 ..1. .... 0100 11.. .... ....
2197 // size<23:22> | Rm<20:16> | Rn<9:5> | Zd<4:0>
2198
2199 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2200 VIXL_ASSERT(static_cast<unsigned>(rn.GetSizeInBits()) >=
2201 zd.GetLaneSizeInBits());
2202 VIXL_ASSERT(static_cast<unsigned>(rm.GetSizeInBits()) >=
2203 zd.GetLaneSizeInBits());
2204
2205 Emit(INDEX_z_rr | SVESize(zd) | Rd(zd) | Rn(rn) | Rm(rm));
2206 }
2207
index(const ZRegister & zd,const Register & rn,int imm5)2208 void Assembler::index(const ZRegister& zd, const Register& rn, int imm5) {
2209 // INDEX <Zd>.<T>, <R><n>, #<imm>
2210 // 0000 0100 ..1. .... 0100 01.. .... ....
2211 // size<23:22> | imm5<20:16> | Rn<9:5> | Zd<4:0>
2212
2213 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2214 VIXL_ASSERT(static_cast<unsigned>(rn.GetSizeInBits()) >=
2215 zd.GetLaneSizeInBits());
2216
2217 Emit(INDEX_z_ri | SVESize(zd) | Rd(zd) | Rn(rn) | ImmField<20, 16>(imm5));
2218 }
2219
index(const ZRegister & zd,int imm5,const Register & rm)2220 void Assembler::index(const ZRegister& zd, int imm5, const Register& rm) {
2221 // INDEX <Zd>.<T>, #<imm>, <R><m>
2222 // 0000 0100 ..1. .... 0100 10.. .... ....
2223 // size<23:22> | Rm<20:16> | imm5<9:5> | Zd<4:0>
2224
2225 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2226 VIXL_ASSERT(static_cast<unsigned>(rm.GetSizeInBits()) >=
2227 zd.GetLaneSizeInBits());
2228
2229 Emit(INDEX_z_ir | SVESize(zd) | Rd(zd) | ImmField<9, 5>(imm5) | Rm(rm));
2230 }
2231
2232 // SVEIntArithmeticUnpredicated.
2233
add(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)2234 void Assembler::add(const ZRegister& zd,
2235 const ZRegister& zn,
2236 const ZRegister& zm) {
2237 // ADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
2238 // 0000 0100 ..1. .... 0000 00.. .... ....
2239 // size<23:22> | Zm<20:16> | opc<12:10> = 000 | Zn<9:5> | Zd<4:0>
2240
2241 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2242 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
2243
2244 Emit(ADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
2245 }
2246
sqadd(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)2247 void Assembler::sqadd(const ZRegister& zd,
2248 const ZRegister& zn,
2249 const ZRegister& zm) {
2250 // SQADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
2251 // 0000 0100 ..1. .... 0001 00.. .... ....
2252 // size<23:22> | Zm<20:16> | opc<12:10> = 100 | Zn<9:5> | Zd<4:0>
2253
2254 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2255 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
2256
2257 Emit(SQADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
2258 }
2259
sqsub(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)2260 void Assembler::sqsub(const ZRegister& zd,
2261 const ZRegister& zn,
2262 const ZRegister& zm) {
2263 // SQSUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
2264 // 0000 0100 ..1. .... 0001 10.. .... ....
2265 // size<23:22> | Zm<20:16> | opc<12:10> = 110 | Zn<9:5> | Zd<4:0>
2266
2267 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2268 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
2269
2270 Emit(SQSUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
2271 }
2272
sub(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)2273 void Assembler::sub(const ZRegister& zd,
2274 const ZRegister& zn,
2275 const ZRegister& zm) {
2276 // SUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
2277 // 0000 0100 ..1. .... 0000 01.. .... ....
2278 // size<23:22> | Zm<20:16> | opc<12:10> = 001 | Zn<9:5> | Zd<4:0>
2279
2280 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2281 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
2282
2283 Emit(SUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
2284 }
2285
uqadd(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)2286 void Assembler::uqadd(const ZRegister& zd,
2287 const ZRegister& zn,
2288 const ZRegister& zm) {
2289 // UQADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
2290 // 0000 0100 ..1. .... 0001 01.. .... ....
2291 // size<23:22> | Zm<20:16> | opc<12:10> = 101 | Zn<9:5> | Zd<4:0>
2292
2293 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2294 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
2295
2296 Emit(UQADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
2297 }
2298
uqsub(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)2299 void Assembler::uqsub(const ZRegister& zd,
2300 const ZRegister& zn,
2301 const ZRegister& zm) {
2302 // UQSUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
2303 // 0000 0100 ..1. .... 0001 11.. .... ....
2304 // size<23:22> | Zm<20:16> | opc<12:10> = 111 | Zn<9:5> | Zd<4:0>
2305
2306 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2307 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
2308
2309 Emit(UQSUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
2310 }
2311
2312 // SVEIntBinaryArithmeticPredicated.
2313
add(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2314 void Assembler::add(const ZRegister& zd,
2315 const PRegisterM& pg,
2316 const ZRegister& zn,
2317 const ZRegister& zm) {
2318 // ADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2319 // 0000 0100 ..00 0000 000. .... .... ....
2320 // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2321
2322 USE(zn);
2323 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2324 VIXL_ASSERT(zd.Is(zn));
2325 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2326
2327 Emit(ADD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2328 }
2329
and_(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2330 void Assembler::and_(const ZRegister& zd,
2331 const PRegisterM& pg,
2332 const ZRegister& zn,
2333 const ZRegister& zm) {
2334 // AND <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2335 // 0000 0100 ..01 1010 000. .... .... ....
2336 // size<23:22> | opc<18:16> = 010 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2337
2338 USE(zn);
2339 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2340 VIXL_ASSERT(zd.Is(zn));
2341 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2342
2343 Emit(AND_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2344 }
2345
bic(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2346 void Assembler::bic(const ZRegister& zd,
2347 const PRegisterM& pg,
2348 const ZRegister& zn,
2349 const ZRegister& zm) {
2350 // BIC <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2351 // 0000 0100 ..01 1011 000. .... .... ....
2352 // size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2353
2354 USE(zn);
2355 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2356 VIXL_ASSERT(zd.Is(zn));
2357 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2358
2359 Emit(BIC_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2360 }
2361
eor(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2362 void Assembler::eor(const ZRegister& zd,
2363 const PRegisterM& pg,
2364 const ZRegister& zn,
2365 const ZRegister& zm) {
2366 // EOR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2367 // 0000 0100 ..01 1001 000. .... .... ....
2368 // size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2369
2370 USE(zn);
2371 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2372 VIXL_ASSERT(zd.Is(zn));
2373 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2374
2375 Emit(EOR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2376 }
2377
mul(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2378 void Assembler::mul(const ZRegister& zd,
2379 const PRegisterM& pg,
2380 const ZRegister& zn,
2381 const ZRegister& zm) {
2382 // MUL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2383 // 0000 0100 ..01 0000 000. .... .... ....
2384 // size<23:22> | H<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2385
2386 USE(zn);
2387 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2388 VIXL_ASSERT(zd.Is(zn));
2389 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2390
2391 Emit(MUL_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2392 }
2393
orr(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2394 void Assembler::orr(const ZRegister& zd,
2395 const PRegisterM& pg,
2396 const ZRegister& zn,
2397 const ZRegister& zm) {
2398 // ORR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2399 // 0000 0100 ..01 1000 000. .... .... ....
2400 // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2401
2402 USE(zn);
2403 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2404 VIXL_ASSERT(zd.Is(zn));
2405 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2406
2407 Emit(ORR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2408 }
2409
sabd(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2410 void Assembler::sabd(const ZRegister& zd,
2411 const PRegisterM& pg,
2412 const ZRegister& zn,
2413 const ZRegister& zm) {
2414 // SABD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2415 // 0000 0100 ..00 1100 000. .... .... ....
2416 // size<23:22> | opc<18:17> = 10 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2417
2418 USE(zn);
2419 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2420 VIXL_ASSERT(zd.Is(zn));
2421 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2422
2423 Emit(SABD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2424 }
2425
sdiv(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2426 void Assembler::sdiv(const ZRegister& zd,
2427 const PRegisterM& pg,
2428 const ZRegister& zn,
2429 const ZRegister& zm) {
2430 // SDIV <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2431 // 0000 0100 ..01 0100 000. .... .... ....
2432 // size<23:22> | R<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2433
2434 USE(zn);
2435 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2436 VIXL_ASSERT(zd.Is(zn));
2437 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2438 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD());
2439
2440 Emit(SDIV_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2441 }
2442
sdivr(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2443 void Assembler::sdivr(const ZRegister& zd,
2444 const PRegisterM& pg,
2445 const ZRegister& zn,
2446 const ZRegister& zm) {
2447 // SDIVR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2448 // 0000 0100 ..01 0110 000. .... .... ....
2449 // size<23:22> | R<17> = 1 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2450
2451 USE(zn);
2452 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2453 VIXL_ASSERT(zd.Is(zn));
2454 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2455 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD());
2456
2457 Emit(SDIVR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2458 }
2459
smax(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2460 void Assembler::smax(const ZRegister& zd,
2461 const PRegisterM& pg,
2462 const ZRegister& zn,
2463 const ZRegister& zm) {
2464 // SMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2465 // 0000 0100 ..00 1000 000. .... .... ....
2466 // size<23:22> | opc<18:17> = 00 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2467
2468 USE(zn);
2469 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2470 VIXL_ASSERT(zd.Is(zn));
2471 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2472
2473 Emit(SMAX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2474 }
2475
smin(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2476 void Assembler::smin(const ZRegister& zd,
2477 const PRegisterM& pg,
2478 const ZRegister& zn,
2479 const ZRegister& zm) {
2480 // SMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2481 // 0000 0100 ..00 1010 000. .... .... ....
2482 // size<23:22> | opc<18:17> = 01 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2483
2484 USE(zn);
2485 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2486 VIXL_ASSERT(zd.Is(zn));
2487 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2488
2489 Emit(SMIN_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2490 }
2491
smulh(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2492 void Assembler::smulh(const ZRegister& zd,
2493 const PRegisterM& pg,
2494 const ZRegister& zn,
2495 const ZRegister& zm) {
2496 // SMULH <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2497 // 0000 0100 ..01 0010 000. .... .... ....
2498 // size<23:22> | H<17> = 1 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2499
2500 USE(zn);
2501 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2502 VIXL_ASSERT(zd.Is(zn));
2503 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2504
2505 Emit(SMULH_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2506 }
2507
sub(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2508 void Assembler::sub(const ZRegister& zd,
2509 const PRegisterM& pg,
2510 const ZRegister& zn,
2511 const ZRegister& zm) {
2512 // SUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2513 // 0000 0100 ..00 0001 000. .... .... ....
2514 // size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2515
2516 USE(zn);
2517 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2518 VIXL_ASSERT(zd.Is(zn));
2519 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2520
2521 Emit(SUB_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2522 }
2523
subr(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2524 void Assembler::subr(const ZRegister& zd,
2525 const PRegisterM& pg,
2526 const ZRegister& zn,
2527 const ZRegister& zm) {
2528 // SUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2529 // 0000 0100 ..00 0011 000. .... .... ....
2530 // size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2531
2532 USE(zn);
2533 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2534 VIXL_ASSERT(zd.Is(zn));
2535 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2536
2537 Emit(SUBR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2538 }
2539
uabd(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2540 void Assembler::uabd(const ZRegister& zd,
2541 const PRegisterM& pg,
2542 const ZRegister& zn,
2543 const ZRegister& zm) {
2544 // UABD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2545 // 0000 0100 ..00 1101 000. .... .... ....
2546 // size<23:22> | opc<18:17> = 10 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2547
2548 USE(zn);
2549 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2550 VIXL_ASSERT(zd.Is(zn));
2551 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2552
2553 Emit(UABD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2554 }
2555
udiv(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2556 void Assembler::udiv(const ZRegister& zd,
2557 const PRegisterM& pg,
2558 const ZRegister& zn,
2559 const ZRegister& zm) {
2560 // UDIV <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2561 // 0000 0100 ..01 0101 000. .... .... ....
2562 // size<23:22> | R<17> = 0 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2563
2564 USE(zn);
2565 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2566 VIXL_ASSERT(zd.Is(zn));
2567 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2568 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD());
2569
2570 Emit(UDIV_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2571 }
2572
udivr(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2573 void Assembler::udivr(const ZRegister& zd,
2574 const PRegisterM& pg,
2575 const ZRegister& zn,
2576 const ZRegister& zm) {
2577 // UDIVR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2578 // 0000 0100 ..01 0111 000. .... .... ....
2579 // size<23:22> | R<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2580
2581 USE(zn);
2582 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2583 VIXL_ASSERT(zd.Is(zn));
2584 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2585 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD());
2586
2587 Emit(UDIVR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2588 }
2589
umax(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2590 void Assembler::umax(const ZRegister& zd,
2591 const PRegisterM& pg,
2592 const ZRegister& zn,
2593 const ZRegister& zm) {
2594 // UMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2595 // 0000 0100 ..00 1001 000. .... .... ....
2596 // size<23:22> | opc<18:17> = 00 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2597
2598 USE(zn);
2599 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2600 VIXL_ASSERT(zd.Is(zn));
2601 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2602
2603 Emit(UMAX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2604 }
2605
umin(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2606 void Assembler::umin(const ZRegister& zd,
2607 const PRegisterM& pg,
2608 const ZRegister& zn,
2609 const ZRegister& zm) {
2610 // UMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2611 // 0000 0100 ..00 1011 000. .... .... ....
2612 // size<23:22> | opc<18:17> = 01 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2613
2614 USE(zn);
2615 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2616 VIXL_ASSERT(zd.Is(zn));
2617 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2618
2619 Emit(UMIN_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2620 }
2621
umulh(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)2622 void Assembler::umulh(const ZRegister& zd,
2623 const PRegisterM& pg,
2624 const ZRegister& zn,
2625 const ZRegister& zm) {
2626 // UMULH <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2627 // 0000 0100 ..01 0011 000. .... .... ....
2628 // size<23:22> | H<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2629
2630 USE(zn);
2631 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2632 VIXL_ASSERT(zd.Is(zn));
2633 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2634
2635 Emit(UMULH_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2636 }
2637
2638 // SVEIntCompareScalars.
2639
ctermeq(const Register & rn,const Register & rm)2640 void Assembler::ctermeq(const Register& rn, const Register& rm) {
2641 // CTERMEQ <R><n>, <R><m>
2642 // 0010 0101 1.1. .... 0010 00.. ...0 0000
2643 // op<23> = 1 | sz<22> | Rm<20:16> | Rn<9:5> | ne<4> = 0
2644
2645 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2646 VIXL_ASSERT(AreSameSizeAndType(rn, rm));
2647 const Instr sz = rn.Is64Bits() ? 0x00400000 : 0x00000000;
2648
2649 Emit(CTERMEQ_rr | sz | Rn(rn) | Rm(rm));
2650 }
2651
ctermne(const Register & rn,const Register & rm)2652 void Assembler::ctermne(const Register& rn, const Register& rm) {
2653 // CTERMNE <R><n>, <R><m>
2654 // 0010 0101 1.1. .... 0010 00.. ...1 0000
2655 // op<23> = 1 | sz<22> | Rm<20:16> | Rn<9:5> | ne<4> = 1
2656
2657 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2658 VIXL_ASSERT(AreSameSizeAndType(rn, rm));
2659 const Instr sz = rn.Is64Bits() ? 0x00400000 : 0x00000000;
2660
2661 Emit(CTERMNE_rr | sz | Rn(rn) | Rm(rm));
2662 }
2663
whilele(const PRegisterWithLaneSize & pd,const Register & rn,const Register & rm)2664 void Assembler::whilele(const PRegisterWithLaneSize& pd,
2665 const Register& rn,
2666 const Register& rm) {
2667 // WHILELE <Pd>.<T>, <R><n>, <R><m>
2668 // 0010 0101 ..1. .... 000. 01.. ...1 ....
2669 // size<23:22> | Rm<20:16> | sf<12> | U<11> = 0 | lt<10> = 1 | Rn<9:5> |
2670 // eq<4> = 1 | Pd<3:0>
2671
2672 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2673 VIXL_ASSERT(AreSameSizeAndType(rn, rm));
2674 const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000;
2675
2676 Emit(WHILELE_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm));
2677 }
2678
whilelo(const PRegisterWithLaneSize & pd,const Register & rn,const Register & rm)2679 void Assembler::whilelo(const PRegisterWithLaneSize& pd,
2680 const Register& rn,
2681 const Register& rm) {
2682 // WHILELO <Pd>.<T>, <R><n>, <R><m>
2683 // 0010 0101 ..1. .... 000. 11.. ...0 ....
2684 // size<23:22> | Rm<20:16> | sf<12> | U<11> = 1 | lt<10> = 1 | Rn<9:5> |
2685 // eq<4> = 0 | Pd<3:0>
2686
2687 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2688 VIXL_ASSERT(AreSameSizeAndType(rn, rm));
2689 const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000;
2690
2691 Emit(WHILELO_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm));
2692 }
2693
whilels(const PRegisterWithLaneSize & pd,const Register & rn,const Register & rm)2694 void Assembler::whilels(const PRegisterWithLaneSize& pd,
2695 const Register& rn,
2696 const Register& rm) {
2697 // WHILELS <Pd>.<T>, <R><n>, <R><m>
2698 // 0010 0101 ..1. .... 000. 11.. ...1 ....
2699 // size<23:22> | Rm<20:16> | sf<12> | U<11> = 1 | lt<10> = 1 | Rn<9:5> |
2700 // eq<4> = 1 | Pd<3:0>
2701
2702 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2703 VIXL_ASSERT(AreSameSizeAndType(rn, rm));
2704 const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000;
2705
2706 Emit(WHILELS_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm));
2707 }
2708
whilelt(const PRegisterWithLaneSize & pd,const Register & rn,const Register & rm)2709 void Assembler::whilelt(const PRegisterWithLaneSize& pd,
2710 const Register& rn,
2711 const Register& rm) {
2712 // WHILELT <Pd>.<T>, <R><n>, <R><m>
2713 // 0010 0101 ..1. .... 000. 01.. ...0 ....
2714 // size<23:22> | Rm<20:16> | sf<12> | U<11> = 0 | lt<10> = 1 | Rn<9:5> |
2715 // eq<4> = 0 | Pd<3:0>
2716
2717 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2718 VIXL_ASSERT(AreSameSizeAndType(rn, rm));
2719 const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000;
2720
2721 Emit(WHILELT_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm));
2722 }
2723
CompareVectors(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm,SVEIntCompareVectorsOp op)2724 void Assembler::CompareVectors(const PRegisterWithLaneSize& pd,
2725 const PRegisterZ& pg,
2726 const ZRegister& zn,
2727 const ZRegister& zm,
2728 SVEIntCompareVectorsOp op) {
2729 Emit(op | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
2730 }
2731
CompareVectors(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,int imm,SVEIntCompareSignedImmOp op)2732 void Assembler::CompareVectors(const PRegisterWithLaneSize& pd,
2733 const PRegisterZ& pg,
2734 const ZRegister& zn,
2735 int imm,
2736 SVEIntCompareSignedImmOp op) {
2737 Emit(op | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm));
2738 }
2739
CompareVectors(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,unsigned imm,SVEIntCompareUnsignedImmOp op)2740 void Assembler::CompareVectors(const PRegisterWithLaneSize& pd,
2741 const PRegisterZ& pg,
2742 const ZRegister& zn,
2743 unsigned imm,
2744 SVEIntCompareUnsignedImmOp op) {
2745 Emit(op | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) |
2746 ImmUnsignedField<20, 14>(imm));
2747 }
2748
cmp(Condition cond,const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)2749 void Assembler::cmp(Condition cond,
2750 const PRegisterWithLaneSize& pd,
2751 const PRegisterZ& pg,
2752 const ZRegister& zn,
2753 const ZRegister& zm) {
2754 switch (cond) {
2755 case eq:
2756 cmpeq(pd, pg, zn, zm);
2757 break;
2758 case ge:
2759 cmpge(pd, pg, zn, zm);
2760 break;
2761 case gt:
2762 cmpgt(pd, pg, zn, zm);
2763 break;
2764 case le:
2765 cmple(pd, pg, zn, zm);
2766 break;
2767 case lt:
2768 cmplt(pd, pg, zn, zm);
2769 break;
2770 case ne:
2771 cmpne(pd, pg, zn, zm);
2772 break;
2773 case hi:
2774 cmphi(pd, pg, zn, zm);
2775 break;
2776 case hs:
2777 cmphs(pd, pg, zn, zm);
2778 break;
2779 case lo:
2780 cmplo(pd, pg, zn, zm);
2781 break;
2782 case ls:
2783 cmpls(pd, pg, zn, zm);
2784 break;
2785 default:
2786 VIXL_UNREACHABLE();
2787 }
2788 }
2789
2790 // SVEIntCompareSignedImm.
2791
cmpeq(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,int imm5)2792 void Assembler::cmpeq(const PRegisterWithLaneSize& pd,
2793 const PRegisterZ& pg,
2794 const ZRegister& zn,
2795 int imm5) {
2796 // CMPEQ <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2797 // 0010 0101 ..0. .... 100. .... ...0 ....
2798 // size<23:22> | imm5<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5>
2799 // | ne<4> = 0 | Pd<3:0>
2800
2801 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2802 VIXL_ASSERT(AreSameLaneSize(pd, zn));
2803
2804 CompareVectors(pd, pg, zn, imm5, CMPEQ_p_p_zi);
2805 }
2806
cmpge(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,int imm5)2807 void Assembler::cmpge(const PRegisterWithLaneSize& pd,
2808 const PRegisterZ& pg,
2809 const ZRegister& zn,
2810 int imm5) {
2811 // CMPGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2812 // 0010 0101 ..0. .... 000. .... ...0 ....
2813 // size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5>
2814 // | ne<4> = 0 | Pd<3:0>
2815
2816 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2817 VIXL_ASSERT(AreSameLaneSize(pd, zn));
2818
2819 CompareVectors(pd, pg, zn, imm5, CMPGE_p_p_zi);
2820 }
2821
cmpgt(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,int imm5)2822 void Assembler::cmpgt(const PRegisterWithLaneSize& pd,
2823 const PRegisterZ& pg,
2824 const ZRegister& zn,
2825 int imm5) {
2826 // CMPGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2827 // 0010 0101 ..0. .... 000. .... ...1 ....
2828 // size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5>
2829 // | ne<4> = 1 | Pd<3:0>
2830
2831 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2832 VIXL_ASSERT(AreSameLaneSize(pd, zn));
2833
2834 CompareVectors(pd, pg, zn, imm5, CMPGT_p_p_zi);
2835 }
2836
cmple(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,int imm5)2837 void Assembler::cmple(const PRegisterWithLaneSize& pd,
2838 const PRegisterZ& pg,
2839 const ZRegister& zn,
2840 int imm5) {
2841 // CMPLE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2842 // 0010 0101 ..0. .... 001. .... ...1 ....
2843 // size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5>
2844 // | ne<4> = 1 | Pd<3:0>
2845
2846 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2847 VIXL_ASSERT(AreSameLaneSize(pd, zn));
2848
2849 CompareVectors(pd, pg, zn, imm5, CMPLE_p_p_zi);
2850 }
2851
cmplt(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,int imm5)2852 void Assembler::cmplt(const PRegisterWithLaneSize& pd,
2853 const PRegisterZ& pg,
2854 const ZRegister& zn,
2855 int imm5) {
2856 // CMPLT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2857 // 0010 0101 ..0. .... 001. .... ...0 ....
2858 // size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5>
2859 // | ne<4> = 0 | Pd<3:0>
2860
2861 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2862 VIXL_ASSERT(AreSameLaneSize(pd, zn));
2863
2864 CompareVectors(pd, pg, zn, imm5, CMPLT_p_p_zi);
2865 }
2866
cmpne(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,int imm5)2867 void Assembler::cmpne(const PRegisterWithLaneSize& pd,
2868 const PRegisterZ& pg,
2869 const ZRegister& zn,
2870 int imm5) {
2871 // CMPNE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2872 // 0010 0101 ..0. .... 100. .... ...1 ....
2873 // size<23:22> | imm5<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5>
2874 // | ne<4> = 1 | Pd<3:0>
2875
2876 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2877 VIXL_ASSERT(AreSameLaneSize(pd, zn));
2878
2879 CompareVectors(pd, pg, zn, imm5, CMPNE_p_p_zi);
2880 }
2881
2882 // SVEIntCompareUnsignedImm.
2883
cmphi(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,unsigned imm7)2884 void Assembler::cmphi(const PRegisterWithLaneSize& pd,
2885 const PRegisterZ& pg,
2886 const ZRegister& zn,
2887 unsigned imm7) {
2888 // CMPHI <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2889 // 0010 0100 ..1. .... ..0. .... ...1 ....
2890 // size<23:22> | imm7<20:14> | lt<13> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 1 |
2891 // Pd<3:0>
2892
2893 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2894 VIXL_ASSERT(AreSameLaneSize(pd, zn));
2895
2896 CompareVectors(pd, pg, zn, imm7, CMPHI_p_p_zi);
2897 }
2898
cmphs(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,unsigned imm7)2899 void Assembler::cmphs(const PRegisterWithLaneSize& pd,
2900 const PRegisterZ& pg,
2901 const ZRegister& zn,
2902 unsigned imm7) {
2903 // CMPHS <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2904 // 0010 0100 ..1. .... ..0. .... ...0 ....
2905 // size<23:22> | imm7<20:14> | lt<13> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 0 |
2906 // Pd<3:0>
2907
2908 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2909 VIXL_ASSERT(AreSameLaneSize(pd, zn));
2910
2911 CompareVectors(pd, pg, zn, imm7, CMPHS_p_p_zi);
2912 }
2913
cmplo(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,unsigned imm7)2914 void Assembler::cmplo(const PRegisterWithLaneSize& pd,
2915 const PRegisterZ& pg,
2916 const ZRegister& zn,
2917 unsigned imm7) {
2918 // CMPLO <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2919 // 0010 0100 ..1. .... ..1. .... ...0 ....
2920 // size<23:22> | imm7<20:14> | lt<13> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 0 |
2921 // Pd<3:0>
2922
2923 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2924 VIXL_ASSERT(AreSameLaneSize(pd, zn));
2925
2926 CompareVectors(pd, pg, zn, imm7, CMPLO_p_p_zi);
2927 }
2928
cmpls(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,unsigned imm7)2929 void Assembler::cmpls(const PRegisterWithLaneSize& pd,
2930 const PRegisterZ& pg,
2931 const ZRegister& zn,
2932 unsigned imm7) {
2933 // CMPLS <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2934 // 0010 0100 ..1. .... ..1. .... ...1 ....
2935 // size<23:22> | imm7<20:14> | lt<13> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 1 |
2936 // Pd<3:0>
2937
2938 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2939 VIXL_ASSERT(AreSameLaneSize(pd, zn));
2940
2941 CompareVectors(pd, pg, zn, imm7, CMPLS_p_p_zi);
2942 }
2943
2944 // SVEIntCompareVectors.
2945
2946 // This prototype maps to 2 instruction encodings:
2947 // CMPEQ_p_p_zw
2948 // CMPEQ_p_p_zz
cmpeq(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)2949 void Assembler::cmpeq(const PRegisterWithLaneSize& pd,
2950 const PRegisterZ& pg,
2951 const ZRegister& zn,
2952 const ZRegister& zm) {
2953 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2954 VIXL_ASSERT(AreSameLaneSize(pd, zn));
2955 SVEIntCompareVectorsOp op = CMPEQ_p_p_zz;
2956 if (!AreSameLaneSize(zn, zm)) {
2957 VIXL_ASSERT(zm.IsLaneSizeD());
2958 op = CMPEQ_p_p_zw;
2959 }
2960 CompareVectors(pd, pg, zn, zm, op);
2961 }
2962
2963 // This prototype maps to 2 instruction encodings:
2964 // CMPGE_p_p_zw
2965 // CMPGE_p_p_zz
cmpge(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)2966 void Assembler::cmpge(const PRegisterWithLaneSize& pd,
2967 const PRegisterZ& pg,
2968 const ZRegister& zn,
2969 const ZRegister& zm) {
2970 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2971 VIXL_ASSERT(AreSameLaneSize(pd, zn));
2972 SVEIntCompareVectorsOp op = CMPGE_p_p_zz;
2973 if (!AreSameLaneSize(zn, zm)) {
2974 VIXL_ASSERT(zm.IsLaneSizeD());
2975 op = CMPGE_p_p_zw;
2976 }
2977 CompareVectors(pd, pg, zn, zm, op);
2978 }
2979
2980 // This prototype maps to 2 instruction encodings:
2981 // CMPGT_p_p_zw
2982 // CMPGT_p_p_zz
cmpgt(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)2983 void Assembler::cmpgt(const PRegisterWithLaneSize& pd,
2984 const PRegisterZ& pg,
2985 const ZRegister& zn,
2986 const ZRegister& zm) {
2987 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2988 VIXL_ASSERT(AreSameLaneSize(pd, zn));
2989 SVEIntCompareVectorsOp op = CMPGT_p_p_zz;
2990 if (!AreSameLaneSize(zn, zm)) {
2991 VIXL_ASSERT(zm.IsLaneSizeD());
2992 op = CMPGT_p_p_zw;
2993 }
2994 CompareVectors(pd, pg, zn, zm, op);
2995 }
2996
2997 // This prototype maps to 2 instruction encodings:
2998 // CMPHI_p_p_zw
2999 // CMPHI_p_p_zz
cmphi(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)3000 void Assembler::cmphi(const PRegisterWithLaneSize& pd,
3001 const PRegisterZ& pg,
3002 const ZRegister& zn,
3003 const ZRegister& zm) {
3004 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3005 VIXL_ASSERT(AreSameLaneSize(pd, zn));
3006 SVEIntCompareVectorsOp op = CMPHI_p_p_zz;
3007 if (!AreSameLaneSize(zn, zm)) {
3008 VIXL_ASSERT(zm.IsLaneSizeD());
3009 op = CMPHI_p_p_zw;
3010 }
3011 CompareVectors(pd, pg, zn, zm, op);
3012 }
3013
3014 // This prototype maps to 2 instruction encodings:
3015 // CMPHS_p_p_zw
3016 // CMPHS_p_p_zz
cmphs(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)3017 void Assembler::cmphs(const PRegisterWithLaneSize& pd,
3018 const PRegisterZ& pg,
3019 const ZRegister& zn,
3020 const ZRegister& zm) {
3021 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3022 VIXL_ASSERT(AreSameLaneSize(pd, zn));
3023 SVEIntCompareVectorsOp op = CMPHS_p_p_zz;
3024 if (!AreSameLaneSize(zn, zm)) {
3025 VIXL_ASSERT(zm.IsLaneSizeD());
3026 op = CMPHS_p_p_zw;
3027 }
3028 CompareVectors(pd, pg, zn, zm, op);
3029 }
3030
cmple(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)3031 void Assembler::cmple(const PRegisterWithLaneSize& pd,
3032 const PRegisterZ& pg,
3033 const ZRegister& zn,
3034 const ZRegister& zm) {
3035 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3036 VIXL_ASSERT(AreSameLaneSize(pd, zn));
3037 if (AreSameLaneSize(zn, zm)) {
3038 cmpge(pd, pg, zm, zn);
3039 return;
3040 }
3041 VIXL_ASSERT(zm.IsLaneSizeD());
3042 VIXL_ASSERT(!zn.IsLaneSizeD());
3043
3044 CompareVectors(pd, pg, zn, zm, CMPLE_p_p_zw);
3045 }
3046
cmplo(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)3047 void Assembler::cmplo(const PRegisterWithLaneSize& pd,
3048 const PRegisterZ& pg,
3049 const ZRegister& zn,
3050 const ZRegister& zm) {
3051 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3052 VIXL_ASSERT(AreSameLaneSize(pd, zn));
3053 if (AreSameLaneSize(zn, zm)) {
3054 cmphi(pd, pg, zm, zn);
3055 return;
3056 }
3057 VIXL_ASSERT(zm.IsLaneSizeD());
3058 VIXL_ASSERT(!zn.IsLaneSizeD());
3059
3060 CompareVectors(pd, pg, zn, zm, CMPLO_p_p_zw);
3061 }
3062
cmpls(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)3063 void Assembler::cmpls(const PRegisterWithLaneSize& pd,
3064 const PRegisterZ& pg,
3065 const ZRegister& zn,
3066 const ZRegister& zm) {
3067 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3068 VIXL_ASSERT(AreSameLaneSize(pd, zn));
3069 if (AreSameLaneSize(zn, zm)) {
3070 cmphs(pd, pg, zm, zn);
3071 return;
3072 }
3073 VIXL_ASSERT(zm.IsLaneSizeD());
3074 VIXL_ASSERT(!zn.IsLaneSizeD());
3075
3076 CompareVectors(pd, pg, zn, zm, CMPLS_p_p_zw);
3077 }
3078
cmplt(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)3079 void Assembler::cmplt(const PRegisterWithLaneSize& pd,
3080 const PRegisterZ& pg,
3081 const ZRegister& zn,
3082 const ZRegister& zm) {
3083 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3084 VIXL_ASSERT(AreSameLaneSize(pd, zn));
3085 if (AreSameLaneSize(zn, zm)) {
3086 cmpgt(pd, pg, zm, zn);
3087 return;
3088 }
3089 VIXL_ASSERT(zm.IsLaneSizeD());
3090 VIXL_ASSERT(!zn.IsLaneSizeD());
3091
3092 CompareVectors(pd, pg, zn, zm, CMPLT_p_p_zw);
3093 }
3094
3095 // This prototype maps to 2 instruction encodings:
3096 // CMPNE_p_p_zw
3097 // CMPNE_p_p_zz
cmpne(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const ZRegister & zn,const ZRegister & zm)3098 void Assembler::cmpne(const PRegisterWithLaneSize& pd,
3099 const PRegisterZ& pg,
3100 const ZRegister& zn,
3101 const ZRegister& zm) {
3102 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3103 VIXL_ASSERT(AreSameLaneSize(pd, zn));
3104 SVEIntCompareVectorsOp op = CMPNE_p_p_zz;
3105 if (!AreSameLaneSize(zn, zm)) {
3106 VIXL_ASSERT(zm.IsLaneSizeD());
3107 op = CMPNE_p_p_zw;
3108 }
3109 CompareVectors(pd, pg, zn, zm, op);
3110 }
3111
3112 // SVEIntMiscUnpredicated.
3113
fexpa(const ZRegister & zd,const ZRegister & zn)3114 void Assembler::fexpa(const ZRegister& zd, const ZRegister& zn) {
3115 // FEXPA <Zd>.<T>, <Zn>.<T>
3116 // 0000 0100 ..10 0000 1011 10.. .... ....
3117 // size<23:22> | opc<20:16> = 00000 | Zn<9:5> | Zd<4:0>
3118
3119 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3120 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3121 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
3122
3123 Emit(FEXPA_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
3124 }
3125
ftssel(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)3126 void Assembler::ftssel(const ZRegister& zd,
3127 const ZRegister& zn,
3128 const ZRegister& zm) {
3129 // FTSSEL <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
3130 // 0000 0100 ..1. .... 1011 00.. .... ....
3131 // size<23:22> | Zm<20:16> | op<10> = 0 | Zn<9:5> | Zd<4:0>
3132
3133 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3134 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
3135 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
3136
3137 Emit(FTSSEL_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
3138 }
3139
movprfx(const ZRegister & zd,const ZRegister & zn)3140 void Assembler::movprfx(const ZRegister& zd, const ZRegister& zn) {
3141 // MOVPRFX <Zd>, <Zn>
3142 // 0000 0100 0010 0000 1011 11.. .... ....
3143 // opc<23:22> = 00 | opc2<20:16> = 00000 | Zn<9:5> | Zd<4:0>
3144
3145 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3146 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3147
3148 Emit(MOVPRFX_z_z | Rd(zd) | Rn(zn));
3149 }
3150
3151 // SVEIntMulAddPredicated.
3152
mad(const ZRegister & zdn,const PRegisterM & pg,const ZRegister & zm,const ZRegister & za)3153 void Assembler::mad(const ZRegister& zdn,
3154 const PRegisterM& pg,
3155 const ZRegister& zm,
3156 const ZRegister& za) {
3157 // MAD <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T>
3158 // 0000 0100 ..0. .... 110. .... .... ....
3159 // size<23:22> | Zm<20:16> | op<13> = 0 | Pg<12:10> | Za<9:5> | Zdn<4:0>
3160
3161 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3162 VIXL_ASSERT(AreSameLaneSize(zdn, zm, za));
3163
3164 Emit(MAD_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rm(zm) | Rn(za));
3165 }
3166
mla(const ZRegister & zda,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)3167 void Assembler::mla(const ZRegister& zda,
3168 const PRegisterM& pg,
3169 const ZRegister& zn,
3170 const ZRegister& zm) {
3171 // MLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>
3172 // 0000 0100 ..0. .... 010. .... .... ....
3173 // size<23:22> | Zm<20:16> | op<13> = 0 | Pg<12:10> | Zn<9:5> | Zda<4:0>
3174
3175 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3176 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
3177
3178 Emit(MLA_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm));
3179 }
3180
mls(const ZRegister & zda,const PRegisterM & pg,const ZRegister & zn,const ZRegister & zm)3181 void Assembler::mls(const ZRegister& zda,
3182 const PRegisterM& pg,
3183 const ZRegister& zn,
3184 const ZRegister& zm) {
3185 // MLS <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>
3186 // 0000 0100 ..0. .... 011. .... .... ....
3187 // size<23:22> | Zm<20:16> | op<13> = 1 | Pg<12:10> | Zn<9:5> | Zda<4:0>
3188
3189 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3190 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
3191
3192 Emit(MLS_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm));
3193 }
3194
msb(const ZRegister & zdn,const PRegisterM & pg,const ZRegister & zm,const ZRegister & za)3195 void Assembler::msb(const ZRegister& zdn,
3196 const PRegisterM& pg,
3197 const ZRegister& zm,
3198 const ZRegister& za) {
3199 // MSB <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T>
3200 // 0000 0100 ..0. .... 111. .... .... ....
3201 // size<23:22> | Zm<20:16> | op<13> = 1 | Pg<12:10> | Za<9:5> | Zdn<4:0>
3202
3203 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3204 VIXL_ASSERT(AreSameLaneSize(zdn, zm, za));
3205
3206 Emit(MSB_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rm(zm) | Rn(za));
3207 }
3208
3209 // SVEIntMulAddUnpredicated.
3210
sdot(const ZRegister & zda,const ZRegister & zn,const ZRegister & zm)3211 void Assembler::sdot(const ZRegister& zda,
3212 const ZRegister& zn,
3213 const ZRegister& zm) {
3214 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3215 VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD());
3216 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4));
3217 VIXL_ASSERT(AreSameLaneSize(zm, zn));
3218
3219 Emit(SDOT_z_zzz | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
3220 }
3221
udot(const ZRegister & zda,const ZRegister & zn,const ZRegister & zm)3222 void Assembler::udot(const ZRegister& zda,
3223 const ZRegister& zn,
3224 const ZRegister& zm) {
3225 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3226 VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD());
3227 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4));
3228 VIXL_ASSERT(AreSameLaneSize(zm, zn));
3229
3230 Emit(UDOT_z_zzz | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
3231 }
3232
3233 // SVEIntReduction.
3234
andv(const VRegister & vd,const PRegister & pg,const ZRegister & zn)3235 void Assembler::andv(const VRegister& vd,
3236 const PRegister& pg,
3237 const ZRegister& zn) {
3238 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3239 VIXL_ASSERT(vd.IsScalar());
3240
3241 Emit(ANDV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
3242 }
3243
eorv(const VRegister & vd,const PRegister & pg,const ZRegister & zn)3244 void Assembler::eorv(const VRegister& vd,
3245 const PRegister& pg,
3246 const ZRegister& zn) {
3247 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3248 VIXL_ASSERT(vd.IsScalar());
3249
3250 Emit(EORV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
3251 }
3252
movprfx(const ZRegister & zd,const PRegister & pg,const ZRegister & zn)3253 void Assembler::movprfx(const ZRegister& zd,
3254 const PRegister& pg,
3255 const ZRegister& zn) {
3256 // MOVPRFX <Zd>.<T>, <Pg>/<ZM>, <Zn>.<T>
3257 // 0000 0100 ..01 000. 001. .... .... ....
3258 // size<23:22> | opc<18:17> = 00 | M<16> | Pg<12:10> | Zn<9:5> | Zd<4:0>
3259
3260 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3261 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3262 VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing());
3263 VIXL_ASSERT(!pg.HasLaneSize());
3264
3265 Instr m = pg.IsMerging() ? 0x00010000 : 0x00000000;
3266 Emit(MOVPRFX_z_p_z | SVESize(zd) | m | Rd(zd) | PgLow8(pg) | Rn(zn));
3267 }
3268
orv(const VRegister & vd,const PRegister & pg,const ZRegister & zn)3269 void Assembler::orv(const VRegister& vd,
3270 const PRegister& pg,
3271 const ZRegister& zn) {
3272 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3273 VIXL_ASSERT(vd.IsScalar());
3274
3275 Emit(ORV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
3276 }
3277
saddv(const VRegister & dd,const PRegister & pg,const ZRegister & zn)3278 void Assembler::saddv(const VRegister& dd,
3279 const PRegister& pg,
3280 const ZRegister& zn) {
3281 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3282 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kDRegSizeInBytes);
3283
3284 Emit(SADDV_r_p_z | SVESize(zn) | Rd(dd) | PgLow8(pg) | Rn(zn));
3285 }
3286
smaxv(const VRegister & vd,const PRegister & pg,const ZRegister & zn)3287 void Assembler::smaxv(const VRegister& vd,
3288 const PRegister& pg,
3289 const ZRegister& zn) {
3290 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3291 VIXL_ASSERT(vd.IsScalar());
3292
3293 Emit(SMAXV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
3294 }
3295
sminv(const VRegister & vd,const PRegister & pg,const ZRegister & zn)3296 void Assembler::sminv(const VRegister& vd,
3297 const PRegister& pg,
3298 const ZRegister& zn) {
3299 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3300 VIXL_ASSERT(vd.IsScalar());
3301
3302 Emit(SMINV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
3303 }
3304
uaddv(const VRegister & dd,const PRegister & pg,const ZRegister & zn)3305 void Assembler::uaddv(const VRegister& dd,
3306 const PRegister& pg,
3307 const ZRegister& zn) {
3308 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3309
3310 Emit(UADDV_r_p_z | SVESize(zn) | Rd(dd) | PgLow8(pg) | Rn(zn));
3311 }
3312
umaxv(const VRegister & vd,const PRegister & pg,const ZRegister & zn)3313 void Assembler::umaxv(const VRegister& vd,
3314 const PRegister& pg,
3315 const ZRegister& zn) {
3316 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3317 VIXL_ASSERT(vd.IsScalar());
3318
3319 Emit(UMAXV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
3320 }
3321
uminv(const VRegister & vd,const PRegister & pg,const ZRegister & zn)3322 void Assembler::uminv(const VRegister& vd,
3323 const PRegister& pg,
3324 const ZRegister& zn) {
3325 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3326 VIXL_ASSERT(vd.IsScalar());
3327
3328 Emit(UMINV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
3329 }
3330
3331 // SVEIntUnaryArithmeticPredicated.
3332
abs(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)3333 void Assembler::abs(const ZRegister& zd,
3334 const PRegisterM& pg,
3335 const ZRegister& zn) {
3336 // ABS <Zd>.<T>, <Pg>/M, <Zn>.<T>
3337 // 0000 0100 ..01 0110 101. .... .... ....
3338 // size<23:22> | opc<18:16> = 110 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3339
3340 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3341 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3342
3343 Emit(ABS_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3344 }
3345
cls(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)3346 void Assembler::cls(const ZRegister& zd,
3347 const PRegisterM& pg,
3348 const ZRegister& zn) {
3349 // CLS <Zd>.<T>, <Pg>/M, <Zn>.<T>
3350 // 0000 0100 ..01 1000 101. .... .... ....
3351 // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3352
3353 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3354 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3355
3356 Emit(CLS_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3357 }
3358
clz(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)3359 void Assembler::clz(const ZRegister& zd,
3360 const PRegisterM& pg,
3361 const ZRegister& zn) {
3362 // CLZ <Zd>.<T>, <Pg>/M, <Zn>.<T>
3363 // 0000 0100 ..01 1001 101. .... .... ....
3364 // size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3365
3366 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3367 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3368
3369 Emit(CLZ_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3370 }
3371
cnot(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)3372 void Assembler::cnot(const ZRegister& zd,
3373 const PRegisterM& pg,
3374 const ZRegister& zn) {
3375 // CNOT <Zd>.<T>, <Pg>/M, <Zn>.<T>
3376 // 0000 0100 ..01 1011 101. .... .... ....
3377 // size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3378
3379 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3380 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3381
3382 Emit(CNOT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3383 }
3384
cnt(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)3385 void Assembler::cnt(const ZRegister& zd,
3386 const PRegisterM& pg,
3387 const ZRegister& zn) {
3388 // CNT <Zd>.<T>, <Pg>/M, <Zn>.<T>
3389 // 0000 0100 ..01 1010 101. .... .... ....
3390 // size<23:22> | opc<18:16> = 010 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3391
3392 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3393 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3394
3395 Emit(CNT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3396 }
3397
fabs(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)3398 void Assembler::fabs(const ZRegister& zd,
3399 const PRegisterM& pg,
3400 const ZRegister& zn) {
3401 // FABS <Zd>.<T>, <Pg>/M, <Zn>.<T>
3402 // 0000 0100 ..01 1100 101. .... .... ....
3403 // size<23:22> | opc<18:16> = 100 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3404
3405 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3406 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3407 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
3408
3409 Emit(FABS_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3410 }
3411
fneg(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)3412 void Assembler::fneg(const ZRegister& zd,
3413 const PRegisterM& pg,
3414 const ZRegister& zn) {
3415 // FNEG <Zd>.<T>, <Pg>/M, <Zn>.<T>
3416 // 0000 0100 ..01 1101 101. .... .... ....
3417 // size<23:22> | opc<18:16> = 101 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3418
3419 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3420 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3421 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
3422
3423 Emit(FNEG_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3424 }
3425
neg(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)3426 void Assembler::neg(const ZRegister& zd,
3427 const PRegisterM& pg,
3428 const ZRegister& zn) {
3429 // NEG <Zd>.<T>, <Pg>/M, <Zn>.<T>
3430 // 0000 0100 ..01 0111 101. .... .... ....
3431 // size<23:22> | opc<18:16> = 111 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3432
3433 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3434 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3435
3436 Emit(NEG_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3437 }
3438
not_(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)3439 void Assembler::not_(const ZRegister& zd,
3440 const PRegisterM& pg,
3441 const ZRegister& zn) {
3442 // NOT <Zd>.<T>, <Pg>/M, <Zn>.<T>
3443 // 0000 0100 ..01 1110 101. .... .... ....
3444 // size<23:22> | opc<18:16> = 110 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3445
3446 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3447 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3448
3449 Emit(NOT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3450 }
3451
sxtb(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)3452 void Assembler::sxtb(const ZRegister& zd,
3453 const PRegisterM& pg,
3454 const ZRegister& zn) {
3455 // SXTB <Zd>.<T>, <Pg>/M, <Zn>.<T>
3456 // 0000 0100 ..01 0000 101. .... .... ....
3457 // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3458
3459 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3460 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3461 VIXL_ASSERT(zd.GetLaneSizeInBytes() > kBRegSizeInBytes);
3462
3463 Emit(SXTB_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3464 }
3465
sxth(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)3466 void Assembler::sxth(const ZRegister& zd,
3467 const PRegisterM& pg,
3468 const ZRegister& zn) {
3469 // SXTH <Zd>.<T>, <Pg>/M, <Zn>.<T>
3470 // 0000 0100 ..01 0010 101. .... .... ....
3471 // size<23:22> | opc<18:16> = 010 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3472
3473 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3474 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3475 VIXL_ASSERT(zd.GetLaneSizeInBytes() > kHRegSizeInBytes);
3476
3477 Emit(SXTH_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3478 }
3479
sxtw(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)3480 void Assembler::sxtw(const ZRegister& zd,
3481 const PRegisterM& pg,
3482 const ZRegister& zn) {
3483 // SXTW <Zd>.D, <Pg>/M, <Zn>.D
3484 // 0000 0100 ..01 0100 101. .... .... ....
3485 // size<23:22> | opc<18:16> = 100 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3486
3487 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3488 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3489 VIXL_ASSERT(zd.GetLaneSizeInBytes() > kSRegSizeInBytes);
3490
3491 Emit(SXTW_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3492 }
3493
uxtb(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)3494 void Assembler::uxtb(const ZRegister& zd,
3495 const PRegisterM& pg,
3496 const ZRegister& zn) {
3497 // UXTB <Zd>.<T>, <Pg>/M, <Zn>.<T>
3498 // 0000 0100 ..01 0001 101. .... .... ....
3499 // size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3500
3501 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3502 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3503 VIXL_ASSERT(zd.GetLaneSizeInBytes() > kBRegSizeInBytes);
3504
3505 Emit(UXTB_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3506 }
3507
uxth(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)3508 void Assembler::uxth(const ZRegister& zd,
3509 const PRegisterM& pg,
3510 const ZRegister& zn) {
3511 // UXTH <Zd>.<T>, <Pg>/M, <Zn>.<T>
3512 // 0000 0100 ..01 0011 101. .... .... ....
3513 // size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3514
3515 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3516 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3517 VIXL_ASSERT(zd.GetLaneSizeInBytes() > kHRegSizeInBytes);
3518
3519 Emit(UXTH_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3520 }
3521
uxtw(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)3522 void Assembler::uxtw(const ZRegister& zd,
3523 const PRegisterM& pg,
3524 const ZRegister& zn) {
3525 // UXTW <Zd>.D, <Pg>/M, <Zn>.D
3526 // 0000 0100 ..01 0101 101. .... .... ....
3527 // size<23:22> | opc<18:16> = 101 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3528
3529 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3530 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3531 VIXL_ASSERT(zd.GetLaneSizeInBytes() > kSRegSizeInBytes);
3532
3533 Emit(UXTW_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3534 }
3535
3536 // SVEIntWideImmPredicated.
3537
cpy(const ZRegister & zd,const PRegister & pg,int imm8,int shift)3538 void Assembler::cpy(const ZRegister& zd,
3539 const PRegister& pg,
3540 int imm8,
3541 int shift) {
3542 // CPY <Zd>.<T>, <Pg>/<ZM>, #<imm>{, <shift>}
3543 // 0000 0101 ..01 .... 0... .... .... ....
3544 // size<23:22> | Pg<19:16> | M<14> | sh<13> | imm8<12:5> | Zd<4:0>
3545
3546 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3547 VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing());
3548
3549 ResolveSVEImm8Shift(&imm8, &shift);
3550
3551 Instr sh = (shift > 0) ? (1 << 13) : 0;
3552 Instr m = pg.IsMerging() ? (1 << 14) : 0;
3553 Emit(CPY_z_p_i | m | sh | SVESize(zd) | Rd(zd) | Pg<19, 16>(pg) |
3554 ImmField<12, 5>(imm8));
3555 }
3556
fcpy(const ZRegister & zd,const PRegisterM & pg,double imm)3557 void Assembler::fcpy(const ZRegister& zd, const PRegisterM& pg, double imm) {
3558 // FCPY <Zd>.<T>, <Pg>/M, #<const>
3559 // 0000 0101 ..01 .... 110. .... .... ....
3560 // size<23:22> | Pg<19:16> | imm8<12:5> | Zd<4:0>
3561
3562 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3563 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
3564
3565 Instr imm_field = ImmUnsignedField<12, 5>(FP64ToImm8(imm));
3566 Emit(FCPY_z_p_i | SVESize(zd) | Rd(zd) | Pg<19, 16>(pg) | imm_field);
3567 }
3568
3569 // SVEIntAddSubtractImmUnpredicated.
3570
SVEIntAddSubtractImmUnpredicatedHelper(SVEIntAddSubtractImm_UnpredicatedOp op,const ZRegister & zd,int imm8,int shift)3571 void Assembler::SVEIntAddSubtractImmUnpredicatedHelper(
3572 SVEIntAddSubtractImm_UnpredicatedOp op,
3573 const ZRegister& zd,
3574 int imm8,
3575 int shift) {
3576 if (shift < 0) {
3577 VIXL_ASSERT(shift == -1);
3578 // Derive the shift amount from the immediate.
3579 if (IsUint8(imm8)) {
3580 shift = 0;
3581 } else if (IsUint16(imm8) && ((imm8 % 256) == 0)) {
3582 imm8 /= 256;
3583 shift = 8;
3584 }
3585 }
3586
3587 VIXL_ASSERT(IsUint8(imm8));
3588 VIXL_ASSERT((shift == 0) || (shift == 8));
3589
3590 Instr shift_bit = (shift > 0) ? (1 << 13) : 0;
3591 Emit(op | SVESize(zd) | Rd(zd) | shift_bit | ImmUnsignedField<12, 5>(imm8));
3592 }
3593
add(const ZRegister & zd,const ZRegister & zn,int imm8,int shift)3594 void Assembler::add(const ZRegister& zd,
3595 const ZRegister& zn,
3596 int imm8,
3597 int shift) {
3598 // ADD <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>}
3599 // 0010 0101 ..10 0000 11.. .... .... ....
3600 // size<23:22> | opc<18:16> = 000 | sh<13> | imm8<12:5> | Zdn<4:0>
3601
3602 USE(zn);
3603 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3604 VIXL_ASSERT(zd.Is(zn));
3605 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3606
3607 SVEIntAddSubtractImmUnpredicatedHelper(ADD_z_zi, zd, imm8, shift);
3608 }
3609
dup(const ZRegister & zd,int imm8,int shift)3610 void Assembler::dup(const ZRegister& zd, int imm8, int shift) {
3611 // DUP <Zd>.<T>, #<imm>{, <shift>}
3612 // 0010 0101 ..11 1000 11.. .... .... ....
3613 // size<23:22> | opc<18:17> = 00 | sh<13> | imm8<12:5> | Zd<4:0>
3614
3615 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3616
3617 ResolveSVEImm8Shift(&imm8, &shift);
3618 VIXL_ASSERT((shift < 8) || !zd.IsLaneSizeB());
3619
3620 Instr shift_bit = (shift > 0) ? (1 << 13) : 0;
3621 Emit(DUP_z_i | SVESize(zd) | Rd(zd) | shift_bit | ImmField<12, 5>(imm8));
3622 }
3623
fdup(const ZRegister & zd,double imm)3624 void Assembler::fdup(const ZRegister& zd, double imm) {
3625 // FDUP <Zd>.<T>, #<const>
3626 // 0010 0101 ..11 1001 110. .... .... ....
3627 // size<23:22> | opc<18:17> = 00 | o2<13> = 0 | imm8<12:5> | Zd<4:0>
3628
3629 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3630 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
3631
3632 Instr encoded_imm = FP64ToImm8(imm) << 5;
3633 Emit(FDUP_z_i | SVESize(zd) | encoded_imm | Rd(zd));
3634 }
3635
mul(const ZRegister & zd,const ZRegister & zn,int imm8)3636 void Assembler::mul(const ZRegister& zd, const ZRegister& zn, int imm8) {
3637 // MUL <Zdn>.<T>, <Zdn>.<T>, #<imm>
3638 // 0010 0101 ..11 0000 110. .... .... ....
3639 // size<23:22> | opc<18:16> = 000 | o2<13> = 0 | imm8<12:5> | Zdn<4:0>
3640
3641 USE(zn);
3642 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3643 VIXL_ASSERT(zd.Is(zn));
3644 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3645
3646 Emit(MUL_z_zi | SVESize(zd) | Rd(zd) | ImmField<12, 5>(imm8));
3647 }
3648
smax(const ZRegister & zd,const ZRegister & zn,int imm8)3649 void Assembler::smax(const ZRegister& zd, const ZRegister& zn, int imm8) {
3650 // SMAX <Zdn>.<T>, <Zdn>.<T>, #<imm>
3651 // 0010 0101 ..10 1000 110. .... .... ....
3652 // size<23:22> | opc<18:16> = 000 | o2<13> = 0 | imm8<12:5> | Zdn<4:0>
3653
3654 USE(zn);
3655 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3656 VIXL_ASSERT(zd.Is(zn));
3657 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3658
3659 Emit(SMAX_z_zi | SVESize(zd) | Rd(zd) | ImmField<12, 5>(imm8));
3660 }
3661
smin(const ZRegister & zd,const ZRegister & zn,int imm8)3662 void Assembler::smin(const ZRegister& zd, const ZRegister& zn, int imm8) {
3663 // SMIN <Zdn>.<T>, <Zdn>.<T>, #<imm>
3664 // 0010 0101 ..10 1010 110. .... .... ....
3665 // size<23:22> | opc<18:16> = 010 | o2<13> = 0 | imm8<12:5> | Zdn<4:0>
3666
3667 USE(zn);
3668 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3669 VIXL_ASSERT(zd.Is(zn));
3670 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3671
3672 Emit(SMIN_z_zi | SVESize(zd) | Rd(zd) | ImmField<12, 5>(imm8));
3673 }
3674
sqadd(const ZRegister & zd,const ZRegister & zn,int imm8,int shift)3675 void Assembler::sqadd(const ZRegister& zd,
3676 const ZRegister& zn,
3677 int imm8,
3678 int shift) {
3679 // SQADD <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>}
3680 // 0010 0101 ..10 0100 11.. .... .... ....
3681 // size<23:22> | opc<18:16> = 100 | sh<13> | imm8<12:5> | Zdn<4:0>
3682
3683 USE(zn);
3684 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3685 VIXL_ASSERT(zd.Is(zn));
3686 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3687
3688 SVEIntAddSubtractImmUnpredicatedHelper(SQADD_z_zi, zd, imm8, shift);
3689 }
3690
sqsub(const ZRegister & zd,const ZRegister & zn,int imm8,int shift)3691 void Assembler::sqsub(const ZRegister& zd,
3692 const ZRegister& zn,
3693 int imm8,
3694 int shift) {
3695 // SQSUB <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>}
3696 // 0010 0101 ..10 0110 11.. .... .... ....
3697 // size<23:22> | opc<18:16> = 110 | sh<13> | imm8<12:5> | Zdn<4:0>
3698
3699 USE(zn);
3700 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3701 VIXL_ASSERT(zd.Is(zn));
3702 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3703
3704 SVEIntAddSubtractImmUnpredicatedHelper(SQSUB_z_zi, zd, imm8, shift);
3705 }
3706
sub(const ZRegister & zd,const ZRegister & zn,int imm8,int shift)3707 void Assembler::sub(const ZRegister& zd,
3708 const ZRegister& zn,
3709 int imm8,
3710 int shift) {
3711 // SUB <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>}
3712 // 0010 0101 ..10 0001 11.. .... .... ....
3713 // size<23:22> | opc<18:16> = 001 | sh<13> | imm8<12:5> | Zdn<4:0>
3714
3715 USE(zn);
3716 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3717 VIXL_ASSERT(zd.Is(zn));
3718 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3719
3720 SVEIntAddSubtractImmUnpredicatedHelper(SUB_z_zi, zd, imm8, shift);
3721 }
3722
subr(const ZRegister & zd,const ZRegister & zn,int imm8,int shift)3723 void Assembler::subr(const ZRegister& zd,
3724 const ZRegister& zn,
3725 int imm8,
3726 int shift) {
3727 // SUBR <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>}
3728 // 0010 0101 ..10 0011 11.. .... .... ....
3729 // size<23:22> | opc<18:16> = 011 | sh<13> | imm8<12:5> | Zdn<4:0>
3730
3731 USE(zn);
3732 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3733 VIXL_ASSERT(zd.Is(zn));
3734 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3735
3736 SVEIntAddSubtractImmUnpredicatedHelper(SUBR_z_zi, zd, imm8, shift);
3737 }
3738
umax(const ZRegister & zd,const ZRegister & zn,int imm8)3739 void Assembler::umax(const ZRegister& zd, const ZRegister& zn, int imm8) {
3740 // UMAX <Zdn>.<T>, <Zdn>.<T>, #<imm>
3741 // 0010 0101 ..10 1001 110. .... .... ....
3742 // size<23:22> | opc<18:16> = 001 | o2<13> = 0 | imm8<12:5> | Zdn<4:0>
3743
3744 USE(zn);
3745 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3746 VIXL_ASSERT(zd.Is(zn));
3747 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3748
3749 Emit(UMAX_z_zi | SVESize(zd) | Rd(zd) | ImmUnsignedField<12, 5>(imm8));
3750 }
3751
umin(const ZRegister & zd,const ZRegister & zn,int imm8)3752 void Assembler::umin(const ZRegister& zd, const ZRegister& zn, int imm8) {
3753 // UMIN <Zdn>.<T>, <Zdn>.<T>, #<imm>
3754 // 0010 0101 ..10 1011 110. .... .... ....
3755 // size<23:22> | opc<18:16> = 011 | o2<13> = 0 | imm8<12:5> | Zdn<4:0>
3756
3757 USE(zn);
3758 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3759 VIXL_ASSERT(zd.Is(zn));
3760 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3761
3762 Emit(UMIN_z_zi | SVESize(zd) | Rd(zd) | ImmUnsignedField<12, 5>(imm8));
3763 }
3764
uqadd(const ZRegister & zd,const ZRegister & zn,int imm8,int shift)3765 void Assembler::uqadd(const ZRegister& zd,
3766 const ZRegister& zn,
3767 int imm8,
3768 int shift) {
3769 // UQADD <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>}
3770 // 0010 0101 ..10 0101 11.. .... .... ....
3771 // size<23:22> | opc<18:16> = 101 | sh<13> | imm8<12:5> | Zdn<4:0>
3772
3773 USE(zn);
3774 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3775 VIXL_ASSERT(zd.Is(zn));
3776 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3777
3778 SVEIntAddSubtractImmUnpredicatedHelper(UQADD_z_zi, zd, imm8, shift);
3779 }
3780
uqsub(const ZRegister & zd,const ZRegister & zn,int imm8,int shift)3781 void Assembler::uqsub(const ZRegister& zd,
3782 const ZRegister& zn,
3783 int imm8,
3784 int shift) {
3785 // UQSUB <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>}
3786 // 0010 0101 ..10 0111 11.. .... .... ....
3787 // size<23:22> | opc<18:16> = 111 | sh<13> | imm8<12:5> | Zdn<4:0>
3788
3789 USE(zn);
3790 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3791 VIXL_ASSERT(zd.Is(zn));
3792 VIXL_ASSERT(AreSameLaneSize(zd, zn));
3793
3794 SVEIntAddSubtractImmUnpredicatedHelper(UQSUB_z_zi, zd, imm8, shift);
3795 }
3796
3797 // SVEMemLoad.
3798
SVELdSt1Helper(unsigned msize_in_bytes_log2,const ZRegister & zt,const PRegister & pg,const SVEMemOperand & addr,bool is_signed,Instr op)3799 void Assembler::SVELdSt1Helper(unsigned msize_in_bytes_log2,
3800 const ZRegister& zt,
3801 const PRegister& pg,
3802 const SVEMemOperand& addr,
3803 bool is_signed,
3804 Instr op) {
3805 VIXL_ASSERT(addr.IsContiguous());
3806
3807 Instr mem_op = SVEMemOperandHelper(msize_in_bytes_log2, 1, addr);
3808 Instr dtype =
3809 SVEDtype(msize_in_bytes_log2, zt.GetLaneSizeInBytesLog2(), is_signed);
3810 Emit(op | mem_op | dtype | Rt(zt) | PgLow8(pg));
3811 }
3812
SVELdSt234Helper(int num_regs,const ZRegister & zt1,const PRegister & pg,const SVEMemOperand & addr,Instr op)3813 void Assembler::SVELdSt234Helper(int num_regs,
3814 const ZRegister& zt1,
3815 const PRegister& pg,
3816 const SVEMemOperand& addr,
3817 Instr op) {
3818 VIXL_ASSERT((num_regs >= 2) && (num_regs <= 4));
3819
3820 unsigned msize_in_bytes_log2 = zt1.GetLaneSizeInBytesLog2();
3821 Instr num = (num_regs - 1) << 21;
3822 Instr msz = msize_in_bytes_log2 << 23;
3823 Instr mem_op = SVEMemOperandHelper(msize_in_bytes_log2, num_regs, addr);
3824 Emit(op | mem_op | msz | num | Rt(zt1) | PgLow8(pg));
3825 }
3826
SVELd1Helper(unsigned msize_in_bytes_log2,const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr,bool is_signed)3827 void Assembler::SVELd1Helper(unsigned msize_in_bytes_log2,
3828 const ZRegister& zt,
3829 const PRegisterZ& pg,
3830 const SVEMemOperand& addr,
3831 bool is_signed) {
3832 VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() >= msize_in_bytes_log2);
3833 if (is_signed) {
3834 // Sign-extension is only possible when the vector elements are larger than
3835 // the elements in memory.
3836 VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() != msize_in_bytes_log2);
3837 }
3838
3839 if (addr.IsScatterGather()) {
3840 bool is_load = true;
3841 bool is_ff = false;
3842 SVEScatterGatherHelper(msize_in_bytes_log2,
3843 zt,
3844 pg,
3845 addr,
3846 is_load,
3847 is_signed,
3848 is_ff);
3849 return;
3850 }
3851
3852 Instr op = 0xffffffff;
3853 if (addr.IsScalarPlusImmediate()) {
3854 op = SVEContiguousLoad_ScalarPlusImmFixed;
3855 } else if (addr.IsScalarPlusScalar()) {
3856 // Rm must not be xzr.
3857 VIXL_ASSERT(!addr.GetScalarOffset().IsZero());
3858 op = SVEContiguousLoad_ScalarPlusScalarFixed;
3859 } else {
3860 VIXL_UNIMPLEMENTED();
3861 }
3862 SVELdSt1Helper(msize_in_bytes_log2, zt, pg, addr, is_signed, op);
3863 }
3864
SVELdff1Helper(unsigned msize_in_bytes_log2,const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr,bool is_signed)3865 void Assembler::SVELdff1Helper(unsigned msize_in_bytes_log2,
3866 const ZRegister& zt,
3867 const PRegisterZ& pg,
3868 const SVEMemOperand& addr,
3869 bool is_signed) {
3870 VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() >= msize_in_bytes_log2);
3871 if (is_signed) {
3872 // Sign-extension is only possible when the vector elements are larger than
3873 // the elements in memory.
3874 VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() != msize_in_bytes_log2);
3875 }
3876
3877 if (addr.IsScatterGather()) {
3878 bool is_load = true;
3879 bool is_ff = true;
3880 SVEScatterGatherHelper(msize_in_bytes_log2,
3881 zt,
3882 pg,
3883 addr,
3884 is_load,
3885 is_signed,
3886 is_ff);
3887 return;
3888 }
3889
3890 if (addr.IsPlainScalar()) {
3891 // SVEMemOperand(x0) is treated as a scalar-plus-immediate form ([x0, #0]).
3892 // In these instructions, we want to treat it as [x0, xzr].
3893 SVEMemOperand addr_scalar_plus_scalar(addr.GetScalarBase(), xzr);
3894 // Guard against infinite recursion.
3895 VIXL_ASSERT(!addr_scalar_plus_scalar.IsPlainScalar());
3896 SVELdff1Helper(msize_in_bytes_log2,
3897 zt,
3898 pg,
3899 addr_scalar_plus_scalar,
3900 is_signed);
3901 return;
3902 }
3903
3904 Instr op = 0xffffffff;
3905 if (addr.IsScalarPlusScalar()) {
3906 op = SVEContiguousFirstFaultLoad_ScalarPlusScalarFixed;
3907 } else {
3908 VIXL_UNIMPLEMENTED();
3909 }
3910 SVELdSt1Helper(msize_in_bytes_log2, zt, pg, addr, is_signed, op);
3911 }
3912
SVEScatterGatherHelper(unsigned msize_in_bytes_log2,const ZRegister & zt,const PRegister & pg,const SVEMemOperand & addr,bool is_load,bool is_signed,bool is_first_fault)3913 void Assembler::SVEScatterGatherHelper(unsigned msize_in_bytes_log2,
3914 const ZRegister& zt,
3915 const PRegister& pg,
3916 const SVEMemOperand& addr,
3917 bool is_load,
3918 bool is_signed,
3919 bool is_first_fault) {
3920 VIXL_ASSERT(addr.IsScatterGather());
3921 VIXL_ASSERT(zt.IsLaneSizeS() || zt.IsLaneSizeD());
3922 VIXL_ASSERT(is_load || !is_first_fault);
3923 VIXL_ASSERT(is_load || !is_signed);
3924
3925 Instr op = 0xffffffff;
3926 if (addr.IsVectorPlusImmediate()) {
3927 VIXL_ASSERT(AreSameLaneSize(zt, addr.GetVectorBase()));
3928 if (is_load) {
3929 if (zt.IsLaneSizeS()) {
3930 op = SVE32BitGatherLoad_VectorPlusImmFixed;
3931 } else {
3932 op = SVE64BitGatherLoad_VectorPlusImmFixed;
3933 }
3934 } else {
3935 if (zt.IsLaneSizeS()) {
3936 op = SVE32BitScatterStore_VectorPlusImmFixed;
3937 } else {
3938 op = SVE64BitScatterStore_VectorPlusImmFixed;
3939 }
3940 }
3941 } else {
3942 VIXL_ASSERT(addr.IsScalarPlusVector());
3943 VIXL_ASSERT(AreSameLaneSize(zt, addr.GetVectorOffset()));
3944 SVEOffsetModifier mod = addr.GetOffsetModifier();
3945 if (zt.IsLaneSizeS()) {
3946 VIXL_ASSERT((mod == SVE_UXTW) || (mod == SVE_SXTW));
3947 unsigned shift_amount = addr.GetShiftAmount();
3948 if (shift_amount == 0) {
3949 if (is_load) {
3950 op = SVE32BitGatherLoad_ScalarPlus32BitUnscaledOffsetsFixed;
3951 } else {
3952 op = SVE32BitScatterStore_ScalarPlus32BitUnscaledOffsetsFixed;
3953 }
3954 } else if (shift_amount == 1) {
3955 VIXL_ASSERT(msize_in_bytes_log2 == kHRegSizeInBytesLog2);
3956 if (is_load) {
3957 op = SVE32BitGatherLoadHalfwords_ScalarPlus32BitScaledOffsetsFixed;
3958 } else {
3959 op = SVE32BitScatterStore_ScalarPlus32BitScaledOffsetsFixed;
3960 }
3961 } else {
3962 VIXL_ASSERT(shift_amount == 2);
3963 VIXL_ASSERT(msize_in_bytes_log2 == kSRegSizeInBytesLog2);
3964 if (is_load) {
3965 op = SVE32BitGatherLoadWords_ScalarPlus32BitScaledOffsetsFixed;
3966 } else {
3967 op = SVE32BitScatterStore_ScalarPlus32BitScaledOffsetsFixed;
3968 }
3969 }
3970 } else if (zt.IsLaneSizeD()) {
3971 switch (mod) {
3972 case NO_SVE_OFFSET_MODIFIER:
3973 if (is_load) {
3974 op = SVE64BitGatherLoad_ScalarPlus64BitUnscaledOffsetsFixed;
3975 } else {
3976 op = SVE64BitScatterStore_ScalarPlus64BitUnscaledOffsetsFixed;
3977 }
3978 break;
3979 case SVE_LSL:
3980 if (is_load) {
3981 op = SVE64BitGatherLoad_ScalarPlus64BitScaledOffsetsFixed;
3982 } else {
3983 op = SVE64BitScatterStore_ScalarPlus64BitScaledOffsetsFixed;
3984 }
3985 break;
3986 case SVE_UXTW:
3987 case SVE_SXTW: {
3988 unsigned shift_amount = addr.GetShiftAmount();
3989 if (shift_amount == 0) {
3990 if (is_load) {
3991 op =
3992 SVE64BitGatherLoad_ScalarPlusUnpacked32BitUnscaledOffsetsFixed;
3993 } else {
3994 op =
3995 SVE64BitScatterStore_ScalarPlusUnpacked32BitUnscaledOffsetsFixed;
3996 }
3997 } else {
3998 VIXL_ASSERT(shift_amount == msize_in_bytes_log2);
3999 if (is_load) {
4000 op = SVE64BitGatherLoad_ScalarPlus32BitUnpackedScaledOffsetsFixed;
4001 } else {
4002 op =
4003 SVE64BitScatterStore_ScalarPlusUnpacked32BitScaledOffsetsFixed;
4004 }
4005 }
4006 break;
4007 }
4008 default:
4009 VIXL_UNIMPLEMENTED();
4010 }
4011 }
4012 }
4013
4014 Instr mem_op = SVEMemOperandHelper(msize_in_bytes_log2, 1, addr, is_load);
4015 Instr msz = ImmUnsignedField<24, 23>(msize_in_bytes_log2);
4016 Instr u = (!is_load || is_signed) ? 0 : (1 << 14);
4017 Instr ff = is_first_fault ? (1 << 13) : 0;
4018 Emit(op | mem_op | msz | u | ff | Rt(zt) | PgLow8(pg));
4019 }
4020
SVELd234Helper(int num_regs,const ZRegister & zt1,const PRegisterZ & pg,const SVEMemOperand & addr)4021 void Assembler::SVELd234Helper(int num_regs,
4022 const ZRegister& zt1,
4023 const PRegisterZ& pg,
4024 const SVEMemOperand& addr) {
4025 if (addr.IsScalarPlusScalar()) {
4026 // Rm must not be xzr.
4027 VIXL_ASSERT(!addr.GetScalarOffset().IsZero());
4028 }
4029
4030 Instr op;
4031 if (addr.IsScalarPlusImmediate()) {
4032 op = SVELoadMultipleStructures_ScalarPlusImmFixed;
4033 } else if (addr.IsScalarPlusScalar()) {
4034 op = SVELoadMultipleStructures_ScalarPlusScalarFixed;
4035 } else {
4036 // These instructions don't support any other addressing modes.
4037 VIXL_ABORT();
4038 }
4039 SVELdSt234Helper(num_regs, zt1, pg, addr, op);
4040 }
4041
4042 // SVEMemContiguousLoad.
4043
4044 #define VIXL_DEFINE_LD1(MSZ, LANE_SIZE) \
4045 void Assembler::ld1##MSZ(const ZRegister& zt, \
4046 const PRegisterZ& pg, \
4047 const SVEMemOperand& addr) { \
4048 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \
4049 SVELd1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, false); \
4050 }
4051 #define VIXL_DEFINE_LD2(MSZ, LANE_SIZE) \
4052 void Assembler::ld2##MSZ(const ZRegister& zt1, \
4053 const ZRegister& zt2, \
4054 const PRegisterZ& pg, \
4055 const SVEMemOperand& addr) { \
4056 USE(zt2); \
4057 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \
4058 VIXL_ASSERT(AreConsecutive(zt1, zt2)); \
4059 VIXL_ASSERT(AreSameFormat(zt1, zt2)); \
4060 VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \
4061 SVELd234Helper(2, zt1, pg, addr); \
4062 }
4063 #define VIXL_DEFINE_LD3(MSZ, LANE_SIZE) \
4064 void Assembler::ld3##MSZ(const ZRegister& zt1, \
4065 const ZRegister& zt2, \
4066 const ZRegister& zt3, \
4067 const PRegisterZ& pg, \
4068 const SVEMemOperand& addr) { \
4069 USE(zt2, zt3); \
4070 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \
4071 VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3)); \
4072 VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3)); \
4073 VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \
4074 SVELd234Helper(3, zt1, pg, addr); \
4075 }
4076 #define VIXL_DEFINE_LD4(MSZ, LANE_SIZE) \
4077 void Assembler::ld4##MSZ(const ZRegister& zt1, \
4078 const ZRegister& zt2, \
4079 const ZRegister& zt3, \
4080 const ZRegister& zt4, \
4081 const PRegisterZ& pg, \
4082 const SVEMemOperand& addr) { \
4083 USE(zt2, zt3, zt4); \
4084 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \
4085 VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3, zt4)); \
4086 VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3, zt4)); \
4087 VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \
4088 SVELd234Helper(4, zt1, pg, addr); \
4089 }
4090
4091 VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD1)
VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD2)4092 VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD2)
4093 VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD3)
4094 VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD4)
4095
4096 #define VIXL_DEFINE_LD1S(MSZ, LANE_SIZE) \
4097 void Assembler::ld1s##MSZ(const ZRegister& zt, \
4098 const PRegisterZ& pg, \
4099 const SVEMemOperand& addr) { \
4100 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \
4101 SVELd1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, true); \
4102 }
4103 VIXL_SVE_LOAD_STORE_SIGNED_VARIANT_LIST(VIXL_DEFINE_LD1S)
4104
4105 // SVEMem32BitGatherAndUnsizedContiguous.
4106
4107 void Assembler::SVELd1BroadcastHelper(unsigned msize_in_bytes_log2,
4108 const ZRegister& zt,
4109 const PRegisterZ& pg,
4110 const SVEMemOperand& addr,
4111 bool is_signed) {
4112 VIXL_ASSERT(addr.IsScalarPlusImmediate());
4113 VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() >= msize_in_bytes_log2);
4114 if (is_signed) {
4115 // Sign-extension is only possible when the vector elements are larger than
4116 // the elements in memory.
4117 VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() != msize_in_bytes_log2);
4118 }
4119
4120 int64_t imm = addr.GetImmediateOffset();
4121 int divisor = 1 << msize_in_bytes_log2;
4122 VIXL_ASSERT(imm % divisor == 0);
4123 Instr dtype = SVEDtypeSplit(msize_in_bytes_log2,
4124 zt.GetLaneSizeInBytesLog2(),
4125 is_signed);
4126
4127 Emit(SVELoadAndBroadcastElementFixed | dtype | RnSP(addr.GetScalarBase()) |
4128 ImmUnsignedField<21, 16>(imm / divisor) | Rt(zt) | PgLow8(pg));
4129 }
4130
4131 // This prototype maps to 4 instruction encodings:
4132 // LD1RB_z_p_bi_u16
4133 // LD1RB_z_p_bi_u32
4134 // LD1RB_z_p_bi_u64
4135 // LD1RB_z_p_bi_u8
ld1rb(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4136 void Assembler::ld1rb(const ZRegister& zt,
4137 const PRegisterZ& pg,
4138 const SVEMemOperand& addr) {
4139 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4140
4141 SVELd1BroadcastHelper(kBRegSizeInBytesLog2, zt, pg, addr, false);
4142 }
4143
4144 // This prototype maps to 3 instruction encodings:
4145 // LD1RH_z_p_bi_u16
4146 // LD1RH_z_p_bi_u32
4147 // LD1RH_z_p_bi_u64
ld1rh(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4148 void Assembler::ld1rh(const ZRegister& zt,
4149 const PRegisterZ& pg,
4150 const SVEMemOperand& addr) {
4151 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4152
4153 SVELd1BroadcastHelper(kHRegSizeInBytesLog2, zt, pg, addr, false);
4154 }
4155
4156 // This prototype maps to 2 instruction encodings:
4157 // LD1RW_z_p_bi_u32
4158 // LD1RW_z_p_bi_u64
ld1rw(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4159 void Assembler::ld1rw(const ZRegister& zt,
4160 const PRegisterZ& pg,
4161 const SVEMemOperand& addr) {
4162 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4163
4164 SVELd1BroadcastHelper(kSRegSizeInBytesLog2, zt, pg, addr, false);
4165 }
4166
ld1rd(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4167 void Assembler::ld1rd(const ZRegister& zt,
4168 const PRegisterZ& pg,
4169 const SVEMemOperand& addr) {
4170 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4171
4172 SVELd1BroadcastHelper(kDRegSizeInBytesLog2, zt, pg, addr, false);
4173 }
4174
4175 // This prototype maps to 3 instruction encodings:
4176 // LD1RSB_z_p_bi_s16
4177 // LD1RSB_z_p_bi_s32
4178 // LD1RSB_z_p_bi_s64
ld1rsb(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4179 void Assembler::ld1rsb(const ZRegister& zt,
4180 const PRegisterZ& pg,
4181 const SVEMemOperand& addr) {
4182 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4183
4184 SVELd1BroadcastHelper(kBRegSizeInBytesLog2, zt, pg, addr, true);
4185 }
4186
4187 // This prototype maps to 2 instruction encodings:
4188 // LD1RSH_z_p_bi_s32
4189 // LD1RSH_z_p_bi_s64
ld1rsh(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4190 void Assembler::ld1rsh(const ZRegister& zt,
4191 const PRegisterZ& pg,
4192 const SVEMemOperand& addr) {
4193 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4194
4195 SVELd1BroadcastHelper(kHRegSizeInBytesLog2, zt, pg, addr, true);
4196 }
4197
ld1rsw(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4198 void Assembler::ld1rsw(const ZRegister& zt,
4199 const PRegisterZ& pg,
4200 const SVEMemOperand& addr) {
4201 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4202
4203 SVELd1BroadcastHelper(kWRegSizeInBytesLog2, zt, pg, addr, true);
4204 }
4205
ldr(const CPURegister & rt,const SVEMemOperand & addr)4206 void Assembler::ldr(const CPURegister& rt, const SVEMemOperand& addr) {
4207 // LDR <Pt/Zt>, [<Xn|SP>{, #<imm>, MUL VL}]
4208
4209 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4210 VIXL_ASSERT(rt.IsPRegister() || rt.IsZRegister());
4211 VIXL_ASSERT(addr.IsPlainScalar() ||
4212 (addr.IsScalarPlusImmediate() &&
4213 (addr.GetOffsetModifier() == SVE_MUL_VL)));
4214 int64_t imm9 = addr.GetImmediateOffset();
4215 VIXL_ASSERT(IsInt9(imm9));
4216 Instr imm9l = ExtractUnsignedBitfield32(2, 0, imm9) << 10;
4217 Instr imm9h = ExtractUnsignedBitfield32(8, 3, imm9) << 16;
4218
4219 Instr op = LDR_z_bi;
4220 if (rt.IsPRegister()) {
4221 op = LDR_p_bi;
4222 }
4223 Emit(op | Rt(rt) | RnSP(addr.GetScalarBase()) | imm9h | imm9l);
4224 }
4225
4226 // SVEMem64BitGather.
4227
4228 // This prototype maps to 3 instruction encodings:
4229 // LDFF1B_z_p_bz_d_64_unscaled
4230 // LDFF1B_z_p_bz_d_x32_unscaled
ldff1b(const ZRegister & zt,const PRegisterZ & pg,const Register & xn,const ZRegister & zm)4231 void Assembler::ldff1b(const ZRegister& zt,
4232 const PRegisterZ& pg,
4233 const Register& xn,
4234 const ZRegister& zm) {
4235 // LDFF1B { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D]
4236 // 1100 0100 010. .... 111. .... .... ....
4237 // msz<24:23> = 00 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5>
4238 // | Zt<4:0>
4239
4240 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4241
4242 Emit(LDFF1B_z_p_bz_d_64_unscaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm));
4243 }
4244
4245 // This prototype maps to 2 instruction encodings:
4246 // LDFF1B_z_p_ai_d
4247 // LDFF1B_z_p_ai_s
ldff1b(const ZRegister & zt,const PRegisterZ & pg,const ZRegister & zn,int imm5)4248 void Assembler::ldff1b(const ZRegister& zt,
4249 const PRegisterZ& pg,
4250 const ZRegister& zn,
4251 int imm5) {
4252 // LDFF1B { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}]
4253 // 1100 0100 001. .... 111. .... .... ....
4254 // msz<24:23> = 00 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> |
4255 // Zn<9:5> | Zt<4:0>
4256
4257 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4258
4259 Emit(LDFF1B_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5));
4260 }
4261
4262 // This prototype maps to 4 instruction encodings:
4263 // LDFF1D_z_p_bz_d_64_scaled
4264 // LDFF1D_z_p_bz_d_64_unscaled
4265 // LDFF1D_z_p_bz_d_x32_scaled
4266 // LDFF1D_z_p_bz_d_x32_unscaled
ldff1d(const ZRegister & zt,const PRegisterZ & pg,const Register & xn,const ZRegister & zm)4267 void Assembler::ldff1d(const ZRegister& zt,
4268 const PRegisterZ& pg,
4269 const Register& xn,
4270 const ZRegister& zm) {
4271 // LDFF1D { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #3]
4272 // 1100 0101 111. .... 111. .... .... ....
4273 // msz<24:23> = 11 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5>
4274 // | Zt<4:0>
4275
4276 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4277
4278 Emit(LDFF1D_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm));
4279 }
4280
ldff1d(const ZRegister & zt,const PRegisterZ & pg,const ZRegister & zn,int imm5)4281 void Assembler::ldff1d(const ZRegister& zt,
4282 const PRegisterZ& pg,
4283 const ZRegister& zn,
4284 int imm5) {
4285 // LDFF1D { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}]
4286 // 1100 0101 101. .... 111. .... .... ....
4287 // msz<24:23> = 11 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> |
4288 // Zn<9:5> | Zt<4:0>
4289
4290 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4291
4292 Emit(LDFF1D_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5));
4293 }
4294
4295 // This prototype maps to 6 instruction encodings:
4296 // LDFF1H_z_p_bz_d_64_scaled
4297 // LDFF1H_z_p_bz_d_64_unscaled
4298 // LDFF1H_z_p_bz_d_x32_scaled
4299 // LDFF1H_z_p_bz_d_x32_unscaled
ldff1h(const ZRegister & zt,const PRegisterZ & pg,const Register & xn,const ZRegister & zm)4300 void Assembler::ldff1h(const ZRegister& zt,
4301 const PRegisterZ& pg,
4302 const Register& xn,
4303 const ZRegister& zm) {
4304 // LDFF1H { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #1]
4305 // 1100 0100 111. .... 111. .... .... ....
4306 // msz<24:23> = 01 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5>
4307 // | Zt<4:0>
4308
4309 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4310
4311 Emit(LDFF1H_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm));
4312 }
4313
4314 // This prototype maps to 2 instruction encodings:
4315 // LDFF1H_z_p_ai_d
4316 // LDFF1H_z_p_ai_s
ldff1h(const ZRegister & zt,const PRegisterZ & pg,const ZRegister & zn,int imm5)4317 void Assembler::ldff1h(const ZRegister& zt,
4318 const PRegisterZ& pg,
4319 const ZRegister& zn,
4320 int imm5) {
4321 // LDFF1H { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}]
4322 // 1100 0100 101. .... 111. .... .... ....
4323 // msz<24:23> = 01 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> |
4324 // Zn<9:5> | Zt<4:0>
4325
4326 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4327
4328 Emit(LDFF1H_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5));
4329 }
4330
4331 // This prototype maps to 3 instruction encodings:
4332 // LDFF1SB_z_p_bz_d_64_unscaled
4333 // LDFF1SB_z_p_bz_d_x32_unscaled
ldff1sb(const ZRegister & zt,const PRegisterZ & pg,const Register & xn,const ZRegister & zm)4334 void Assembler::ldff1sb(const ZRegister& zt,
4335 const PRegisterZ& pg,
4336 const Register& xn,
4337 const ZRegister& zm) {
4338 // LDFF1SB { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D]
4339 // 1100 0100 010. .... 101. .... .... ....
4340 // msz<24:23> = 00 | Zm<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | Rn<9:5>
4341 // | Zt<4:0>
4342
4343 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4344
4345 Emit(LDFF1SB_z_p_bz_d_64_unscaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm));
4346 }
4347
4348 // This prototype maps to 2 instruction encodings:
4349 // LDFF1SB_z_p_ai_d
4350 // LDFF1SB_z_p_ai_s
ldff1sb(const ZRegister & zt,const PRegisterZ & pg,const ZRegister & zn,int imm5)4351 void Assembler::ldff1sb(const ZRegister& zt,
4352 const PRegisterZ& pg,
4353 const ZRegister& zn,
4354 int imm5) {
4355 // LDFF1SB { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}]
4356 // 1100 0100 001. .... 101. .... .... ....
4357 // msz<24:23> = 00 | imm5<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> |
4358 // Zn<9:5> | Zt<4:0>
4359
4360 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4361
4362 Emit(LDFF1SB_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) |
4363 ImmField<20, 16>(imm5));
4364 }
4365
4366 // This prototype maps to 6 instruction encodings:
4367 // LDFF1SH_z_p_bz_d_64_scaled
4368 // LDFF1SH_z_p_bz_d_64_unscaled
4369 // LDFF1SH_z_p_bz_d_x32_scaled
4370 // LDFF1SH_z_p_bz_d_x32_unscaled
ldff1sh(const ZRegister & zt,const PRegisterZ & pg,const Register & xn,const ZRegister & zm)4371 void Assembler::ldff1sh(const ZRegister& zt,
4372 const PRegisterZ& pg,
4373 const Register& xn,
4374 const ZRegister& zm) {
4375 // LDFF1SH { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #1]
4376 // 1100 0100 111. .... 101. .... .... ....
4377 // msz<24:23> = 01 | Zm<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | Rn<9:5>
4378 // | Zt<4:0>
4379
4380 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4381
4382 Emit(LDFF1SH_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm));
4383 }
4384
4385 // This prototype maps to 2 instruction encodings:
4386 // LDFF1SH_z_p_ai_d
4387 // LDFF1SH_z_p_ai_s
ldff1sh(const ZRegister & zt,const PRegisterZ & pg,const ZRegister & zn,int imm5)4388 void Assembler::ldff1sh(const ZRegister& zt,
4389 const PRegisterZ& pg,
4390 const ZRegister& zn,
4391 int imm5) {
4392 // LDFF1SH { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}]
4393 // 1100 0100 101. .... 101. .... .... ....
4394 // msz<24:23> = 01 | imm5<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> |
4395 // Zn<9:5> | Zt<4:0>
4396
4397 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4398
4399 Emit(LDFF1SH_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) |
4400 ImmField<20, 16>(imm5));
4401 }
4402
4403 // This prototype maps to 4 instruction encodings:
4404 // LDFF1SW_z_p_bz_d_64_scaled
4405 // LDFF1SW_z_p_bz_d_64_unscaled
4406 // LDFF1SW_z_p_bz_d_x32_scaled
4407 // LDFF1SW_z_p_bz_d_x32_unscaled
ldff1sw(const ZRegister & zt,const PRegisterZ & pg,const Register & xn,const ZRegister & zm)4408 void Assembler::ldff1sw(const ZRegister& zt,
4409 const PRegisterZ& pg,
4410 const Register& xn,
4411 const ZRegister& zm) {
4412 // LDFF1SW { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #2]
4413 // 1100 0101 011. .... 101. .... .... ....
4414 // msz<24:23> = 10 | Zm<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | Rn<9:5>
4415 // | Zt<4:0>
4416
4417 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4418
4419 Emit(LDFF1SW_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm));
4420 }
4421
ldff1sw(const ZRegister & zt,const PRegisterZ & pg,const ZRegister & zn,int imm5)4422 void Assembler::ldff1sw(const ZRegister& zt,
4423 const PRegisterZ& pg,
4424 const ZRegister& zn,
4425 int imm5) {
4426 // LDFF1SW { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}]
4427 // 1100 0101 001. .... 101. .... .... ....
4428 // msz<24:23> = 10 | imm5<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> |
4429 // Zn<9:5> | Zt<4:0>
4430
4431 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4432
4433 Emit(LDFF1SW_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) |
4434 ImmField<20, 16>(imm5));
4435 }
4436
4437 // This prototype maps to 6 instruction encodings:
4438 // LDFF1W_z_p_bz_d_64_scaled
4439 // LDFF1W_z_p_bz_d_64_unscaled
4440 // LDFF1W_z_p_bz_d_x32_scaled
4441 // LDFF1W_z_p_bz_d_x32_unscaled
ldff1w(const ZRegister & zt,const PRegisterZ & pg,const Register & xn,const ZRegister & zm)4442 void Assembler::ldff1w(const ZRegister& zt,
4443 const PRegisterZ& pg,
4444 const Register& xn,
4445 const ZRegister& zm) {
4446 // LDFF1W { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #2]
4447 // 1100 0101 011. .... 111. .... .... ....
4448 // msz<24:23> = 10 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5>
4449 // | Zt<4:0>
4450
4451 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4452
4453 Emit(LDFF1W_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm));
4454 }
4455
4456 // This prototype maps to 2 instruction encodings:
4457 // LDFF1W_z_p_ai_d
4458 // LDFF1W_z_p_ai_s
ldff1w(const ZRegister & zt,const PRegisterZ & pg,const ZRegister & zn,int imm5)4459 void Assembler::ldff1w(const ZRegister& zt,
4460 const PRegisterZ& pg,
4461 const ZRegister& zn,
4462 int imm5) {
4463 // LDFF1W { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}]
4464 // 1100 0101 001. .... 111. .... .... ....
4465 // msz<24:23> = 10 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> |
4466 // Zn<9:5> | Zt<4:0>
4467
4468 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4469
4470 Emit(LDFF1W_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5));
4471 }
4472
SVEGatherPrefetchVectorPlusImmediateHelper(PrefetchOperation prfop,const PRegister & pg,const SVEMemOperand & addr,int prefetch_size)4473 void Assembler::SVEGatherPrefetchVectorPlusImmediateHelper(
4474 PrefetchOperation prfop,
4475 const PRegister& pg,
4476 const SVEMemOperand& addr,
4477 int prefetch_size) {
4478 VIXL_ASSERT(addr.IsVectorPlusImmediate());
4479 ZRegister zn = addr.GetVectorBase();
4480 VIXL_ASSERT(zn.IsLaneSizeS() || zn.IsLaneSizeD());
4481
4482 Instr op = 0xffffffff;
4483 switch (prefetch_size) {
4484 case kBRegSize:
4485 op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFB_i_p_ai_s)
4486 : static_cast<Instr>(PRFB_i_p_ai_d);
4487 break;
4488 case kHRegSize:
4489 op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFH_i_p_ai_s)
4490 : static_cast<Instr>(PRFH_i_p_ai_d);
4491 break;
4492 case kSRegSize:
4493 op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFW_i_p_ai_s)
4494 : static_cast<Instr>(PRFW_i_p_ai_d);
4495 break;
4496 case kDRegSize:
4497 op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFD_i_p_ai_s)
4498 : static_cast<Instr>(PRFD_i_p_ai_d);
4499 break;
4500 default:
4501 VIXL_UNIMPLEMENTED();
4502 break;
4503 }
4504
4505 int64_t imm5 = addr.GetImmediateOffset();
4506 Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) | Rn(zn) |
4507 ImmUnsignedField<20, 16>(imm5));
4508 }
4509
SVEGatherPrefetchScalarPlusImmediateHelper(PrefetchOperation prfop,const PRegister & pg,const SVEMemOperand & addr,int prefetch_size)4510 void Assembler::SVEGatherPrefetchScalarPlusImmediateHelper(
4511 PrefetchOperation prfop,
4512 const PRegister& pg,
4513 const SVEMemOperand& addr,
4514 int prefetch_size) {
4515 VIXL_ASSERT(addr.IsScalarPlusImmediate());
4516 int64_t imm6 = addr.GetImmediateOffset();
4517
4518 Instr op = 0xffffffff;
4519 switch (prefetch_size) {
4520 case kBRegSize:
4521 op = PRFB_i_p_bi_s;
4522 break;
4523 case kHRegSize:
4524 op = PRFH_i_p_bi_s;
4525 break;
4526 case kSRegSize:
4527 op = PRFW_i_p_bi_s;
4528 break;
4529 case kDRegSize:
4530 op = PRFD_i_p_bi_s;
4531 break;
4532 default:
4533 VIXL_UNIMPLEMENTED();
4534 break;
4535 }
4536
4537 Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) |
4538 RnSP(addr.GetScalarBase()) | ImmField<21, 16>(imm6));
4539 }
4540
SVEContiguousPrefetchScalarPlusScalarHelper(PrefetchOperation prfop,const PRegister & pg,const SVEMemOperand & addr,int prefetch_size)4541 void Assembler::SVEContiguousPrefetchScalarPlusScalarHelper(
4542 PrefetchOperation prfop,
4543 const PRegister& pg,
4544 const SVEMemOperand& addr,
4545 int prefetch_size) {
4546 VIXL_ASSERT(addr.IsScalarPlusScalar());
4547 Instr op = 0xffffffff;
4548
4549 switch (prefetch_size) {
4550 case kBRegSize:
4551 VIXL_ASSERT(addr.GetOffsetModifier() == NO_SVE_OFFSET_MODIFIER);
4552 op = PRFB_i_p_br_s;
4553 break;
4554 case kHRegSize:
4555 VIXL_ASSERT(addr.GetOffsetModifier() == SVE_LSL);
4556 VIXL_ASSERT(addr.GetShiftAmount() == kHRegSizeInBytesLog2);
4557 op = PRFH_i_p_br_s;
4558 break;
4559 case kSRegSize:
4560 VIXL_ASSERT(addr.GetOffsetModifier() == SVE_LSL);
4561 VIXL_ASSERT(addr.GetShiftAmount() == kSRegSizeInBytesLog2);
4562 op = PRFW_i_p_br_s;
4563 break;
4564 case kDRegSize:
4565 VIXL_ASSERT(addr.GetOffsetModifier() == SVE_LSL);
4566 VIXL_ASSERT(addr.GetShiftAmount() == kDRegSizeInBytesLog2);
4567 op = PRFD_i_p_br_s;
4568 break;
4569 default:
4570 VIXL_UNIMPLEMENTED();
4571 break;
4572 }
4573
4574 VIXL_ASSERT(!addr.GetScalarOffset().IsZero());
4575 Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) |
4576 RnSP(addr.GetScalarBase()) | Rm(addr.GetScalarOffset()));
4577 }
4578
SVEContiguousPrefetchScalarPlusVectorHelper(PrefetchOperation prfop,const PRegister & pg,const SVEMemOperand & addr,int prefetch_size)4579 void Assembler::SVEContiguousPrefetchScalarPlusVectorHelper(
4580 PrefetchOperation prfop,
4581 const PRegister& pg,
4582 const SVEMemOperand& addr,
4583 int prefetch_size) {
4584 VIXL_ASSERT(addr.IsScalarPlusVector());
4585 ZRegister zm = addr.GetVectorOffset();
4586 SVEOffsetModifier mod = addr.GetOffsetModifier();
4587
4588 // All prefetch scalar-plus-vector addressing modes use a shift corresponding
4589 // to the element size.
4590 switch (prefetch_size) {
4591 case kBRegSize:
4592 VIXL_ASSERT(addr.GetShiftAmount() == kBRegSizeInBytesLog2);
4593 break;
4594 case kHRegSize:
4595 VIXL_ASSERT(addr.GetShiftAmount() == kHRegSizeInBytesLog2);
4596 break;
4597 case kSRegSize:
4598 VIXL_ASSERT(addr.GetShiftAmount() == kSRegSizeInBytesLog2);
4599 break;
4600 case kDRegSize:
4601 VIXL_ASSERT(addr.GetShiftAmount() == kDRegSizeInBytesLog2);
4602 break;
4603 default:
4604 VIXL_UNIMPLEMENTED();
4605 break;
4606 }
4607
4608 Instr sx = 0;
4609 Instr op = 0xffffffff;
4610 if ((mod == NO_SVE_OFFSET_MODIFIER) || (mod == SVE_LSL)) {
4611 VIXL_ASSERT(zm.IsLaneSizeD());
4612
4613 switch (prefetch_size) {
4614 case kBRegSize:
4615 VIXL_ASSERT(mod == NO_SVE_OFFSET_MODIFIER);
4616 op = PRFB_i_p_bz_d_64_scaled;
4617 break;
4618 case kHRegSize:
4619 VIXL_ASSERT(mod == SVE_LSL);
4620 op = PRFH_i_p_bz_d_64_scaled;
4621 break;
4622 case kSRegSize:
4623 VIXL_ASSERT(mod == SVE_LSL);
4624 op = PRFW_i_p_bz_d_64_scaled;
4625 break;
4626 case kDRegSize:
4627 VIXL_ASSERT(mod == SVE_LSL);
4628 op = PRFD_i_p_bz_d_64_scaled;
4629 break;
4630 default:
4631 VIXL_UNIMPLEMENTED();
4632 break;
4633 }
4634 } else {
4635 VIXL_ASSERT((mod == SVE_SXTW) || (mod == SVE_UXTW));
4636 VIXL_ASSERT(zm.IsLaneSizeS() || zm.IsLaneSizeD());
4637
4638 switch (prefetch_size) {
4639 case kBRegSize:
4640 op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFB_i_p_bz_s_x32_scaled)
4641 : static_cast<Instr>(PRFB_i_p_bz_d_x32_scaled);
4642 break;
4643 case kHRegSize:
4644 op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFH_i_p_bz_s_x32_scaled)
4645 : static_cast<Instr>(PRFH_i_p_bz_d_x32_scaled);
4646 break;
4647 case kSRegSize:
4648 op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFW_i_p_bz_s_x32_scaled)
4649 : static_cast<Instr>(PRFW_i_p_bz_d_x32_scaled);
4650 break;
4651 case kDRegSize:
4652 op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFD_i_p_bz_s_x32_scaled)
4653 : static_cast<Instr>(PRFD_i_p_bz_d_x32_scaled);
4654 break;
4655 default:
4656 VIXL_UNIMPLEMENTED();
4657 break;
4658 }
4659
4660 if (mod == SVE_SXTW) {
4661 sx = 1 << 22;
4662 }
4663 }
4664
4665 Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) | sx |
4666 RnSP(addr.GetScalarBase()) | Rm(zm));
4667 }
4668
SVEPrefetchHelper(PrefetchOperation prfop,const PRegister & pg,const SVEMemOperand & addr,int prefetch_size)4669 void Assembler::SVEPrefetchHelper(PrefetchOperation prfop,
4670 const PRegister& pg,
4671 const SVEMemOperand& addr,
4672 int prefetch_size) {
4673 if (addr.IsVectorPlusImmediate()) {
4674 // For example:
4675 // [z0.s, #0]
4676 SVEGatherPrefetchVectorPlusImmediateHelper(prfop, pg, addr, prefetch_size);
4677
4678 } else if (addr.IsScalarPlusImmediate()) {
4679 // For example:
4680 // [x0, #42, mul vl]
4681 SVEGatherPrefetchScalarPlusImmediateHelper(prfop, pg, addr, prefetch_size);
4682
4683 } else if (addr.IsScalarPlusVector()) {
4684 // For example:
4685 // [x0, z0.s, sxtw]
4686 SVEContiguousPrefetchScalarPlusVectorHelper(prfop, pg, addr, prefetch_size);
4687
4688 } else if (addr.IsScalarPlusScalar()) {
4689 // For example:
4690 // [x0, x1]
4691 SVEContiguousPrefetchScalarPlusScalarHelper(prfop, pg, addr, prefetch_size);
4692
4693 } else {
4694 VIXL_UNIMPLEMENTED();
4695 }
4696 }
4697
prfb(PrefetchOperation prfop,const PRegister & pg,const SVEMemOperand & addr)4698 void Assembler::prfb(PrefetchOperation prfop,
4699 const PRegister& pg,
4700 const SVEMemOperand& addr) {
4701 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4702 SVEPrefetchHelper(prfop, pg, addr, kBRegSize);
4703 }
4704
prfd(PrefetchOperation prfop,const PRegister & pg,const SVEMemOperand & addr)4705 void Assembler::prfd(PrefetchOperation prfop,
4706 const PRegister& pg,
4707 const SVEMemOperand& addr) {
4708 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4709 SVEPrefetchHelper(prfop, pg, addr, kDRegSize);
4710 }
4711
prfh(PrefetchOperation prfop,const PRegister & pg,const SVEMemOperand & addr)4712 void Assembler::prfh(PrefetchOperation prfop,
4713 const PRegister& pg,
4714 const SVEMemOperand& addr) {
4715 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4716 SVEPrefetchHelper(prfop, pg, addr, kHRegSize);
4717 }
4718
prfw(PrefetchOperation prfop,const PRegister & pg,const SVEMemOperand & addr)4719 void Assembler::prfw(PrefetchOperation prfop,
4720 const PRegister& pg,
4721 const SVEMemOperand& addr) {
4722 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4723 SVEPrefetchHelper(prfop, pg, addr, kSRegSize);
4724 }
4725
SVELd1St1ScaImmHelper(const ZRegister & zt,const PRegister & pg,const SVEMemOperand & addr,Instr regoffset_op,Instr immoffset_op,int imm_divisor)4726 void Assembler::SVELd1St1ScaImmHelper(const ZRegister& zt,
4727 const PRegister& pg,
4728 const SVEMemOperand& addr,
4729 Instr regoffset_op,
4730 Instr immoffset_op,
4731 int imm_divisor) {
4732 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4733 VIXL_ASSERT(addr.IsScalarPlusScalar() || addr.IsScalarPlusImmediate());
4734
4735 Instr op;
4736 if (addr.IsScalarPlusScalar()) {
4737 op = regoffset_op | Rm(addr.GetScalarOffset());
4738 } else {
4739 int64_t imm = addr.GetImmediateOffset();
4740 VIXL_ASSERT(((imm % imm_divisor) == 0) && IsInt4(imm / imm_divisor));
4741 op = immoffset_op | ImmField<19, 16>(imm / imm_divisor);
4742 }
4743 Emit(op | Rt(zt) | PgLow8(pg) | RnSP(addr.GetScalarBase()));
4744 }
4745
ld1rqb(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4746 void Assembler::ld1rqb(const ZRegister& zt,
4747 const PRegisterZ& pg,
4748 const SVEMemOperand& addr) {
4749 VIXL_ASSERT(addr.IsScalarPlusImmediate() || addr.IsEquivalentToLSL(0));
4750 VIXL_ASSERT(zt.IsLaneSizeB());
4751 SVELd1St1ScaImmHelper(zt,
4752 pg,
4753 addr,
4754 LD1RQB_z_p_br_contiguous,
4755 LD1RQB_z_p_bi_u8,
4756 16);
4757 }
4758
ld1rqd(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4759 void Assembler::ld1rqd(const ZRegister& zt,
4760 const PRegisterZ& pg,
4761 const SVEMemOperand& addr) {
4762 VIXL_ASSERT(addr.IsScalarPlusImmediate() || addr.IsEquivalentToLSL(3));
4763 VIXL_ASSERT(zt.IsLaneSizeD());
4764 SVELd1St1ScaImmHelper(zt,
4765 pg,
4766 addr,
4767 LD1RQD_z_p_br_contiguous,
4768 LD1RQD_z_p_bi_u64,
4769 16);
4770 }
4771
ld1rqh(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4772 void Assembler::ld1rqh(const ZRegister& zt,
4773 const PRegisterZ& pg,
4774 const SVEMemOperand& addr) {
4775 VIXL_ASSERT(addr.IsScalarPlusImmediate() || addr.IsEquivalentToLSL(1));
4776 VIXL_ASSERT(zt.IsLaneSizeH());
4777 SVELd1St1ScaImmHelper(zt,
4778 pg,
4779 addr,
4780 LD1RQH_z_p_br_contiguous,
4781 LD1RQH_z_p_bi_u16,
4782 16);
4783 }
4784
ld1rqw(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4785 void Assembler::ld1rqw(const ZRegister& zt,
4786 const PRegisterZ& pg,
4787 const SVEMemOperand& addr) {
4788 VIXL_ASSERT(addr.IsScalarPlusImmediate() || addr.IsEquivalentToLSL(2));
4789 VIXL_ASSERT(zt.IsLaneSizeS());
4790 SVELd1St1ScaImmHelper(zt,
4791 pg,
4792 addr,
4793 LD1RQW_z_p_br_contiguous,
4794 LD1RQW_z_p_bi_u32,
4795 16);
4796 }
4797
4798 #define VIXL_DEFINE_LDFF1(MSZ, LANE_SIZE) \
4799 void Assembler::ldff1##MSZ(const ZRegister& zt, \
4800 const PRegisterZ& pg, \
4801 const SVEMemOperand& addr) { \
4802 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \
4803 SVELdff1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, false); \
4804 }
4805 VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LDFF1)
4806
4807 #define VIXL_DEFINE_LDFF1S(MSZ, LANE_SIZE) \
4808 void Assembler::ldff1s##MSZ(const ZRegister& zt, \
4809 const PRegisterZ& pg, \
4810 const SVEMemOperand& addr) { \
4811 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \
4812 SVELdff1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, true); \
4813 }
VIXL_SVE_LOAD_STORE_SIGNED_VARIANT_LIST(VIXL_DEFINE_LDFF1S)4814 VIXL_SVE_LOAD_STORE_SIGNED_VARIANT_LIST(VIXL_DEFINE_LDFF1S)
4815
4816 void Assembler::ldnf1b(const ZRegister& zt,
4817 const PRegisterZ& pg,
4818 const SVEMemOperand& addr) {
4819 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4820 VIXL_ASSERT(addr.IsPlainRegister() ||
4821 (addr.IsScalarPlusImmediate() &&
4822 (addr.GetOffsetModifier() == SVE_MUL_VL)));
4823
4824 SVELdSt1Helper(0,
4825 zt,
4826 pg,
4827 addr,
4828 /* is_signed = */ false,
4829 SVEContiguousNonFaultLoad_ScalarPlusImmFixed);
4830 }
4831
ldnf1d(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4832 void Assembler::ldnf1d(const ZRegister& zt,
4833 const PRegisterZ& pg,
4834 const SVEMemOperand& addr) {
4835 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4836 VIXL_ASSERT(addr.IsPlainRegister() ||
4837 (addr.IsScalarPlusImmediate() &&
4838 (addr.GetOffsetModifier() == SVE_MUL_VL)));
4839
4840 SVELdSt1Helper(3,
4841 zt,
4842 pg,
4843 addr,
4844 /* is_signed = */ false,
4845 SVEContiguousNonFaultLoad_ScalarPlusImmFixed);
4846 }
4847
ldnf1h(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4848 void Assembler::ldnf1h(const ZRegister& zt,
4849 const PRegisterZ& pg,
4850 const SVEMemOperand& addr) {
4851 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4852 VIXL_ASSERT(addr.IsPlainRegister() ||
4853 (addr.IsScalarPlusImmediate() &&
4854 (addr.GetOffsetModifier() == SVE_MUL_VL)));
4855
4856 SVELdSt1Helper(1,
4857 zt,
4858 pg,
4859 addr,
4860 /* is_signed = */ false,
4861 SVEContiguousNonFaultLoad_ScalarPlusImmFixed);
4862 }
4863
ldnf1sb(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4864 void Assembler::ldnf1sb(const ZRegister& zt,
4865 const PRegisterZ& pg,
4866 const SVEMemOperand& addr) {
4867 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4868 VIXL_ASSERT(addr.IsPlainRegister() ||
4869 (addr.IsScalarPlusImmediate() &&
4870 (addr.GetOffsetModifier() == SVE_MUL_VL)));
4871
4872 SVELdSt1Helper(0,
4873 zt,
4874 pg,
4875 addr,
4876 /* is_signed = */ true,
4877 SVEContiguousNonFaultLoad_ScalarPlusImmFixed);
4878 }
4879
ldnf1sh(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4880 void Assembler::ldnf1sh(const ZRegister& zt,
4881 const PRegisterZ& pg,
4882 const SVEMemOperand& addr) {
4883 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4884 VIXL_ASSERT(addr.IsPlainRegister() ||
4885 (addr.IsScalarPlusImmediate() &&
4886 (addr.GetOffsetModifier() == SVE_MUL_VL)));
4887
4888 SVELdSt1Helper(1,
4889 zt,
4890 pg,
4891 addr,
4892 /* is_signed = */ true,
4893 SVEContiguousNonFaultLoad_ScalarPlusImmFixed);
4894 }
4895
ldnf1sw(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4896 void Assembler::ldnf1sw(const ZRegister& zt,
4897 const PRegisterZ& pg,
4898 const SVEMemOperand& addr) {
4899 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4900 VIXL_ASSERT(addr.IsPlainRegister() ||
4901 (addr.IsScalarPlusImmediate() &&
4902 (addr.GetOffsetModifier() == SVE_MUL_VL)));
4903
4904 SVELdSt1Helper(2,
4905 zt,
4906 pg,
4907 addr,
4908 /* is_signed = */ true,
4909 SVEContiguousNonFaultLoad_ScalarPlusImmFixed);
4910 }
4911
ldnf1w(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4912 void Assembler::ldnf1w(const ZRegister& zt,
4913 const PRegisterZ& pg,
4914 const SVEMemOperand& addr) {
4915 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4916 VIXL_ASSERT(addr.IsPlainRegister() ||
4917 (addr.IsScalarPlusImmediate() &&
4918 (addr.GetOffsetModifier() == SVE_MUL_VL)));
4919
4920 SVELdSt1Helper(2,
4921 zt,
4922 pg,
4923 addr,
4924 /* is_signed = */ false,
4925 SVEContiguousNonFaultLoad_ScalarPlusImmFixed);
4926 }
4927
ldnt1b(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4928 void Assembler::ldnt1b(const ZRegister& zt,
4929 const PRegisterZ& pg,
4930 const SVEMemOperand& addr) {
4931 VIXL_ASSERT(addr.IsPlainScalar() ||
4932 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
4933 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(0)));
4934 SVELd1St1ScaImmHelper(zt,
4935 pg,
4936 addr,
4937 LDNT1B_z_p_br_contiguous,
4938 LDNT1B_z_p_bi_contiguous);
4939 }
4940
ldnt1d(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4941 void Assembler::ldnt1d(const ZRegister& zt,
4942 const PRegisterZ& pg,
4943 const SVEMemOperand& addr) {
4944 VIXL_ASSERT(addr.IsPlainScalar() ||
4945 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
4946 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(3)));
4947 SVELd1St1ScaImmHelper(zt,
4948 pg,
4949 addr,
4950 LDNT1D_z_p_br_contiguous,
4951 LDNT1D_z_p_bi_contiguous);
4952 }
4953
ldnt1h(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4954 void Assembler::ldnt1h(const ZRegister& zt,
4955 const PRegisterZ& pg,
4956 const SVEMemOperand& addr) {
4957 VIXL_ASSERT(addr.IsPlainScalar() ||
4958 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
4959 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(1)));
4960 SVELd1St1ScaImmHelper(zt,
4961 pg,
4962 addr,
4963 LDNT1H_z_p_br_contiguous,
4964 LDNT1H_z_p_bi_contiguous);
4965 }
4966
ldnt1w(const ZRegister & zt,const PRegisterZ & pg,const SVEMemOperand & addr)4967 void Assembler::ldnt1w(const ZRegister& zt,
4968 const PRegisterZ& pg,
4969 const SVEMemOperand& addr) {
4970 VIXL_ASSERT(addr.IsPlainScalar() ||
4971 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
4972 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(2)));
4973 SVELd1St1ScaImmHelper(zt,
4974 pg,
4975 addr,
4976 LDNT1W_z_p_br_contiguous,
4977 LDNT1W_z_p_bi_contiguous);
4978 }
4979
SVEMemOperandHelper(unsigned msize_in_bytes_log2,int num_regs,const SVEMemOperand & addr,bool is_load)4980 Instr Assembler::SVEMemOperandHelper(unsigned msize_in_bytes_log2,
4981 int num_regs,
4982 const SVEMemOperand& addr,
4983 bool is_load) {
4984 VIXL_ASSERT((num_regs >= 1) && (num_regs <= 4));
4985
4986 Instr op = 0xfffffff;
4987 if (addr.IsScalarPlusImmediate()) {
4988 VIXL_ASSERT((addr.GetImmediateOffset() == 0) || addr.IsMulVl());
4989 int64_t imm = addr.GetImmediateOffset();
4990 VIXL_ASSERT((imm % num_regs) == 0);
4991 op = RnSP(addr.GetScalarBase()) | ImmField<19, 16>(imm / num_regs);
4992
4993 } else if (addr.IsScalarPlusScalar()) {
4994 VIXL_ASSERT(addr.GetScalarOffset().IsZero() ||
4995 addr.IsEquivalentToLSL(msize_in_bytes_log2));
4996 op = RnSP(addr.GetScalarBase()) | Rm(addr.GetScalarOffset());
4997
4998 } else if (addr.IsVectorPlusImmediate()) {
4999 ZRegister zn = addr.GetVectorBase();
5000 uint64_t imm = addr.GetImmediateOffset();
5001 VIXL_ASSERT(num_regs == 1);
5002 VIXL_ASSERT(zn.IsLaneSizeS() || zn.IsLaneSizeD());
5003 VIXL_ASSERT(IsMultiple(imm, (1 << msize_in_bytes_log2)));
5004 op = Rn(zn) | ImmUnsignedField<20, 16>(imm >> msize_in_bytes_log2);
5005
5006 } else if (addr.IsScalarPlusVector()) {
5007 // We have to support several different addressing modes. Some instructions
5008 // support a subset of these, but the SVEMemOperand encoding is consistent.
5009 Register xn = addr.GetScalarBase();
5010 ZRegister zm = addr.GetVectorOffset();
5011 SVEOffsetModifier mod = addr.GetOffsetModifier();
5012 Instr modifier_bit = 1 << (is_load ? 22 : 14);
5013 Instr xs = (mod == SVE_SXTW) ? modifier_bit : 0;
5014 VIXL_ASSERT(num_regs == 1);
5015
5016 if (mod == SVE_LSL) {
5017 // 64-bit scaled offset: [<Xn|SP>, <Zm>.D, LSL #<shift>]
5018 VIXL_ASSERT(zm.IsLaneSizeD());
5019 VIXL_ASSERT(addr.GetShiftAmount() == msize_in_bytes_log2);
5020 } else if (mod == NO_SVE_OFFSET_MODIFIER) {
5021 // 64-bit unscaled offset: [<Xn|SP>, <Zm>.D]
5022 VIXL_ASSERT(zm.IsLaneSizeD());
5023 VIXL_ASSERT(addr.GetShiftAmount() == 0);
5024 } else {
5025 // 32-bit scaled offset: [<Xn|SP>, <Zm>.S, <mod> #<shift>]
5026 // 32-bit unscaled offset: [<Xn|SP>, <Zm>.S, <mod>]
5027 // 32-bit unpacked scaled offset: [<Xn|SP>, <Zm>.D, <mod> #<shift>]
5028 // 32-bit unpacked unscaled offset: [<Xn|SP>, <Zm>.D, <mod>]
5029 VIXL_ASSERT(zm.IsLaneSizeS() || zm.IsLaneSizeD());
5030 VIXL_ASSERT((mod == SVE_SXTW) || (mod == SVE_UXTW));
5031 VIXL_ASSERT((addr.GetShiftAmount() == 0) ||
5032 (addr.GetShiftAmount() == msize_in_bytes_log2));
5033 }
5034
5035 // The form itself is encoded in the instruction opcode.
5036 op = RnSP(xn) | Rm(zm) | xs;
5037 } else {
5038 VIXL_UNIMPLEMENTED();
5039 }
5040
5041 return op;
5042 }
5043
5044 // SVEMemStore.
5045
SVESt1Helper(unsigned msize_in_bytes_log2,const ZRegister & zt,const PRegister & pg,const SVEMemOperand & addr)5046 void Assembler::SVESt1Helper(unsigned msize_in_bytes_log2,
5047 const ZRegister& zt,
5048 const PRegister& pg,
5049 const SVEMemOperand& addr) {
5050 if (addr.IsScalarPlusScalar()) {
5051 // Rm must not be xzr.
5052 VIXL_ASSERT(!addr.GetScalarOffset().IsZero());
5053 }
5054
5055 if (addr.IsScatterGather()) {
5056 bool is_load = false;
5057 bool is_signed = false;
5058 bool is_ff = false;
5059 SVEScatterGatherHelper(msize_in_bytes_log2,
5060 zt,
5061 pg,
5062 addr,
5063 is_load,
5064 is_signed,
5065 is_ff);
5066 return;
5067 }
5068
5069 Instr op;
5070 if (addr.IsScalarPlusImmediate()) {
5071 op = SVEContiguousStore_ScalarPlusImmFixed;
5072 } else if (addr.IsScalarPlusScalar()) {
5073 op = SVEContiguousStore_ScalarPlusScalarFixed;
5074 } else {
5075 VIXL_UNIMPLEMENTED();
5076 op = 0xffffffff;
5077 }
5078 SVELdSt1Helper(msize_in_bytes_log2, zt, pg, addr, false, op);
5079 }
5080
SVESt234Helper(int num_regs,const ZRegister & zt1,const PRegister & pg,const SVEMemOperand & addr)5081 void Assembler::SVESt234Helper(int num_regs,
5082 const ZRegister& zt1,
5083 const PRegister& pg,
5084 const SVEMemOperand& addr) {
5085 if (addr.IsScalarPlusScalar()) {
5086 // Rm must not be xzr.
5087 VIXL_ASSERT(!addr.GetScalarOffset().IsZero());
5088 }
5089
5090 Instr op;
5091 if (addr.IsScalarPlusImmediate()) {
5092 op = SVEStoreMultipleStructures_ScalarPlusImmFixed;
5093 } else if (addr.IsScalarPlusScalar()) {
5094 op = SVEStoreMultipleStructures_ScalarPlusScalarFixed;
5095 } else {
5096 // These instructions don't support any other addressing modes.
5097 VIXL_ABORT();
5098 }
5099 SVELdSt234Helper(num_regs, zt1, pg, addr, op);
5100 }
5101
5102 #define VIXL_DEFINE_ST1(MSZ, LANE_SIZE) \
5103 void Assembler::st1##MSZ(const ZRegister& zt, \
5104 const PRegister& pg, \
5105 const SVEMemOperand& addr) { \
5106 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \
5107 SVESt1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr); \
5108 }
5109 #define VIXL_DEFINE_ST2(MSZ, LANE_SIZE) \
5110 void Assembler::st2##MSZ(const ZRegister& zt1, \
5111 const ZRegister& zt2, \
5112 const PRegister& pg, \
5113 const SVEMemOperand& addr) { \
5114 USE(zt2); \
5115 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \
5116 VIXL_ASSERT(AreConsecutive(zt1, zt2)); \
5117 VIXL_ASSERT(AreSameFormat(zt1, zt2)); \
5118 VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \
5119 SVESt234Helper(2, zt1, pg, addr); \
5120 }
5121 #define VIXL_DEFINE_ST3(MSZ, LANE_SIZE) \
5122 void Assembler::st3##MSZ(const ZRegister& zt1, \
5123 const ZRegister& zt2, \
5124 const ZRegister& zt3, \
5125 const PRegister& pg, \
5126 const SVEMemOperand& addr) { \
5127 USE(zt2, zt3); \
5128 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \
5129 VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3)); \
5130 VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3)); \
5131 VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \
5132 SVESt234Helper(3, zt1, pg, addr); \
5133 }
5134 #define VIXL_DEFINE_ST4(MSZ, LANE_SIZE) \
5135 void Assembler::st4##MSZ(const ZRegister& zt1, \
5136 const ZRegister& zt2, \
5137 const ZRegister& zt3, \
5138 const ZRegister& zt4, \
5139 const PRegister& pg, \
5140 const SVEMemOperand& addr) { \
5141 USE(zt2, zt3, zt4); \
5142 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \
5143 VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3, zt4)); \
5144 VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3, zt4)); \
5145 VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \
5146 SVESt234Helper(4, zt1, pg, addr); \
5147 }
5148
5149 VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST1)
VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST2)5150 VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST2)
5151 VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST3)
5152 VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST4)
5153
5154 void Assembler::stnt1b(const ZRegister& zt,
5155 const PRegister& pg,
5156 const SVEMemOperand& addr) {
5157 VIXL_ASSERT(addr.IsPlainScalar() ||
5158 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
5159 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(0)));
5160 SVELd1St1ScaImmHelper(zt,
5161 pg,
5162 addr,
5163 STNT1B_z_p_br_contiguous,
5164 STNT1B_z_p_bi_contiguous);
5165 }
5166
stnt1d(const ZRegister & zt,const PRegister & pg,const SVEMemOperand & addr)5167 void Assembler::stnt1d(const ZRegister& zt,
5168 const PRegister& pg,
5169 const SVEMemOperand& addr) {
5170 VIXL_ASSERT(addr.IsPlainScalar() ||
5171 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
5172 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(3)));
5173 SVELd1St1ScaImmHelper(zt,
5174 pg,
5175 addr,
5176 STNT1D_z_p_br_contiguous,
5177 STNT1D_z_p_bi_contiguous);
5178 }
5179
stnt1h(const ZRegister & zt,const PRegister & pg,const SVEMemOperand & addr)5180 void Assembler::stnt1h(const ZRegister& zt,
5181 const PRegister& pg,
5182 const SVEMemOperand& addr) {
5183 VIXL_ASSERT(addr.IsPlainScalar() ||
5184 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
5185 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(1)));
5186 SVELd1St1ScaImmHelper(zt,
5187 pg,
5188 addr,
5189 STNT1H_z_p_br_contiguous,
5190 STNT1H_z_p_bi_contiguous);
5191 }
5192
stnt1w(const ZRegister & zt,const PRegister & pg,const SVEMemOperand & addr)5193 void Assembler::stnt1w(const ZRegister& zt,
5194 const PRegister& pg,
5195 const SVEMemOperand& addr) {
5196 VIXL_ASSERT(addr.IsPlainScalar() ||
5197 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
5198 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(2)));
5199 SVELd1St1ScaImmHelper(zt,
5200 pg,
5201 addr,
5202 STNT1W_z_p_br_contiguous,
5203 STNT1W_z_p_bi_contiguous);
5204 }
5205
str(const CPURegister & rt,const SVEMemOperand & addr)5206 void Assembler::str(const CPURegister& rt, const SVEMemOperand& addr) {
5207 // STR <Pt/Zt>, [<Xn|SP>{, #<imm>, MUL VL}]
5208
5209 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5210 VIXL_ASSERT(rt.IsPRegister() || rt.IsZRegister());
5211 VIXL_ASSERT(addr.IsPlainScalar() ||
5212 (addr.IsScalarPlusImmediate() &&
5213 (addr.GetOffsetModifier() == SVE_MUL_VL)));
5214 int64_t imm9 = addr.GetImmediateOffset();
5215 VIXL_ASSERT(IsInt9(imm9));
5216 Instr imm9l = ExtractUnsignedBitfield32(2, 0, imm9) << 10;
5217 Instr imm9h = ExtractUnsignedBitfield32(8, 3, imm9) << 16;
5218
5219 Instr op = STR_z_bi;
5220 if (rt.IsPRegister()) {
5221 op = STR_p_bi;
5222 }
5223 Emit(op | Rt(rt) | RnSP(addr.GetScalarBase()) | imm9h | imm9l);
5224 }
5225
5226 // SVEMulIndex.
5227
sdot(const ZRegister & zda,const ZRegister & zn,const ZRegister & zm,int index)5228 void Assembler::sdot(const ZRegister& zda,
5229 const ZRegister& zn,
5230 const ZRegister& zm,
5231 int index) {
5232 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5233 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4));
5234 VIXL_ASSERT(AreSameLaneSize(zn, zm));
5235
5236 Instr op = 0xffffffff;
5237 switch (zda.GetLaneSizeInBits()) {
5238 case kSRegSize:
5239 VIXL_ASSERT(IsUint2(index));
5240 op = SDOT_z_zzzi_s | Rx<18, 16>(zm) | (index << 19) | Rd(zda) | Rn(zn);
5241 break;
5242 case kDRegSize:
5243 VIXL_ASSERT(IsUint1(index));
5244 op = SDOT_z_zzzi_d | Rx<19, 16>(zm) | (index << 20) | Rd(zda) | Rn(zn);
5245 break;
5246 default:
5247 VIXL_UNIMPLEMENTED();
5248 break;
5249 }
5250
5251 Emit(op);
5252 }
5253
udot(const ZRegister & zda,const ZRegister & zn,const ZRegister & zm,int index)5254 void Assembler::udot(const ZRegister& zda,
5255 const ZRegister& zn,
5256 const ZRegister& zm,
5257 int index) {
5258 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5259 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4));
5260 VIXL_ASSERT(AreSameLaneSize(zn, zm));
5261
5262 Instr op = 0xffffffff;
5263 switch (zda.GetLaneSizeInBits()) {
5264 case kSRegSize:
5265 VIXL_ASSERT(IsUint2(index));
5266 op = UDOT_z_zzzi_s | Rx<18, 16>(zm) | (index << 19) | Rd(zda) | Rn(zn);
5267 break;
5268 case kDRegSize:
5269 VIXL_ASSERT(IsUint1(index));
5270 op = UDOT_z_zzzi_d | Rx<19, 16>(zm) | (index << 20) | Rd(zda) | Rn(zn);
5271 break;
5272 default:
5273 VIXL_UNIMPLEMENTED();
5274 break;
5275 }
5276
5277 Emit(op);
5278 }
5279
5280 // SVEPartitionBreak.
5281
brka(const PRegisterWithLaneSize & pd,const PRegister & pg,const PRegisterWithLaneSize & pn)5282 void Assembler::brka(const PRegisterWithLaneSize& pd,
5283 const PRegister& pg,
5284 const PRegisterWithLaneSize& pn) {
5285 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5286 VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing());
5287 VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB());
5288
5289 Instr m = pg.IsMerging() ? 0x00000010 : 0x00000000;
5290 Emit(BRKA_p_p_p | Pd(pd) | Pg<13, 10>(pg) | m | Pn(pn));
5291 }
5292
brkas(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn)5293 void Assembler::brkas(const PRegisterWithLaneSize& pd,
5294 const PRegisterZ& pg,
5295 const PRegisterWithLaneSize& pn) {
5296 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5297 VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB());
5298
5299 Emit(BRKAS_p_p_p_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn));
5300 }
5301
brkb(const PRegisterWithLaneSize & pd,const PRegister & pg,const PRegisterWithLaneSize & pn)5302 void Assembler::brkb(const PRegisterWithLaneSize& pd,
5303 const PRegister& pg,
5304 const PRegisterWithLaneSize& pn) {
5305 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5306 VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing());
5307 VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB());
5308
5309 Instr m = pg.IsMerging() ? 0x00000010 : 0x00000000;
5310 Emit(BRKB_p_p_p | Pd(pd) | Pg<13, 10>(pg) | m | Pn(pn));
5311 }
5312
brkbs(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn)5313 void Assembler::brkbs(const PRegisterWithLaneSize& pd,
5314 const PRegisterZ& pg,
5315 const PRegisterWithLaneSize& pn) {
5316 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5317 VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB());
5318
5319 Emit(BRKBS_p_p_p_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn));
5320 }
5321
brkn(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)5322 void Assembler::brkn(const PRegisterWithLaneSize& pd,
5323 const PRegisterZ& pg,
5324 const PRegisterWithLaneSize& pn,
5325 const PRegisterWithLaneSize& pm) {
5326 USE(pm);
5327 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5328 VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB());
5329 VIXL_ASSERT(pd.Is(pm));
5330
5331 Emit(BRKN_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn));
5332 }
5333
brkns(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)5334 void Assembler::brkns(const PRegisterWithLaneSize& pd,
5335 const PRegisterZ& pg,
5336 const PRegisterWithLaneSize& pn,
5337 const PRegisterWithLaneSize& pm) {
5338 USE(pm);
5339 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5340 VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB());
5341 VIXL_ASSERT(pd.Is(pm));
5342
5343 Emit(BRKNS_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn));
5344 }
5345
5346 // SVEPermutePredicate.
5347
punpkhi(const PRegisterWithLaneSize & pd,const PRegisterWithLaneSize & pn)5348 void Assembler::punpkhi(const PRegisterWithLaneSize& pd,
5349 const PRegisterWithLaneSize& pn) {
5350 // PUNPKHI <Pd>.H, <Pn>.B
5351 // 0000 0101 0011 0001 0100 000. ...0 ....
5352 // H<16> = 1 | Pn<8:5> | Pd<3:0>
5353
5354 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5355 VIXL_ASSERT(pd.IsLaneSizeH());
5356 VIXL_ASSERT(pn.IsLaneSizeB());
5357
5358 Emit(PUNPKHI_p_p | Pd(pd) | Pn(pn));
5359 }
5360
punpklo(const PRegisterWithLaneSize & pd,const PRegisterWithLaneSize & pn)5361 void Assembler::punpklo(const PRegisterWithLaneSize& pd,
5362 const PRegisterWithLaneSize& pn) {
5363 // PUNPKLO <Pd>.H, <Pn>.B
5364 // 0000 0101 0011 0000 0100 000. ...0 ....
5365 // H<16> = 0 | Pn<8:5> | Pd<3:0>
5366
5367 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5368 VIXL_ASSERT(pd.IsLaneSizeH());
5369 VIXL_ASSERT(pn.IsLaneSizeB());
5370
5371 Emit(PUNPKLO_p_p | Pd(pd) | Pn(pn));
5372 }
5373
rev(const PRegisterWithLaneSize & pd,const PRegisterWithLaneSize & pn)5374 void Assembler::rev(const PRegisterWithLaneSize& pd,
5375 const PRegisterWithLaneSize& pn) {
5376 // REV <Pd>.<T>, <Pn>.<T>
5377 // 0000 0101 ..11 0100 0100 000. ...0 ....
5378 // size<23:22> | Pn<8:5> | Pd<3:0>
5379
5380 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5381 VIXL_ASSERT(AreSameLaneSize(pd, pn));
5382
5383 Emit(REV_p_p | SVESize(pd) | Pd(pd) | Rx<8, 5>(pn));
5384 }
5385
trn1(const PRegisterWithLaneSize & pd,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)5386 void Assembler::trn1(const PRegisterWithLaneSize& pd,
5387 const PRegisterWithLaneSize& pn,
5388 const PRegisterWithLaneSize& pm) {
5389 // TRN1 <Pd>.<T>, <Pn>.<T>, <Pm>.<T>
5390 // 0000 0101 ..10 .... 0101 000. ...0 ....
5391 // size<23:22> | Pm<19:16> | opc<12:11> = 10 | H<10> = 0 | Pn<8:5> | Pd<3:0>
5392
5393 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5394 VIXL_ASSERT(AreSameLaneSize(pd, pn, pm));
5395
5396 Emit(TRN1_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm));
5397 }
5398
trn2(const PRegisterWithLaneSize & pd,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)5399 void Assembler::trn2(const PRegisterWithLaneSize& pd,
5400 const PRegisterWithLaneSize& pn,
5401 const PRegisterWithLaneSize& pm) {
5402 // TRN2 <Pd>.<T>, <Pn>.<T>, <Pm>.<T>
5403 // 0000 0101 ..10 .... 0101 010. ...0 ....
5404 // size<23:22> | Pm<19:16> | opc<12:11> = 10 | H<10> = 1 | Pn<8:5> | Pd<3:0>
5405
5406 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5407 VIXL_ASSERT(AreSameLaneSize(pd, pn, pm));
5408
5409 Emit(TRN2_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm));
5410 }
5411
uzp1(const PRegisterWithLaneSize & pd,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)5412 void Assembler::uzp1(const PRegisterWithLaneSize& pd,
5413 const PRegisterWithLaneSize& pn,
5414 const PRegisterWithLaneSize& pm) {
5415 // UZP1 <Pd>.<T>, <Pn>.<T>, <Pm>.<T>
5416 // 0000 0101 ..10 .... 0100 100. ...0 ....
5417 // size<23:22> | Pm<19:16> | opc<12:11> = 01 | H<10> = 0 | Pn<8:5> | Pd<3:0>
5418
5419 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5420 VIXL_ASSERT(AreSameLaneSize(pd, pn, pm));
5421
5422 Emit(UZP1_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm));
5423 }
5424
uzp2(const PRegisterWithLaneSize & pd,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)5425 void Assembler::uzp2(const PRegisterWithLaneSize& pd,
5426 const PRegisterWithLaneSize& pn,
5427 const PRegisterWithLaneSize& pm) {
5428 // UZP2 <Pd>.<T>, <Pn>.<T>, <Pm>.<T>
5429 // 0000 0101 ..10 .... 0100 110. ...0 ....
5430 // size<23:22> | Pm<19:16> | opc<12:11> = 01 | H<10> = 1 | Pn<8:5> | Pd<3:0>
5431
5432 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5433 VIXL_ASSERT(AreSameLaneSize(pd, pn, pm));
5434
5435 Emit(UZP2_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm));
5436 }
5437
zip1(const PRegisterWithLaneSize & pd,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)5438 void Assembler::zip1(const PRegisterWithLaneSize& pd,
5439 const PRegisterWithLaneSize& pn,
5440 const PRegisterWithLaneSize& pm) {
5441 // ZIP1 <Pd>.<T>, <Pn>.<T>, <Pm>.<T>
5442 // 0000 0101 ..10 .... 0100 000. ...0 ....
5443 // size<23:22> | Pm<19:16> | opc<12:11> = 00 | H<10> = 0 | Pn<8:5> | Pd<3:0>
5444
5445 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5446 VIXL_ASSERT(AreSameLaneSize(pd, pn, pm));
5447
5448 Emit(ZIP1_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm));
5449 }
5450
zip2(const PRegisterWithLaneSize & pd,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)5451 void Assembler::zip2(const PRegisterWithLaneSize& pd,
5452 const PRegisterWithLaneSize& pn,
5453 const PRegisterWithLaneSize& pm) {
5454 // ZIP2 <Pd>.<T>, <Pn>.<T>, <Pm>.<T>
5455 // 0000 0101 ..10 .... 0100 010. ...0 ....
5456 // size<23:22> | Pm<19:16> | opc<12:11> = 00 | H<10> = 1 | Pn<8:5> | Pd<3:0>
5457
5458 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5459 VIXL_ASSERT(AreSameLaneSize(pd, pn, pm));
5460
5461 Emit(ZIP2_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm));
5462 }
5463
5464 // SVEPermuteVectorExtract.
5465
ext(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm,unsigned offset)5466 void Assembler::ext(const ZRegister& zd,
5467 const ZRegister& zn,
5468 const ZRegister& zm,
5469 unsigned offset) {
5470 // EXT <Zdn>.B, <Zdn>.B, <Zm>.B, #<imm>
5471 // 0000 0101 001. .... 000. .... .... ....
5472 // imm8h<20:16> | imm8l<12:10> | Zm<9:5> | Zdn<4:0>
5473
5474 USE(zn);
5475 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5476 VIXL_ASSERT(zd.Is(zn));
5477 VIXL_ASSERT(IsUint8(offset));
5478
5479 int imm8h = ExtractUnsignedBitfield32(7, 3, offset);
5480 int imm8l = ExtractUnsignedBitfield32(2, 0, offset);
5481 Emit(EXT_z_zi_des | Rd(zd) | Rn(zm) | ImmUnsignedField<20, 16>(imm8h) |
5482 ImmUnsignedField<12, 10>(imm8l));
5483 }
5484
5485 // SVEPermuteVectorInterleaving.
5486
trn1(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)5487 void Assembler::trn1(const ZRegister& zd,
5488 const ZRegister& zn,
5489 const ZRegister& zm) {
5490 // TRN1 <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
5491 // 0000 0101 ..1. .... 0111 00.. .... ....
5492 // size<23:22> | Zm<20:16> | opc<12:10> = 100 | Zn<9:5> | Zd<4:0>
5493
5494 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5495 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5496
5497 Emit(TRN1_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
5498 }
5499
trn2(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)5500 void Assembler::trn2(const ZRegister& zd,
5501 const ZRegister& zn,
5502 const ZRegister& zm) {
5503 // TRN2 <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
5504 // 0000 0101 ..1. .... 0111 01.. .... ....
5505 // size<23:22> | Zm<20:16> | opc<12:10> = 101 | Zn<9:5> | Zd<4:0>
5506
5507 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5508 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5509
5510 Emit(TRN2_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
5511 }
5512
uzp1(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)5513 void Assembler::uzp1(const ZRegister& zd,
5514 const ZRegister& zn,
5515 const ZRegister& zm) {
5516 // UZP1 <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
5517 // 0000 0101 ..1. .... 0110 10.. .... ....
5518 // size<23:22> | Zm<20:16> | opc<12:10> = 010 | Zn<9:5> | Zd<4:0>
5519
5520 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5521 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5522
5523 Emit(UZP1_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
5524 }
5525
uzp2(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)5526 void Assembler::uzp2(const ZRegister& zd,
5527 const ZRegister& zn,
5528 const ZRegister& zm) {
5529 // UZP2 <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
5530 // 0000 0101 ..1. .... 0110 11.. .... ....
5531 // size<23:22> | Zm<20:16> | opc<12:10> = 011 | Zn<9:5> | Zd<4:0>
5532
5533 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5534 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5535
5536 Emit(UZP2_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
5537 }
5538
zip1(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)5539 void Assembler::zip1(const ZRegister& zd,
5540 const ZRegister& zn,
5541 const ZRegister& zm) {
5542 // ZIP1 <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
5543 // 0000 0101 ..1. .... 0110 00.. .... ....
5544 // size<23:22> | Zm<20:16> | opc<12:10> = 000 | Zn<9:5> | Zd<4:0>
5545
5546 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5547 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5548
5549 Emit(ZIP1_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
5550 }
5551
zip2(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)5552 void Assembler::zip2(const ZRegister& zd,
5553 const ZRegister& zn,
5554 const ZRegister& zm) {
5555 // ZIP2 <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
5556 // 0000 0101 ..1. .... 0110 01.. .... ....
5557 // size<23:22> | Zm<20:16> | opc<12:10> = 001 | Zn<9:5> | Zd<4:0>
5558
5559 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5560 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5561
5562 Emit(ZIP2_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
5563 }
5564
5565 // SVEPermuteVectorPredicated.
5566
clasta(const Register & rd,const PRegister & pg,const Register & rn,const ZRegister & zm)5567 void Assembler::clasta(const Register& rd,
5568 const PRegister& pg,
5569 const Register& rn,
5570 const ZRegister& zm) {
5571 // CLASTA <R><dn>, <Pg>, <R><dn>, <Zm>.<T>
5572 // 0000 0101 ..11 0000 101. .... .... ....
5573 // size<23:22> | B<16> = 0 | Pg<12:10> | Zm<9:5> | Rdn<4:0>
5574
5575 USE(rn);
5576 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5577 VIXL_ASSERT(rd.Is(rn));
5578
5579 Emit(CLASTA_r_p_z | SVESize(zm) | Rd(rd) | PgLow8(pg) | Rn(zm));
5580 }
5581
clasta(const VRegister & vd,const PRegister & pg,const VRegister & vn,const ZRegister & zm)5582 void Assembler::clasta(const VRegister& vd,
5583 const PRegister& pg,
5584 const VRegister& vn,
5585 const ZRegister& zm) {
5586 // CLASTA <V><dn>, <Pg>, <V><dn>, <Zm>.<T>
5587 // 0000 0101 ..10 1010 100. .... .... ....
5588 // size<23:22> | B<16> = 0 | Pg<12:10> | Zm<9:5> | Vdn<4:0>
5589
5590 USE(vn);
5591 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5592 VIXL_ASSERT(vd.Is(vn));
5593 VIXL_ASSERT(vd.IsScalar());
5594 VIXL_ASSERT(AreSameLaneSize(vd, zm));
5595
5596 Emit(CLASTA_v_p_z | SVESize(zm) | Rd(vd) | PgLow8(pg) | Rn(zm));
5597 }
5598
clasta(const ZRegister & zd,const PRegister & pg,const ZRegister & zn,const ZRegister & zm)5599 void Assembler::clasta(const ZRegister& zd,
5600 const PRegister& pg,
5601 const ZRegister& zn,
5602 const ZRegister& zm) {
5603 // CLASTA <Zdn>.<T>, <Pg>, <Zdn>.<T>, <Zm>.<T>
5604 // 0000 0101 ..10 1000 100. .... .... ....
5605 // size<23:22> | B<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
5606
5607 USE(zn);
5608 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5609 VIXL_ASSERT(zd.Is(zn));
5610 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5611
5612 Emit(CLASTA_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
5613 }
5614
clastb(const Register & rd,const PRegister & pg,const Register & rn,const ZRegister & zm)5615 void Assembler::clastb(const Register& rd,
5616 const PRegister& pg,
5617 const Register& rn,
5618 const ZRegister& zm) {
5619 // CLASTB <R><dn>, <Pg>, <R><dn>, <Zm>.<T>
5620 // 0000 0101 ..11 0001 101. .... .... ....
5621 // size<23:22> | B<16> = 1 | Pg<12:10> | Zm<9:5> | Rdn<4:0>
5622
5623 USE(rn);
5624 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5625 VIXL_ASSERT(rd.Is(rn));
5626
5627 Emit(CLASTB_r_p_z | SVESize(zm) | Rd(rd) | PgLow8(pg) | Rn(zm));
5628 }
5629
clastb(const VRegister & vd,const PRegister & pg,const VRegister & vn,const ZRegister & zm)5630 void Assembler::clastb(const VRegister& vd,
5631 const PRegister& pg,
5632 const VRegister& vn,
5633 const ZRegister& zm) {
5634 // CLASTB <V><dn>, <Pg>, <V><dn>, <Zm>.<T>
5635 // 0000 0101 ..10 1011 100. .... .... ....
5636 // size<23:22> | B<16> = 1 | Pg<12:10> | Zm<9:5> | Vdn<4:0>
5637
5638 USE(vn);
5639 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5640 VIXL_ASSERT(vd.Is(vn));
5641 VIXL_ASSERT(vd.IsScalar());
5642 VIXL_ASSERT(AreSameLaneSize(vd, zm));
5643
5644 Emit(CLASTB_v_p_z | SVESize(zm) | Rd(vd) | PgLow8(pg) | Rn(zm));
5645 }
5646
clastb(const ZRegister & zd,const PRegister & pg,const ZRegister & zn,const ZRegister & zm)5647 void Assembler::clastb(const ZRegister& zd,
5648 const PRegister& pg,
5649 const ZRegister& zn,
5650 const ZRegister& zm) {
5651 // CLASTB <Zdn>.<T>, <Pg>, <Zdn>.<T>, <Zm>.<T>
5652 // 0000 0101 ..10 1001 100. .... .... ....
5653 // size<23:22> | B<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
5654
5655 USE(zn);
5656 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5657 VIXL_ASSERT(zd.Is(zn));
5658 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5659
5660 Emit(CLASTB_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
5661 }
5662
compact(const ZRegister & zd,const PRegister & pg,const ZRegister & zn)5663 void Assembler::compact(const ZRegister& zd,
5664 const PRegister& pg,
5665 const ZRegister& zn) {
5666 // COMPACT <Zd>.<T>, <Pg>, <Zn>.<T>
5667 // 0000 0101 1.10 0001 100. .... .... ....
5668 // sz<22> | Pg<12:10> | Zn<9:5> | Zd<4:0>
5669
5670 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5671 VIXL_ASSERT(AreSameLaneSize(zd, zn));
5672 VIXL_ASSERT((zd.GetLaneSizeInBits() == kSRegSize) ||
5673 (zd.GetLaneSizeInBits() == kDRegSize));
5674
5675 Instr sz = (zd.GetLaneSizeInBits() == kDRegSize) ? (1 << 22) : 0;
5676 Emit(COMPACT_z_p_z | sz | Rd(zd) | PgLow8(pg) | Rn(zn));
5677 }
5678
cpy(const ZRegister & zd,const PRegisterM & pg,const Register & rn)5679 void Assembler::cpy(const ZRegister& zd,
5680 const PRegisterM& pg,
5681 const Register& rn) {
5682 // CPY <Zd>.<T>, <Pg>/M, <R><n|SP>
5683 // 0000 0101 ..10 1000 101. .... .... ....
5684 // size<23:22> | Pg<12:10> | Rn<9:5> | Zd<4:0>
5685
5686 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5687 VIXL_ASSERT(static_cast<unsigned>(rn.GetSizeInBits()) >=
5688 zd.GetLaneSizeInBits());
5689
5690 Emit(CPY_z_p_r | SVESize(zd) | Rd(zd) | PgLow8(pg) | RnSP(rn));
5691 }
5692
cpy(const ZRegister & zd,const PRegisterM & pg,const VRegister & vn)5693 void Assembler::cpy(const ZRegister& zd,
5694 const PRegisterM& pg,
5695 const VRegister& vn) {
5696 // CPY <Zd>.<T>, <Pg>/M, <V><n>
5697 // 0000 0101 ..10 0000 100. .... .... ....
5698 // size<23:22> | Pg<12:10> | Vn<9:5> | Zd<4:0>
5699
5700 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5701 VIXL_ASSERT(vn.IsScalar());
5702 VIXL_ASSERT(static_cast<unsigned>(vn.GetSizeInBits()) ==
5703 zd.GetLaneSizeInBits());
5704
5705 Emit(CPY_z_p_v | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(vn));
5706 }
5707
lasta(const Register & rd,const PRegister & pg,const ZRegister & zn)5708 void Assembler::lasta(const Register& rd,
5709 const PRegister& pg,
5710 const ZRegister& zn) {
5711 // LASTA <R><d>, <Pg>, <Zn>.<T>
5712 // 0000 0101 ..10 0000 101. .... .... ....
5713 // size<23:22> | B<16> = 0 | Pg<12:10> | Zn<9:5> | Rd<4:0>
5714
5715 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5716
5717 Emit(LASTA_r_p_z | SVESize(zn) | Rd(rd) | PgLow8(pg) | Rn(zn));
5718 }
5719
lasta(const VRegister & vd,const PRegister & pg,const ZRegister & zn)5720 void Assembler::lasta(const VRegister& vd,
5721 const PRegister& pg,
5722 const ZRegister& zn) {
5723 // LASTA <V><d>, <Pg>, <Zn>.<T>
5724 // 0000 0101 ..10 0010 100. .... .... ....
5725 // size<23:22> | B<16> = 0 | Pg<12:10> | Zn<9:5> | Vd<4:0>
5726
5727 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5728 VIXL_ASSERT(vd.IsScalar());
5729
5730 Emit(LASTA_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
5731 }
5732
lastb(const Register & rd,const PRegister & pg,const ZRegister & zn)5733 void Assembler::lastb(const Register& rd,
5734 const PRegister& pg,
5735 const ZRegister& zn) {
5736 // LASTB <R><d>, <Pg>, <Zn>.<T>
5737 // 0000 0101 ..10 0001 101. .... .... ....
5738 // size<23:22> | B<16> = 1 | Pg<12:10> | Zn<9:5> | Rd<4:0>
5739
5740 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5741
5742 Emit(LASTB_r_p_z | SVESize(zn) | Rd(rd) | PgLow8(pg) | Rn(zn));
5743 }
5744
lastb(const VRegister & vd,const PRegister & pg,const ZRegister & zn)5745 void Assembler::lastb(const VRegister& vd,
5746 const PRegister& pg,
5747 const ZRegister& zn) {
5748 // LASTB <V><d>, <Pg>, <Zn>.<T>
5749 // 0000 0101 ..10 0011 100. .... .... ....
5750 // size<23:22> | B<16> = 1 | Pg<12:10> | Zn<9:5> | Vd<4:0>
5751
5752 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5753 VIXL_ASSERT(vd.IsScalar());
5754
5755 Emit(LASTB_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
5756 }
5757
rbit(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)5758 void Assembler::rbit(const ZRegister& zd,
5759 const PRegisterM& pg,
5760 const ZRegister& zn) {
5761 // RBIT <Zd>.<T>, <Pg>/M, <Zn>.<T>
5762 // 0000 0101 ..10 0111 100. .... .... ....
5763 // size<23:22> | opc<17:16> = 11 | Pg<12:10> | Zn<9:5> | Zd<4:0>
5764
5765 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5766 VIXL_ASSERT(AreSameLaneSize(zd, zn));
5767
5768 Emit(RBIT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
5769 }
5770
revb(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)5771 void Assembler::revb(const ZRegister& zd,
5772 const PRegisterM& pg,
5773 const ZRegister& zn) {
5774 // REVB <Zd>.<T>, <Pg>/M, <Zn>.<T>
5775 // 0000 0101 ..10 0100 100. .... .... ....
5776 // size<23:22> | opc<17:16> = 00 | Pg<12:10> | Zn<9:5> | Zd<4:0>
5777
5778 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5779 VIXL_ASSERT(AreSameLaneSize(zd, zn));
5780 VIXL_ASSERT(zd.IsLaneSizeH() || zd.IsLaneSizeS() || zd.IsLaneSizeD());
5781
5782 Emit(REVB_z_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
5783 }
5784
revh(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)5785 void Assembler::revh(const ZRegister& zd,
5786 const PRegisterM& pg,
5787 const ZRegister& zn) {
5788 // REVH <Zd>.<T>, <Pg>/M, <Zn>.<T>
5789 // 0000 0101 ..10 0101 100. .... .... ....
5790 // size<23:22> | opc<17:16> = 01 | Pg<12:10> | Zn<9:5> | Zd<4:0>
5791
5792 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5793 VIXL_ASSERT(AreSameLaneSize(zd, zn));
5794 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD());
5795
5796 Emit(REVH_z_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
5797 }
5798
revw(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)5799 void Assembler::revw(const ZRegister& zd,
5800 const PRegisterM& pg,
5801 const ZRegister& zn) {
5802 // REVW <Zd>.D, <Pg>/M, <Zn>.D
5803 // 0000 0101 ..10 0110 100. .... .... ....
5804 // size<23:22> | opc<17:16> = 10 | Pg<12:10> | Zn<9:5> | Zd<4:0>
5805
5806 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5807 VIXL_ASSERT(AreSameLaneSize(zd, zn));
5808 VIXL_ASSERT(zd.IsLaneSizeD());
5809
5810 Emit(REVW_z_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
5811 }
5812
splice(const ZRegister & zd,const PRegister & pg,const ZRegister & zn,const ZRegister & zm)5813 void Assembler::splice(const ZRegister& zd,
5814 const PRegister& pg,
5815 const ZRegister& zn,
5816 const ZRegister& zm) {
5817 // SPLICE <Zdn>.<T>, <Pg>, <Zdn>.<T>, <Zm>.<T>
5818 // 0000 0101 ..10 1100 100. .... .... ....
5819 // size<23:22> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
5820
5821 USE(zn);
5822 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5823 VIXL_ASSERT(zd.Is(zn));
5824 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5825
5826 Emit(SPLICE_z_p_zz_des | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
5827 }
5828
5829 // SVEPermuteVectorUnpredicated.
5830
dup(const ZRegister & zd,const Register & xn)5831 void Assembler::dup(const ZRegister& zd, const Register& xn) {
5832 // DUP <Zd>.<T>, <R><n|SP>
5833 // 0000 0101 ..10 0000 0011 10.. .... ....
5834 // size<23:22> | Rn<9:5> | Zd<4:0>
5835
5836 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5837
5838 Emit(DUP_z_r | SVESize(zd) | Rd(zd) | RnSP(xn));
5839 }
5840
dup(const ZRegister & zd,const ZRegister & zn,unsigned index)5841 void Assembler::dup(const ZRegister& zd, const ZRegister& zn, unsigned index) {
5842 // DUP <Zd>.<T>, <Zn>.<T>[<imm>]
5843 // 0000 0101 ..1. .... 0010 00.. .... ....
5844 // imm2<23:22> | tsz<20:16> | Zn<9:5> | Zd<4:0>
5845
5846 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5847 VIXL_ASSERT(AreSameFormat(zd, zn));
5848 VIXL_ASSERT((index * zd.GetLaneSizeInBits()) < 512);
5849 int n = zd.GetLaneSizeInBytesLog2();
5850 unsigned imm_7 = (index << (n + 1)) | (1 << n);
5851 VIXL_ASSERT(IsUint7(imm_7));
5852 unsigned imm_2 = ExtractUnsignedBitfield32(6, 5, imm_7);
5853 unsigned tsz_5 = ExtractUnsignedBitfield32(4, 0, imm_7);
5854
5855 Emit(DUP_z_zi | ImmUnsignedField<23, 22>(imm_2) |
5856 ImmUnsignedField<20, 16>(tsz_5) | Rd(zd) | Rn(zn));
5857 }
5858
insr(const ZRegister & zdn,const Register & rm)5859 void Assembler::insr(const ZRegister& zdn, const Register& rm) {
5860 // INSR <Zdn>.<T>, <R><m>
5861 // 0000 0101 ..10 0100 0011 10.. .... ....
5862 // size<23:22> | Rm<9:5> | Zdn<4:0>
5863
5864 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5865
5866 Emit(INSR_z_r | SVESize(zdn) | Rd(zdn) | Rn(rm));
5867 }
5868
insr(const ZRegister & zdn,const VRegister & vm)5869 void Assembler::insr(const ZRegister& zdn, const VRegister& vm) {
5870 // INSR <Zdn>.<T>, <V><m>
5871 // 0000 0101 ..11 0100 0011 10.. .... ....
5872 // size<23:22> | Vm<9:5> | Zdn<4:0>
5873
5874 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5875 VIXL_ASSERT(vm.IsScalar());
5876
5877 Emit(INSR_z_v | SVESize(zdn) | Rd(zdn) | Rn(vm));
5878 }
5879
rev(const ZRegister & zd,const ZRegister & zn)5880 void Assembler::rev(const ZRegister& zd, const ZRegister& zn) {
5881 // REV <Zd>.<T>, <Zn>.<T>
5882 // 0000 0101 ..11 1000 0011 10.. .... ....
5883 // size<23:22> | Zn<9:5> | Zd<4:0>
5884
5885 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5886 VIXL_ASSERT(AreSameFormat(zd, zn));
5887
5888 Emit(REV_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
5889 }
5890
sunpkhi(const ZRegister & zd,const ZRegister & zn)5891 void Assembler::sunpkhi(const ZRegister& zd, const ZRegister& zn) {
5892 // SUNPKHI <Zd>.<T>, <Zn>.<Tb>
5893 // 0000 0101 ..11 0001 0011 10.. .... ....
5894 // size<23:22> | U<17> = 0 | H<16> = 1 | Zn<9:5> | Zd<4:0>
5895
5896 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5897 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
5898 VIXL_ASSERT(!zd.IsLaneSizeB());
5899
5900 Emit(SUNPKHI_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
5901 }
5902
sunpklo(const ZRegister & zd,const ZRegister & zn)5903 void Assembler::sunpklo(const ZRegister& zd, const ZRegister& zn) {
5904 // SUNPKLO <Zd>.<T>, <Zn>.<Tb>
5905 // 0000 0101 ..11 0000 0011 10.. .... ....
5906 // size<23:22> | U<17> = 0 | H<16> = 0 | Zn<9:5> | Zd<4:0>
5907
5908 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5909 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
5910 VIXL_ASSERT(!zd.IsLaneSizeB());
5911
5912 Emit(SUNPKLO_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
5913 }
5914
tbl(const ZRegister & zd,const ZRegister & zn,const ZRegister & zm)5915 void Assembler::tbl(const ZRegister& zd,
5916 const ZRegister& zn,
5917 const ZRegister& zm) {
5918 // TBL <Zd>.<T>, { <Zn>.<T> }, <Zm>.<T>
5919 // 0000 0101 ..1. .... 0011 00.. .... ....
5920 // size<23:22> | Zm<20:16> | Zn<9:5> | Zd<4:0>
5921
5922 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5923 VIXL_ASSERT(AreSameFormat(zd, zn, zm));
5924
5925 Emit(TBL_z_zz_1 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
5926 }
5927
uunpkhi(const ZRegister & zd,const ZRegister & zn)5928 void Assembler::uunpkhi(const ZRegister& zd, const ZRegister& zn) {
5929 // UUNPKHI <Zd>.<T>, <Zn>.<Tb>
5930 // 0000 0101 ..11 0011 0011 10.. .... ....
5931 // size<23:22> | U<17> = 1 | H<16> = 1 | Zn<9:5> | Zd<4:0>
5932
5933 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5934 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
5935 VIXL_ASSERT(!zd.IsLaneSizeB());
5936
5937 Emit(UUNPKHI_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
5938 }
5939
uunpklo(const ZRegister & zd,const ZRegister & zn)5940 void Assembler::uunpklo(const ZRegister& zd, const ZRegister& zn) {
5941 // UUNPKLO <Zd>.<T>, <Zn>.<Tb>
5942 // 0000 0101 ..11 0010 0011 10.. .... ....
5943 // size<23:22> | U<17> = 1 | H<16> = 0 | Zn<9:5> | Zd<4:0>
5944
5945 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5946 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
5947 VIXL_ASSERT(!zd.IsLaneSizeB());
5948
5949 Emit(UUNPKLO_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
5950 }
5951
5952 // SVEPredicateCount.
5953
cntp(const Register & xd,const PRegister & pg,const PRegisterWithLaneSize & pn)5954 void Assembler::cntp(const Register& xd,
5955 const PRegister& pg,
5956 const PRegisterWithLaneSize& pn) {
5957 // CNTP <Xd>, <Pg>, <Pn>.<T>
5958 // 0010 0101 ..10 0000 10.. ..0. .... ....
5959 // size<23:22> | opc<18:16> = 000 | Pg<13:10> | o2<9> = 0 | Pn<8:5> | Rd<4:0>
5960
5961 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5962 VIXL_ASSERT(xd.IsX());
5963 VIXL_ASSERT(pg.IsUnqualified());
5964 if (pg.HasLaneSize()) VIXL_ASSERT(AreSameFormat(pg, pn));
5965
5966 Emit(CNTP_r_p_p | SVESize(pn) | Rd(xd) | Pg<13, 10>(pg) | Pn(pn));
5967 }
5968
5969 // SVEPredicateLogicalOp.
and_(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)5970 void Assembler::and_(const PRegisterWithLaneSize& pd,
5971 const PRegisterZ& pg,
5972 const PRegisterWithLaneSize& pn,
5973 const PRegisterWithLaneSize& pm) {
5974 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5975 VIXL_ASSERT(AreSameFormat(pd, pn, pm));
5976 VIXL_ASSERT(pd.IsLaneSizeB());
5977 Emit(AND_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
5978 }
5979
ands(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)5980 void Assembler::ands(const PRegisterWithLaneSize& pd,
5981 const PRegisterZ& pg,
5982 const PRegisterWithLaneSize& pn,
5983 const PRegisterWithLaneSize& pm) {
5984 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5985 VIXL_ASSERT(AreSameFormat(pd, pn, pm));
5986 VIXL_ASSERT(pd.IsLaneSizeB());
5987 Emit(ANDS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
5988 }
5989
bic(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)5990 void Assembler::bic(const PRegisterWithLaneSize& pd,
5991 const PRegisterZ& pg,
5992 const PRegisterWithLaneSize& pn,
5993 const PRegisterWithLaneSize& pm) {
5994 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5995 VIXL_ASSERT(AreSameFormat(pd, pn, pm));
5996 VIXL_ASSERT(pd.IsLaneSizeB());
5997 Emit(BIC_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
5998 }
5999
bics(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6000 void Assembler::bics(const PRegisterWithLaneSize& pd,
6001 const PRegisterZ& pg,
6002 const PRegisterWithLaneSize& pn,
6003 const PRegisterWithLaneSize& pm) {
6004 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6005 VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6006 VIXL_ASSERT(pd.IsLaneSizeB());
6007 Emit(BICS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6008 }
6009
eor(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6010 void Assembler::eor(const PRegisterWithLaneSize& pd,
6011 const PRegisterZ& pg,
6012 const PRegisterWithLaneSize& pn,
6013 const PRegisterWithLaneSize& pm) {
6014 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6015 VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6016 VIXL_ASSERT(pd.IsLaneSizeB());
6017 Emit(EOR_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6018 }
6019
eors(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6020 void Assembler::eors(const PRegisterWithLaneSize& pd,
6021 const PRegisterZ& pg,
6022 const PRegisterWithLaneSize& pn,
6023 const PRegisterWithLaneSize& pm) {
6024 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6025 VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6026 VIXL_ASSERT(pd.IsLaneSizeB());
6027 Emit(EORS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6028 }
6029
nand(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6030 void Assembler::nand(const PRegisterWithLaneSize& pd,
6031 const PRegisterZ& pg,
6032 const PRegisterWithLaneSize& pn,
6033 const PRegisterWithLaneSize& pm) {
6034 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6035 VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6036 VIXL_ASSERT(pd.IsLaneSizeB());
6037 Emit(NAND_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6038 }
6039
nands(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6040 void Assembler::nands(const PRegisterWithLaneSize& pd,
6041 const PRegisterZ& pg,
6042 const PRegisterWithLaneSize& pn,
6043 const PRegisterWithLaneSize& pm) {
6044 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6045 VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6046 VIXL_ASSERT(pd.IsLaneSizeB());
6047 Emit(NANDS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6048 }
6049
nor(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6050 void Assembler::nor(const PRegisterWithLaneSize& pd,
6051 const PRegisterZ& pg,
6052 const PRegisterWithLaneSize& pn,
6053 const PRegisterWithLaneSize& pm) {
6054 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6055 VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6056 VIXL_ASSERT(pd.IsLaneSizeB());
6057 Emit(NOR_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6058 }
6059
nors(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6060 void Assembler::nors(const PRegisterWithLaneSize& pd,
6061 const PRegisterZ& pg,
6062 const PRegisterWithLaneSize& pn,
6063 const PRegisterWithLaneSize& pm) {
6064 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6065 VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6066 VIXL_ASSERT(pd.IsLaneSizeB());
6067 Emit(NORS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6068 }
6069
orn(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6070 void Assembler::orn(const PRegisterWithLaneSize& pd,
6071 const PRegisterZ& pg,
6072 const PRegisterWithLaneSize& pn,
6073 const PRegisterWithLaneSize& pm) {
6074 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6075 VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6076 VIXL_ASSERT(pd.IsLaneSizeB());
6077 Emit(ORN_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6078 }
6079
orns(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6080 void Assembler::orns(const PRegisterWithLaneSize& pd,
6081 const PRegisterZ& pg,
6082 const PRegisterWithLaneSize& pn,
6083 const PRegisterWithLaneSize& pm) {
6084 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6085 VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6086 VIXL_ASSERT(pd.IsLaneSizeB());
6087 Emit(ORNS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6088 }
6089
orr(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6090 void Assembler::orr(const PRegisterWithLaneSize& pd,
6091 const PRegisterZ& pg,
6092 const PRegisterWithLaneSize& pn,
6093 const PRegisterWithLaneSize& pm) {
6094 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6095 VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6096 VIXL_ASSERT(pd.IsLaneSizeB());
6097 Emit(ORR_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6098 }
6099
orrs(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6100 void Assembler::orrs(const PRegisterWithLaneSize& pd,
6101 const PRegisterZ& pg,
6102 const PRegisterWithLaneSize& pn,
6103 const PRegisterWithLaneSize& pm) {
6104 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6105 VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6106 VIXL_ASSERT(pd.IsLaneSizeB());
6107 Emit(ORRS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6108 }
6109
sel(const PRegisterWithLaneSize & pd,const PRegister & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6110 void Assembler::sel(const PRegisterWithLaneSize& pd,
6111 const PRegister& pg,
6112 const PRegisterWithLaneSize& pn,
6113 const PRegisterWithLaneSize& pm) {
6114 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6115 Emit(SEL_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6116 }
6117
6118 // SVEPredicateMisc.
6119
pfalse(const PRegisterWithLaneSize & pd)6120 void Assembler::pfalse(const PRegisterWithLaneSize& pd) {
6121 // PFALSE <Pd>.B
6122 // 0010 0101 0001 1000 1110 0100 0000 ....
6123 // op<23> = 0 | S<22> = 0 | Pd<3:0>
6124
6125 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6126 // Ignore the lane size, since it makes no difference to the operation.
6127
6128 Emit(PFALSE_p | Pd(pd));
6129 }
6130
pfirst(const PRegisterWithLaneSize & pd,const PRegister & pg,const PRegisterWithLaneSize & pn)6131 void Assembler::pfirst(const PRegisterWithLaneSize& pd,
6132 const PRegister& pg,
6133 const PRegisterWithLaneSize& pn) {
6134 // PFIRST <Pdn>.B, <Pg>, <Pdn>.B
6135 // 0010 0101 0101 1000 1100 000. ...0 ....
6136 // op<23> = 0 | S<22> = 1 | Pg<8:5> | Pdn<3:0>
6137
6138 USE(pn);
6139 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6140 VIXL_ASSERT(pd.Is(pn));
6141 VIXL_ASSERT(pd.IsLaneSizeB());
6142
6143 Emit(PFIRST_p_p_p | Pd(pd) | Pg<8, 5>(pg));
6144 }
6145
pnext(const PRegisterWithLaneSize & pd,const PRegister & pg,const PRegisterWithLaneSize & pn)6146 void Assembler::pnext(const PRegisterWithLaneSize& pd,
6147 const PRegister& pg,
6148 const PRegisterWithLaneSize& pn) {
6149 // PNEXT <Pdn>.<T>, <Pg>, <Pdn>.<T>
6150 // 0010 0101 ..01 1001 1100 010. ...0 ....
6151 // size<23:22> | Pg<8:5> | Pdn<3:0>
6152
6153 USE(pn);
6154 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6155 VIXL_ASSERT(pd.Is(pn));
6156
6157 Emit(PNEXT_p_p_p | SVESize(pd) | Pd(pd) | Pg<8, 5>(pg));
6158 }
6159
ptest(const PRegister & pg,const PRegisterWithLaneSize & pn)6160 void Assembler::ptest(const PRegister& pg, const PRegisterWithLaneSize& pn) {
6161 // PTEST <Pg>, <Pn>.B
6162 // 0010 0101 0101 0000 11.. ..0. ...0 0000
6163 // op<23> = 0 | S<22> = 1 | Pg<13:10> | Pn<8:5> | opc2<3:0> = 0000
6164
6165 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6166 VIXL_ASSERT(pn.IsLaneSizeB());
6167
6168 Emit(PTEST_p_p | Pg<13, 10>(pg) | Rx<8, 5>(pn));
6169 }
6170
ptrue(const PRegisterWithLaneSize & pd,int pattern)6171 void Assembler::ptrue(const PRegisterWithLaneSize& pd, int pattern) {
6172 // PTRUE <Pd>.<T>{, <pattern>}
6173 // 0010 0101 ..01 1000 1110 00.. ...0 ....
6174 // size<23:22> | S<16> = 0 | pattern<9:5> | Pd<3:0>
6175
6176 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6177
6178 Emit(PTRUE_p_s | SVESize(pd) | Pd(pd) | ImmSVEPredicateConstraint(pattern));
6179 }
6180
ptrues(const PRegisterWithLaneSize & pd,int pattern)6181 void Assembler::ptrues(const PRegisterWithLaneSize& pd, int pattern) {
6182 // PTRUES <Pd>.<T>{, <pattern>}
6183 // 0010 0101 ..01 1001 1110 00.. ...0 ....
6184 // size<23:22> | S<16> = 1 | pattern<9:5> | Pd<3:0>
6185
6186 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6187
6188 Emit(PTRUES_p_s | SVESize(pd) | Pd(pd) | ImmSVEPredicateConstraint(pattern));
6189 }
6190
rdffr(const PRegisterWithLaneSize & pd)6191 void Assembler::rdffr(const PRegisterWithLaneSize& pd) {
6192 // RDFFR <Pd>.B
6193 // 0010 0101 0001 1001 1111 0000 0000 ....
6194 // op<23> = 0 | S<22> = 0 | Pd<3:0>
6195
6196 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6197
6198 Emit(RDFFR_p_f | Pd(pd));
6199 }
6200
rdffr(const PRegisterWithLaneSize & pd,const PRegisterZ & pg)6201 void Assembler::rdffr(const PRegisterWithLaneSize& pd, const PRegisterZ& pg) {
6202 // RDFFR <Pd>.B, <Pg>/Z
6203 // 0010 0101 0001 1000 1111 000. ...0 ....
6204 // op<23> = 0 | S<22> = 0 | Pg<8:5> | Pd<3:0>
6205
6206 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6207
6208 Emit(RDFFR_p_p_f | Pd(pd) | Pg<8, 5>(pg));
6209 }
6210
rdffrs(const PRegisterWithLaneSize & pd,const PRegisterZ & pg)6211 void Assembler::rdffrs(const PRegisterWithLaneSize& pd, const PRegisterZ& pg) {
6212 // RDFFRS <Pd>.B, <Pg>/Z
6213 // 0010 0101 0101 1000 1111 000. ...0 ....
6214 // op<23> = 0 | S<22> = 1 | Pg<8:5> | Pd<3:0>
6215
6216 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6217
6218 Emit(RDFFRS_p_p_f | Pd(pd) | Pg<8, 5>(pg));
6219 }
6220
6221 // SVEPropagateBreak.
6222
brkpa(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6223 void Assembler::brkpa(const PRegisterWithLaneSize& pd,
6224 const PRegisterZ& pg,
6225 const PRegisterWithLaneSize& pn,
6226 const PRegisterWithLaneSize& pm) {
6227 // BRKPA <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B
6228 // 0010 0101 0000 .... 11.. ..0. ...0 ....
6229 // op<23> = 0 | S<22> = 0 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 0 |
6230 // Pd<3:0>
6231
6232 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6233
6234 Emit(BRKPA_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6235 }
6236
brkpas(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6237 void Assembler::brkpas(const PRegisterWithLaneSize& pd,
6238 const PRegisterZ& pg,
6239 const PRegisterWithLaneSize& pn,
6240 const PRegisterWithLaneSize& pm) {
6241 // BRKPAS <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B
6242 // 0010 0101 0100 .... 11.. ..0. ...0 ....
6243 // op<23> = 0 | S<22> = 1 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 0 |
6244 // Pd<3:0>
6245
6246 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6247
6248 Emit(BRKPAS_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6249 }
6250
brkpb(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6251 void Assembler::brkpb(const PRegisterWithLaneSize& pd,
6252 const PRegisterZ& pg,
6253 const PRegisterWithLaneSize& pn,
6254 const PRegisterWithLaneSize& pm) {
6255 // BRKPB <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B
6256 // 0010 0101 0000 .... 11.. ..0. ...1 ....
6257 // op<23> = 0 | S<22> = 0 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 1 |
6258 // Pd<3:0>
6259
6260 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6261
6262 Emit(BRKPB_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6263 }
6264
brkpbs(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn,const PRegisterWithLaneSize & pm)6265 void Assembler::brkpbs(const PRegisterWithLaneSize& pd,
6266 const PRegisterZ& pg,
6267 const PRegisterWithLaneSize& pn,
6268 const PRegisterWithLaneSize& pm) {
6269 // BRKPBS <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B
6270 // 0010 0101 0100 .... 11.. ..0. ...1 ....
6271 // op<23> = 0 | S<22> = 1 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 1 |
6272 // Pd<3:0>
6273
6274 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6275
6276 Emit(BRKPBS_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6277 }
6278
6279 // SVEStackFrameAdjustment.
6280
addpl(const Register & xd,const Register & xn,int imm6)6281 void Assembler::addpl(const Register& xd, const Register& xn, int imm6) {
6282 // ADDPL <Xd|SP>, <Xn|SP>, #<imm>
6283 // 0000 0100 011. .... 0101 0... .... ....
6284 // op<22> = 1 | Rn<20:16> | imm6<10:5> | Rd<4:0>
6285
6286 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6287 VIXL_ASSERT(xd.IsX());
6288 VIXL_ASSERT(xn.IsX());
6289
6290 Emit(ADDPL_r_ri | RdSP(xd) | RmSP(xn) | ImmField<10, 5>(imm6));
6291 }
6292
addvl(const Register & xd,const Register & xn,int imm6)6293 void Assembler::addvl(const Register& xd, const Register& xn, int imm6) {
6294 // ADDVL <Xd|SP>, <Xn|SP>, #<imm>
6295 // 0000 0100 001. .... 0101 0... .... ....
6296 // op<22> = 0 | Rn<20:16> | imm6<10:5> | Rd<4:0>
6297
6298 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6299 VIXL_ASSERT(xd.IsX());
6300 VIXL_ASSERT(xn.IsX());
6301
6302 Emit(ADDVL_r_ri | RdSP(xd) | RmSP(xn) | ImmField<10, 5>(imm6));
6303 }
6304
6305 // SVEStackFrameSize.
6306
rdvl(const Register & xd,int imm6)6307 void Assembler::rdvl(const Register& xd, int imm6) {
6308 // RDVL <Xd>, #<imm>
6309 // 0000 0100 1011 1111 0101 0... .... ....
6310 // op<22> = 0 | opc2<20:16> = 11111 | imm6<10:5> | Rd<4:0>
6311
6312 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6313 VIXL_ASSERT(xd.IsX());
6314
6315 Emit(RDVL_r_i | Rd(xd) | ImmField<10, 5>(imm6));
6316 }
6317
6318 // SVEVectorSelect.
6319
sel(const ZRegister & zd,const PRegister & pg,const ZRegister & zn,const ZRegister & zm)6320 void Assembler::sel(const ZRegister& zd,
6321 const PRegister& pg,
6322 const ZRegister& zn,
6323 const ZRegister& zm) {
6324 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6325 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
6326
6327 Emit(SEL_z_p_zz | SVESize(zd) | Rd(zd) | Pg<13, 10>(pg) | Rn(zn) | Rm(zm));
6328 }
6329
6330 // SVEWriteFFR.
6331
setffr()6332 void Assembler::setffr() {
6333 // SETFFR
6334 // 0010 0101 0010 1100 1001 0000 0000 0000
6335 // opc<23:22> = 00
6336
6337 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6338
6339 Emit(SETFFR_f);
6340 }
6341
wrffr(const PRegisterWithLaneSize & pn)6342 void Assembler::wrffr(const PRegisterWithLaneSize& pn) {
6343 // WRFFR <Pn>.B
6344 // 0010 0101 0010 1000 1001 000. ...0 0000
6345 // opc<23:22> = 00 | Pn<8:5>
6346
6347 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6348
6349 Emit(WRFFR_f_p | Rx<8, 5>(pn));
6350 }
6351
6352 // Aliases.
6353
bic(const ZRegister & zd,const ZRegister & zn,uint64_t imm)6354 void Assembler::bic(const ZRegister& zd, const ZRegister& zn, uint64_t imm) {
6355 and_(zd, zn, ~imm);
6356 }
6357
eon(const ZRegister & zd,const ZRegister & zn,uint64_t imm)6358 void Assembler::eon(const ZRegister& zd, const ZRegister& zn, uint64_t imm) {
6359 eor(zd, zn, ~imm);
6360 }
6361
orn(const ZRegister & zd,const ZRegister & zn,uint64_t imm)6362 void Assembler::orn(const ZRegister& zd, const ZRegister& zn, uint64_t imm) {
6363 orr(zd, zn, ~imm);
6364 }
6365
6366
fmov(const ZRegister & zd,const PRegisterM & pg,double imm)6367 void Assembler::fmov(const ZRegister& zd, const PRegisterM& pg, double imm) {
6368 if (IsPositiveZero(imm)) {
6369 cpy(zd, pg, 0);
6370 } else {
6371 fcpy(zd, pg, imm);
6372 }
6373 }
6374
fmov(const ZRegister & zd,double imm)6375 void Assembler::fmov(const ZRegister& zd, double imm) {
6376 if (IsPositiveZero(imm)) {
6377 dup(zd, imm);
6378 } else {
6379 fdup(zd, imm);
6380 }
6381 }
6382
mov(const PRegister & pd,const PRegister & pn)6383 void Assembler::mov(const PRegister& pd, const PRegister& pn) {
6384 // If the inputs carry a lane size, they must match.
6385 VIXL_ASSERT((!pd.HasLaneSize() && !pn.HasLaneSize()) ||
6386 AreSameLaneSize(pd, pn));
6387 orr(pd.VnB(), pn.Zeroing(), pn.VnB(), pn.VnB());
6388 }
6389
mov(const PRegisterWithLaneSize & pd,const PRegisterM & pg,const PRegisterWithLaneSize & pn)6390 void Assembler::mov(const PRegisterWithLaneSize& pd,
6391 const PRegisterM& pg,
6392 const PRegisterWithLaneSize& pn) {
6393 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6394 sel(pd, pg, pn, pd);
6395 }
6396
mov(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn)6397 void Assembler::mov(const PRegisterWithLaneSize& pd,
6398 const PRegisterZ& pg,
6399 const PRegisterWithLaneSize& pn) {
6400 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6401 and_(pd, pg, pn, pn);
6402 }
6403
mov(const ZRegister & zd,const PRegister & pg,int imm8,int shift)6404 void Assembler::mov(const ZRegister& zd,
6405 const PRegister& pg,
6406 int imm8,
6407 int shift) {
6408 VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing());
6409 cpy(zd, pg, imm8, shift);
6410 }
6411
mov(const ZRegister & zd,const Register & xn)6412 void Assembler::mov(const ZRegister& zd, const Register& xn) { dup(zd, xn); }
6413
mov(const ZRegister & zd,const VRegister & vn)6414 void Assembler::mov(const ZRegister& zd, const VRegister& vn) {
6415 VIXL_ASSERT(vn.IsScalar());
6416 VIXL_ASSERT(AreSameLaneSize(zd, vn));
6417 dup(zd, vn.Z().WithSameLaneSizeAs(vn), 0);
6418 }
6419
mov(const ZRegister & zd,const ZRegister & zn)6420 void Assembler::mov(const ZRegister& zd, const ZRegister& zn) {
6421 VIXL_ASSERT(AreSameLaneSize(zd, zn));
6422 orr(zd.VnD(), zn.VnD(), zn.VnD());
6423 }
6424
mov(const ZRegister & zd,const ZRegister & zn,unsigned index)6425 void Assembler::mov(const ZRegister& zd, const ZRegister& zn, unsigned index) {
6426 VIXL_ASSERT(AreSameLaneSize(zd, zn));
6427 dup(zd, zn, index);
6428 }
6429
mov(const ZRegister & zd,const PRegisterM & pg,const Register & rn)6430 void Assembler::mov(const ZRegister& zd,
6431 const PRegisterM& pg,
6432 const Register& rn) {
6433 cpy(zd, pg, rn);
6434 }
6435
mov(const ZRegister & zd,const PRegisterM & pg,const VRegister & vn)6436 void Assembler::mov(const ZRegister& zd,
6437 const PRegisterM& pg,
6438 const VRegister& vn) {
6439 VIXL_ASSERT(vn.IsScalar());
6440 VIXL_ASSERT(AreSameLaneSize(zd, vn));
6441 cpy(zd, pg, vn);
6442 }
6443
mov(const ZRegister & zd,const PRegisterM & pg,const ZRegister & zn)6444 void Assembler::mov(const ZRegister& zd,
6445 const PRegisterM& pg,
6446 const ZRegister& zn) {
6447 VIXL_ASSERT(AreSameLaneSize(zd, zn));
6448 sel(zd, pg, zn, zd);
6449 }
6450
mov(const ZRegister & zd,uint64_t imm)6451 void Assembler::mov(const ZRegister& zd, uint64_t imm) {
6452 // Mov is an alias of dupm for certain values of imm. Whilst this matters in
6453 // the disassembler, for the assembler, we don't distinguish between the
6454 // two mnemonics, and simply call dupm.
6455 dupm(zd, imm);
6456 }
6457
mov(const ZRegister & zd,int imm8,int shift)6458 void Assembler::mov(const ZRegister& zd, int imm8, int shift) {
6459 dup(zd, imm8, shift);
6460 }
6461
movs(const PRegister & pd,const PRegister & pn)6462 void Assembler::movs(const PRegister& pd, const PRegister& pn) {
6463 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6464 orrs(pd.VnB(), pn.Zeroing(), pn.VnB(), pn.VnB());
6465 }
6466
movs(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn)6467 void Assembler::movs(const PRegisterWithLaneSize& pd,
6468 const PRegisterZ& pg,
6469 const PRegisterWithLaneSize& pn) {
6470 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6471 ands(pd, pg, pn, pn);
6472 }
6473
not_(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn)6474 void Assembler::not_(const PRegisterWithLaneSize& pd,
6475 const PRegisterZ& pg,
6476 const PRegisterWithLaneSize& pn) {
6477 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6478 eor(pd, pg, pn, pg.VnB());
6479 }
6480
nots(const PRegisterWithLaneSize & pd,const PRegisterZ & pg,const PRegisterWithLaneSize & pn)6481 void Assembler::nots(const PRegisterWithLaneSize& pd,
6482 const PRegisterZ& pg,
6483 const PRegisterWithLaneSize& pn) {
6484 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6485 eors(pd, pg, pn, pg.VnB());
6486 }
6487
6488 } // namespace aarch64
6489 } // namespace vixl
6490