1 /*
2  * Copyright (C) 2023 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <inttypes.h>
18 #include <sys/mman.h>
19 
20 #include "berberis/base/bit_util.h"
21 #include "berberis/base/mmap.h"
22 #include "berberis/base/struct_check.h"
23 
24 #include "berberis/intrinsics/macro_assembler.h"
25 
26 namespace berberis::constants_pool {
27 
28 // All constants we refer in macroinstructions are collected in MacroAssemblerConstants.
29 // This allows us:
30 //   1. To save memory (they are not duplicated when reused).
31 //   2. Make it possible to access in text assembler without complex dance with hash-tables.
32 //   3. Allocate below-2GB-copy in x86_64 mode easily.
33 struct MacroAssemblerConstants {
34   alignas(16) const uint64_t kNanBoxFloat32[2] = {0xffff'ffff'0000'0000, 0xffff'ffff'0000'0000};
35   alignas(16) const uint64_t kNanBoxedNansFloat32[2] = {0xffff'ffff'7fc0'0000,
36                                                         0xffff'ffff'7fc0'0000};
37   alignas(16) const uint32_t kCanonicalNansFloat32[4] = {0x7fc'00000,
38                                                          0x7fc'00000,
39                                                          0x7fc'00000,
40                                                          0x7fc'00000};
41   alignas(16) const uint64_t kCanonicalNansFloat64[2] = {0x7ff8'0000'0000'0000,
42                                                          0x7ff8'0000'0000'0000};
43   alignas(16) const uint32_t kFloat32One[4] = {0x3f80'0000, 0x3f80'0000, 0x3f80'0000, 0x3f80'0000};
44   alignas(16) const uint64_t kFloat64One[2] = {0x3ff0'0000'0000'0000, 0x3ff0'0000'0000'0000};
45   alignas(16) const uint32_t kFloat32PInf[4] = {0x7f80'0000, 0x7f80'0000, 0x7f80'0000, 0x7f80'0000};
46   alignas(16) const uint32_t kFloat32NInf[4] = {0xff80'0000, 0xff80'0000, 0xff80'0000, 0xff80'0000};
47   alignas(16) const uint64_t kFloat64PInf[2] = {0x7ff0'0000'0000'0000, 0x7ff0'0000'0000'0000};
48   alignas(16) const uint64_t kFloat64NInf[2] = {0xfff0'0000'0000'0000, 0xfff0'0000'0000'0000};
49   alignas(16) const int8_t kMinInt8[16] = {
50       -128,
51       -128,
52       -128,
53       -128,
54       -128,
55       -128,
56       -128,
57       -128,
58       -128,
59       -128,
60       -128,
61       -128,
62       -128,
63       -128,
64       -128,
65       -128,
66   };
67   alignas(16) const int8_t kMaxInt8[16] =
68       {127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127};
69   alignas(16) const int16_t kMinInt16[8] =
70       {-0x8000, -0x8000, -0x8000, -0x8000, -0x8000, -0x8000, -0x8000, -0x8000};
71   alignas(16)
72       const int16_t kMaxInt16[8] = {0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff};
73   alignas(16) const int32_t kMinInt32[4] = {
74       static_cast<int32_t>(-0x8000'0000),
75       static_cast<int32_t>(-0x8000'0000),
76       static_cast<int32_t>(-0x8000'0000),
77       static_cast<int32_t>(-0x8000'0000),
78   };
79   alignas(16) const int32_t kMaxInt32[4] = {0x7fff'ffff, 0x7fff'ffff, 0x7fff'ffff, 0x7fff'ffff};
80   alignas(16) const int64_t kMinInt64[2] = {
81       static_cast<int64_t>(-0x8000'0000'0000'0000),
82       static_cast<int64_t>(-0x8000'0000'0000'0000),
83   };
84   alignas(16) const int64_t kMaxInt64[2] = {0x7fff'ffff'ffff'ffff, 0x7fff'ffff'ffff'ffff};
85   int64_t kBsrToClzInt64 = 127;
86   int64_t kWidthInBits64 = 64;
87   int32_t kBsrToClzInt32 = 63;
88   int32_t kWidthInBits32 = 32;
89   // 64 bit constants for use with arithmetic operations.
90   // Used because only 32 bit immediates are supported on x86-64.
91   int64_t k0x8000_0000_0000_00ff = 0x8000'0000'0000'00ff;
92   alignas(16) const int8_t kPMovmskwToPMovmskb[16] =
93       {0, 2, 4, 6, 8, 10, 12, 14, -63, -24, -19, -27, -28, -128, -128, -128};
94   alignas(16) const int8_t kPMovmskdToPMovmskb[16] =
95       {0, 4, 8, 12, -128, -128, -128, -128, -51, -17, -24, -31, -19, -27, -28, -128};
96   alignas(16) const int8_t kPMovmskqToPMovmskb[16] =
97       {0, 8, -128, -128, -128, -128, -128, -128, -57, -24, -31, -6, -7, -128, -128, -128};
98   alignas(16) const uint8_t kRiscVToX87Exceptions[32] = {
99       0x00, 0x20, 0x10, 0x30, 0x08, 0x28, 0x18, 0x38,
100       0x04, 0x24, 0x14, 0x34, 0x0c, 0x2c, 0x1c, 0x3c,
101       0x01, 0x21, 0x11, 0x31, 0x09, 0x29, 0x19, 0x39,
102       0x05, 0x25, 0x15, 0x35, 0x0d, 0x2d, 0x1d, 0x3d};
103   alignas(16) const uint8_t kX87ToRiscVExceptions[64] = {
104       0x00, 0x10, 0x00, 0x10, 0x08, 0x18, 0x08, 0x18,
105       0x04, 0x14, 0x04, 0x14, 0x0c, 0x1c, 0x0c, 0x1c,
106       0x02, 0x12, 0x02, 0x12, 0x0a, 0x1a, 0x0a, 0x1a,
107       0x06, 0x16, 0x06, 0x16, 0x0e, 0x1e, 0x0e, 0x1e,
108       0x01, 0x11, 0x01, 0x11, 0x09, 0x19, 0x09, 0x19,
109       0x05, 0x15, 0x05, 0x15, 0x0d, 0x1d, 0x0d, 0x1d,
110       0x03, 0x13, 0x03, 0x13, 0x0b, 0x1b, 0x0b, 0x1b,
111       0x07, 0x17, 0x07, 0x17, 0x0f, 0x1f, 0x0f, 0x1f};
112   // This table represents exactly what you see: ������ + �� unset bits and then ������ - �� set bits for ��
113   // in range from �� to ��.  The last ������ bits from line �� then it's mask for ���� equal to �� and if
114   // you shift start address down by ��  bytes then you get mask for �� * �� + �� bits.
115   // Using this approach we may load appropriate bitmask from memory with one load instruction and
116   // the table itself takes ������ bytes.
117   // Since valid values for ���� are from �� to ���� �� ������ (including ������), �� may be from �� to ����,
118   // that's why we need 16 zero bytes in that table.
119   // On AMD CPUs with misalignsse feature we may access data from that table without using movups,
120   // and on all CPUs we may use 2KiB table instead.
121   // But for now we are using movups and small table as probably-adequate compromise.
122   alignas(16) const uint64_t kBitMaskTable[8][4] = {
123       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'ffff, 0xffff'ffff'ffff'ffff},
124       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'fffe, 0xffff'ffff'ffff'ffff},
125       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'fffc, 0xffff'ffff'ffff'ffff},
126       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'fff8, 0xffff'ffff'ffff'ffff},
127       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'fff0, 0xffff'ffff'ffff'ffff},
128       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'ffe0, 0xffff'ffff'ffff'ffff},
129       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'ffc0, 0xffff'ffff'ffff'ffff},
130       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'ff80, 0xffff'ffff'ffff'ffff},
131   };
132   // RISC-V manual strongly implies that vid.v may be implemented similarly to viota.m
133   // This may be true for hardware implementation, but in software vid.v may be implemented with a
134   // simple precomputed table which implementation of viota.m is much more tricky and slow.
135   // Here are precomputed values for Vid.v
136   alignas(16) __m128i kVid64Bit[8] = {
137       {0, 1},
138       {2, 3},
139       {4, 5},
140       {6, 7},
141       {8, 9},
142       {10, 11},
143       {12, 13},
144       {14, 15},
145   };
146   alignas(16) __v4si kVid32Bit[8] = {
147       {0, 1, 2, 3},
148       {4, 5, 6, 7},
149       {8, 9, 10, 11},
150       {12, 13, 14, 15},
151       {16, 17, 18, 19},
152       {20, 21, 22, 23},
153       {24, 25, 26, 27},
154       {28, 29, 30, 31},
155   };
156   alignas(16) __v8hi kVid16Bit[8] = {
157       {0, 1, 2, 3, 4, 5, 6, 7},
158       {8, 9, 10, 11, 12, 13, 14, 15},
159       {16, 17, 18, 19, 20, 21, 22, 23},
160       {24, 25, 26, 27, 28, 29, 30, 31},
161       {32, 33, 34, 35, 36, 37, 38, 39},
162       {40, 41, 42, 43, 44, 45, 46, 47},
163       {48, 49, 50, 51, 52, 53, 54, 55},
164       {56, 57, 58, 59, 60, 61, 62, 63},
165   };
166   alignas(16) __v16qi kVid8Bit[8] = {
167       {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
168       {16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31},
169       {32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47},
170       {48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63},
171       {64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79},
172       {80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95},
173       {96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111},
174       {112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127},
175   };
176   alignas(16) const uint64_t kBitMaskTo32bitMask[4] = {
177       0x0000'0000'0000'0000,
178       0x0000'0000'ffff'ffff,
179       0xffff'ffff'0000'0000,
180       0xffff'ffff'ffff'ffff,
181   };
182   alignas(16) const uint64_t kBitMaskTo16bitMask[16] = {
183       0x0000'0000'0000'0000,
184       0x0000'0000'0000'ffff,
185       0x0000'0000'ffff'0000,
186       0x0000'0000'ffff'ffff,
187       0x0000'ffff'0000'0000,
188       0x0000'ffff'0000'ffff,
189       0x0000'ffff'ffff'0000,
190       0x0000'ffff'ffff'ffff,
191       0xffff'0000'0000'0000,
192       0xffff'0000'0000'ffff,
193       0xffff'0000'ffff'0000,
194       0xffff'0000'ffff'ffff,
195       0xffff'ffff'0000'0000,
196       0xffff'ffff'0000'ffff,
197       0xffff'ffff'ffff'0000,
198       0xffff'ffff'ffff'ffff,
199   };
200   alignas(16) const uint64_t kBitMaskTo8bitMask[256] = {
201       0x0000'0000'0000'0000, 0x0000'0000'0000'00ff, 0x0000'0000'0000'ff00, 0x0000'0000'0000'ffff,
202       0x0000'0000'00ff'0000, 0x0000'0000'00ff'00ff, 0x0000'0000'00ff'ff00, 0x0000'0000'00ff'ffff,
203       0x0000'0000'ff00'0000, 0x0000'0000'ff00'00ff, 0x0000'0000'ff00'ff00, 0x0000'0000'ff00'ffff,
204       0x0000'0000'ffff'0000, 0x0000'0000'ffff'00ff, 0x0000'0000'ffff'ff00, 0x0000'0000'ffff'ffff,
205       0x0000'00ff'0000'0000, 0x0000'00ff'0000'00ff, 0x0000'00ff'0000'ff00, 0x0000'00ff'0000'ffff,
206       0x0000'00ff'00ff'0000, 0x0000'00ff'00ff'00ff, 0x0000'00ff'00ff'ff00, 0x0000'00ff'00ff'ffff,
207       0x0000'00ff'ff00'0000, 0x0000'00ff'ff00'00ff, 0x0000'00ff'ff00'ff00, 0x0000'00ff'ff00'ffff,
208       0x0000'00ff'ffff'0000, 0x0000'00ff'ffff'00ff, 0x0000'00ff'ffff'ff00, 0x0000'00ff'ffff'ffff,
209       0x0000'ff00'0000'0000, 0x0000'ff00'0000'00ff, 0x0000'ff00'0000'ff00, 0x0000'ff00'0000'ffff,
210       0x0000'ff00'00ff'0000, 0x0000'ff00'00ff'00ff, 0x0000'ff00'00ff'ff00, 0x0000'ff00'00ff'ffff,
211       0x0000'ff00'ff00'0000, 0x0000'ff00'ff00'00ff, 0x0000'ff00'ff00'ff00, 0x0000'ff00'ff00'ffff,
212       0x0000'ff00'ffff'0000, 0x0000'ff00'ffff'00ff, 0x0000'ff00'ffff'ff00, 0x0000'ff00'ffff'ffff,
213       0x0000'ffff'0000'0000, 0x0000'ffff'0000'00ff, 0x0000'ffff'0000'ff00, 0x0000'ffff'0000'ffff,
214       0x0000'ffff'00ff'0000, 0x0000'ffff'00ff'00ff, 0x0000'ffff'00ff'ff00, 0x0000'ffff'00ff'ffff,
215       0x0000'ffff'ff00'0000, 0x0000'ffff'ff00'00ff, 0x0000'ffff'ff00'ff00, 0x0000'ffff'ff00'ffff,
216       0x0000'ffff'ffff'0000, 0x0000'ffff'ffff'00ff, 0x0000'ffff'ffff'ff00, 0x0000'ffff'ffff'ffff,
217       0x00ff'0000'0000'0000, 0x00ff'0000'0000'00ff, 0x00ff'0000'0000'ff00, 0x00ff'0000'0000'ffff,
218       0x00ff'0000'00ff'0000, 0x00ff'0000'00ff'00ff, 0x00ff'0000'00ff'ff00, 0x00ff'0000'00ff'ffff,
219       0x00ff'0000'ff00'0000, 0x00ff'0000'ff00'00ff, 0x00ff'0000'ff00'ff00, 0x00ff'0000'ff00'ffff,
220       0x00ff'0000'ffff'0000, 0x00ff'0000'ffff'00ff, 0x00ff'0000'ffff'ff00, 0x00ff'0000'ffff'ffff,
221       0x00ff'00ff'0000'0000, 0x00ff'00ff'0000'00ff, 0x00ff'00ff'0000'ff00, 0x00ff'00ff'0000'ffff,
222       0x00ff'00ff'00ff'0000, 0x00ff'00ff'00ff'00ff, 0x00ff'00ff'00ff'ff00, 0x00ff'00ff'00ff'ffff,
223       0x00ff'00ff'ff00'0000, 0x00ff'00ff'ff00'00ff, 0x00ff'00ff'ff00'ff00, 0x00ff'00ff'ff00'ffff,
224       0x00ff'00ff'ffff'0000, 0x00ff'00ff'ffff'00ff, 0x00ff'00ff'ffff'ff00, 0x00ff'00ff'ffff'ffff,
225       0x00ff'ff00'0000'0000, 0x00ff'ff00'0000'00ff, 0x00ff'ff00'0000'ff00, 0x00ff'ff00'0000'ffff,
226       0x00ff'ff00'00ff'0000, 0x00ff'ff00'00ff'00ff, 0x00ff'ff00'00ff'ff00, 0x00ff'ff00'00ff'ffff,
227       0x00ff'ff00'ff00'0000, 0x00ff'ff00'ff00'00ff, 0x00ff'ff00'ff00'ff00, 0x00ff'ff00'ff00'ffff,
228       0x00ff'ff00'ffff'0000, 0x00ff'ff00'ffff'00ff, 0x00ff'ff00'ffff'ff00, 0x00ff'ff00'ffff'ffff,
229       0x00ff'ffff'0000'0000, 0x00ff'ffff'0000'00ff, 0x00ff'ffff'0000'ff00, 0x00ff'ffff'0000'ffff,
230       0x00ff'ffff'00ff'0000, 0x00ff'ffff'00ff'00ff, 0x00ff'ffff'00ff'ff00, 0x00ff'ffff'00ff'ffff,
231       0x00ff'ffff'ff00'0000, 0x00ff'ffff'ff00'00ff, 0x00ff'ffff'ff00'ff00, 0x00ff'ffff'ff00'ffff,
232       0x00ff'ffff'ffff'0000, 0x00ff'ffff'ffff'00ff, 0x00ff'ffff'ffff'ff00, 0x00ff'ffff'ffff'ffff,
233       0xff00'0000'0000'0000, 0xff00'0000'0000'00ff, 0xff00'0000'0000'ff00, 0xff00'0000'0000'ffff,
234       0xff00'0000'00ff'0000, 0xff00'0000'00ff'00ff, 0xff00'0000'00ff'ff00, 0xff00'0000'00ff'ffff,
235       0xff00'0000'ff00'0000, 0xff00'0000'ff00'00ff, 0xff00'0000'ff00'ff00, 0xff00'0000'ff00'ffff,
236       0xff00'0000'ffff'0000, 0xff00'0000'ffff'00ff, 0xff00'0000'ffff'ff00, 0xff00'0000'ffff'ffff,
237       0xff00'00ff'0000'0000, 0xff00'00ff'0000'00ff, 0xff00'00ff'0000'ff00, 0xff00'00ff'0000'ffff,
238       0xff00'00ff'00ff'0000, 0xff00'00ff'00ff'00ff, 0xff00'00ff'00ff'ff00, 0xff00'00ff'00ff'ffff,
239       0xff00'00ff'ff00'0000, 0xff00'00ff'ff00'00ff, 0xff00'00ff'ff00'ff00, 0xff00'00ff'ff00'ffff,
240       0xff00'00ff'ffff'0000, 0xff00'00ff'ffff'00ff, 0xff00'00ff'ffff'ff00, 0xff00'00ff'ffff'ffff,
241       0xff00'ff00'0000'0000, 0xff00'ff00'0000'00ff, 0xff00'ff00'0000'ff00, 0xff00'ff00'0000'ffff,
242       0xff00'ff00'00ff'0000, 0xff00'ff00'00ff'00ff, 0xff00'ff00'00ff'ff00, 0xff00'ff00'00ff'ffff,
243       0xff00'ff00'ff00'0000, 0xff00'ff00'ff00'00ff, 0xff00'ff00'ff00'ff00, 0xff00'ff00'ff00'ffff,
244       0xff00'ff00'ffff'0000, 0xff00'ff00'ffff'00ff, 0xff00'ff00'ffff'ff00, 0xff00'ff00'ffff'ffff,
245       0xff00'ffff'0000'0000, 0xff00'ffff'0000'00ff, 0xff00'ffff'0000'ff00, 0xff00'ffff'0000'ffff,
246       0xff00'ffff'00ff'0000, 0xff00'ffff'00ff'00ff, 0xff00'ffff'00ff'ff00, 0xff00'ffff'00ff'ffff,
247       0xff00'ffff'ff00'0000, 0xff00'ffff'ff00'00ff, 0xff00'ffff'ff00'ff00, 0xff00'ffff'ff00'ffff,
248       0xff00'ffff'ffff'0000, 0xff00'ffff'ffff'00ff, 0xff00'ffff'ffff'ff00, 0xff00'ffff'ffff'ffff,
249       0xffff'0000'0000'0000, 0xffff'0000'0000'00ff, 0xffff'0000'0000'ff00, 0xffff'0000'0000'ffff,
250       0xffff'0000'00ff'0000, 0xffff'0000'00ff'00ff, 0xffff'0000'00ff'ff00, 0xffff'0000'00ff'ffff,
251       0xffff'0000'ff00'0000, 0xffff'0000'ff00'00ff, 0xffff'0000'ff00'ff00, 0xffff'0000'ff00'ffff,
252       0xffff'0000'ffff'0000, 0xffff'0000'ffff'00ff, 0xffff'0000'ffff'ff00, 0xffff'0000'ffff'ffff,
253       0xffff'00ff'0000'0000, 0xffff'00ff'0000'00ff, 0xffff'00ff'0000'ff00, 0xffff'00ff'0000'ffff,
254       0xffff'00ff'00ff'0000, 0xffff'00ff'00ff'00ff, 0xffff'00ff'00ff'ff00, 0xffff'00ff'00ff'ffff,
255       0xffff'00ff'ff00'0000, 0xffff'00ff'ff00'00ff, 0xffff'00ff'ff00'ff00, 0xffff'00ff'ff00'ffff,
256       0xffff'00ff'ffff'0000, 0xffff'00ff'ffff'00ff, 0xffff'00ff'ffff'ff00, 0xffff'00ff'ffff'ffff,
257       0xffff'ff00'0000'0000, 0xffff'ff00'0000'00ff, 0xffff'ff00'0000'ff00, 0xffff'ff00'0000'ffff,
258       0xffff'ff00'00ff'0000, 0xffff'ff00'00ff'00ff, 0xffff'ff00'00ff'ff00, 0xffff'ff00'00ff'ffff,
259       0xffff'ff00'ff00'0000, 0xffff'ff00'ff00'00ff, 0xffff'ff00'ff00'ff00, 0xffff'ff00'ff00'ffff,
260       0xffff'ff00'ffff'0000, 0xffff'ff00'ffff'00ff, 0xffff'ff00'ffff'ff00, 0xffff'ff00'ffff'ffff,
261       0xffff'ffff'0000'0000, 0xffff'ffff'0000'00ff, 0xffff'ffff'0000'ff00, 0xffff'ffff'0000'ffff,
262       0xffff'ffff'00ff'0000, 0xffff'ffff'00ff'00ff, 0xffff'ffff'00ff'ff00, 0xffff'ffff'00ff'ffff,
263       0xffff'ffff'ff00'0000, 0xffff'ffff'ff00'00ff, 0xffff'ffff'ff00'ff00, 0xffff'ffff'ff00'ffff,
264       0xffff'ffff'ffff'0000, 0xffff'ffff'ffff'00ff, 0xffff'ffff'ffff'ff00, 0xffff'ffff'ffff'ffff,
265   };
266 };
267 
268 // Make sure Layout is the same in 32-bit mode and 64-bit mode.
269 CHECK_STRUCT_LAYOUT(MacroAssemblerConstants, 27520, 128);
270 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kNanBoxFloat32, 0, 128);
271 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kNanBoxedNansFloat32, 128, 128);
272 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kCanonicalNansFloat32, 256, 128);
273 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kCanonicalNansFloat64, 384, 128);
274 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kFloat32One, 512, 128);
275 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kFloat64One, 640, 128);
276 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kFloat32PInf, 768, 128);
277 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kFloat32NInf, 896, 128);
278 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kFloat64PInf, 1024, 128);
279 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kFloat64NInf, 1152, 128);
280 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMinInt8, 1280, 128);
281 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMaxInt8, 1408, 128);
282 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMinInt16, 1536, 128);
283 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMaxInt16, 1664, 128);
284 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMinInt32, 1792, 128);
285 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMaxInt32, 1920, 128);
286 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMinInt64, 2048, 128);
287 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMaxInt64, 2176, 128);
288 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kBsrToClzInt64, 2304, 64);
289 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kWidthInBits64, 2368, 64);
290 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kBsrToClzInt32, 2432, 32);
291 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kWidthInBits32, 2464, 32);
292 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, k0x8000_0000_0000_00ff, 2496, 64);
293 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kRiscVToX87Exceptions, 2944, 256);
294 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kX87ToRiscVExceptions, 3200, 512);
295 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kBitMaskTable, 3712, 2048);
296 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kVid64Bit, 5760, 1024);
297 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kVid32Bit, 6784, 1024);
298 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kVid16Bit, 7808, 1024);
299 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kVid8Bit, 8832, 1024);
300 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kBitMaskTo32bitMask, 9856, 256);
301 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kBitMaskTo16bitMask, 10112, 1024);
302 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kBitMaskTo8bitMask, 11136, 16384);
303 
304 // Note: because we have aligned fields and thus padding in that data structure
305 // value-initialization is both slower and larger than copy-initialization for
306 // that structure.
307 //
308 // Also assembler intrinsics for interpreter use kMacroAssemblerConstants
309 // directly (they couldn't use consts below because these addresses are only
310 // known during runtime)
311 extern const MacroAssemblerConstants kBerberisMacroAssemblerConstants
312     __attribute__((visibility("hidden")));
313 const MacroAssemblerConstants kBerberisMacroAssemblerConstants;
314 
315 namespace {
316 
GetConstants()317 int32_t GetConstants() {
318   static const MacroAssemblerConstants* Constants =
319       new (mmap(nullptr,
320                 AlignUpPageSize(sizeof(MacroAssemblerConstants)),
321                 PROT_READ | PROT_WRITE,
322                 MAP_PRIVATE | MAP_ANONYMOUS | MAP_32BIT,
323                 -1,
324                 0)) MacroAssemblerConstants(kBerberisMacroAssemblerConstants);
325   // Note that we are returning only 32-bit address here, but it's guaranteed to
326   // be enough since struct is in low memory because of MAP_32BIT flag.
327   return bit_cast<intptr_t>(Constants);
328 }
329 
330 }  // namespace
331 
332 extern const int32_t kBerberisMacroAssemblerConstantsRelocated;
333 const int32_t kBerberisMacroAssemblerConstantsRelocated = GetConstants();
334 template <>
335 extern const int32_t kVectorConst<int8_t{-128}> =
336     GetConstants() + offsetof(MacroAssemblerConstants, kMinInt8);
337 template <>
338 extern const int32_t kVectorConst<int8_t{127}> =
339     GetConstants() + offsetof(MacroAssemblerConstants, kMaxInt8);
340 template <>
341 extern const int32_t kVectorConst<int16_t{-0x8000}> =
342     GetConstants() + offsetof(MacroAssemblerConstants, kMinInt16);
343 template <>
344 extern const int32_t kVectorConst<int16_t{0x7fff}> =
345     GetConstants() + offsetof(MacroAssemblerConstants, kMaxInt16);
346 template <>
347 extern const int32_t kVectorConst<int32_t{static_cast<int32_t>(-0x8000'0000)}> =
348     GetConstants() + offsetof(MacroAssemblerConstants, kMinInt32);
349 template <>
350 extern const int32_t kVectorConst<int32_t{0x3f80'0000}> =
351     GetConstants() + offsetof(MacroAssemblerConstants, kFloat32One);
352 template <>
353 extern const int32_t kVectorConst<int32_t{0x7f80'0000}> =
354     GetConstants() + offsetof(MacroAssemblerConstants, kFloat32PInf);
355 template <>
356 extern const int32_t kVectorConst<int32_t{0x7fff'ffff}> =
357     GetConstants() + offsetof(MacroAssemblerConstants, kMaxInt32);
358 template <>
359 extern const int32_t kVectorConst<int32_t{-0x0080'0000}> =
360     GetConstants() + offsetof(MacroAssemblerConstants, kFloat32NInf);
361 template <>
362 extern const int32_t kVectorConst<int64_t{static_cast<int64_t>(-0x8000'0000'0000'0000)}> =
363     GetConstants() + offsetof(MacroAssemblerConstants, kMinInt64);
364 template <>
365 extern const int32_t kVectorConst<int64_t{0x3ff0'0000'0000'0000}> =
366     GetConstants() + offsetof(MacroAssemblerConstants, kFloat64One);
367 template <>
368 extern const int32_t kVectorConst<int64_t{0x7ff0'0000'0000'0000}> =
369     GetConstants() + offsetof(MacroAssemblerConstants, kFloat64PInf);
370 template <>
371 extern const int32_t kVectorConst<int64_t{0x7fff'ffff'ffff'ffff}> =
372     GetConstants() + offsetof(MacroAssemblerConstants, kMaxInt64);
373 template <>
374 extern const int32_t kVectorConst<int64_t{-0x0010'0000'0000'0000}> =
375     GetConstants() + offsetof(MacroAssemblerConstants, kFloat64NInf);
376 template <>
377 const int32_t kVectorConst<uint64_t{0x0000'0000'0000'0000}> =
378     GetConstants() + offsetof(MacroAssemblerConstants, kBitMaskTable);
379 template <>
380 const int32_t kVectorConst<uint64_t{0xffff'ffff'ffff'ffff}> =
381     GetConstants() + offsetof(MacroAssemblerConstants, kBitMaskTable) + 16;
382 template <>
383 const int32_t kVectorConst<uint64_t{0xffff'ffff'0000'0000}> =
384     GetConstants() + offsetof(MacroAssemblerConstants, kNanBoxFloat32);
385 template <>
386 const int32_t kVectorConst<uint64_t{0xffff'ffff'7fc0'0000}> =
387     GetConstants() + offsetof(MacroAssemblerConstants, kNanBoxedNansFloat32);
388 template <>
389 const int32_t kVectorConst<uint64_t{0x7fc0'0000'7fc0'0000}> =
390     GetConstants() + offsetof(MacroAssemblerConstants, kCanonicalNansFloat32);
391 template <>
392 const int32_t kVectorConst<uint64_t{0x7ff8'0000'0000'0000}> =
393     GetConstants() + offsetof(MacroAssemblerConstants, kCanonicalNansFloat64);
394 template <>
395 const int32_t kConst<uint64_t{127}> =
396     GetConstants() + offsetof(MacroAssemblerConstants, kBsrToClzInt64);
397 template <>
398 const int32_t kConst<uint64_t{64}> =
399     GetConstants() + offsetof(MacroAssemblerConstants, kWidthInBits64);
400 template <>
401 const int32_t kConst<uint32_t{63}> =
402     GetConstants() + offsetof(MacroAssemblerConstants, kBsrToClzInt32);
403 template <>
404 const int32_t kConst<uint32_t{32}> =
405     GetConstants() + offsetof(MacroAssemblerConstants, kWidthInBits32);
406 template <>
407 const int32_t kConst<uint64_t{0x8000'0000'0000'00ff}> =
408     GetConstants() + offsetof(MacroAssemblerConstants, k0x8000_0000_0000_00ff);
409 const int32_t kRiscVToX87Exceptions =
410     GetConstants() + offsetof(MacroAssemblerConstants, kRiscVToX87Exceptions);
411 const int32_t kX87ToRiscVExceptions =
412     GetConstants() + offsetof(MacroAssemblerConstants, kX87ToRiscVExceptions);
413 const int32_t kBitMaskTable = GetConstants() + offsetof(MacroAssemblerConstants, kBitMaskTable);
414 const int32_t kVid64Bit = GetConstants() + offsetof(MacroAssemblerConstants, kVid64Bit);
415 const int32_t kVid32Bit = GetConstants() + offsetof(MacroAssemblerConstants, kVid32Bit);
416 const int32_t kVid16Bit = GetConstants() + offsetof(MacroAssemblerConstants, kVid16Bit);
417 const int32_t kVid8Bit = GetConstants() + offsetof(MacroAssemblerConstants, kVid8Bit);
418 const int32_t kBitMaskTo32bitMask =
419     GetConstants() + offsetof(MacroAssemblerConstants, kBitMaskTo32bitMask);
420 const int32_t kBitMaskTo16bitMask =
421     GetConstants() + offsetof(MacroAssemblerConstants, kBitMaskTo16bitMask);
422 const int32_t kBitMaskTo8bitMask =
423     GetConstants() + offsetof(MacroAssemblerConstants, kBitMaskTo8bitMask);
424 const int32_t kPMovmskwToPMovmskb =
425     GetConstants() + offsetof(MacroAssemblerConstants, kPMovmskwToPMovmskb);
426 const int32_t kPMovmskdToPMovmskb =
427     GetConstants() + offsetof(MacroAssemblerConstants, kPMovmskdToPMovmskb);
428 const int32_t kPMovmskqToPMovmskb =
429     GetConstants() + offsetof(MacroAssemblerConstants, kPMovmskqToPMovmskb);
430 
431 }  // namespace berberis::constants_pool
432