1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3 * rseq-ppc.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
7 */
8
9 #define RSEQ_SIG 0x53053053
10
11 #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc")
12 #define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc")
13 #define rseq_smp_rmb() rseq_smp_lwsync()
14 #define rseq_smp_wmb() rseq_smp_lwsync()
15
16 #define rseq_smp_load_acquire(p) \
17 __extension__ ({ \
18 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
19 rseq_smp_lwsync(); \
20 ____p1; \
21 })
22
23 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync()
24
25 #define rseq_smp_store_release(p, v) \
26 do { \
27 rseq_smp_lwsync(); \
28 RSEQ_WRITE_ONCE(*p, v); \
29 } while (0)
30
31 #ifdef RSEQ_SKIP_FASTPATH
32 #include "rseq-skip.h"
33 #else /* !RSEQ_SKIP_FASTPATH */
34
35 /*
36 * The __rseq_table section can be used by debuggers to better handle
37 * single-stepping through the restartable critical sections.
38 */
39
40 #ifdef __PPC64__
41
42 #define STORE_WORD "std "
43 #define LOAD_WORD "ld "
44 #define LOADX_WORD "ldx "
45 #define CMP_WORD "cmpd "
46
47 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
48 start_ip, post_commit_offset, abort_ip) \
49 ".pushsection __rseq_table, \"aw\"\n\t" \
50 ".balign 32\n\t" \
51 __rseq_str(label) ":\n\t" \
52 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
53 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
54 ".popsection\n\t"
55
56 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
57 RSEQ_INJECT_ASM(1) \
58 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \
59 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \
60 "rldicr %%r17, %%r17, 32, 31\n\t" \
61 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \
62 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
63 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
64 __rseq_str(label) ":\n\t"
65
66 #else /* #ifdef __PPC64__ */
67
68 #define STORE_WORD "stw "
69 #define LOAD_WORD "lwz "
70 #define LOADX_WORD "lwzx "
71 #define CMP_WORD "cmpw "
72
73 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
74 start_ip, post_commit_offset, abort_ip) \
75 ".pushsection __rseq_table, \"aw\"\n\t" \
76 ".balign 32\n\t" \
77 __rseq_str(label) ":\n\t" \
78 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
79 /* 32-bit only supported on BE */ \
80 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
81 ".popsection\n\t"
82
83 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
84 RSEQ_INJECT_ASM(1) \
85 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \
86 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
87 "stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
88 __rseq_str(label) ":\n\t"
89
90 #endif /* #ifdef __PPC64__ */
91
92 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
93 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
94 (post_commit_ip - start_ip), abort_ip)
95
96 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
97 RSEQ_INJECT_ASM(2) \
98 "lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
99 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \
100 "bne- cr7, " __rseq_str(label) "\n\t"
101
102 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
103 ".pushsection __rseq_failure, \"ax\"\n\t" \
104 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
105 __rseq_str(label) ":\n\t" \
106 "b %l[" __rseq_str(abort_label) "]\n\t" \
107 ".popsection\n\t"
108
109 /*
110 * RSEQ_ASM_OPs: asm operations for rseq
111 * RSEQ_ASM_OP_R_*: has hard-code registers in it
112 * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
113 */
114 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
115 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
116 CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \
117 "bne- cr7, " __rseq_str(label) "\n\t"
118
119 #define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \
120 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
121 CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \
122 "beq- cr7, " __rseq_str(label) "\n\t"
123
124 #define RSEQ_ASM_OP_STORE(value, var) \
125 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
126
127 /* Load @var to r17 */
128 #define RSEQ_ASM_OP_R_LOAD(var) \
129 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
130
131 /* Store r17 to @var */
132 #define RSEQ_ASM_OP_R_STORE(var) \
133 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
134
135 /* Add @count to r17 */
136 #define RSEQ_ASM_OP_R_ADD(count) \
137 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
138
139 /* Load (r17 + voffp) to r17 */
140 #define RSEQ_ASM_OP_R_LOADX(voffp) \
141 LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
142
143 /* TODO: implement a faster memcpy. */
144 #define RSEQ_ASM_OP_R_MEMCPY() \
145 "cmpdi %%r19, 0\n\t" \
146 "beq 333f\n\t" \
147 "addi %%r20, %%r20, -1\n\t" \
148 "addi %%r21, %%r21, -1\n\t" \
149 "222:\n\t" \
150 "lbzu %%r18, 1(%%r20)\n\t" \
151 "stbu %%r18, 1(%%r21)\n\t" \
152 "addi %%r19, %%r19, -1\n\t" \
153 "cmpdi %%r19, 0\n\t" \
154 "bne 222b\n\t" \
155 "333:\n\t" \
156
157 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
158 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
159 __rseq_str(post_commit_label) ":\n\t"
160
161 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
162 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
163 __rseq_str(post_commit_label) ":\n\t"
164
165 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)166 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
167 {
168 RSEQ_INJECT_C(9)
169
170 __asm__ __volatile__ goto (
171 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
172 /* Start rseq by storing table entry pointer into rseq_cs. */
173 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
174 /* cmp cpuid */
175 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
176 RSEQ_INJECT_ASM(3)
177 /* cmp @v equal to @expect */
178 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
179 RSEQ_INJECT_ASM(4)
180 #ifdef RSEQ_COMPARE_TWICE
181 /* cmp cpuid */
182 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
183 /* cmp @v equal to @expect */
184 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
185 #endif
186 /* final store */
187 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
188 RSEQ_INJECT_ASM(5)
189 RSEQ_ASM_DEFINE_ABORT(4, abort)
190 : /* gcc asm goto does not allow outputs */
191 : [cpu_id] "r" (cpu),
192 [current_cpu_id] "m" (__rseq_abi.cpu_id),
193 [rseq_cs] "m" (__rseq_abi.rseq_cs),
194 [v] "m" (*v),
195 [expect] "r" (expect),
196 [newv] "r" (newv)
197 RSEQ_INJECT_INPUT
198 : "memory", "cc", "r17"
199 RSEQ_INJECT_CLOBBER
200 : abort, cmpfail
201 #ifdef RSEQ_COMPARE_TWICE
202 , error1, error2
203 #endif
204 );
205 return 0;
206 abort:
207 RSEQ_INJECT_FAILED
208 return -1;
209 cmpfail:
210 return 1;
211 #ifdef RSEQ_COMPARE_TWICE
212 error1:
213 rseq_bug("cpu_id comparison failed");
214 error2:
215 rseq_bug("expected value comparison failed");
216 #endif
217 }
218
219 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,off_t voffp,intptr_t * load,int cpu)220 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
221 off_t voffp, intptr_t *load, int cpu)
222 {
223 RSEQ_INJECT_C(9)
224
225 __asm__ __volatile__ goto (
226 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
227 /* Start rseq by storing table entry pointer into rseq_cs. */
228 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
229 /* cmp cpuid */
230 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
231 RSEQ_INJECT_ASM(3)
232 /* cmp @v not equal to @expectnot */
233 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
234 RSEQ_INJECT_ASM(4)
235 #ifdef RSEQ_COMPARE_TWICE
236 /* cmp cpuid */
237 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
238 /* cmp @v not equal to @expectnot */
239 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
240 #endif
241 /* load the value of @v */
242 RSEQ_ASM_OP_R_LOAD(v)
243 /* store it in @load */
244 RSEQ_ASM_OP_R_STORE(load)
245 /* dereference voffp(v) */
246 RSEQ_ASM_OP_R_LOADX(voffp)
247 /* final store the value at voffp(v) */
248 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
249 RSEQ_INJECT_ASM(5)
250 RSEQ_ASM_DEFINE_ABORT(4, abort)
251 : /* gcc asm goto does not allow outputs */
252 : [cpu_id] "r" (cpu),
253 [current_cpu_id] "m" (__rseq_abi.cpu_id),
254 [rseq_cs] "m" (__rseq_abi.rseq_cs),
255 /* final store input */
256 [v] "m" (*v),
257 [expectnot] "r" (expectnot),
258 [voffp] "b" (voffp),
259 [load] "m" (*load)
260 RSEQ_INJECT_INPUT
261 : "memory", "cc", "r17"
262 RSEQ_INJECT_CLOBBER
263 : abort, cmpfail
264 #ifdef RSEQ_COMPARE_TWICE
265 , error1, error2
266 #endif
267 );
268 return 0;
269 abort:
270 RSEQ_INJECT_FAILED
271 return -1;
272 cmpfail:
273 return 1;
274 #ifdef RSEQ_COMPARE_TWICE
275 error1:
276 rseq_bug("cpu_id comparison failed");
277 error2:
278 rseq_bug("expected value comparison failed");
279 #endif
280 }
281
282 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)283 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
284 {
285 RSEQ_INJECT_C(9)
286
287 __asm__ __volatile__ goto (
288 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
289 /* Start rseq by storing table entry pointer into rseq_cs. */
290 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
291 /* cmp cpuid */
292 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
293 RSEQ_INJECT_ASM(3)
294 #ifdef RSEQ_COMPARE_TWICE
295 /* cmp cpuid */
296 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
297 #endif
298 /* load the value of @v */
299 RSEQ_ASM_OP_R_LOAD(v)
300 /* add @count to it */
301 RSEQ_ASM_OP_R_ADD(count)
302 /* final store */
303 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
304 RSEQ_INJECT_ASM(4)
305 RSEQ_ASM_DEFINE_ABORT(4, abort)
306 : /* gcc asm goto does not allow outputs */
307 : [cpu_id] "r" (cpu),
308 [current_cpu_id] "m" (__rseq_abi.cpu_id),
309 [rseq_cs] "m" (__rseq_abi.rseq_cs),
310 /* final store input */
311 [v] "m" (*v),
312 [count] "r" (count)
313 RSEQ_INJECT_INPUT
314 : "memory", "cc", "r17"
315 RSEQ_INJECT_CLOBBER
316 : abort
317 #ifdef RSEQ_COMPARE_TWICE
318 , error1
319 #endif
320 );
321 return 0;
322 abort:
323 RSEQ_INJECT_FAILED
324 return -1;
325 #ifdef RSEQ_COMPARE_TWICE
326 error1:
327 rseq_bug("cpu_id comparison failed");
328 #endif
329 }
330
331 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)332 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
333 intptr_t *v2, intptr_t newv2,
334 intptr_t newv, int cpu)
335 {
336 RSEQ_INJECT_C(9)
337
338 __asm__ __volatile__ goto (
339 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
340 /* Start rseq by storing table entry pointer into rseq_cs. */
341 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
342 /* cmp cpuid */
343 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
344 RSEQ_INJECT_ASM(3)
345 /* cmp @v equal to @expect */
346 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
347 RSEQ_INJECT_ASM(4)
348 #ifdef RSEQ_COMPARE_TWICE
349 /* cmp cpuid */
350 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
351 /* cmp @v equal to @expect */
352 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
353 #endif
354 /* try store */
355 RSEQ_ASM_OP_STORE(newv2, v2)
356 RSEQ_INJECT_ASM(5)
357 /* final store */
358 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
359 RSEQ_INJECT_ASM(6)
360 RSEQ_ASM_DEFINE_ABORT(4, abort)
361 : /* gcc asm goto does not allow outputs */
362 : [cpu_id] "r" (cpu),
363 [current_cpu_id] "m" (__rseq_abi.cpu_id),
364 [rseq_cs] "m" (__rseq_abi.rseq_cs),
365 /* try store input */
366 [v2] "m" (*v2),
367 [newv2] "r" (newv2),
368 /* final store input */
369 [v] "m" (*v),
370 [expect] "r" (expect),
371 [newv] "r" (newv)
372 RSEQ_INJECT_INPUT
373 : "memory", "cc", "r17"
374 RSEQ_INJECT_CLOBBER
375 : abort, cmpfail
376 #ifdef RSEQ_COMPARE_TWICE
377 , error1, error2
378 #endif
379 );
380 return 0;
381 abort:
382 RSEQ_INJECT_FAILED
383 return -1;
384 cmpfail:
385 return 1;
386 #ifdef RSEQ_COMPARE_TWICE
387 error1:
388 rseq_bug("cpu_id comparison failed");
389 error2:
390 rseq_bug("expected value comparison failed");
391 #endif
392 }
393
394 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)395 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
396 intptr_t *v2, intptr_t newv2,
397 intptr_t newv, int cpu)
398 {
399 RSEQ_INJECT_C(9)
400
401 __asm__ __volatile__ goto (
402 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
403 /* Start rseq by storing table entry pointer into rseq_cs. */
404 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
405 /* cmp cpuid */
406 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
407 RSEQ_INJECT_ASM(3)
408 /* cmp @v equal to @expect */
409 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
410 RSEQ_INJECT_ASM(4)
411 #ifdef RSEQ_COMPARE_TWICE
412 /* cmp cpuid */
413 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
414 /* cmp @v equal to @expect */
415 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
416 #endif
417 /* try store */
418 RSEQ_ASM_OP_STORE(newv2, v2)
419 RSEQ_INJECT_ASM(5)
420 /* for 'release' */
421 "lwsync\n\t"
422 /* final store */
423 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
424 RSEQ_INJECT_ASM(6)
425 RSEQ_ASM_DEFINE_ABORT(4, abort)
426 : /* gcc asm goto does not allow outputs */
427 : [cpu_id] "r" (cpu),
428 [current_cpu_id] "m" (__rseq_abi.cpu_id),
429 [rseq_cs] "m" (__rseq_abi.rseq_cs),
430 /* try store input */
431 [v2] "m" (*v2),
432 [newv2] "r" (newv2),
433 /* final store input */
434 [v] "m" (*v),
435 [expect] "r" (expect),
436 [newv] "r" (newv)
437 RSEQ_INJECT_INPUT
438 : "memory", "cc", "r17"
439 RSEQ_INJECT_CLOBBER
440 : abort, cmpfail
441 #ifdef RSEQ_COMPARE_TWICE
442 , error1, error2
443 #endif
444 );
445 return 0;
446 abort:
447 RSEQ_INJECT_FAILED
448 return -1;
449 cmpfail:
450 return 1;
451 #ifdef RSEQ_COMPARE_TWICE
452 error1:
453 rseq_bug("cpu_id comparison failed");
454 error2:
455 rseq_bug("expected value comparison failed");
456 #endif
457 }
458
459 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)460 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
461 intptr_t *v2, intptr_t expect2,
462 intptr_t newv, int cpu)
463 {
464 RSEQ_INJECT_C(9)
465
466 __asm__ __volatile__ goto (
467 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
468 /* Start rseq by storing table entry pointer into rseq_cs. */
469 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
470 /* cmp cpuid */
471 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
472 RSEQ_INJECT_ASM(3)
473 /* cmp @v equal to @expect */
474 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
475 RSEQ_INJECT_ASM(4)
476 /* cmp @v2 equal to @expct2 */
477 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
478 RSEQ_INJECT_ASM(5)
479 #ifdef RSEQ_COMPARE_TWICE
480 /* cmp cpuid */
481 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
482 /* cmp @v equal to @expect */
483 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
484 /* cmp @v2 equal to @expct2 */
485 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
486 #endif
487 /* final store */
488 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
489 RSEQ_INJECT_ASM(6)
490 RSEQ_ASM_DEFINE_ABORT(4, abort)
491 : /* gcc asm goto does not allow outputs */
492 : [cpu_id] "r" (cpu),
493 [current_cpu_id] "m" (__rseq_abi.cpu_id),
494 [rseq_cs] "m" (__rseq_abi.rseq_cs),
495 /* cmp2 input */
496 [v2] "m" (*v2),
497 [expect2] "r" (expect2),
498 /* final store input */
499 [v] "m" (*v),
500 [expect] "r" (expect),
501 [newv] "r" (newv)
502 RSEQ_INJECT_INPUT
503 : "memory", "cc", "r17"
504 RSEQ_INJECT_CLOBBER
505 : abort, cmpfail
506 #ifdef RSEQ_COMPARE_TWICE
507 , error1, error2, error3
508 #endif
509 );
510 return 0;
511 abort:
512 RSEQ_INJECT_FAILED
513 return -1;
514 cmpfail:
515 return 1;
516 #ifdef RSEQ_COMPARE_TWICE
517 error1:
518 rseq_bug("cpu_id comparison failed");
519 error2:
520 rseq_bug("1st expected value comparison failed");
521 error3:
522 rseq_bug("2nd expected value comparison failed");
523 #endif
524 }
525
526 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)527 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
528 void *dst, void *src, size_t len,
529 intptr_t newv, int cpu)
530 {
531 RSEQ_INJECT_C(9)
532
533 __asm__ __volatile__ goto (
534 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
535 /* setup for mempcy */
536 "mr %%r19, %[len]\n\t"
537 "mr %%r20, %[src]\n\t"
538 "mr %%r21, %[dst]\n\t"
539 /* Start rseq by storing table entry pointer into rseq_cs. */
540 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
541 /* cmp cpuid */
542 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
543 RSEQ_INJECT_ASM(3)
544 /* cmp @v equal to @expect */
545 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
546 RSEQ_INJECT_ASM(4)
547 #ifdef RSEQ_COMPARE_TWICE
548 /* cmp cpuid */
549 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
550 /* cmp @v equal to @expect */
551 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
552 #endif
553 /* try memcpy */
554 RSEQ_ASM_OP_R_MEMCPY()
555 RSEQ_INJECT_ASM(5)
556 /* final store */
557 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
558 RSEQ_INJECT_ASM(6)
559 /* teardown */
560 RSEQ_ASM_DEFINE_ABORT(4, abort)
561 : /* gcc asm goto does not allow outputs */
562 : [cpu_id] "r" (cpu),
563 [current_cpu_id] "m" (__rseq_abi.cpu_id),
564 [rseq_cs] "m" (__rseq_abi.rseq_cs),
565 /* final store input */
566 [v] "m" (*v),
567 [expect] "r" (expect),
568 [newv] "r" (newv),
569 /* try memcpy input */
570 [dst] "r" (dst),
571 [src] "r" (src),
572 [len] "r" (len)
573 RSEQ_INJECT_INPUT
574 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
575 RSEQ_INJECT_CLOBBER
576 : abort, cmpfail
577 #ifdef RSEQ_COMPARE_TWICE
578 , error1, error2
579 #endif
580 );
581 return 0;
582 abort:
583 RSEQ_INJECT_FAILED
584 return -1;
585 cmpfail:
586 return 1;
587 #ifdef RSEQ_COMPARE_TWICE
588 error1:
589 rseq_bug("cpu_id comparison failed");
590 error2:
591 rseq_bug("expected value comparison failed");
592 #endif
593 }
594
595 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)596 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
597 void *dst, void *src, size_t len,
598 intptr_t newv, int cpu)
599 {
600 RSEQ_INJECT_C(9)
601
602 __asm__ __volatile__ goto (
603 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
604 /* setup for mempcy */
605 "mr %%r19, %[len]\n\t"
606 "mr %%r20, %[src]\n\t"
607 "mr %%r21, %[dst]\n\t"
608 /* Start rseq by storing table entry pointer into rseq_cs. */
609 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
610 /* cmp cpuid */
611 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
612 RSEQ_INJECT_ASM(3)
613 /* cmp @v equal to @expect */
614 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
615 RSEQ_INJECT_ASM(4)
616 #ifdef RSEQ_COMPARE_TWICE
617 /* cmp cpuid */
618 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
619 /* cmp @v equal to @expect */
620 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
621 #endif
622 /* try memcpy */
623 RSEQ_ASM_OP_R_MEMCPY()
624 RSEQ_INJECT_ASM(5)
625 /* for 'release' */
626 "lwsync\n\t"
627 /* final store */
628 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
629 RSEQ_INJECT_ASM(6)
630 /* teardown */
631 RSEQ_ASM_DEFINE_ABORT(4, abort)
632 : /* gcc asm goto does not allow outputs */
633 : [cpu_id] "r" (cpu),
634 [current_cpu_id] "m" (__rseq_abi.cpu_id),
635 [rseq_cs] "m" (__rseq_abi.rseq_cs),
636 /* final store input */
637 [v] "m" (*v),
638 [expect] "r" (expect),
639 [newv] "r" (newv),
640 /* try memcpy input */
641 [dst] "r" (dst),
642 [src] "r" (src),
643 [len] "r" (len)
644 RSEQ_INJECT_INPUT
645 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
646 RSEQ_INJECT_CLOBBER
647 : abort, cmpfail
648 #ifdef RSEQ_COMPARE_TWICE
649 , error1, error2
650 #endif
651 );
652 return 0;
653 abort:
654 RSEQ_INJECT_FAILED
655 return -1;
656 cmpfail:
657 return 1;
658 #ifdef RSEQ_COMPARE_TWICE
659 error1:
660 rseq_bug("cpu_id comparison failed");
661 error2:
662 rseq_bug("expected value comparison failed");
663 #endif
664 }
665
666 #undef STORE_WORD
667 #undef LOAD_WORD
668 #undef LOADX_WORD
669 #undef CMP_WORD
670
671 #endif /* !RSEQ_SKIP_FASTPATH */
672