1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3 * rseq-x86.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8 #include <stdint.h>
9
10 #define RSEQ_SIG 0x53053053
11
12 #ifdef __x86_64__
13
14 #define rseq_smp_mb() \
15 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
16 #define rseq_smp_rmb() rseq_barrier()
17 #define rseq_smp_wmb() rseq_barrier()
18
19 #define rseq_smp_load_acquire(p) \
20 __extension__ ({ \
21 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
22 rseq_barrier(); \
23 ____p1; \
24 })
25
26 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
27
28 #define rseq_smp_store_release(p, v) \
29 do { \
30 rseq_barrier(); \
31 RSEQ_WRITE_ONCE(*p, v); \
32 } while (0)
33
34 #ifdef RSEQ_SKIP_FASTPATH
35 #include "rseq-skip.h"
36 #else /* !RSEQ_SKIP_FASTPATH */
37
38 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
39 start_ip, post_commit_offset, abort_ip) \
40 ".pushsection __rseq_table, \"aw\"\n\t" \
41 ".balign 32\n\t" \
42 __rseq_str(label) ":\n\t" \
43 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
44 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
45 ".popsection\n\t"
46
47 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
48 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
49 (post_commit_ip - start_ip), abort_ip)
50
51 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
52 RSEQ_INJECT_ASM(1) \
53 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
54 "movq %%rax, %[" __rseq_str(rseq_cs) "]\n\t" \
55 __rseq_str(label) ":\n\t"
56
57 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
58 RSEQ_INJECT_ASM(2) \
59 "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
60 "jnz " __rseq_str(label) "\n\t"
61
62 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
63 ".pushsection __rseq_failure, \"ax\"\n\t" \
64 /* Disassembler-friendly signature: nopl <sig>(%rip). */\
65 ".byte 0x0f, 0x1f, 0x05\n\t" \
66 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
67 __rseq_str(label) ":\n\t" \
68 teardown \
69 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
70 ".popsection\n\t"
71
72 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
73 ".pushsection __rseq_failure, \"ax\"\n\t" \
74 __rseq_str(label) ":\n\t" \
75 teardown \
76 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
77 ".popsection\n\t"
78
79 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)80 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
81 {
82 RSEQ_INJECT_C(9)
83
84 __asm__ __volatile__ goto (
85 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
86 /* Start rseq by storing table entry pointer into rseq_cs. */
87 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
88 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
89 RSEQ_INJECT_ASM(3)
90 "cmpq %[v], %[expect]\n\t"
91 "jnz %l[cmpfail]\n\t"
92 RSEQ_INJECT_ASM(4)
93 #ifdef RSEQ_COMPARE_TWICE
94 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
95 "cmpq %[v], %[expect]\n\t"
96 "jnz %l[error2]\n\t"
97 #endif
98 /* final store */
99 "movq %[newv], %[v]\n\t"
100 "2:\n\t"
101 RSEQ_INJECT_ASM(5)
102 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
103 : /* gcc asm goto does not allow outputs */
104 : [cpu_id] "r" (cpu),
105 [current_cpu_id] "m" (__rseq_abi.cpu_id),
106 [rseq_cs] "m" (__rseq_abi.rseq_cs),
107 [v] "m" (*v),
108 [expect] "r" (expect),
109 [newv] "r" (newv)
110 : "memory", "cc", "rax"
111 RSEQ_INJECT_CLOBBER
112 : abort, cmpfail
113 #ifdef RSEQ_COMPARE_TWICE
114 , error1, error2
115 #endif
116 );
117 return 0;
118 abort:
119 RSEQ_INJECT_FAILED
120 return -1;
121 cmpfail:
122 return 1;
123 #ifdef RSEQ_COMPARE_TWICE
124 error1:
125 rseq_bug("cpu_id comparison failed");
126 error2:
127 rseq_bug("expected value comparison failed");
128 #endif
129 }
130
131 /*
132 * Compare @v against @expectnot. When it does _not_ match, load @v
133 * into @load, and store the content of *@v + voffp into @v.
134 */
135 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,off_t voffp,intptr_t * load,int cpu)136 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
137 off_t voffp, intptr_t *load, int cpu)
138 {
139 RSEQ_INJECT_C(9)
140
141 __asm__ __volatile__ goto (
142 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
143 /* Start rseq by storing table entry pointer into rseq_cs. */
144 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
145 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
146 RSEQ_INJECT_ASM(3)
147 "movq %[v], %%rbx\n\t"
148 "cmpq %%rbx, %[expectnot]\n\t"
149 "je %l[cmpfail]\n\t"
150 RSEQ_INJECT_ASM(4)
151 #ifdef RSEQ_COMPARE_TWICE
152 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
153 "movq %[v], %%rbx\n\t"
154 "cmpq %%rbx, %[expectnot]\n\t"
155 "je %l[error2]\n\t"
156 #endif
157 "movq %%rbx, %[load]\n\t"
158 "addq %[voffp], %%rbx\n\t"
159 "movq (%%rbx), %%rbx\n\t"
160 /* final store */
161 "movq %%rbx, %[v]\n\t"
162 "2:\n\t"
163 RSEQ_INJECT_ASM(5)
164 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
165 : /* gcc asm goto does not allow outputs */
166 : [cpu_id] "r" (cpu),
167 [current_cpu_id] "m" (__rseq_abi.cpu_id),
168 [rseq_cs] "m" (__rseq_abi.rseq_cs),
169 /* final store input */
170 [v] "m" (*v),
171 [expectnot] "r" (expectnot),
172 [voffp] "er" (voffp),
173 [load] "m" (*load)
174 : "memory", "cc", "rax", "rbx"
175 RSEQ_INJECT_CLOBBER
176 : abort, cmpfail
177 #ifdef RSEQ_COMPARE_TWICE
178 , error1, error2
179 #endif
180 );
181 return 0;
182 abort:
183 RSEQ_INJECT_FAILED
184 return -1;
185 cmpfail:
186 return 1;
187 #ifdef RSEQ_COMPARE_TWICE
188 error1:
189 rseq_bug("cpu_id comparison failed");
190 error2:
191 rseq_bug("expected value comparison failed");
192 #endif
193 }
194
195 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)196 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
197 {
198 RSEQ_INJECT_C(9)
199
200 __asm__ __volatile__ goto (
201 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
202 /* Start rseq by storing table entry pointer into rseq_cs. */
203 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
204 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
205 RSEQ_INJECT_ASM(3)
206 #ifdef RSEQ_COMPARE_TWICE
207 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
208 #endif
209 /* final store */
210 "addq %[count], %[v]\n\t"
211 "2:\n\t"
212 RSEQ_INJECT_ASM(4)
213 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
214 : /* gcc asm goto does not allow outputs */
215 : [cpu_id] "r" (cpu),
216 [current_cpu_id] "m" (__rseq_abi.cpu_id),
217 [rseq_cs] "m" (__rseq_abi.rseq_cs),
218 /* final store input */
219 [v] "m" (*v),
220 [count] "er" (count)
221 : "memory", "cc", "rax"
222 RSEQ_INJECT_CLOBBER
223 : abort
224 #ifdef RSEQ_COMPARE_TWICE
225 , error1
226 #endif
227 );
228 return 0;
229 abort:
230 RSEQ_INJECT_FAILED
231 return -1;
232 #ifdef RSEQ_COMPARE_TWICE
233 error1:
234 rseq_bug("cpu_id comparison failed");
235 #endif
236 }
237
238 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)239 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
240 intptr_t *v2, intptr_t newv2,
241 intptr_t newv, int cpu)
242 {
243 RSEQ_INJECT_C(9)
244
245 __asm__ __volatile__ goto (
246 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
247 /* Start rseq by storing table entry pointer into rseq_cs. */
248 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
249 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
250 RSEQ_INJECT_ASM(3)
251 "cmpq %[v], %[expect]\n\t"
252 "jnz %l[cmpfail]\n\t"
253 RSEQ_INJECT_ASM(4)
254 #ifdef RSEQ_COMPARE_TWICE
255 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
256 "cmpq %[v], %[expect]\n\t"
257 "jnz %l[error2]\n\t"
258 #endif
259 /* try store */
260 "movq %[newv2], %[v2]\n\t"
261 RSEQ_INJECT_ASM(5)
262 /* final store */
263 "movq %[newv], %[v]\n\t"
264 "2:\n\t"
265 RSEQ_INJECT_ASM(6)
266 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
267 : /* gcc asm goto does not allow outputs */
268 : [cpu_id] "r" (cpu),
269 [current_cpu_id] "m" (__rseq_abi.cpu_id),
270 [rseq_cs] "m" (__rseq_abi.rseq_cs),
271 /* try store input */
272 [v2] "m" (*v2),
273 [newv2] "r" (newv2),
274 /* final store input */
275 [v] "m" (*v),
276 [expect] "r" (expect),
277 [newv] "r" (newv)
278 : "memory", "cc", "rax"
279 RSEQ_INJECT_CLOBBER
280 : abort, cmpfail
281 #ifdef RSEQ_COMPARE_TWICE
282 , error1, error2
283 #endif
284 );
285 return 0;
286 abort:
287 RSEQ_INJECT_FAILED
288 return -1;
289 cmpfail:
290 return 1;
291 #ifdef RSEQ_COMPARE_TWICE
292 error1:
293 rseq_bug("cpu_id comparison failed");
294 error2:
295 rseq_bug("expected value comparison failed");
296 #endif
297 }
298
299 /* x86-64 is TSO. */
300 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)301 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
302 intptr_t *v2, intptr_t newv2,
303 intptr_t newv, int cpu)
304 {
305 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
306 }
307
308 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)309 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
310 intptr_t *v2, intptr_t expect2,
311 intptr_t newv, int cpu)
312 {
313 RSEQ_INJECT_C(9)
314
315 __asm__ __volatile__ goto (
316 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
317 /* Start rseq by storing table entry pointer into rseq_cs. */
318 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
319 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
320 RSEQ_INJECT_ASM(3)
321 "cmpq %[v], %[expect]\n\t"
322 "jnz %l[cmpfail]\n\t"
323 RSEQ_INJECT_ASM(4)
324 "cmpq %[v2], %[expect2]\n\t"
325 "jnz %l[cmpfail]\n\t"
326 RSEQ_INJECT_ASM(5)
327 #ifdef RSEQ_COMPARE_TWICE
328 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
329 "cmpq %[v], %[expect]\n\t"
330 "jnz %l[error2]\n\t"
331 "cmpq %[v2], %[expect2]\n\t"
332 "jnz %l[error3]\n\t"
333 #endif
334 /* final store */
335 "movq %[newv], %[v]\n\t"
336 "2:\n\t"
337 RSEQ_INJECT_ASM(6)
338 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
339 : /* gcc asm goto does not allow outputs */
340 : [cpu_id] "r" (cpu),
341 [current_cpu_id] "m" (__rseq_abi.cpu_id),
342 [rseq_cs] "m" (__rseq_abi.rseq_cs),
343 /* cmp2 input */
344 [v2] "m" (*v2),
345 [expect2] "r" (expect2),
346 /* final store input */
347 [v] "m" (*v),
348 [expect] "r" (expect),
349 [newv] "r" (newv)
350 : "memory", "cc", "rax"
351 RSEQ_INJECT_CLOBBER
352 : abort, cmpfail
353 #ifdef RSEQ_COMPARE_TWICE
354 , error1, error2, error3
355 #endif
356 );
357 return 0;
358 abort:
359 RSEQ_INJECT_FAILED
360 return -1;
361 cmpfail:
362 return 1;
363 #ifdef RSEQ_COMPARE_TWICE
364 error1:
365 rseq_bug("cpu_id comparison failed");
366 error2:
367 rseq_bug("1st expected value comparison failed");
368 error3:
369 rseq_bug("2nd expected value comparison failed");
370 #endif
371 }
372
373 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)374 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
375 void *dst, void *src, size_t len,
376 intptr_t newv, int cpu)
377 {
378 uint64_t rseq_scratch[3];
379
380 RSEQ_INJECT_C(9)
381
382 __asm__ __volatile__ goto (
383 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
384 "movq %[src], %[rseq_scratch0]\n\t"
385 "movq %[dst], %[rseq_scratch1]\n\t"
386 "movq %[len], %[rseq_scratch2]\n\t"
387 /* Start rseq by storing table entry pointer into rseq_cs. */
388 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
389 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
390 RSEQ_INJECT_ASM(3)
391 "cmpq %[v], %[expect]\n\t"
392 "jnz 5f\n\t"
393 RSEQ_INJECT_ASM(4)
394 #ifdef RSEQ_COMPARE_TWICE
395 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
396 "cmpq %[v], %[expect]\n\t"
397 "jnz 7f\n\t"
398 #endif
399 /* try memcpy */
400 "test %[len], %[len]\n\t" \
401 "jz 333f\n\t" \
402 "222:\n\t" \
403 "movb (%[src]), %%al\n\t" \
404 "movb %%al, (%[dst])\n\t" \
405 "inc %[src]\n\t" \
406 "inc %[dst]\n\t" \
407 "dec %[len]\n\t" \
408 "jnz 222b\n\t" \
409 "333:\n\t" \
410 RSEQ_INJECT_ASM(5)
411 /* final store */
412 "movq %[newv], %[v]\n\t"
413 "2:\n\t"
414 RSEQ_INJECT_ASM(6)
415 /* teardown */
416 "movq %[rseq_scratch2], %[len]\n\t"
417 "movq %[rseq_scratch1], %[dst]\n\t"
418 "movq %[rseq_scratch0], %[src]\n\t"
419 RSEQ_ASM_DEFINE_ABORT(4,
420 "movq %[rseq_scratch2], %[len]\n\t"
421 "movq %[rseq_scratch1], %[dst]\n\t"
422 "movq %[rseq_scratch0], %[src]\n\t",
423 abort)
424 RSEQ_ASM_DEFINE_CMPFAIL(5,
425 "movq %[rseq_scratch2], %[len]\n\t"
426 "movq %[rseq_scratch1], %[dst]\n\t"
427 "movq %[rseq_scratch0], %[src]\n\t",
428 cmpfail)
429 #ifdef RSEQ_COMPARE_TWICE
430 RSEQ_ASM_DEFINE_CMPFAIL(6,
431 "movq %[rseq_scratch2], %[len]\n\t"
432 "movq %[rseq_scratch1], %[dst]\n\t"
433 "movq %[rseq_scratch0], %[src]\n\t",
434 error1)
435 RSEQ_ASM_DEFINE_CMPFAIL(7,
436 "movq %[rseq_scratch2], %[len]\n\t"
437 "movq %[rseq_scratch1], %[dst]\n\t"
438 "movq %[rseq_scratch0], %[src]\n\t",
439 error2)
440 #endif
441 : /* gcc asm goto does not allow outputs */
442 : [cpu_id] "r" (cpu),
443 [current_cpu_id] "m" (__rseq_abi.cpu_id),
444 [rseq_cs] "m" (__rseq_abi.rseq_cs),
445 /* final store input */
446 [v] "m" (*v),
447 [expect] "r" (expect),
448 [newv] "r" (newv),
449 /* try memcpy input */
450 [dst] "r" (dst),
451 [src] "r" (src),
452 [len] "r" (len),
453 [rseq_scratch0] "m" (rseq_scratch[0]),
454 [rseq_scratch1] "m" (rseq_scratch[1]),
455 [rseq_scratch2] "m" (rseq_scratch[2])
456 : "memory", "cc", "rax"
457 RSEQ_INJECT_CLOBBER
458 : abort, cmpfail
459 #ifdef RSEQ_COMPARE_TWICE
460 , error1, error2
461 #endif
462 );
463 return 0;
464 abort:
465 RSEQ_INJECT_FAILED
466 return -1;
467 cmpfail:
468 return 1;
469 #ifdef RSEQ_COMPARE_TWICE
470 error1:
471 rseq_bug("cpu_id comparison failed");
472 error2:
473 rseq_bug("expected value comparison failed");
474 #endif
475 }
476
477 /* x86-64 is TSO. */
478 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)479 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
480 void *dst, void *src, size_t len,
481 intptr_t newv, int cpu)
482 {
483 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
484 newv, cpu);
485 }
486
487 #endif /* !RSEQ_SKIP_FASTPATH */
488
489 #elif __i386__
490
491 #define rseq_smp_mb() \
492 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
493 #define rseq_smp_rmb() \
494 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
495 #define rseq_smp_wmb() \
496 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
497
498 #define rseq_smp_load_acquire(p) \
499 __extension__ ({ \
500 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
501 rseq_smp_mb(); \
502 ____p1; \
503 })
504
505 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
506
507 #define rseq_smp_store_release(p, v) \
508 do { \
509 rseq_smp_mb(); \
510 RSEQ_WRITE_ONCE(*p, v); \
511 } while (0)
512
513 #ifdef RSEQ_SKIP_FASTPATH
514 #include "rseq-skip.h"
515 #else /* !RSEQ_SKIP_FASTPATH */
516
517 /*
518 * Use eax as scratch register and take memory operands as input to
519 * lessen register pressure. Especially needed when compiling in O0.
520 */
521 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
522 start_ip, post_commit_offset, abort_ip) \
523 ".pushsection __rseq_table, \"aw\"\n\t" \
524 ".balign 32\n\t" \
525 __rseq_str(label) ":\n\t" \
526 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
527 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
528 ".popsection\n\t"
529
530 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
531 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
532 (post_commit_ip - start_ip), abort_ip)
533
534 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
535 RSEQ_INJECT_ASM(1) \
536 "movl $" __rseq_str(cs_label) ", %[rseq_cs]\n\t" \
537 __rseq_str(label) ":\n\t"
538
539 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
540 RSEQ_INJECT_ASM(2) \
541 "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
542 "jnz " __rseq_str(label) "\n\t"
543
544 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
545 ".pushsection __rseq_failure, \"ax\"\n\t" \
546 /* Disassembler-friendly signature: nopl <sig>. */ \
547 ".byte 0x0f, 0x1f, 0x05\n\t" \
548 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
549 __rseq_str(label) ":\n\t" \
550 teardown \
551 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
552 ".popsection\n\t"
553
554 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
555 ".pushsection __rseq_failure, \"ax\"\n\t" \
556 __rseq_str(label) ":\n\t" \
557 teardown \
558 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
559 ".popsection\n\t"
560
561 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)562 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
563 {
564 RSEQ_INJECT_C(9)
565
566 __asm__ __volatile__ goto (
567 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
568 /* Start rseq by storing table entry pointer into rseq_cs. */
569 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
570 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
571 RSEQ_INJECT_ASM(3)
572 "cmpl %[v], %[expect]\n\t"
573 "jnz %l[cmpfail]\n\t"
574 RSEQ_INJECT_ASM(4)
575 #ifdef RSEQ_COMPARE_TWICE
576 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
577 "cmpl %[v], %[expect]\n\t"
578 "jnz %l[error2]\n\t"
579 #endif
580 /* final store */
581 "movl %[newv], %[v]\n\t"
582 "2:\n\t"
583 RSEQ_INJECT_ASM(5)
584 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
585 : /* gcc asm goto does not allow outputs */
586 : [cpu_id] "r" (cpu),
587 [current_cpu_id] "m" (__rseq_abi.cpu_id),
588 [rseq_cs] "m" (__rseq_abi.rseq_cs),
589 [v] "m" (*v),
590 [expect] "r" (expect),
591 [newv] "r" (newv)
592 : "memory", "cc", "eax"
593 RSEQ_INJECT_CLOBBER
594 : abort, cmpfail
595 #ifdef RSEQ_COMPARE_TWICE
596 , error1, error2
597 #endif
598 );
599 return 0;
600 abort:
601 RSEQ_INJECT_FAILED
602 return -1;
603 cmpfail:
604 return 1;
605 #ifdef RSEQ_COMPARE_TWICE
606 error1:
607 rseq_bug("cpu_id comparison failed");
608 error2:
609 rseq_bug("expected value comparison failed");
610 #endif
611 }
612
613 /*
614 * Compare @v against @expectnot. When it does _not_ match, load @v
615 * into @load, and store the content of *@v + voffp into @v.
616 */
617 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,off_t voffp,intptr_t * load,int cpu)618 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
619 off_t voffp, intptr_t *load, int cpu)
620 {
621 RSEQ_INJECT_C(9)
622
623 __asm__ __volatile__ goto (
624 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
625 /* Start rseq by storing table entry pointer into rseq_cs. */
626 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
627 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
628 RSEQ_INJECT_ASM(3)
629 "movl %[v], %%ebx\n\t"
630 "cmpl %%ebx, %[expectnot]\n\t"
631 "je %l[cmpfail]\n\t"
632 RSEQ_INJECT_ASM(4)
633 #ifdef RSEQ_COMPARE_TWICE
634 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
635 "movl %[v], %%ebx\n\t"
636 "cmpl %%ebx, %[expectnot]\n\t"
637 "je %l[error2]\n\t"
638 #endif
639 "movl %%ebx, %[load]\n\t"
640 "addl %[voffp], %%ebx\n\t"
641 "movl (%%ebx), %%ebx\n\t"
642 /* final store */
643 "movl %%ebx, %[v]\n\t"
644 "2:\n\t"
645 RSEQ_INJECT_ASM(5)
646 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
647 : /* gcc asm goto does not allow outputs */
648 : [cpu_id] "r" (cpu),
649 [current_cpu_id] "m" (__rseq_abi.cpu_id),
650 [rseq_cs] "m" (__rseq_abi.rseq_cs),
651 /* final store input */
652 [v] "m" (*v),
653 [expectnot] "r" (expectnot),
654 [voffp] "ir" (voffp),
655 [load] "m" (*load)
656 : "memory", "cc", "eax", "ebx"
657 RSEQ_INJECT_CLOBBER
658 : abort, cmpfail
659 #ifdef RSEQ_COMPARE_TWICE
660 , error1, error2
661 #endif
662 );
663 return 0;
664 abort:
665 RSEQ_INJECT_FAILED
666 return -1;
667 cmpfail:
668 return 1;
669 #ifdef RSEQ_COMPARE_TWICE
670 error1:
671 rseq_bug("cpu_id comparison failed");
672 error2:
673 rseq_bug("expected value comparison failed");
674 #endif
675 }
676
677 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)678 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
679 {
680 RSEQ_INJECT_C(9)
681
682 __asm__ __volatile__ goto (
683 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
684 /* Start rseq by storing table entry pointer into rseq_cs. */
685 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
686 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
687 RSEQ_INJECT_ASM(3)
688 #ifdef RSEQ_COMPARE_TWICE
689 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
690 #endif
691 /* final store */
692 "addl %[count], %[v]\n\t"
693 "2:\n\t"
694 RSEQ_INJECT_ASM(4)
695 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
696 : /* gcc asm goto does not allow outputs */
697 : [cpu_id] "r" (cpu),
698 [current_cpu_id] "m" (__rseq_abi.cpu_id),
699 [rseq_cs] "m" (__rseq_abi.rseq_cs),
700 /* final store input */
701 [v] "m" (*v),
702 [count] "ir" (count)
703 : "memory", "cc", "eax"
704 RSEQ_INJECT_CLOBBER
705 : abort
706 #ifdef RSEQ_COMPARE_TWICE
707 , error1
708 #endif
709 );
710 return 0;
711 abort:
712 RSEQ_INJECT_FAILED
713 return -1;
714 #ifdef RSEQ_COMPARE_TWICE
715 error1:
716 rseq_bug("cpu_id comparison failed");
717 #endif
718 }
719
720 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)721 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
722 intptr_t *v2, intptr_t newv2,
723 intptr_t newv, int cpu)
724 {
725 RSEQ_INJECT_C(9)
726
727 __asm__ __volatile__ goto (
728 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
729 /* Start rseq by storing table entry pointer into rseq_cs. */
730 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
731 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
732 RSEQ_INJECT_ASM(3)
733 "cmpl %[v], %[expect]\n\t"
734 "jnz %l[cmpfail]\n\t"
735 RSEQ_INJECT_ASM(4)
736 #ifdef RSEQ_COMPARE_TWICE
737 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
738 "cmpl %[v], %[expect]\n\t"
739 "jnz %l[error2]\n\t"
740 #endif
741 /* try store */
742 "movl %[newv2], %%eax\n\t"
743 "movl %%eax, %[v2]\n\t"
744 RSEQ_INJECT_ASM(5)
745 /* final store */
746 "movl %[newv], %[v]\n\t"
747 "2:\n\t"
748 RSEQ_INJECT_ASM(6)
749 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
750 : /* gcc asm goto does not allow outputs */
751 : [cpu_id] "r" (cpu),
752 [current_cpu_id] "m" (__rseq_abi.cpu_id),
753 [rseq_cs] "m" (__rseq_abi.rseq_cs),
754 /* try store input */
755 [v2] "m" (*v2),
756 [newv2] "m" (newv2),
757 /* final store input */
758 [v] "m" (*v),
759 [expect] "r" (expect),
760 [newv] "r" (newv)
761 : "memory", "cc", "eax"
762 RSEQ_INJECT_CLOBBER
763 : abort, cmpfail
764 #ifdef RSEQ_COMPARE_TWICE
765 , error1, error2
766 #endif
767 );
768 return 0;
769 abort:
770 RSEQ_INJECT_FAILED
771 return -1;
772 cmpfail:
773 return 1;
774 #ifdef RSEQ_COMPARE_TWICE
775 error1:
776 rseq_bug("cpu_id comparison failed");
777 error2:
778 rseq_bug("expected value comparison failed");
779 #endif
780 }
781
782 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)783 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
784 intptr_t *v2, intptr_t newv2,
785 intptr_t newv, int cpu)
786 {
787 RSEQ_INJECT_C(9)
788
789 __asm__ __volatile__ goto (
790 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
791 /* Start rseq by storing table entry pointer into rseq_cs. */
792 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
793 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
794 RSEQ_INJECT_ASM(3)
795 "movl %[expect], %%eax\n\t"
796 "cmpl %[v], %%eax\n\t"
797 "jnz %l[cmpfail]\n\t"
798 RSEQ_INJECT_ASM(4)
799 #ifdef RSEQ_COMPARE_TWICE
800 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
801 "movl %[expect], %%eax\n\t"
802 "cmpl %[v], %%eax\n\t"
803 "jnz %l[error2]\n\t"
804 #endif
805 /* try store */
806 "movl %[newv2], %[v2]\n\t"
807 RSEQ_INJECT_ASM(5)
808 "lock; addl $0,-128(%%esp)\n\t"
809 /* final store */
810 "movl %[newv], %[v]\n\t"
811 "2:\n\t"
812 RSEQ_INJECT_ASM(6)
813 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
814 : /* gcc asm goto does not allow outputs */
815 : [cpu_id] "r" (cpu),
816 [current_cpu_id] "m" (__rseq_abi.cpu_id),
817 [rseq_cs] "m" (__rseq_abi.rseq_cs),
818 /* try store input */
819 [v2] "m" (*v2),
820 [newv2] "r" (newv2),
821 /* final store input */
822 [v] "m" (*v),
823 [expect] "m" (expect),
824 [newv] "r" (newv)
825 : "memory", "cc", "eax"
826 RSEQ_INJECT_CLOBBER
827 : abort, cmpfail
828 #ifdef RSEQ_COMPARE_TWICE
829 , error1, error2
830 #endif
831 );
832 return 0;
833 abort:
834 RSEQ_INJECT_FAILED
835 return -1;
836 cmpfail:
837 return 1;
838 #ifdef RSEQ_COMPARE_TWICE
839 error1:
840 rseq_bug("cpu_id comparison failed");
841 error2:
842 rseq_bug("expected value comparison failed");
843 #endif
844
845 }
846
847 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)848 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
849 intptr_t *v2, intptr_t expect2,
850 intptr_t newv, int cpu)
851 {
852 RSEQ_INJECT_C(9)
853
854 __asm__ __volatile__ goto (
855 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
856 /* Start rseq by storing table entry pointer into rseq_cs. */
857 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
858 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
859 RSEQ_INJECT_ASM(3)
860 "cmpl %[v], %[expect]\n\t"
861 "jnz %l[cmpfail]\n\t"
862 RSEQ_INJECT_ASM(4)
863 "cmpl %[expect2], %[v2]\n\t"
864 "jnz %l[cmpfail]\n\t"
865 RSEQ_INJECT_ASM(5)
866 #ifdef RSEQ_COMPARE_TWICE
867 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
868 "cmpl %[v], %[expect]\n\t"
869 "jnz %l[error2]\n\t"
870 "cmpl %[expect2], %[v2]\n\t"
871 "jnz %l[error3]\n\t"
872 #endif
873 "movl %[newv], %%eax\n\t"
874 /* final store */
875 "movl %%eax, %[v]\n\t"
876 "2:\n\t"
877 RSEQ_INJECT_ASM(6)
878 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
879 : /* gcc asm goto does not allow outputs */
880 : [cpu_id] "r" (cpu),
881 [current_cpu_id] "m" (__rseq_abi.cpu_id),
882 [rseq_cs] "m" (__rseq_abi.rseq_cs),
883 /* cmp2 input */
884 [v2] "m" (*v2),
885 [expect2] "r" (expect2),
886 /* final store input */
887 [v] "m" (*v),
888 [expect] "r" (expect),
889 [newv] "m" (newv)
890 : "memory", "cc", "eax"
891 RSEQ_INJECT_CLOBBER
892 : abort, cmpfail
893 #ifdef RSEQ_COMPARE_TWICE
894 , error1, error2, error3
895 #endif
896 );
897 return 0;
898 abort:
899 RSEQ_INJECT_FAILED
900 return -1;
901 cmpfail:
902 return 1;
903 #ifdef RSEQ_COMPARE_TWICE
904 error1:
905 rseq_bug("cpu_id comparison failed");
906 error2:
907 rseq_bug("1st expected value comparison failed");
908 error3:
909 rseq_bug("2nd expected value comparison failed");
910 #endif
911 }
912
913 /* TODO: implement a faster memcpy. */
914 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)915 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
916 void *dst, void *src, size_t len,
917 intptr_t newv, int cpu)
918 {
919 uint32_t rseq_scratch[3];
920
921 RSEQ_INJECT_C(9)
922
923 __asm__ __volatile__ goto (
924 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
925 "movl %[src], %[rseq_scratch0]\n\t"
926 "movl %[dst], %[rseq_scratch1]\n\t"
927 "movl %[len], %[rseq_scratch2]\n\t"
928 /* Start rseq by storing table entry pointer into rseq_cs. */
929 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
930 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
931 RSEQ_INJECT_ASM(3)
932 "movl %[expect], %%eax\n\t"
933 "cmpl %%eax, %[v]\n\t"
934 "jnz 5f\n\t"
935 RSEQ_INJECT_ASM(4)
936 #ifdef RSEQ_COMPARE_TWICE
937 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
938 "movl %[expect], %%eax\n\t"
939 "cmpl %%eax, %[v]\n\t"
940 "jnz 7f\n\t"
941 #endif
942 /* try memcpy */
943 "test %[len], %[len]\n\t" \
944 "jz 333f\n\t" \
945 "222:\n\t" \
946 "movb (%[src]), %%al\n\t" \
947 "movb %%al, (%[dst])\n\t" \
948 "inc %[src]\n\t" \
949 "inc %[dst]\n\t" \
950 "dec %[len]\n\t" \
951 "jnz 222b\n\t" \
952 "333:\n\t" \
953 RSEQ_INJECT_ASM(5)
954 "movl %[newv], %%eax\n\t"
955 /* final store */
956 "movl %%eax, %[v]\n\t"
957 "2:\n\t"
958 RSEQ_INJECT_ASM(6)
959 /* teardown */
960 "movl %[rseq_scratch2], %[len]\n\t"
961 "movl %[rseq_scratch1], %[dst]\n\t"
962 "movl %[rseq_scratch0], %[src]\n\t"
963 RSEQ_ASM_DEFINE_ABORT(4,
964 "movl %[rseq_scratch2], %[len]\n\t"
965 "movl %[rseq_scratch1], %[dst]\n\t"
966 "movl %[rseq_scratch0], %[src]\n\t",
967 abort)
968 RSEQ_ASM_DEFINE_CMPFAIL(5,
969 "movl %[rseq_scratch2], %[len]\n\t"
970 "movl %[rseq_scratch1], %[dst]\n\t"
971 "movl %[rseq_scratch0], %[src]\n\t",
972 cmpfail)
973 #ifdef RSEQ_COMPARE_TWICE
974 RSEQ_ASM_DEFINE_CMPFAIL(6,
975 "movl %[rseq_scratch2], %[len]\n\t"
976 "movl %[rseq_scratch1], %[dst]\n\t"
977 "movl %[rseq_scratch0], %[src]\n\t",
978 error1)
979 RSEQ_ASM_DEFINE_CMPFAIL(7,
980 "movl %[rseq_scratch2], %[len]\n\t"
981 "movl %[rseq_scratch1], %[dst]\n\t"
982 "movl %[rseq_scratch0], %[src]\n\t",
983 error2)
984 #endif
985 : /* gcc asm goto does not allow outputs */
986 : [cpu_id] "r" (cpu),
987 [current_cpu_id] "m" (__rseq_abi.cpu_id),
988 [rseq_cs] "m" (__rseq_abi.rseq_cs),
989 /* final store input */
990 [v] "m" (*v),
991 [expect] "m" (expect),
992 [newv] "m" (newv),
993 /* try memcpy input */
994 [dst] "r" (dst),
995 [src] "r" (src),
996 [len] "r" (len),
997 [rseq_scratch0] "m" (rseq_scratch[0]),
998 [rseq_scratch1] "m" (rseq_scratch[1]),
999 [rseq_scratch2] "m" (rseq_scratch[2])
1000 : "memory", "cc", "eax"
1001 RSEQ_INJECT_CLOBBER
1002 : abort, cmpfail
1003 #ifdef RSEQ_COMPARE_TWICE
1004 , error1, error2
1005 #endif
1006 );
1007 return 0;
1008 abort:
1009 RSEQ_INJECT_FAILED
1010 return -1;
1011 cmpfail:
1012 return 1;
1013 #ifdef RSEQ_COMPARE_TWICE
1014 error1:
1015 rseq_bug("cpu_id comparison failed");
1016 error2:
1017 rseq_bug("expected value comparison failed");
1018 #endif
1019 }
1020
1021 /* TODO: implement a faster memcpy. */
1022 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)1023 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1024 void *dst, void *src, size_t len,
1025 intptr_t newv, int cpu)
1026 {
1027 uint32_t rseq_scratch[3];
1028
1029 RSEQ_INJECT_C(9)
1030
1031 __asm__ __volatile__ goto (
1032 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1033 "movl %[src], %[rseq_scratch0]\n\t"
1034 "movl %[dst], %[rseq_scratch1]\n\t"
1035 "movl %[len], %[rseq_scratch2]\n\t"
1036 /* Start rseq by storing table entry pointer into rseq_cs. */
1037 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
1038 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
1039 RSEQ_INJECT_ASM(3)
1040 "movl %[expect], %%eax\n\t"
1041 "cmpl %%eax, %[v]\n\t"
1042 "jnz 5f\n\t"
1043 RSEQ_INJECT_ASM(4)
1044 #ifdef RSEQ_COMPARE_TWICE
1045 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
1046 "movl %[expect], %%eax\n\t"
1047 "cmpl %%eax, %[v]\n\t"
1048 "jnz 7f\n\t"
1049 #endif
1050 /* try memcpy */
1051 "test %[len], %[len]\n\t" \
1052 "jz 333f\n\t" \
1053 "222:\n\t" \
1054 "movb (%[src]), %%al\n\t" \
1055 "movb %%al, (%[dst])\n\t" \
1056 "inc %[src]\n\t" \
1057 "inc %[dst]\n\t" \
1058 "dec %[len]\n\t" \
1059 "jnz 222b\n\t" \
1060 "333:\n\t" \
1061 RSEQ_INJECT_ASM(5)
1062 "lock; addl $0,-128(%%esp)\n\t"
1063 "movl %[newv], %%eax\n\t"
1064 /* final store */
1065 "movl %%eax, %[v]\n\t"
1066 "2:\n\t"
1067 RSEQ_INJECT_ASM(6)
1068 /* teardown */
1069 "movl %[rseq_scratch2], %[len]\n\t"
1070 "movl %[rseq_scratch1], %[dst]\n\t"
1071 "movl %[rseq_scratch0], %[src]\n\t"
1072 RSEQ_ASM_DEFINE_ABORT(4,
1073 "movl %[rseq_scratch2], %[len]\n\t"
1074 "movl %[rseq_scratch1], %[dst]\n\t"
1075 "movl %[rseq_scratch0], %[src]\n\t",
1076 abort)
1077 RSEQ_ASM_DEFINE_CMPFAIL(5,
1078 "movl %[rseq_scratch2], %[len]\n\t"
1079 "movl %[rseq_scratch1], %[dst]\n\t"
1080 "movl %[rseq_scratch0], %[src]\n\t",
1081 cmpfail)
1082 #ifdef RSEQ_COMPARE_TWICE
1083 RSEQ_ASM_DEFINE_CMPFAIL(6,
1084 "movl %[rseq_scratch2], %[len]\n\t"
1085 "movl %[rseq_scratch1], %[dst]\n\t"
1086 "movl %[rseq_scratch0], %[src]\n\t",
1087 error1)
1088 RSEQ_ASM_DEFINE_CMPFAIL(7,
1089 "movl %[rseq_scratch2], %[len]\n\t"
1090 "movl %[rseq_scratch1], %[dst]\n\t"
1091 "movl %[rseq_scratch0], %[src]\n\t",
1092 error2)
1093 #endif
1094 : /* gcc asm goto does not allow outputs */
1095 : [cpu_id] "r" (cpu),
1096 [current_cpu_id] "m" (__rseq_abi.cpu_id),
1097 [rseq_cs] "m" (__rseq_abi.rseq_cs),
1098 /* final store input */
1099 [v] "m" (*v),
1100 [expect] "m" (expect),
1101 [newv] "m" (newv),
1102 /* try memcpy input */
1103 [dst] "r" (dst),
1104 [src] "r" (src),
1105 [len] "r" (len),
1106 [rseq_scratch0] "m" (rseq_scratch[0]),
1107 [rseq_scratch1] "m" (rseq_scratch[1]),
1108 [rseq_scratch2] "m" (rseq_scratch[2])
1109 : "memory", "cc", "eax"
1110 RSEQ_INJECT_CLOBBER
1111 : abort, cmpfail
1112 #ifdef RSEQ_COMPARE_TWICE
1113 , error1, error2
1114 #endif
1115 );
1116 return 0;
1117 abort:
1118 RSEQ_INJECT_FAILED
1119 return -1;
1120 cmpfail:
1121 return 1;
1122 #ifdef RSEQ_COMPARE_TWICE
1123 error1:
1124 rseq_bug("cpu_id comparison failed");
1125 error2:
1126 rseq_bug("expected value comparison failed");
1127 #endif
1128 }
1129
1130 #endif /* !RSEQ_SKIP_FASTPATH */
1131
1132 #endif
1133