1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-x86.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8 #include <stdint.h> 9 10 #define RSEQ_SIG 0x53053053 11 12 #ifdef __x86_64__ 13 14 #define rseq_smp_mb() \ 15 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") 16 #define rseq_smp_rmb() rseq_barrier() 17 #define rseq_smp_wmb() rseq_barrier() 18 19 #define rseq_smp_load_acquire(p) \ 20 __extension__ ({ \ 21 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 22 rseq_barrier(); \ 23 ____p1; \ 24 }) 25 26 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 27 28 #define rseq_smp_store_release(p, v) \ 29 do { \ 30 rseq_barrier(); \ 31 RSEQ_WRITE_ONCE(*p, v); \ 32 } while (0) 33 34 #ifdef RSEQ_SKIP_FASTPATH 35 #include "rseq-skip.h" 36 #else /* !RSEQ_SKIP_FASTPATH */ 37 38 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 39 start_ip, post_commit_offset, abort_ip) \ 40 ".pushsection __rseq_table, \"aw\"\n\t" \ 41 ".balign 32\n\t" \ 42 __rseq_str(label) ":\n\t" \ 43 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 44 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 45 ".popsection\n\t" 46 47 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 48 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 49 (post_commit_ip - start_ip), abort_ip) 50 51 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 52 RSEQ_INJECT_ASM(1) \ 53 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ 54 "movq %%rax, %[" __rseq_str(rseq_cs) "]\n\t" \ 55 __rseq_str(label) ":\n\t" 56 57 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 58 RSEQ_INJECT_ASM(2) \ 59 "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \ 60 "jnz " __rseq_str(label) "\n\t" 61 62 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 63 ".pushsection __rseq_failure, \"ax\"\n\t" \ 64 /* Disassembler-friendly signature: nopl <sig>(%rip). */\ 65 ".byte 0x0f, 0x1f, 0x05\n\t" \ 66 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 67 __rseq_str(label) ":\n\t" \ 68 teardown \ 69 "jmp %l[" __rseq_str(abort_label) "]\n\t" \ 70 ".popsection\n\t" 71 72 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 73 ".pushsection __rseq_failure, \"ax\"\n\t" \ 74 __rseq_str(label) ":\n\t" \ 75 teardown \ 76 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ 77 ".popsection\n\t" 78 79 static inline __attribute__((always_inline)) 80 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 81 { 82 RSEQ_INJECT_C(9) 83 84 __asm__ __volatile__ goto ( 85 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 86 /* Start rseq by storing table entry pointer into rseq_cs. */ 87 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 88 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 89 RSEQ_INJECT_ASM(3) 90 "cmpq %[v], %[expect]\n\t" 91 "jnz %l[cmpfail]\n\t" 92 RSEQ_INJECT_ASM(4) 93 #ifdef RSEQ_COMPARE_TWICE 94 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 95 "cmpq %[v], %[expect]\n\t" 96 "jnz %l[error2]\n\t" 97 #endif 98 /* final store */ 99 "movq %[newv], %[v]\n\t" 100 "2:\n\t" 101 RSEQ_INJECT_ASM(5) 102 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 103 : /* gcc asm goto does not allow outputs */ 104 : [cpu_id] "r" (cpu), 105 [current_cpu_id] "m" (__rseq_abi.cpu_id), 106 [rseq_cs] "m" (__rseq_abi.rseq_cs), 107 [v] "m" (*v), 108 [expect] "r" (expect), 109 [newv] "r" (newv) 110 : "memory", "cc", "rax" 111 RSEQ_INJECT_CLOBBER 112 : abort, cmpfail 113 #ifdef RSEQ_COMPARE_TWICE 114 , error1, error2 115 #endif 116 ); 117 return 0; 118 abort: 119 RSEQ_INJECT_FAILED 120 return -1; 121 cmpfail: 122 return 1; 123 #ifdef RSEQ_COMPARE_TWICE 124 error1: 125 rseq_bug("cpu_id comparison failed"); 126 error2: 127 rseq_bug("expected value comparison failed"); 128 #endif 129 } 130 131 /* 132 * Compare @v against @expectnot. When it does _not_ match, load @v 133 * into @load, and store the content of *@v + voffp into @v. 134 */ 135 static inline __attribute__((always_inline)) 136 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 137 off_t voffp, intptr_t *load, int cpu) 138 { 139 RSEQ_INJECT_C(9) 140 141 __asm__ __volatile__ goto ( 142 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 143 /* Start rseq by storing table entry pointer into rseq_cs. */ 144 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 145 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 146 RSEQ_INJECT_ASM(3) 147 "movq %[v], %%rbx\n\t" 148 "cmpq %%rbx, %[expectnot]\n\t" 149 "je %l[cmpfail]\n\t" 150 RSEQ_INJECT_ASM(4) 151 #ifdef RSEQ_COMPARE_TWICE 152 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 153 "movq %[v], %%rbx\n\t" 154 "cmpq %%rbx, %[expectnot]\n\t" 155 "je %l[error2]\n\t" 156 #endif 157 "movq %%rbx, %[load]\n\t" 158 "addq %[voffp], %%rbx\n\t" 159 "movq (%%rbx), %%rbx\n\t" 160 /* final store */ 161 "movq %%rbx, %[v]\n\t" 162 "2:\n\t" 163 RSEQ_INJECT_ASM(5) 164 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 165 : /* gcc asm goto does not allow outputs */ 166 : [cpu_id] "r" (cpu), 167 [current_cpu_id] "m" (__rseq_abi.cpu_id), 168 [rseq_cs] "m" (__rseq_abi.rseq_cs), 169 /* final store input */ 170 [v] "m" (*v), 171 [expectnot] "r" (expectnot), 172 [voffp] "er" (voffp), 173 [load] "m" (*load) 174 : "memory", "cc", "rax", "rbx" 175 RSEQ_INJECT_CLOBBER 176 : abort, cmpfail 177 #ifdef RSEQ_COMPARE_TWICE 178 , error1, error2 179 #endif 180 ); 181 return 0; 182 abort: 183 RSEQ_INJECT_FAILED 184 return -1; 185 cmpfail: 186 return 1; 187 #ifdef RSEQ_COMPARE_TWICE 188 error1: 189 rseq_bug("cpu_id comparison failed"); 190 error2: 191 rseq_bug("expected value comparison failed"); 192 #endif 193 } 194 195 static inline __attribute__((always_inline)) 196 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 197 { 198 RSEQ_INJECT_C(9) 199 200 __asm__ __volatile__ goto ( 201 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 202 /* Start rseq by storing table entry pointer into rseq_cs. */ 203 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 204 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 205 RSEQ_INJECT_ASM(3) 206 #ifdef RSEQ_COMPARE_TWICE 207 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 208 #endif 209 /* final store */ 210 "addq %[count], %[v]\n\t" 211 "2:\n\t" 212 RSEQ_INJECT_ASM(4) 213 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 214 : /* gcc asm goto does not allow outputs */ 215 : [cpu_id] "r" (cpu), 216 [current_cpu_id] "m" (__rseq_abi.cpu_id), 217 [rseq_cs] "m" (__rseq_abi.rseq_cs), 218 /* final store input */ 219 [v] "m" (*v), 220 [count] "er" (count) 221 : "memory", "cc", "rax" 222 RSEQ_INJECT_CLOBBER 223 : abort 224 #ifdef RSEQ_COMPARE_TWICE 225 , error1 226 #endif 227 ); 228 return 0; 229 abort: 230 RSEQ_INJECT_FAILED 231 return -1; 232 #ifdef RSEQ_COMPARE_TWICE 233 error1: 234 rseq_bug("cpu_id comparison failed"); 235 #endif 236 } 237 238 static inline __attribute__((always_inline)) 239 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 240 intptr_t *v2, intptr_t newv2, 241 intptr_t newv, int cpu) 242 { 243 RSEQ_INJECT_C(9) 244 245 __asm__ __volatile__ goto ( 246 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 247 /* Start rseq by storing table entry pointer into rseq_cs. */ 248 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 249 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 250 RSEQ_INJECT_ASM(3) 251 "cmpq %[v], %[expect]\n\t" 252 "jnz %l[cmpfail]\n\t" 253 RSEQ_INJECT_ASM(4) 254 #ifdef RSEQ_COMPARE_TWICE 255 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 256 "cmpq %[v], %[expect]\n\t" 257 "jnz %l[error2]\n\t" 258 #endif 259 /* try store */ 260 "movq %[newv2], %[v2]\n\t" 261 RSEQ_INJECT_ASM(5) 262 /* final store */ 263 "movq %[newv], %[v]\n\t" 264 "2:\n\t" 265 RSEQ_INJECT_ASM(6) 266 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 267 : /* gcc asm goto does not allow outputs */ 268 : [cpu_id] "r" (cpu), 269 [current_cpu_id] "m" (__rseq_abi.cpu_id), 270 [rseq_cs] "m" (__rseq_abi.rseq_cs), 271 /* try store input */ 272 [v2] "m" (*v2), 273 [newv2] "r" (newv2), 274 /* final store input */ 275 [v] "m" (*v), 276 [expect] "r" (expect), 277 [newv] "r" (newv) 278 : "memory", "cc", "rax" 279 RSEQ_INJECT_CLOBBER 280 : abort, cmpfail 281 #ifdef RSEQ_COMPARE_TWICE 282 , error1, error2 283 #endif 284 ); 285 return 0; 286 abort: 287 RSEQ_INJECT_FAILED 288 return -1; 289 cmpfail: 290 return 1; 291 #ifdef RSEQ_COMPARE_TWICE 292 error1: 293 rseq_bug("cpu_id comparison failed"); 294 error2: 295 rseq_bug("expected value comparison failed"); 296 #endif 297 } 298 299 /* x86-64 is TSO. */ 300 static inline __attribute__((always_inline)) 301 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 302 intptr_t *v2, intptr_t newv2, 303 intptr_t newv, int cpu) 304 { 305 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu); 306 } 307 308 static inline __attribute__((always_inline)) 309 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 310 intptr_t *v2, intptr_t expect2, 311 intptr_t newv, int cpu) 312 { 313 RSEQ_INJECT_C(9) 314 315 __asm__ __volatile__ goto ( 316 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 317 /* Start rseq by storing table entry pointer into rseq_cs. */ 318 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 319 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 320 RSEQ_INJECT_ASM(3) 321 "cmpq %[v], %[expect]\n\t" 322 "jnz %l[cmpfail]\n\t" 323 RSEQ_INJECT_ASM(4) 324 "cmpq %[v2], %[expect2]\n\t" 325 "jnz %l[cmpfail]\n\t" 326 RSEQ_INJECT_ASM(5) 327 #ifdef RSEQ_COMPARE_TWICE 328 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 329 "cmpq %[v], %[expect]\n\t" 330 "jnz %l[error2]\n\t" 331 "cmpq %[v2], %[expect2]\n\t" 332 "jnz %l[error3]\n\t" 333 #endif 334 /* final store */ 335 "movq %[newv], %[v]\n\t" 336 "2:\n\t" 337 RSEQ_INJECT_ASM(6) 338 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 339 : /* gcc asm goto does not allow outputs */ 340 : [cpu_id] "r" (cpu), 341 [current_cpu_id] "m" (__rseq_abi.cpu_id), 342 [rseq_cs] "m" (__rseq_abi.rseq_cs), 343 /* cmp2 input */ 344 [v2] "m" (*v2), 345 [expect2] "r" (expect2), 346 /* final store input */ 347 [v] "m" (*v), 348 [expect] "r" (expect), 349 [newv] "r" (newv) 350 : "memory", "cc", "rax" 351 RSEQ_INJECT_CLOBBER 352 : abort, cmpfail 353 #ifdef RSEQ_COMPARE_TWICE 354 , error1, error2, error3 355 #endif 356 ); 357 return 0; 358 abort: 359 RSEQ_INJECT_FAILED 360 return -1; 361 cmpfail: 362 return 1; 363 #ifdef RSEQ_COMPARE_TWICE 364 error1: 365 rseq_bug("cpu_id comparison failed"); 366 error2: 367 rseq_bug("1st expected value comparison failed"); 368 error3: 369 rseq_bug("2nd expected value comparison failed"); 370 #endif 371 } 372 373 static inline __attribute__((always_inline)) 374 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 375 void *dst, void *src, size_t len, 376 intptr_t newv, int cpu) 377 { 378 uint64_t rseq_scratch[3]; 379 380 RSEQ_INJECT_C(9) 381 382 __asm__ __volatile__ goto ( 383 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 384 "movq %[src], %[rseq_scratch0]\n\t" 385 "movq %[dst], %[rseq_scratch1]\n\t" 386 "movq %[len], %[rseq_scratch2]\n\t" 387 /* Start rseq by storing table entry pointer into rseq_cs. */ 388 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 389 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 390 RSEQ_INJECT_ASM(3) 391 "cmpq %[v], %[expect]\n\t" 392 "jnz 5f\n\t" 393 RSEQ_INJECT_ASM(4) 394 #ifdef RSEQ_COMPARE_TWICE 395 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 396 "cmpq %[v], %[expect]\n\t" 397 "jnz 7f\n\t" 398 #endif 399 /* try memcpy */ 400 "test %[len], %[len]\n\t" \ 401 "jz 333f\n\t" \ 402 "222:\n\t" \ 403 "movb (%[src]), %%al\n\t" \ 404 "movb %%al, (%[dst])\n\t" \ 405 "inc %[src]\n\t" \ 406 "inc %[dst]\n\t" \ 407 "dec %[len]\n\t" \ 408 "jnz 222b\n\t" \ 409 "333:\n\t" \ 410 RSEQ_INJECT_ASM(5) 411 /* final store */ 412 "movq %[newv], %[v]\n\t" 413 "2:\n\t" 414 RSEQ_INJECT_ASM(6) 415 /* teardown */ 416 "movq %[rseq_scratch2], %[len]\n\t" 417 "movq %[rseq_scratch1], %[dst]\n\t" 418 "movq %[rseq_scratch0], %[src]\n\t" 419 RSEQ_ASM_DEFINE_ABORT(4, 420 "movq %[rseq_scratch2], %[len]\n\t" 421 "movq %[rseq_scratch1], %[dst]\n\t" 422 "movq %[rseq_scratch0], %[src]\n\t", 423 abort) 424 RSEQ_ASM_DEFINE_CMPFAIL(5, 425 "movq %[rseq_scratch2], %[len]\n\t" 426 "movq %[rseq_scratch1], %[dst]\n\t" 427 "movq %[rseq_scratch0], %[src]\n\t", 428 cmpfail) 429 #ifdef RSEQ_COMPARE_TWICE 430 RSEQ_ASM_DEFINE_CMPFAIL(6, 431 "movq %[rseq_scratch2], %[len]\n\t" 432 "movq %[rseq_scratch1], %[dst]\n\t" 433 "movq %[rseq_scratch0], %[src]\n\t", 434 error1) 435 RSEQ_ASM_DEFINE_CMPFAIL(7, 436 "movq %[rseq_scratch2], %[len]\n\t" 437 "movq %[rseq_scratch1], %[dst]\n\t" 438 "movq %[rseq_scratch0], %[src]\n\t", 439 error2) 440 #endif 441 : /* gcc asm goto does not allow outputs */ 442 : [cpu_id] "r" (cpu), 443 [current_cpu_id] "m" (__rseq_abi.cpu_id), 444 [rseq_cs] "m" (__rseq_abi.rseq_cs), 445 /* final store input */ 446 [v] "m" (*v), 447 [expect] "r" (expect), 448 [newv] "r" (newv), 449 /* try memcpy input */ 450 [dst] "r" (dst), 451 [src] "r" (src), 452 [len] "r" (len), 453 [rseq_scratch0] "m" (rseq_scratch[0]), 454 [rseq_scratch1] "m" (rseq_scratch[1]), 455 [rseq_scratch2] "m" (rseq_scratch[2]) 456 : "memory", "cc", "rax" 457 RSEQ_INJECT_CLOBBER 458 : abort, cmpfail 459 #ifdef RSEQ_COMPARE_TWICE 460 , error1, error2 461 #endif 462 ); 463 return 0; 464 abort: 465 RSEQ_INJECT_FAILED 466 return -1; 467 cmpfail: 468 return 1; 469 #ifdef RSEQ_COMPARE_TWICE 470 error1: 471 rseq_bug("cpu_id comparison failed"); 472 error2: 473 rseq_bug("expected value comparison failed"); 474 #endif 475 } 476 477 /* x86-64 is TSO. */ 478 static inline __attribute__((always_inline)) 479 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 480 void *dst, void *src, size_t len, 481 intptr_t newv, int cpu) 482 { 483 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, 484 newv, cpu); 485 } 486 487 #endif /* !RSEQ_SKIP_FASTPATH */ 488 489 #elif __i386__ 490 491 #define rseq_smp_mb() \ 492 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 493 #define rseq_smp_rmb() \ 494 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 495 #define rseq_smp_wmb() \ 496 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 497 498 #define rseq_smp_load_acquire(p) \ 499 __extension__ ({ \ 500 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 501 rseq_smp_mb(); \ 502 ____p1; \ 503 }) 504 505 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 506 507 #define rseq_smp_store_release(p, v) \ 508 do { \ 509 rseq_smp_mb(); \ 510 RSEQ_WRITE_ONCE(*p, v); \ 511 } while (0) 512 513 #ifdef RSEQ_SKIP_FASTPATH 514 #include "rseq-skip.h" 515 #else /* !RSEQ_SKIP_FASTPATH */ 516 517 /* 518 * Use eax as scratch register and take memory operands as input to 519 * lessen register pressure. Especially needed when compiling in O0. 520 */ 521 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 522 start_ip, post_commit_offset, abort_ip) \ 523 ".pushsection __rseq_table, \"aw\"\n\t" \ 524 ".balign 32\n\t" \ 525 __rseq_str(label) ":\n\t" \ 526 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 527 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 528 ".popsection\n\t" 529 530 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 531 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 532 (post_commit_ip - start_ip), abort_ip) 533 534 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 535 RSEQ_INJECT_ASM(1) \ 536 "movl $" __rseq_str(cs_label) ", %[rseq_cs]\n\t" \ 537 __rseq_str(label) ":\n\t" 538 539 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 540 RSEQ_INJECT_ASM(2) \ 541 "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \ 542 "jnz " __rseq_str(label) "\n\t" 543 544 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 545 ".pushsection __rseq_failure, \"ax\"\n\t" \ 546 /* Disassembler-friendly signature: nopl <sig>. */ \ 547 ".byte 0x0f, 0x1f, 0x05\n\t" \ 548 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 549 __rseq_str(label) ":\n\t" \ 550 teardown \ 551 "jmp %l[" __rseq_str(abort_label) "]\n\t" \ 552 ".popsection\n\t" 553 554 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 555 ".pushsection __rseq_failure, \"ax\"\n\t" \ 556 __rseq_str(label) ":\n\t" \ 557 teardown \ 558 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ 559 ".popsection\n\t" 560 561 static inline __attribute__((always_inline)) 562 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 563 { 564 RSEQ_INJECT_C(9) 565 566 __asm__ __volatile__ goto ( 567 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 568 /* Start rseq by storing table entry pointer into rseq_cs. */ 569 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 570 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 571 RSEQ_INJECT_ASM(3) 572 "cmpl %[v], %[expect]\n\t" 573 "jnz %l[cmpfail]\n\t" 574 RSEQ_INJECT_ASM(4) 575 #ifdef RSEQ_COMPARE_TWICE 576 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 577 "cmpl %[v], %[expect]\n\t" 578 "jnz %l[error2]\n\t" 579 #endif 580 /* final store */ 581 "movl %[newv], %[v]\n\t" 582 "2:\n\t" 583 RSEQ_INJECT_ASM(5) 584 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 585 : /* gcc asm goto does not allow outputs */ 586 : [cpu_id] "r" (cpu), 587 [current_cpu_id] "m" (__rseq_abi.cpu_id), 588 [rseq_cs] "m" (__rseq_abi.rseq_cs), 589 [v] "m" (*v), 590 [expect] "r" (expect), 591 [newv] "r" (newv) 592 : "memory", "cc", "eax" 593 RSEQ_INJECT_CLOBBER 594 : abort, cmpfail 595 #ifdef RSEQ_COMPARE_TWICE 596 , error1, error2 597 #endif 598 ); 599 return 0; 600 abort: 601 RSEQ_INJECT_FAILED 602 return -1; 603 cmpfail: 604 return 1; 605 #ifdef RSEQ_COMPARE_TWICE 606 error1: 607 rseq_bug("cpu_id comparison failed"); 608 error2: 609 rseq_bug("expected value comparison failed"); 610 #endif 611 } 612 613 /* 614 * Compare @v against @expectnot. When it does _not_ match, load @v 615 * into @load, and store the content of *@v + voffp into @v. 616 */ 617 static inline __attribute__((always_inline)) 618 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 619 off_t voffp, intptr_t *load, int cpu) 620 { 621 RSEQ_INJECT_C(9) 622 623 __asm__ __volatile__ goto ( 624 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 625 /* Start rseq by storing table entry pointer into rseq_cs. */ 626 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 627 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 628 RSEQ_INJECT_ASM(3) 629 "movl %[v], %%ebx\n\t" 630 "cmpl %%ebx, %[expectnot]\n\t" 631 "je %l[cmpfail]\n\t" 632 RSEQ_INJECT_ASM(4) 633 #ifdef RSEQ_COMPARE_TWICE 634 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 635 "movl %[v], %%ebx\n\t" 636 "cmpl %%ebx, %[expectnot]\n\t" 637 "je %l[error2]\n\t" 638 #endif 639 "movl %%ebx, %[load]\n\t" 640 "addl %[voffp], %%ebx\n\t" 641 "movl (%%ebx), %%ebx\n\t" 642 /* final store */ 643 "movl %%ebx, %[v]\n\t" 644 "2:\n\t" 645 RSEQ_INJECT_ASM(5) 646 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 647 : /* gcc asm goto does not allow outputs */ 648 : [cpu_id] "r" (cpu), 649 [current_cpu_id] "m" (__rseq_abi.cpu_id), 650 [rseq_cs] "m" (__rseq_abi.rseq_cs), 651 /* final store input */ 652 [v] "m" (*v), 653 [expectnot] "r" (expectnot), 654 [voffp] "ir" (voffp), 655 [load] "m" (*load) 656 : "memory", "cc", "eax", "ebx" 657 RSEQ_INJECT_CLOBBER 658 : abort, cmpfail 659 #ifdef RSEQ_COMPARE_TWICE 660 , error1, error2 661 #endif 662 ); 663 return 0; 664 abort: 665 RSEQ_INJECT_FAILED 666 return -1; 667 cmpfail: 668 return 1; 669 #ifdef RSEQ_COMPARE_TWICE 670 error1: 671 rseq_bug("cpu_id comparison failed"); 672 error2: 673 rseq_bug("expected value comparison failed"); 674 #endif 675 } 676 677 static inline __attribute__((always_inline)) 678 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 679 { 680 RSEQ_INJECT_C(9) 681 682 __asm__ __volatile__ goto ( 683 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 684 /* Start rseq by storing table entry pointer into rseq_cs. */ 685 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 686 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 687 RSEQ_INJECT_ASM(3) 688 #ifdef RSEQ_COMPARE_TWICE 689 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 690 #endif 691 /* final store */ 692 "addl %[count], %[v]\n\t" 693 "2:\n\t" 694 RSEQ_INJECT_ASM(4) 695 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 696 : /* gcc asm goto does not allow outputs */ 697 : [cpu_id] "r" (cpu), 698 [current_cpu_id] "m" (__rseq_abi.cpu_id), 699 [rseq_cs] "m" (__rseq_abi.rseq_cs), 700 /* final store input */ 701 [v] "m" (*v), 702 [count] "ir" (count) 703 : "memory", "cc", "eax" 704 RSEQ_INJECT_CLOBBER 705 : abort 706 #ifdef RSEQ_COMPARE_TWICE 707 , error1 708 #endif 709 ); 710 return 0; 711 abort: 712 RSEQ_INJECT_FAILED 713 return -1; 714 #ifdef RSEQ_COMPARE_TWICE 715 error1: 716 rseq_bug("cpu_id comparison failed"); 717 #endif 718 } 719 720 static inline __attribute__((always_inline)) 721 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 722 intptr_t *v2, intptr_t newv2, 723 intptr_t newv, int cpu) 724 { 725 RSEQ_INJECT_C(9) 726 727 __asm__ __volatile__ goto ( 728 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 729 /* Start rseq by storing table entry pointer into rseq_cs. */ 730 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 731 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 732 RSEQ_INJECT_ASM(3) 733 "cmpl %[v], %[expect]\n\t" 734 "jnz %l[cmpfail]\n\t" 735 RSEQ_INJECT_ASM(4) 736 #ifdef RSEQ_COMPARE_TWICE 737 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 738 "cmpl %[v], %[expect]\n\t" 739 "jnz %l[error2]\n\t" 740 #endif 741 /* try store */ 742 "movl %[newv2], %%eax\n\t" 743 "movl %%eax, %[v2]\n\t" 744 RSEQ_INJECT_ASM(5) 745 /* final store */ 746 "movl %[newv], %[v]\n\t" 747 "2:\n\t" 748 RSEQ_INJECT_ASM(6) 749 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 750 : /* gcc asm goto does not allow outputs */ 751 : [cpu_id] "r" (cpu), 752 [current_cpu_id] "m" (__rseq_abi.cpu_id), 753 [rseq_cs] "m" (__rseq_abi.rseq_cs), 754 /* try store input */ 755 [v2] "m" (*v2), 756 [newv2] "m" (newv2), 757 /* final store input */ 758 [v] "m" (*v), 759 [expect] "r" (expect), 760 [newv] "r" (newv) 761 : "memory", "cc", "eax" 762 RSEQ_INJECT_CLOBBER 763 : abort, cmpfail 764 #ifdef RSEQ_COMPARE_TWICE 765 , error1, error2 766 #endif 767 ); 768 return 0; 769 abort: 770 RSEQ_INJECT_FAILED 771 return -1; 772 cmpfail: 773 return 1; 774 #ifdef RSEQ_COMPARE_TWICE 775 error1: 776 rseq_bug("cpu_id comparison failed"); 777 error2: 778 rseq_bug("expected value comparison failed"); 779 #endif 780 } 781 782 static inline __attribute__((always_inline)) 783 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 784 intptr_t *v2, intptr_t newv2, 785 intptr_t newv, int cpu) 786 { 787 RSEQ_INJECT_C(9) 788 789 __asm__ __volatile__ goto ( 790 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 791 /* Start rseq by storing table entry pointer into rseq_cs. */ 792 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 793 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 794 RSEQ_INJECT_ASM(3) 795 "movl %[expect], %%eax\n\t" 796 "cmpl %[v], %%eax\n\t" 797 "jnz %l[cmpfail]\n\t" 798 RSEQ_INJECT_ASM(4) 799 #ifdef RSEQ_COMPARE_TWICE 800 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 801 "movl %[expect], %%eax\n\t" 802 "cmpl %[v], %%eax\n\t" 803 "jnz %l[error2]\n\t" 804 #endif 805 /* try store */ 806 "movl %[newv2], %[v2]\n\t" 807 RSEQ_INJECT_ASM(5) 808 "lock; addl $0,-128(%%esp)\n\t" 809 /* final store */ 810 "movl %[newv], %[v]\n\t" 811 "2:\n\t" 812 RSEQ_INJECT_ASM(6) 813 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 814 : /* gcc asm goto does not allow outputs */ 815 : [cpu_id] "r" (cpu), 816 [current_cpu_id] "m" (__rseq_abi.cpu_id), 817 [rseq_cs] "m" (__rseq_abi.rseq_cs), 818 /* try store input */ 819 [v2] "m" (*v2), 820 [newv2] "r" (newv2), 821 /* final store input */ 822 [v] "m" (*v), 823 [expect] "m" (expect), 824 [newv] "r" (newv) 825 : "memory", "cc", "eax" 826 RSEQ_INJECT_CLOBBER 827 : abort, cmpfail 828 #ifdef RSEQ_COMPARE_TWICE 829 , error1, error2 830 #endif 831 ); 832 return 0; 833 abort: 834 RSEQ_INJECT_FAILED 835 return -1; 836 cmpfail: 837 return 1; 838 #ifdef RSEQ_COMPARE_TWICE 839 error1: 840 rseq_bug("cpu_id comparison failed"); 841 error2: 842 rseq_bug("expected value comparison failed"); 843 #endif 844 845 } 846 847 static inline __attribute__((always_inline)) 848 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 849 intptr_t *v2, intptr_t expect2, 850 intptr_t newv, int cpu) 851 { 852 RSEQ_INJECT_C(9) 853 854 __asm__ __volatile__ goto ( 855 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 856 /* Start rseq by storing table entry pointer into rseq_cs. */ 857 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 858 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 859 RSEQ_INJECT_ASM(3) 860 "cmpl %[v], %[expect]\n\t" 861 "jnz %l[cmpfail]\n\t" 862 RSEQ_INJECT_ASM(4) 863 "cmpl %[expect2], %[v2]\n\t" 864 "jnz %l[cmpfail]\n\t" 865 RSEQ_INJECT_ASM(5) 866 #ifdef RSEQ_COMPARE_TWICE 867 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 868 "cmpl %[v], %[expect]\n\t" 869 "jnz %l[error2]\n\t" 870 "cmpl %[expect2], %[v2]\n\t" 871 "jnz %l[error3]\n\t" 872 #endif 873 "movl %[newv], %%eax\n\t" 874 /* final store */ 875 "movl %%eax, %[v]\n\t" 876 "2:\n\t" 877 RSEQ_INJECT_ASM(6) 878 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 879 : /* gcc asm goto does not allow outputs */ 880 : [cpu_id] "r" (cpu), 881 [current_cpu_id] "m" (__rseq_abi.cpu_id), 882 [rseq_cs] "m" (__rseq_abi.rseq_cs), 883 /* cmp2 input */ 884 [v2] "m" (*v2), 885 [expect2] "r" (expect2), 886 /* final store input */ 887 [v] "m" (*v), 888 [expect] "r" (expect), 889 [newv] "m" (newv) 890 : "memory", "cc", "eax" 891 RSEQ_INJECT_CLOBBER 892 : abort, cmpfail 893 #ifdef RSEQ_COMPARE_TWICE 894 , error1, error2, error3 895 #endif 896 ); 897 return 0; 898 abort: 899 RSEQ_INJECT_FAILED 900 return -1; 901 cmpfail: 902 return 1; 903 #ifdef RSEQ_COMPARE_TWICE 904 error1: 905 rseq_bug("cpu_id comparison failed"); 906 error2: 907 rseq_bug("1st expected value comparison failed"); 908 error3: 909 rseq_bug("2nd expected value comparison failed"); 910 #endif 911 } 912 913 /* TODO: implement a faster memcpy. */ 914 static inline __attribute__((always_inline)) 915 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 916 void *dst, void *src, size_t len, 917 intptr_t newv, int cpu) 918 { 919 uint32_t rseq_scratch[3]; 920 921 RSEQ_INJECT_C(9) 922 923 __asm__ __volatile__ goto ( 924 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 925 "movl %[src], %[rseq_scratch0]\n\t" 926 "movl %[dst], %[rseq_scratch1]\n\t" 927 "movl %[len], %[rseq_scratch2]\n\t" 928 /* Start rseq by storing table entry pointer into rseq_cs. */ 929 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 930 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 931 RSEQ_INJECT_ASM(3) 932 "movl %[expect], %%eax\n\t" 933 "cmpl %%eax, %[v]\n\t" 934 "jnz 5f\n\t" 935 RSEQ_INJECT_ASM(4) 936 #ifdef RSEQ_COMPARE_TWICE 937 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 938 "movl %[expect], %%eax\n\t" 939 "cmpl %%eax, %[v]\n\t" 940 "jnz 7f\n\t" 941 #endif 942 /* try memcpy */ 943 "test %[len], %[len]\n\t" \ 944 "jz 333f\n\t" \ 945 "222:\n\t" \ 946 "movb (%[src]), %%al\n\t" \ 947 "movb %%al, (%[dst])\n\t" \ 948 "inc %[src]\n\t" \ 949 "inc %[dst]\n\t" \ 950 "dec %[len]\n\t" \ 951 "jnz 222b\n\t" \ 952 "333:\n\t" \ 953 RSEQ_INJECT_ASM(5) 954 "movl %[newv], %%eax\n\t" 955 /* final store */ 956 "movl %%eax, %[v]\n\t" 957 "2:\n\t" 958 RSEQ_INJECT_ASM(6) 959 /* teardown */ 960 "movl %[rseq_scratch2], %[len]\n\t" 961 "movl %[rseq_scratch1], %[dst]\n\t" 962 "movl %[rseq_scratch0], %[src]\n\t" 963 RSEQ_ASM_DEFINE_ABORT(4, 964 "movl %[rseq_scratch2], %[len]\n\t" 965 "movl %[rseq_scratch1], %[dst]\n\t" 966 "movl %[rseq_scratch0], %[src]\n\t", 967 abort) 968 RSEQ_ASM_DEFINE_CMPFAIL(5, 969 "movl %[rseq_scratch2], %[len]\n\t" 970 "movl %[rseq_scratch1], %[dst]\n\t" 971 "movl %[rseq_scratch0], %[src]\n\t", 972 cmpfail) 973 #ifdef RSEQ_COMPARE_TWICE 974 RSEQ_ASM_DEFINE_CMPFAIL(6, 975 "movl %[rseq_scratch2], %[len]\n\t" 976 "movl %[rseq_scratch1], %[dst]\n\t" 977 "movl %[rseq_scratch0], %[src]\n\t", 978 error1) 979 RSEQ_ASM_DEFINE_CMPFAIL(7, 980 "movl %[rseq_scratch2], %[len]\n\t" 981 "movl %[rseq_scratch1], %[dst]\n\t" 982 "movl %[rseq_scratch0], %[src]\n\t", 983 error2) 984 #endif 985 : /* gcc asm goto does not allow outputs */ 986 : [cpu_id] "r" (cpu), 987 [current_cpu_id] "m" (__rseq_abi.cpu_id), 988 [rseq_cs] "m" (__rseq_abi.rseq_cs), 989 /* final store input */ 990 [v] "m" (*v), 991 [expect] "m" (expect), 992 [newv] "m" (newv), 993 /* try memcpy input */ 994 [dst] "r" (dst), 995 [src] "r" (src), 996 [len] "r" (len), 997 [rseq_scratch0] "m" (rseq_scratch[0]), 998 [rseq_scratch1] "m" (rseq_scratch[1]), 999 [rseq_scratch2] "m" (rseq_scratch[2]) 1000 : "memory", "cc", "eax" 1001 RSEQ_INJECT_CLOBBER 1002 : abort, cmpfail 1003 #ifdef RSEQ_COMPARE_TWICE 1004 , error1, error2 1005 #endif 1006 ); 1007 return 0; 1008 abort: 1009 RSEQ_INJECT_FAILED 1010 return -1; 1011 cmpfail: 1012 return 1; 1013 #ifdef RSEQ_COMPARE_TWICE 1014 error1: 1015 rseq_bug("cpu_id comparison failed"); 1016 error2: 1017 rseq_bug("expected value comparison failed"); 1018 #endif 1019 } 1020 1021 /* TODO: implement a faster memcpy. */ 1022 static inline __attribute__((always_inline)) 1023 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 1024 void *dst, void *src, size_t len, 1025 intptr_t newv, int cpu) 1026 { 1027 uint32_t rseq_scratch[3]; 1028 1029 RSEQ_INJECT_C(9) 1030 1031 __asm__ __volatile__ goto ( 1032 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 1033 "movl %[src], %[rseq_scratch0]\n\t" 1034 "movl %[dst], %[rseq_scratch1]\n\t" 1035 "movl %[len], %[rseq_scratch2]\n\t" 1036 /* Start rseq by storing table entry pointer into rseq_cs. */ 1037 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 1038 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 1039 RSEQ_INJECT_ASM(3) 1040 "movl %[expect], %%eax\n\t" 1041 "cmpl %%eax, %[v]\n\t" 1042 "jnz 5f\n\t" 1043 RSEQ_INJECT_ASM(4) 1044 #ifdef RSEQ_COMPARE_TWICE 1045 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 1046 "movl %[expect], %%eax\n\t" 1047 "cmpl %%eax, %[v]\n\t" 1048 "jnz 7f\n\t" 1049 #endif 1050 /* try memcpy */ 1051 "test %[len], %[len]\n\t" \ 1052 "jz 333f\n\t" \ 1053 "222:\n\t" \ 1054 "movb (%[src]), %%al\n\t" \ 1055 "movb %%al, (%[dst])\n\t" \ 1056 "inc %[src]\n\t" \ 1057 "inc %[dst]\n\t" \ 1058 "dec %[len]\n\t" \ 1059 "jnz 222b\n\t" \ 1060 "333:\n\t" \ 1061 RSEQ_INJECT_ASM(5) 1062 "lock; addl $0,-128(%%esp)\n\t" 1063 "movl %[newv], %%eax\n\t" 1064 /* final store */ 1065 "movl %%eax, %[v]\n\t" 1066 "2:\n\t" 1067 RSEQ_INJECT_ASM(6) 1068 /* teardown */ 1069 "movl %[rseq_scratch2], %[len]\n\t" 1070 "movl %[rseq_scratch1], %[dst]\n\t" 1071 "movl %[rseq_scratch0], %[src]\n\t" 1072 RSEQ_ASM_DEFINE_ABORT(4, 1073 "movl %[rseq_scratch2], %[len]\n\t" 1074 "movl %[rseq_scratch1], %[dst]\n\t" 1075 "movl %[rseq_scratch0], %[src]\n\t", 1076 abort) 1077 RSEQ_ASM_DEFINE_CMPFAIL(5, 1078 "movl %[rseq_scratch2], %[len]\n\t" 1079 "movl %[rseq_scratch1], %[dst]\n\t" 1080 "movl %[rseq_scratch0], %[src]\n\t", 1081 cmpfail) 1082 #ifdef RSEQ_COMPARE_TWICE 1083 RSEQ_ASM_DEFINE_CMPFAIL(6, 1084 "movl %[rseq_scratch2], %[len]\n\t" 1085 "movl %[rseq_scratch1], %[dst]\n\t" 1086 "movl %[rseq_scratch0], %[src]\n\t", 1087 error1) 1088 RSEQ_ASM_DEFINE_CMPFAIL(7, 1089 "movl %[rseq_scratch2], %[len]\n\t" 1090 "movl %[rseq_scratch1], %[dst]\n\t" 1091 "movl %[rseq_scratch0], %[src]\n\t", 1092 error2) 1093 #endif 1094 : /* gcc asm goto does not allow outputs */ 1095 : [cpu_id] "r" (cpu), 1096 [current_cpu_id] "m" (__rseq_abi.cpu_id), 1097 [rseq_cs] "m" (__rseq_abi.rseq_cs), 1098 /* final store input */ 1099 [v] "m" (*v), 1100 [expect] "m" (expect), 1101 [newv] "m" (newv), 1102 /* try memcpy input */ 1103 [dst] "r" (dst), 1104 [src] "r" (src), 1105 [len] "r" (len), 1106 [rseq_scratch0] "m" (rseq_scratch[0]), 1107 [rseq_scratch1] "m" (rseq_scratch[1]), 1108 [rseq_scratch2] "m" (rseq_scratch[2]) 1109 : "memory", "cc", "eax" 1110 RSEQ_INJECT_CLOBBER 1111 : abort, cmpfail 1112 #ifdef RSEQ_COMPARE_TWICE 1113 , error1, error2 1114 #endif 1115 ); 1116 return 0; 1117 abort: 1118 RSEQ_INJECT_FAILED 1119 return -1; 1120 cmpfail: 1121 return 1; 1122 #ifdef RSEQ_COMPARE_TWICE 1123 error1: 1124 rseq_bug("cpu_id comparison failed"); 1125 error2: 1126 rseq_bug("expected value comparison failed"); 1127 #endif 1128 } 1129 1130 #endif /* !RSEQ_SKIP_FASTPATH */ 1131 1132 #endif 1133