1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-x86.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8 #include <stdint.h> 9 10 /* 11 * RSEQ_SIG is used with the following reserved undefined instructions, which 12 * trap in user-space: 13 * 14 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi 15 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi 16 */ 17 #define RSEQ_SIG 0x53053053 18 19 /* 20 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input 21 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi 22 * address through a "r" input operand. 23 */ 24 25 /* Offset of cpu_id and rseq_cs fields in struct rseq. */ 26 #define RSEQ_CPU_ID_OFFSET 4 27 #define RSEQ_CS_OFFSET 8 28 29 #ifdef __x86_64__ 30 31 #define rseq_smp_mb() \ 32 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") 33 #define rseq_smp_rmb() rseq_barrier() 34 #define rseq_smp_wmb() rseq_barrier() 35 36 #define rseq_smp_load_acquire(p) \ 37 __extension__ ({ \ 38 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 39 rseq_barrier(); \ 40 ____p1; \ 41 }) 42 43 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 44 45 #define rseq_smp_store_release(p, v) \ 46 do { \ 47 rseq_barrier(); \ 48 RSEQ_WRITE_ONCE(*p, v); \ 49 } while (0) 50 51 #ifdef RSEQ_SKIP_FASTPATH 52 #include "rseq-skip.h" 53 #else /* !RSEQ_SKIP_FASTPATH */ 54 55 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 56 start_ip, post_commit_offset, abort_ip) \ 57 ".pushsection __rseq_cs, \"aw\"\n\t" \ 58 ".balign 32\n\t" \ 59 __rseq_str(label) ":\n\t" \ 60 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 61 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 62 ".popsection\n\t" \ 63 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 64 ".quad " __rseq_str(label) "b\n\t" \ 65 ".popsection\n\t" 66 67 68 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 69 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 70 (post_commit_ip - start_ip), abort_ip) 71 72 /* 73 * Exit points of a rseq critical section consist of all instructions outside 74 * of the critical section where a critical section can either branch to or 75 * reach through the normal course of its execution. The abort IP and the 76 * post-commit IP are already part of the __rseq_cs section and should not be 77 * explicitly defined as additional exit points. Knowing all exit points is 78 * useful to assist debuggers stepping over the critical section. 79 */ 80 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 81 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 82 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ 83 ".popsection\n\t" 84 85 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 86 RSEQ_INJECT_ASM(1) \ 87 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ 88 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \ 89 __rseq_str(label) ":\n\t" 90 91 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 92 RSEQ_INJECT_ASM(2) \ 93 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ 94 "jnz " __rseq_str(label) "\n\t" 95 96 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 97 ".pushsection __rseq_failure, \"ax\"\n\t" \ 98 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \ 99 ".byte 0x0f, 0xb9, 0x3d\n\t" \ 100 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 101 __rseq_str(label) ":\n\t" \ 102 teardown \ 103 "jmp %l[" __rseq_str(abort_label) "]\n\t" \ 104 ".popsection\n\t" 105 106 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 107 ".pushsection __rseq_failure, \"ax\"\n\t" \ 108 __rseq_str(label) ":\n\t" \ 109 teardown \ 110 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ 111 ".popsection\n\t" 112 113 static inline __attribute__((always_inline)) 114 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 115 { 116 RSEQ_INJECT_C(9) 117 118 __asm__ __volatile__ goto ( 119 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 120 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 121 #ifdef RSEQ_COMPARE_TWICE 122 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 123 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 124 #endif 125 /* Start rseq by storing table entry pointer into rseq_cs. */ 126 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 127 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 128 RSEQ_INJECT_ASM(3) 129 "cmpq %[v], %[expect]\n\t" 130 "jnz %l[cmpfail]\n\t" 131 RSEQ_INJECT_ASM(4) 132 #ifdef RSEQ_COMPARE_TWICE 133 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 134 "cmpq %[v], %[expect]\n\t" 135 "jnz %l[error2]\n\t" 136 #endif 137 /* final store */ 138 "movq %[newv], %[v]\n\t" 139 "2:\n\t" 140 RSEQ_INJECT_ASM(5) 141 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 142 : /* gcc asm goto does not allow outputs */ 143 : [cpu_id] "r" (cpu), 144 [rseq_abi] "r" (&__rseq_abi), 145 [v] "m" (*v), 146 [expect] "r" (expect), 147 [newv] "r" (newv) 148 : "memory", "cc", "rax" 149 RSEQ_INJECT_CLOBBER 150 : abort, cmpfail 151 #ifdef RSEQ_COMPARE_TWICE 152 , error1, error2 153 #endif 154 ); 155 return 0; 156 abort: 157 RSEQ_INJECT_FAILED 158 return -1; 159 cmpfail: 160 return 1; 161 #ifdef RSEQ_COMPARE_TWICE 162 error1: 163 rseq_bug("cpu_id comparison failed"); 164 error2: 165 rseq_bug("expected value comparison failed"); 166 #endif 167 } 168 169 /* 170 * Compare @v against @expectnot. When it does _not_ match, load @v 171 * into @load, and store the content of *@v + voffp into @v. 172 */ 173 static inline __attribute__((always_inline)) 174 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 175 off_t voffp, intptr_t *load, int cpu) 176 { 177 RSEQ_INJECT_C(9) 178 179 __asm__ __volatile__ goto ( 180 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 181 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 182 #ifdef RSEQ_COMPARE_TWICE 183 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 184 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 185 #endif 186 /* Start rseq by storing table entry pointer into rseq_cs. */ 187 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 188 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 189 RSEQ_INJECT_ASM(3) 190 "movq %[v], %%rbx\n\t" 191 "cmpq %%rbx, %[expectnot]\n\t" 192 "je %l[cmpfail]\n\t" 193 RSEQ_INJECT_ASM(4) 194 #ifdef RSEQ_COMPARE_TWICE 195 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 196 "movq %[v], %%rbx\n\t" 197 "cmpq %%rbx, %[expectnot]\n\t" 198 "je %l[error2]\n\t" 199 #endif 200 "movq %%rbx, %[load]\n\t" 201 "addq %[voffp], %%rbx\n\t" 202 "movq (%%rbx), %%rbx\n\t" 203 /* final store */ 204 "movq %%rbx, %[v]\n\t" 205 "2:\n\t" 206 RSEQ_INJECT_ASM(5) 207 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 208 : /* gcc asm goto does not allow outputs */ 209 : [cpu_id] "r" (cpu), 210 [rseq_abi] "r" (&__rseq_abi), 211 /* final store input */ 212 [v] "m" (*v), 213 [expectnot] "r" (expectnot), 214 [voffp] "er" (voffp), 215 [load] "m" (*load) 216 : "memory", "cc", "rax", "rbx" 217 RSEQ_INJECT_CLOBBER 218 : abort, cmpfail 219 #ifdef RSEQ_COMPARE_TWICE 220 , error1, error2 221 #endif 222 ); 223 return 0; 224 abort: 225 RSEQ_INJECT_FAILED 226 return -1; 227 cmpfail: 228 return 1; 229 #ifdef RSEQ_COMPARE_TWICE 230 error1: 231 rseq_bug("cpu_id comparison failed"); 232 error2: 233 rseq_bug("expected value comparison failed"); 234 #endif 235 } 236 237 static inline __attribute__((always_inline)) 238 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 239 { 240 RSEQ_INJECT_C(9) 241 242 __asm__ __volatile__ goto ( 243 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 244 #ifdef RSEQ_COMPARE_TWICE 245 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 246 #endif 247 /* Start rseq by storing table entry pointer into rseq_cs. */ 248 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 249 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 250 RSEQ_INJECT_ASM(3) 251 #ifdef RSEQ_COMPARE_TWICE 252 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 253 #endif 254 /* final store */ 255 "addq %[count], %[v]\n\t" 256 "2:\n\t" 257 RSEQ_INJECT_ASM(4) 258 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 259 : /* gcc asm goto does not allow outputs */ 260 : [cpu_id] "r" (cpu), 261 [rseq_abi] "r" (&__rseq_abi), 262 /* final store input */ 263 [v] "m" (*v), 264 [count] "er" (count) 265 : "memory", "cc", "rax" 266 RSEQ_INJECT_CLOBBER 267 : abort 268 #ifdef RSEQ_COMPARE_TWICE 269 , error1 270 #endif 271 ); 272 return 0; 273 abort: 274 RSEQ_INJECT_FAILED 275 return -1; 276 #ifdef RSEQ_COMPARE_TWICE 277 error1: 278 rseq_bug("cpu_id comparison failed"); 279 #endif 280 } 281 282 static inline __attribute__((always_inline)) 283 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 284 intptr_t *v2, intptr_t newv2, 285 intptr_t newv, int cpu) 286 { 287 RSEQ_INJECT_C(9) 288 289 __asm__ __volatile__ goto ( 290 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 291 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 292 #ifdef RSEQ_COMPARE_TWICE 293 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 294 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 295 #endif 296 /* Start rseq by storing table entry pointer into rseq_cs. */ 297 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 298 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 299 RSEQ_INJECT_ASM(3) 300 "cmpq %[v], %[expect]\n\t" 301 "jnz %l[cmpfail]\n\t" 302 RSEQ_INJECT_ASM(4) 303 #ifdef RSEQ_COMPARE_TWICE 304 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 305 "cmpq %[v], %[expect]\n\t" 306 "jnz %l[error2]\n\t" 307 #endif 308 /* try store */ 309 "movq %[newv2], %[v2]\n\t" 310 RSEQ_INJECT_ASM(5) 311 /* final store */ 312 "movq %[newv], %[v]\n\t" 313 "2:\n\t" 314 RSEQ_INJECT_ASM(6) 315 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 316 : /* gcc asm goto does not allow outputs */ 317 : [cpu_id] "r" (cpu), 318 [rseq_abi] "r" (&__rseq_abi), 319 /* try store input */ 320 [v2] "m" (*v2), 321 [newv2] "r" (newv2), 322 /* final store input */ 323 [v] "m" (*v), 324 [expect] "r" (expect), 325 [newv] "r" (newv) 326 : "memory", "cc", "rax" 327 RSEQ_INJECT_CLOBBER 328 : abort, cmpfail 329 #ifdef RSEQ_COMPARE_TWICE 330 , error1, error2 331 #endif 332 ); 333 return 0; 334 abort: 335 RSEQ_INJECT_FAILED 336 return -1; 337 cmpfail: 338 return 1; 339 #ifdef RSEQ_COMPARE_TWICE 340 error1: 341 rseq_bug("cpu_id comparison failed"); 342 error2: 343 rseq_bug("expected value comparison failed"); 344 #endif 345 } 346 347 /* x86-64 is TSO. */ 348 static inline __attribute__((always_inline)) 349 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 350 intptr_t *v2, intptr_t newv2, 351 intptr_t newv, int cpu) 352 { 353 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu); 354 } 355 356 static inline __attribute__((always_inline)) 357 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 358 intptr_t *v2, intptr_t expect2, 359 intptr_t newv, int cpu) 360 { 361 RSEQ_INJECT_C(9) 362 363 __asm__ __volatile__ goto ( 364 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 365 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 366 #ifdef RSEQ_COMPARE_TWICE 367 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 368 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 369 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 370 #endif 371 /* Start rseq by storing table entry pointer into rseq_cs. */ 372 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 373 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 374 RSEQ_INJECT_ASM(3) 375 "cmpq %[v], %[expect]\n\t" 376 "jnz %l[cmpfail]\n\t" 377 RSEQ_INJECT_ASM(4) 378 "cmpq %[v2], %[expect2]\n\t" 379 "jnz %l[cmpfail]\n\t" 380 RSEQ_INJECT_ASM(5) 381 #ifdef RSEQ_COMPARE_TWICE 382 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 383 "cmpq %[v], %[expect]\n\t" 384 "jnz %l[error2]\n\t" 385 "cmpq %[v2], %[expect2]\n\t" 386 "jnz %l[error3]\n\t" 387 #endif 388 /* final store */ 389 "movq %[newv], %[v]\n\t" 390 "2:\n\t" 391 RSEQ_INJECT_ASM(6) 392 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 393 : /* gcc asm goto does not allow outputs */ 394 : [cpu_id] "r" (cpu), 395 [rseq_abi] "r" (&__rseq_abi), 396 /* cmp2 input */ 397 [v2] "m" (*v2), 398 [expect2] "r" (expect2), 399 /* final store input */ 400 [v] "m" (*v), 401 [expect] "r" (expect), 402 [newv] "r" (newv) 403 : "memory", "cc", "rax" 404 RSEQ_INJECT_CLOBBER 405 : abort, cmpfail 406 #ifdef RSEQ_COMPARE_TWICE 407 , error1, error2, error3 408 #endif 409 ); 410 return 0; 411 abort: 412 RSEQ_INJECT_FAILED 413 return -1; 414 cmpfail: 415 return 1; 416 #ifdef RSEQ_COMPARE_TWICE 417 error1: 418 rseq_bug("cpu_id comparison failed"); 419 error2: 420 rseq_bug("1st expected value comparison failed"); 421 error3: 422 rseq_bug("2nd expected value comparison failed"); 423 #endif 424 } 425 426 static inline __attribute__((always_inline)) 427 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 428 void *dst, void *src, size_t len, 429 intptr_t newv, int cpu) 430 { 431 uint64_t rseq_scratch[3]; 432 433 RSEQ_INJECT_C(9) 434 435 __asm__ __volatile__ goto ( 436 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 437 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 438 #ifdef RSEQ_COMPARE_TWICE 439 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 440 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 441 #endif 442 "movq %[src], %[rseq_scratch0]\n\t" 443 "movq %[dst], %[rseq_scratch1]\n\t" 444 "movq %[len], %[rseq_scratch2]\n\t" 445 /* Start rseq by storing table entry pointer into rseq_cs. */ 446 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 447 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 448 RSEQ_INJECT_ASM(3) 449 "cmpq %[v], %[expect]\n\t" 450 "jnz 5f\n\t" 451 RSEQ_INJECT_ASM(4) 452 #ifdef RSEQ_COMPARE_TWICE 453 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) 454 "cmpq %[v], %[expect]\n\t" 455 "jnz 7f\n\t" 456 #endif 457 /* try memcpy */ 458 "test %[len], %[len]\n\t" \ 459 "jz 333f\n\t" \ 460 "222:\n\t" \ 461 "movb (%[src]), %%al\n\t" \ 462 "movb %%al, (%[dst])\n\t" \ 463 "inc %[src]\n\t" \ 464 "inc %[dst]\n\t" \ 465 "dec %[len]\n\t" \ 466 "jnz 222b\n\t" \ 467 "333:\n\t" \ 468 RSEQ_INJECT_ASM(5) 469 /* final store */ 470 "movq %[newv], %[v]\n\t" 471 "2:\n\t" 472 RSEQ_INJECT_ASM(6) 473 /* teardown */ 474 "movq %[rseq_scratch2], %[len]\n\t" 475 "movq %[rseq_scratch1], %[dst]\n\t" 476 "movq %[rseq_scratch0], %[src]\n\t" 477 RSEQ_ASM_DEFINE_ABORT(4, 478 "movq %[rseq_scratch2], %[len]\n\t" 479 "movq %[rseq_scratch1], %[dst]\n\t" 480 "movq %[rseq_scratch0], %[src]\n\t", 481 abort) 482 RSEQ_ASM_DEFINE_CMPFAIL(5, 483 "movq %[rseq_scratch2], %[len]\n\t" 484 "movq %[rseq_scratch1], %[dst]\n\t" 485 "movq %[rseq_scratch0], %[src]\n\t", 486 cmpfail) 487 #ifdef RSEQ_COMPARE_TWICE 488 RSEQ_ASM_DEFINE_CMPFAIL(6, 489 "movq %[rseq_scratch2], %[len]\n\t" 490 "movq %[rseq_scratch1], %[dst]\n\t" 491 "movq %[rseq_scratch0], %[src]\n\t", 492 error1) 493 RSEQ_ASM_DEFINE_CMPFAIL(7, 494 "movq %[rseq_scratch2], %[len]\n\t" 495 "movq %[rseq_scratch1], %[dst]\n\t" 496 "movq %[rseq_scratch0], %[src]\n\t", 497 error2) 498 #endif 499 : /* gcc asm goto does not allow outputs */ 500 : [cpu_id] "r" (cpu), 501 [rseq_abi] "r" (&__rseq_abi), 502 /* final store input */ 503 [v] "m" (*v), 504 [expect] "r" (expect), 505 [newv] "r" (newv), 506 /* try memcpy input */ 507 [dst] "r" (dst), 508 [src] "r" (src), 509 [len] "r" (len), 510 [rseq_scratch0] "m" (rseq_scratch[0]), 511 [rseq_scratch1] "m" (rseq_scratch[1]), 512 [rseq_scratch2] "m" (rseq_scratch[2]) 513 : "memory", "cc", "rax" 514 RSEQ_INJECT_CLOBBER 515 : abort, cmpfail 516 #ifdef RSEQ_COMPARE_TWICE 517 , error1, error2 518 #endif 519 ); 520 return 0; 521 abort: 522 RSEQ_INJECT_FAILED 523 return -1; 524 cmpfail: 525 return 1; 526 #ifdef RSEQ_COMPARE_TWICE 527 error1: 528 rseq_bug("cpu_id comparison failed"); 529 error2: 530 rseq_bug("expected value comparison failed"); 531 #endif 532 } 533 534 /* x86-64 is TSO. */ 535 static inline __attribute__((always_inline)) 536 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 537 void *dst, void *src, size_t len, 538 intptr_t newv, int cpu) 539 { 540 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, 541 newv, cpu); 542 } 543 544 #endif /* !RSEQ_SKIP_FASTPATH */ 545 546 #elif __i386__ 547 548 #define rseq_smp_mb() \ 549 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 550 #define rseq_smp_rmb() \ 551 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 552 #define rseq_smp_wmb() \ 553 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 554 555 #define rseq_smp_load_acquire(p) \ 556 __extension__ ({ \ 557 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 558 rseq_smp_mb(); \ 559 ____p1; \ 560 }) 561 562 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 563 564 #define rseq_smp_store_release(p, v) \ 565 do { \ 566 rseq_smp_mb(); \ 567 RSEQ_WRITE_ONCE(*p, v); \ 568 } while (0) 569 570 #ifdef RSEQ_SKIP_FASTPATH 571 #include "rseq-skip.h" 572 #else /* !RSEQ_SKIP_FASTPATH */ 573 574 /* 575 * Use eax as scratch register and take memory operands as input to 576 * lessen register pressure. Especially needed when compiling in O0. 577 */ 578 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 579 start_ip, post_commit_offset, abort_ip) \ 580 ".pushsection __rseq_cs, \"aw\"\n\t" \ 581 ".balign 32\n\t" \ 582 __rseq_str(label) ":\n\t" \ 583 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 584 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 585 ".popsection\n\t" \ 586 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 587 ".long " __rseq_str(label) "b, 0x0\n\t" \ 588 ".popsection\n\t" 589 590 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 591 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 592 (post_commit_ip - start_ip), abort_ip) 593 594 /* 595 * Exit points of a rseq critical section consist of all instructions outside 596 * of the critical section where a critical section can either branch to or 597 * reach through the normal course of its execution. The abort IP and the 598 * post-commit IP are already part of the __rseq_cs section and should not be 599 * explicitly defined as additional exit points. Knowing all exit points is 600 * useful to assist debuggers stepping over the critical section. 601 */ 602 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 603 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 604 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ 605 ".popsection\n\t" 606 607 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 608 RSEQ_INJECT_ASM(1) \ 609 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \ 610 __rseq_str(label) ":\n\t" 611 612 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 613 RSEQ_INJECT_ASM(2) \ 614 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ 615 "jnz " __rseq_str(label) "\n\t" 616 617 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 618 ".pushsection __rseq_failure, \"ax\"\n\t" \ 619 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \ 620 ".byte 0x0f, 0xb9, 0x3d\n\t" \ 621 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 622 __rseq_str(label) ":\n\t" \ 623 teardown \ 624 "jmp %l[" __rseq_str(abort_label) "]\n\t" \ 625 ".popsection\n\t" 626 627 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 628 ".pushsection __rseq_failure, \"ax\"\n\t" \ 629 __rseq_str(label) ":\n\t" \ 630 teardown \ 631 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ 632 ".popsection\n\t" 633 634 static inline __attribute__((always_inline)) 635 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 636 { 637 RSEQ_INJECT_C(9) 638 639 __asm__ __volatile__ goto ( 640 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 641 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 642 #ifdef RSEQ_COMPARE_TWICE 643 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 644 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 645 #endif 646 /* Start rseq by storing table entry pointer into rseq_cs. */ 647 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 648 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 649 RSEQ_INJECT_ASM(3) 650 "cmpl %[v], %[expect]\n\t" 651 "jnz %l[cmpfail]\n\t" 652 RSEQ_INJECT_ASM(4) 653 #ifdef RSEQ_COMPARE_TWICE 654 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 655 "cmpl %[v], %[expect]\n\t" 656 "jnz %l[error2]\n\t" 657 #endif 658 /* final store */ 659 "movl %[newv], %[v]\n\t" 660 "2:\n\t" 661 RSEQ_INJECT_ASM(5) 662 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 663 : /* gcc asm goto does not allow outputs */ 664 : [cpu_id] "r" (cpu), 665 [rseq_abi] "r" (&__rseq_abi), 666 [v] "m" (*v), 667 [expect] "r" (expect), 668 [newv] "r" (newv) 669 : "memory", "cc", "eax" 670 RSEQ_INJECT_CLOBBER 671 : abort, cmpfail 672 #ifdef RSEQ_COMPARE_TWICE 673 , error1, error2 674 #endif 675 ); 676 return 0; 677 abort: 678 RSEQ_INJECT_FAILED 679 return -1; 680 cmpfail: 681 return 1; 682 #ifdef RSEQ_COMPARE_TWICE 683 error1: 684 rseq_bug("cpu_id comparison failed"); 685 error2: 686 rseq_bug("expected value comparison failed"); 687 #endif 688 } 689 690 /* 691 * Compare @v against @expectnot. When it does _not_ match, load @v 692 * into @load, and store the content of *@v + voffp into @v. 693 */ 694 static inline __attribute__((always_inline)) 695 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 696 off_t voffp, intptr_t *load, int cpu) 697 { 698 RSEQ_INJECT_C(9) 699 700 __asm__ __volatile__ goto ( 701 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 702 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 703 #ifdef RSEQ_COMPARE_TWICE 704 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 705 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 706 #endif 707 /* Start rseq by storing table entry pointer into rseq_cs. */ 708 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 709 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 710 RSEQ_INJECT_ASM(3) 711 "movl %[v], %%ebx\n\t" 712 "cmpl %%ebx, %[expectnot]\n\t" 713 "je %l[cmpfail]\n\t" 714 RSEQ_INJECT_ASM(4) 715 #ifdef RSEQ_COMPARE_TWICE 716 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 717 "movl %[v], %%ebx\n\t" 718 "cmpl %%ebx, %[expectnot]\n\t" 719 "je %l[error2]\n\t" 720 #endif 721 "movl %%ebx, %[load]\n\t" 722 "addl %[voffp], %%ebx\n\t" 723 "movl (%%ebx), %%ebx\n\t" 724 /* final store */ 725 "movl %%ebx, %[v]\n\t" 726 "2:\n\t" 727 RSEQ_INJECT_ASM(5) 728 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 729 : /* gcc asm goto does not allow outputs */ 730 : [cpu_id] "r" (cpu), 731 [rseq_abi] "r" (&__rseq_abi), 732 /* final store input */ 733 [v] "m" (*v), 734 [expectnot] "r" (expectnot), 735 [voffp] "ir" (voffp), 736 [load] "m" (*load) 737 : "memory", "cc", "eax", "ebx" 738 RSEQ_INJECT_CLOBBER 739 : abort, cmpfail 740 #ifdef RSEQ_COMPARE_TWICE 741 , error1, error2 742 #endif 743 ); 744 return 0; 745 abort: 746 RSEQ_INJECT_FAILED 747 return -1; 748 cmpfail: 749 return 1; 750 #ifdef RSEQ_COMPARE_TWICE 751 error1: 752 rseq_bug("cpu_id comparison failed"); 753 error2: 754 rseq_bug("expected value comparison failed"); 755 #endif 756 } 757 758 static inline __attribute__((always_inline)) 759 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 760 { 761 RSEQ_INJECT_C(9) 762 763 __asm__ __volatile__ goto ( 764 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 765 #ifdef RSEQ_COMPARE_TWICE 766 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 767 #endif 768 /* Start rseq by storing table entry pointer into rseq_cs. */ 769 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 770 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 771 RSEQ_INJECT_ASM(3) 772 #ifdef RSEQ_COMPARE_TWICE 773 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 774 #endif 775 /* final store */ 776 "addl %[count], %[v]\n\t" 777 "2:\n\t" 778 RSEQ_INJECT_ASM(4) 779 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 780 : /* gcc asm goto does not allow outputs */ 781 : [cpu_id] "r" (cpu), 782 [rseq_abi] "r" (&__rseq_abi), 783 /* final store input */ 784 [v] "m" (*v), 785 [count] "ir" (count) 786 : "memory", "cc", "eax" 787 RSEQ_INJECT_CLOBBER 788 : abort 789 #ifdef RSEQ_COMPARE_TWICE 790 , error1 791 #endif 792 ); 793 return 0; 794 abort: 795 RSEQ_INJECT_FAILED 796 return -1; 797 #ifdef RSEQ_COMPARE_TWICE 798 error1: 799 rseq_bug("cpu_id comparison failed"); 800 #endif 801 } 802 803 static inline __attribute__((always_inline)) 804 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 805 intptr_t *v2, intptr_t newv2, 806 intptr_t newv, int cpu) 807 { 808 RSEQ_INJECT_C(9) 809 810 __asm__ __volatile__ goto ( 811 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 812 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 813 #ifdef RSEQ_COMPARE_TWICE 814 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 815 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 816 #endif 817 /* Start rseq by storing table entry pointer into rseq_cs. */ 818 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 819 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 820 RSEQ_INJECT_ASM(3) 821 "cmpl %[v], %[expect]\n\t" 822 "jnz %l[cmpfail]\n\t" 823 RSEQ_INJECT_ASM(4) 824 #ifdef RSEQ_COMPARE_TWICE 825 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 826 "cmpl %[v], %[expect]\n\t" 827 "jnz %l[error2]\n\t" 828 #endif 829 /* try store */ 830 "movl %[newv2], %%eax\n\t" 831 "movl %%eax, %[v2]\n\t" 832 RSEQ_INJECT_ASM(5) 833 /* final store */ 834 "movl %[newv], %[v]\n\t" 835 "2:\n\t" 836 RSEQ_INJECT_ASM(6) 837 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 838 : /* gcc asm goto does not allow outputs */ 839 : [cpu_id] "r" (cpu), 840 [rseq_abi] "r" (&__rseq_abi), 841 /* try store input */ 842 [v2] "m" (*v2), 843 [newv2] "m" (newv2), 844 /* final store input */ 845 [v] "m" (*v), 846 [expect] "r" (expect), 847 [newv] "r" (newv) 848 : "memory", "cc", "eax" 849 RSEQ_INJECT_CLOBBER 850 : abort, cmpfail 851 #ifdef RSEQ_COMPARE_TWICE 852 , error1, error2 853 #endif 854 ); 855 return 0; 856 abort: 857 RSEQ_INJECT_FAILED 858 return -1; 859 cmpfail: 860 return 1; 861 #ifdef RSEQ_COMPARE_TWICE 862 error1: 863 rseq_bug("cpu_id comparison failed"); 864 error2: 865 rseq_bug("expected value comparison failed"); 866 #endif 867 } 868 869 static inline __attribute__((always_inline)) 870 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 871 intptr_t *v2, intptr_t newv2, 872 intptr_t newv, int cpu) 873 { 874 RSEQ_INJECT_C(9) 875 876 __asm__ __volatile__ goto ( 877 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 878 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 879 #ifdef RSEQ_COMPARE_TWICE 880 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 881 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 882 #endif 883 /* Start rseq by storing table entry pointer into rseq_cs. */ 884 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 885 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 886 RSEQ_INJECT_ASM(3) 887 "movl %[expect], %%eax\n\t" 888 "cmpl %[v], %%eax\n\t" 889 "jnz %l[cmpfail]\n\t" 890 RSEQ_INJECT_ASM(4) 891 #ifdef RSEQ_COMPARE_TWICE 892 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 893 "movl %[expect], %%eax\n\t" 894 "cmpl %[v], %%eax\n\t" 895 "jnz %l[error2]\n\t" 896 #endif 897 /* try store */ 898 "movl %[newv2], %[v2]\n\t" 899 RSEQ_INJECT_ASM(5) 900 "lock; addl $0,-128(%%esp)\n\t" 901 /* final store */ 902 "movl %[newv], %[v]\n\t" 903 "2:\n\t" 904 RSEQ_INJECT_ASM(6) 905 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 906 : /* gcc asm goto does not allow outputs */ 907 : [cpu_id] "r" (cpu), 908 [rseq_abi] "r" (&__rseq_abi), 909 /* try store input */ 910 [v2] "m" (*v2), 911 [newv2] "r" (newv2), 912 /* final store input */ 913 [v] "m" (*v), 914 [expect] "m" (expect), 915 [newv] "r" (newv) 916 : "memory", "cc", "eax" 917 RSEQ_INJECT_CLOBBER 918 : abort, cmpfail 919 #ifdef RSEQ_COMPARE_TWICE 920 , error1, error2 921 #endif 922 ); 923 return 0; 924 abort: 925 RSEQ_INJECT_FAILED 926 return -1; 927 cmpfail: 928 return 1; 929 #ifdef RSEQ_COMPARE_TWICE 930 error1: 931 rseq_bug("cpu_id comparison failed"); 932 error2: 933 rseq_bug("expected value comparison failed"); 934 #endif 935 936 } 937 938 static inline __attribute__((always_inline)) 939 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 940 intptr_t *v2, intptr_t expect2, 941 intptr_t newv, int cpu) 942 { 943 RSEQ_INJECT_C(9) 944 945 __asm__ __volatile__ goto ( 946 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 947 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 948 #ifdef RSEQ_COMPARE_TWICE 949 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 950 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 951 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 952 #endif 953 /* Start rseq by storing table entry pointer into rseq_cs. */ 954 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 955 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 956 RSEQ_INJECT_ASM(3) 957 "cmpl %[v], %[expect]\n\t" 958 "jnz %l[cmpfail]\n\t" 959 RSEQ_INJECT_ASM(4) 960 "cmpl %[expect2], %[v2]\n\t" 961 "jnz %l[cmpfail]\n\t" 962 RSEQ_INJECT_ASM(5) 963 #ifdef RSEQ_COMPARE_TWICE 964 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 965 "cmpl %[v], %[expect]\n\t" 966 "jnz %l[error2]\n\t" 967 "cmpl %[expect2], %[v2]\n\t" 968 "jnz %l[error3]\n\t" 969 #endif 970 "movl %[newv], %%eax\n\t" 971 /* final store */ 972 "movl %%eax, %[v]\n\t" 973 "2:\n\t" 974 RSEQ_INJECT_ASM(6) 975 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 976 : /* gcc asm goto does not allow outputs */ 977 : [cpu_id] "r" (cpu), 978 [rseq_abi] "r" (&__rseq_abi), 979 /* cmp2 input */ 980 [v2] "m" (*v2), 981 [expect2] "r" (expect2), 982 /* final store input */ 983 [v] "m" (*v), 984 [expect] "r" (expect), 985 [newv] "m" (newv) 986 : "memory", "cc", "eax" 987 RSEQ_INJECT_CLOBBER 988 : abort, cmpfail 989 #ifdef RSEQ_COMPARE_TWICE 990 , error1, error2, error3 991 #endif 992 ); 993 return 0; 994 abort: 995 RSEQ_INJECT_FAILED 996 return -1; 997 cmpfail: 998 return 1; 999 #ifdef RSEQ_COMPARE_TWICE 1000 error1: 1001 rseq_bug("cpu_id comparison failed"); 1002 error2: 1003 rseq_bug("1st expected value comparison failed"); 1004 error3: 1005 rseq_bug("2nd expected value comparison failed"); 1006 #endif 1007 } 1008 1009 /* TODO: implement a faster memcpy. */ 1010 static inline __attribute__((always_inline)) 1011 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 1012 void *dst, void *src, size_t len, 1013 intptr_t newv, int cpu) 1014 { 1015 uint32_t rseq_scratch[3]; 1016 1017 RSEQ_INJECT_C(9) 1018 1019 __asm__ __volatile__ goto ( 1020 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 1021 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 1022 #ifdef RSEQ_COMPARE_TWICE 1023 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 1024 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 1025 #endif 1026 "movl %[src], %[rseq_scratch0]\n\t" 1027 "movl %[dst], %[rseq_scratch1]\n\t" 1028 "movl %[len], %[rseq_scratch2]\n\t" 1029 /* Start rseq by storing table entry pointer into rseq_cs. */ 1030 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 1031 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 1032 RSEQ_INJECT_ASM(3) 1033 "movl %[expect], %%eax\n\t" 1034 "cmpl %%eax, %[v]\n\t" 1035 "jnz 5f\n\t" 1036 RSEQ_INJECT_ASM(4) 1037 #ifdef RSEQ_COMPARE_TWICE 1038 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) 1039 "movl %[expect], %%eax\n\t" 1040 "cmpl %%eax, %[v]\n\t" 1041 "jnz 7f\n\t" 1042 #endif 1043 /* try memcpy */ 1044 "test %[len], %[len]\n\t" \ 1045 "jz 333f\n\t" \ 1046 "222:\n\t" \ 1047 "movb (%[src]), %%al\n\t" \ 1048 "movb %%al, (%[dst])\n\t" \ 1049 "inc %[src]\n\t" \ 1050 "inc %[dst]\n\t" \ 1051 "dec %[len]\n\t" \ 1052 "jnz 222b\n\t" \ 1053 "333:\n\t" \ 1054 RSEQ_INJECT_ASM(5) 1055 "movl %[newv], %%eax\n\t" 1056 /* final store */ 1057 "movl %%eax, %[v]\n\t" 1058 "2:\n\t" 1059 RSEQ_INJECT_ASM(6) 1060 /* teardown */ 1061 "movl %[rseq_scratch2], %[len]\n\t" 1062 "movl %[rseq_scratch1], %[dst]\n\t" 1063 "movl %[rseq_scratch0], %[src]\n\t" 1064 RSEQ_ASM_DEFINE_ABORT(4, 1065 "movl %[rseq_scratch2], %[len]\n\t" 1066 "movl %[rseq_scratch1], %[dst]\n\t" 1067 "movl %[rseq_scratch0], %[src]\n\t", 1068 abort) 1069 RSEQ_ASM_DEFINE_CMPFAIL(5, 1070 "movl %[rseq_scratch2], %[len]\n\t" 1071 "movl %[rseq_scratch1], %[dst]\n\t" 1072 "movl %[rseq_scratch0], %[src]\n\t", 1073 cmpfail) 1074 #ifdef RSEQ_COMPARE_TWICE 1075 RSEQ_ASM_DEFINE_CMPFAIL(6, 1076 "movl %[rseq_scratch2], %[len]\n\t" 1077 "movl %[rseq_scratch1], %[dst]\n\t" 1078 "movl %[rseq_scratch0], %[src]\n\t", 1079 error1) 1080 RSEQ_ASM_DEFINE_CMPFAIL(7, 1081 "movl %[rseq_scratch2], %[len]\n\t" 1082 "movl %[rseq_scratch1], %[dst]\n\t" 1083 "movl %[rseq_scratch0], %[src]\n\t", 1084 error2) 1085 #endif 1086 : /* gcc asm goto does not allow outputs */ 1087 : [cpu_id] "r" (cpu), 1088 [rseq_abi] "r" (&__rseq_abi), 1089 /* final store input */ 1090 [v] "m" (*v), 1091 [expect] "m" (expect), 1092 [newv] "m" (newv), 1093 /* try memcpy input */ 1094 [dst] "r" (dst), 1095 [src] "r" (src), 1096 [len] "r" (len), 1097 [rseq_scratch0] "m" (rseq_scratch[0]), 1098 [rseq_scratch1] "m" (rseq_scratch[1]), 1099 [rseq_scratch2] "m" (rseq_scratch[2]) 1100 : "memory", "cc", "eax" 1101 RSEQ_INJECT_CLOBBER 1102 : abort, cmpfail 1103 #ifdef RSEQ_COMPARE_TWICE 1104 , error1, error2 1105 #endif 1106 ); 1107 return 0; 1108 abort: 1109 RSEQ_INJECT_FAILED 1110 return -1; 1111 cmpfail: 1112 return 1; 1113 #ifdef RSEQ_COMPARE_TWICE 1114 error1: 1115 rseq_bug("cpu_id comparison failed"); 1116 error2: 1117 rseq_bug("expected value comparison failed"); 1118 #endif 1119 } 1120 1121 /* TODO: implement a faster memcpy. */ 1122 static inline __attribute__((always_inline)) 1123 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 1124 void *dst, void *src, size_t len, 1125 intptr_t newv, int cpu) 1126 { 1127 uint32_t rseq_scratch[3]; 1128 1129 RSEQ_INJECT_C(9) 1130 1131 __asm__ __volatile__ goto ( 1132 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 1133 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 1134 #ifdef RSEQ_COMPARE_TWICE 1135 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 1136 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 1137 #endif 1138 "movl %[src], %[rseq_scratch0]\n\t" 1139 "movl %[dst], %[rseq_scratch1]\n\t" 1140 "movl %[len], %[rseq_scratch2]\n\t" 1141 /* Start rseq by storing table entry pointer into rseq_cs. */ 1142 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 1143 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 1144 RSEQ_INJECT_ASM(3) 1145 "movl %[expect], %%eax\n\t" 1146 "cmpl %%eax, %[v]\n\t" 1147 "jnz 5f\n\t" 1148 RSEQ_INJECT_ASM(4) 1149 #ifdef RSEQ_COMPARE_TWICE 1150 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) 1151 "movl %[expect], %%eax\n\t" 1152 "cmpl %%eax, %[v]\n\t" 1153 "jnz 7f\n\t" 1154 #endif 1155 /* try memcpy */ 1156 "test %[len], %[len]\n\t" \ 1157 "jz 333f\n\t" \ 1158 "222:\n\t" \ 1159 "movb (%[src]), %%al\n\t" \ 1160 "movb %%al, (%[dst])\n\t" \ 1161 "inc %[src]\n\t" \ 1162 "inc %[dst]\n\t" \ 1163 "dec %[len]\n\t" \ 1164 "jnz 222b\n\t" \ 1165 "333:\n\t" \ 1166 RSEQ_INJECT_ASM(5) 1167 "lock; addl $0,-128(%%esp)\n\t" 1168 "movl %[newv], %%eax\n\t" 1169 /* final store */ 1170 "movl %%eax, %[v]\n\t" 1171 "2:\n\t" 1172 RSEQ_INJECT_ASM(6) 1173 /* teardown */ 1174 "movl %[rseq_scratch2], %[len]\n\t" 1175 "movl %[rseq_scratch1], %[dst]\n\t" 1176 "movl %[rseq_scratch0], %[src]\n\t" 1177 RSEQ_ASM_DEFINE_ABORT(4, 1178 "movl %[rseq_scratch2], %[len]\n\t" 1179 "movl %[rseq_scratch1], %[dst]\n\t" 1180 "movl %[rseq_scratch0], %[src]\n\t", 1181 abort) 1182 RSEQ_ASM_DEFINE_CMPFAIL(5, 1183 "movl %[rseq_scratch2], %[len]\n\t" 1184 "movl %[rseq_scratch1], %[dst]\n\t" 1185 "movl %[rseq_scratch0], %[src]\n\t", 1186 cmpfail) 1187 #ifdef RSEQ_COMPARE_TWICE 1188 RSEQ_ASM_DEFINE_CMPFAIL(6, 1189 "movl %[rseq_scratch2], %[len]\n\t" 1190 "movl %[rseq_scratch1], %[dst]\n\t" 1191 "movl %[rseq_scratch0], %[src]\n\t", 1192 error1) 1193 RSEQ_ASM_DEFINE_CMPFAIL(7, 1194 "movl %[rseq_scratch2], %[len]\n\t" 1195 "movl %[rseq_scratch1], %[dst]\n\t" 1196 "movl %[rseq_scratch0], %[src]\n\t", 1197 error2) 1198 #endif 1199 : /* gcc asm goto does not allow outputs */ 1200 : [cpu_id] "r" (cpu), 1201 [rseq_abi] "r" (&__rseq_abi), 1202 /* final store input */ 1203 [v] "m" (*v), 1204 [expect] "m" (expect), 1205 [newv] "m" (newv), 1206 /* try memcpy input */ 1207 [dst] "r" (dst), 1208 [src] "r" (src), 1209 [len] "r" (len), 1210 [rseq_scratch0] "m" (rseq_scratch[0]), 1211 [rseq_scratch1] "m" (rseq_scratch[1]), 1212 [rseq_scratch2] "m" (rseq_scratch[2]) 1213 : "memory", "cc", "eax" 1214 RSEQ_INJECT_CLOBBER 1215 : abort, cmpfail 1216 #ifdef RSEQ_COMPARE_TWICE 1217 , error1, error2 1218 #endif 1219 ); 1220 return 0; 1221 abort: 1222 RSEQ_INJECT_FAILED 1223 return -1; 1224 cmpfail: 1225 return 1; 1226 #ifdef RSEQ_COMPARE_TWICE 1227 error1: 1228 rseq_bug("cpu_id comparison failed"); 1229 error2: 1230 rseq_bug("expected value comparison failed"); 1231 #endif 1232 } 1233 1234 #endif /* !RSEQ_SKIP_FASTPATH */ 1235 1236 #endif 1237