1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 3 /* 4 * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the 5 * access-register mode nor the linkage stack this instruction will always 6 * cause a special-operation exception (the trap-enabled bit in the DUCT 7 * is and will stay 0). The instruction pattern is 8 * b2 ff 0f ff trap4 4095(%r0) 9 */ 10 #define RSEQ_SIG 0xB2FF0FFF 11 12 #define rseq_smp_mb() __asm__ __volatile__ ("bcr 15,0" ::: "memory") 13 #define rseq_smp_rmb() rseq_smp_mb() 14 #define rseq_smp_wmb() rseq_smp_mb() 15 16 #define rseq_smp_load_acquire(p) \ 17 __extension__ ({ \ 18 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 19 rseq_barrier(); \ 20 ____p1; \ 21 }) 22 23 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 24 25 #define rseq_smp_store_release(p, v) \ 26 do { \ 27 rseq_barrier(); \ 28 RSEQ_WRITE_ONCE(*p, v); \ 29 } while (0) 30 31 #ifdef RSEQ_SKIP_FASTPATH 32 #include "rseq-skip.h" 33 #else /* !RSEQ_SKIP_FASTPATH */ 34 35 #ifdef __s390x__ 36 37 #define LONG_L "lg" 38 #define LONG_S "stg" 39 #define LONG_LT_R "ltgr" 40 #define LONG_CMP "cg" 41 #define LONG_CMP_R "cgr" 42 #define LONG_ADDI "aghi" 43 #define LONG_ADD_R "agr" 44 45 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 46 start_ip, post_commit_offset, abort_ip) \ 47 ".pushsection __rseq_cs, \"aw\"\n\t" \ 48 ".balign 32\n\t" \ 49 __rseq_str(label) ":\n\t" \ 50 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 51 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 52 ".popsection\n\t" \ 53 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 54 ".quad " __rseq_str(label) "b\n\t" \ 55 ".popsection\n\t" 56 57 /* 58 * Exit points of a rseq critical section consist of all instructions outside 59 * of the critical section where a critical section can either branch to or 60 * reach through the normal course of its execution. The abort IP and the 61 * post-commit IP are already part of the __rseq_cs section and should not be 62 * explicitly defined as additional exit points. Knowing all exit points is 63 * useful to assist debuggers stepping over the critical section. 64 */ 65 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 66 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 67 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ 68 ".popsection\n\t" 69 70 #elif __s390__ 71 72 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 73 start_ip, post_commit_offset, abort_ip) \ 74 ".pushsection __rseq_cs, \"aw\"\n\t" \ 75 ".balign 32\n\t" \ 76 __rseq_str(label) ":\n\t" \ 77 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 78 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \ 79 ".popsection\n\t" \ 80 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 81 ".long 0x0, " __rseq_str(label) "b\n\t" \ 82 ".popsection\n\t" 83 84 /* 85 * Exit points of a rseq critical section consist of all instructions outside 86 * of the critical section where a critical section can either branch to or 87 * reach through the normal course of its execution. The abort IP and the 88 * post-commit IP are already part of the __rseq_cs section and should not be 89 * explicitly defined as additional exit points. Knowing all exit points is 90 * useful to assist debuggers stepping over the critical section. 91 */ 92 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 93 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 94 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \ 95 ".popsection\n\t" 96 97 #define LONG_L "l" 98 #define LONG_S "st" 99 #define LONG_LT_R "ltr" 100 #define LONG_CMP "c" 101 #define LONG_CMP_R "cr" 102 #define LONG_ADDI "ahi" 103 #define LONG_ADD_R "ar" 104 105 #endif 106 107 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 108 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 109 (post_commit_ip - start_ip), abort_ip) 110 111 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 112 RSEQ_INJECT_ASM(1) \ 113 "larl %%r0, " __rseq_str(cs_label) "\n\t" \ 114 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \ 115 __rseq_str(label) ":\n\t" 116 117 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 118 RSEQ_INJECT_ASM(2) \ 119 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \ 120 "jnz " __rseq_str(label) "\n\t" 121 122 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 123 ".pushsection __rseq_failure, \"ax\"\n\t" \ 124 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 125 __rseq_str(label) ":\n\t" \ 126 teardown \ 127 "jg %l[" __rseq_str(abort_label) "]\n\t" \ 128 ".popsection\n\t" 129 130 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 131 ".pushsection __rseq_failure, \"ax\"\n\t" \ 132 __rseq_str(label) ":\n\t" \ 133 teardown \ 134 "jg %l[" __rseq_str(cmpfail_label) "]\n\t" \ 135 ".popsection\n\t" 136 137 static inline __attribute__((always_inline)) 138 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 139 { 140 RSEQ_INJECT_C(9) 141 142 __asm__ __volatile__ goto ( 143 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 144 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 145 #ifdef RSEQ_COMPARE_TWICE 146 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 147 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 148 #endif 149 /* Start rseq by storing table entry pointer into rseq_cs. */ 150 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 151 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 152 RSEQ_INJECT_ASM(3) 153 LONG_CMP " %[expect], %[v]\n\t" 154 "jnz %l[cmpfail]\n\t" 155 RSEQ_INJECT_ASM(4) 156 #ifdef RSEQ_COMPARE_TWICE 157 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 158 LONG_CMP " %[expect], %[v]\n\t" 159 "jnz %l[error2]\n\t" 160 #endif 161 /* final store */ 162 LONG_S " %[newv], %[v]\n\t" 163 "2:\n\t" 164 RSEQ_INJECT_ASM(5) 165 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 166 : /* gcc asm goto does not allow outputs */ 167 : [cpu_id] "r" (cpu), 168 [current_cpu_id] "m" (__rseq_abi.cpu_id), 169 [rseq_cs] "m" (__rseq_abi.rseq_cs), 170 [v] "m" (*v), 171 [expect] "r" (expect), 172 [newv] "r" (newv) 173 RSEQ_INJECT_INPUT 174 : "memory", "cc", "r0" 175 RSEQ_INJECT_CLOBBER 176 : abort, cmpfail 177 #ifdef RSEQ_COMPARE_TWICE 178 , error1, error2 179 #endif 180 ); 181 return 0; 182 abort: 183 RSEQ_INJECT_FAILED 184 return -1; 185 cmpfail: 186 return 1; 187 #ifdef RSEQ_COMPARE_TWICE 188 error1: 189 rseq_bug("cpu_id comparison failed"); 190 error2: 191 rseq_bug("expected value comparison failed"); 192 #endif 193 } 194 195 /* 196 * Compare @v against @expectnot. When it does _not_ match, load @v 197 * into @load, and store the content of *@v + voffp into @v. 198 */ 199 static inline __attribute__((always_inline)) 200 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 201 off_t voffp, intptr_t *load, int cpu) 202 { 203 RSEQ_INJECT_C(9) 204 205 __asm__ __volatile__ goto ( 206 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 207 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 208 #ifdef RSEQ_COMPARE_TWICE 209 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 210 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 211 #endif 212 /* Start rseq by storing table entry pointer into rseq_cs. */ 213 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 214 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 215 RSEQ_INJECT_ASM(3) 216 LONG_L " %%r1, %[v]\n\t" 217 LONG_CMP_R " %%r1, %[expectnot]\n\t" 218 "je %l[cmpfail]\n\t" 219 RSEQ_INJECT_ASM(4) 220 #ifdef RSEQ_COMPARE_TWICE 221 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 222 LONG_L " %%r1, %[v]\n\t" 223 LONG_CMP_R " %%r1, %[expectnot]\n\t" 224 "je %l[error2]\n\t" 225 #endif 226 LONG_S " %%r1, %[load]\n\t" 227 LONG_ADD_R " %%r1, %[voffp]\n\t" 228 LONG_L " %%r1, 0(%%r1)\n\t" 229 /* final store */ 230 LONG_S " %%r1, %[v]\n\t" 231 "2:\n\t" 232 RSEQ_INJECT_ASM(5) 233 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 234 : /* gcc asm goto does not allow outputs */ 235 : [cpu_id] "r" (cpu), 236 [current_cpu_id] "m" (__rseq_abi.cpu_id), 237 [rseq_cs] "m" (__rseq_abi.rseq_cs), 238 /* final store input */ 239 [v] "m" (*v), 240 [expectnot] "r" (expectnot), 241 [voffp] "r" (voffp), 242 [load] "m" (*load) 243 RSEQ_INJECT_INPUT 244 : "memory", "cc", "r0", "r1" 245 RSEQ_INJECT_CLOBBER 246 : abort, cmpfail 247 #ifdef RSEQ_COMPARE_TWICE 248 , error1, error2 249 #endif 250 ); 251 return 0; 252 abort: 253 RSEQ_INJECT_FAILED 254 return -1; 255 cmpfail: 256 return 1; 257 #ifdef RSEQ_COMPARE_TWICE 258 error1: 259 rseq_bug("cpu_id comparison failed"); 260 error2: 261 rseq_bug("expected value comparison failed"); 262 #endif 263 } 264 265 static inline __attribute__((always_inline)) 266 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 267 { 268 RSEQ_INJECT_C(9) 269 270 __asm__ __volatile__ goto ( 271 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 272 #ifdef RSEQ_COMPARE_TWICE 273 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 274 #endif 275 /* Start rseq by storing table entry pointer into rseq_cs. */ 276 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 277 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 278 RSEQ_INJECT_ASM(3) 279 #ifdef RSEQ_COMPARE_TWICE 280 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 281 #endif 282 LONG_L " %%r0, %[v]\n\t" 283 LONG_ADD_R " %%r0, %[count]\n\t" 284 /* final store */ 285 LONG_S " %%r0, %[v]\n\t" 286 "2:\n\t" 287 RSEQ_INJECT_ASM(4) 288 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 289 : /* gcc asm goto does not allow outputs */ 290 : [cpu_id] "r" (cpu), 291 [current_cpu_id] "m" (__rseq_abi.cpu_id), 292 [rseq_cs] "m" (__rseq_abi.rseq_cs), 293 /* final store input */ 294 [v] "m" (*v), 295 [count] "r" (count) 296 RSEQ_INJECT_INPUT 297 : "memory", "cc", "r0" 298 RSEQ_INJECT_CLOBBER 299 : abort 300 #ifdef RSEQ_COMPARE_TWICE 301 , error1 302 #endif 303 ); 304 return 0; 305 abort: 306 RSEQ_INJECT_FAILED 307 return -1; 308 #ifdef RSEQ_COMPARE_TWICE 309 error1: 310 rseq_bug("cpu_id comparison failed"); 311 #endif 312 } 313 314 static inline __attribute__((always_inline)) 315 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 316 intptr_t *v2, intptr_t newv2, 317 intptr_t newv, int cpu) 318 { 319 RSEQ_INJECT_C(9) 320 321 __asm__ __volatile__ goto ( 322 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 323 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 324 #ifdef RSEQ_COMPARE_TWICE 325 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 326 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 327 #endif 328 /* Start rseq by storing table entry pointer into rseq_cs. */ 329 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 330 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 331 RSEQ_INJECT_ASM(3) 332 LONG_CMP " %[expect], %[v]\n\t" 333 "jnz %l[cmpfail]\n\t" 334 RSEQ_INJECT_ASM(4) 335 #ifdef RSEQ_COMPARE_TWICE 336 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 337 LONG_CMP " %[expect], %[v]\n\t" 338 "jnz %l[error2]\n\t" 339 #endif 340 /* try store */ 341 LONG_S " %[newv2], %[v2]\n\t" 342 RSEQ_INJECT_ASM(5) 343 /* final store */ 344 LONG_S " %[newv], %[v]\n\t" 345 "2:\n\t" 346 RSEQ_INJECT_ASM(6) 347 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 348 : /* gcc asm goto does not allow outputs */ 349 : [cpu_id] "r" (cpu), 350 [current_cpu_id] "m" (__rseq_abi.cpu_id), 351 [rseq_cs] "m" (__rseq_abi.rseq_cs), 352 /* try store input */ 353 [v2] "m" (*v2), 354 [newv2] "r" (newv2), 355 /* final store input */ 356 [v] "m" (*v), 357 [expect] "r" (expect), 358 [newv] "r" (newv) 359 RSEQ_INJECT_INPUT 360 : "memory", "cc", "r0" 361 RSEQ_INJECT_CLOBBER 362 : abort, cmpfail 363 #ifdef RSEQ_COMPARE_TWICE 364 , error1, error2 365 #endif 366 ); 367 return 0; 368 abort: 369 RSEQ_INJECT_FAILED 370 return -1; 371 cmpfail: 372 return 1; 373 #ifdef RSEQ_COMPARE_TWICE 374 error1: 375 rseq_bug("cpu_id comparison failed"); 376 error2: 377 rseq_bug("expected value comparison failed"); 378 #endif 379 } 380 381 /* s390 is TSO. */ 382 static inline __attribute__((always_inline)) 383 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 384 intptr_t *v2, intptr_t newv2, 385 intptr_t newv, int cpu) 386 { 387 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu); 388 } 389 390 static inline __attribute__((always_inline)) 391 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 392 intptr_t *v2, intptr_t expect2, 393 intptr_t newv, int cpu) 394 { 395 RSEQ_INJECT_C(9) 396 397 __asm__ __volatile__ goto ( 398 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 399 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 400 #ifdef RSEQ_COMPARE_TWICE 401 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 402 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 403 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 404 #endif 405 /* Start rseq by storing table entry pointer into rseq_cs. */ 406 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 407 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 408 RSEQ_INJECT_ASM(3) 409 LONG_CMP " %[expect], %[v]\n\t" 410 "jnz %l[cmpfail]\n\t" 411 RSEQ_INJECT_ASM(4) 412 LONG_CMP " %[expect2], %[v2]\n\t" 413 "jnz %l[cmpfail]\n\t" 414 RSEQ_INJECT_ASM(5) 415 #ifdef RSEQ_COMPARE_TWICE 416 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 417 LONG_CMP " %[expect], %[v]\n\t" 418 "jnz %l[error2]\n\t" 419 LONG_CMP " %[expect2], %[v2]\n\t" 420 "jnz %l[error3]\n\t" 421 #endif 422 /* final store */ 423 LONG_S " %[newv], %[v]\n\t" 424 "2:\n\t" 425 RSEQ_INJECT_ASM(6) 426 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 427 : /* gcc asm goto does not allow outputs */ 428 : [cpu_id] "r" (cpu), 429 [current_cpu_id] "m" (__rseq_abi.cpu_id), 430 [rseq_cs] "m" (__rseq_abi.rseq_cs), 431 /* cmp2 input */ 432 [v2] "m" (*v2), 433 [expect2] "r" (expect2), 434 /* final store input */ 435 [v] "m" (*v), 436 [expect] "r" (expect), 437 [newv] "r" (newv) 438 RSEQ_INJECT_INPUT 439 : "memory", "cc", "r0" 440 RSEQ_INJECT_CLOBBER 441 : abort, cmpfail 442 #ifdef RSEQ_COMPARE_TWICE 443 , error1, error2, error3 444 #endif 445 ); 446 return 0; 447 abort: 448 RSEQ_INJECT_FAILED 449 return -1; 450 cmpfail: 451 return 1; 452 #ifdef RSEQ_COMPARE_TWICE 453 error1: 454 rseq_bug("cpu_id comparison failed"); 455 error2: 456 rseq_bug("1st expected value comparison failed"); 457 error3: 458 rseq_bug("2nd expected value comparison failed"); 459 #endif 460 } 461 462 static inline __attribute__((always_inline)) 463 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 464 void *dst, void *src, size_t len, 465 intptr_t newv, int cpu) 466 { 467 uint64_t rseq_scratch[3]; 468 469 RSEQ_INJECT_C(9) 470 471 __asm__ __volatile__ goto ( 472 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 473 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 474 #ifdef RSEQ_COMPARE_TWICE 475 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 476 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 477 #endif 478 LONG_S " %[src], %[rseq_scratch0]\n\t" 479 LONG_S " %[dst], %[rseq_scratch1]\n\t" 480 LONG_S " %[len], %[rseq_scratch2]\n\t" 481 /* Start rseq by storing table entry pointer into rseq_cs. */ 482 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 483 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 484 RSEQ_INJECT_ASM(3) 485 LONG_CMP " %[expect], %[v]\n\t" 486 "jnz 5f\n\t" 487 RSEQ_INJECT_ASM(4) 488 #ifdef RSEQ_COMPARE_TWICE 489 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 490 LONG_CMP " %[expect], %[v]\n\t" 491 "jnz 7f\n\t" 492 #endif 493 /* try memcpy */ 494 LONG_LT_R " %[len], %[len]\n\t" 495 "jz 333f\n\t" 496 "222:\n\t" 497 "ic %%r0,0(%[src])\n\t" 498 "stc %%r0,0(%[dst])\n\t" 499 LONG_ADDI " %[src], 1\n\t" 500 LONG_ADDI " %[dst], 1\n\t" 501 LONG_ADDI " %[len], -1\n\t" 502 "jnz 222b\n\t" 503 "333:\n\t" 504 RSEQ_INJECT_ASM(5) 505 /* final store */ 506 LONG_S " %[newv], %[v]\n\t" 507 "2:\n\t" 508 RSEQ_INJECT_ASM(6) 509 /* teardown */ 510 LONG_L " %[len], %[rseq_scratch2]\n\t" 511 LONG_L " %[dst], %[rseq_scratch1]\n\t" 512 LONG_L " %[src], %[rseq_scratch0]\n\t" 513 RSEQ_ASM_DEFINE_ABORT(4, 514 LONG_L " %[len], %[rseq_scratch2]\n\t" 515 LONG_L " %[dst], %[rseq_scratch1]\n\t" 516 LONG_L " %[src], %[rseq_scratch0]\n\t", 517 abort) 518 RSEQ_ASM_DEFINE_CMPFAIL(5, 519 LONG_L " %[len], %[rseq_scratch2]\n\t" 520 LONG_L " %[dst], %[rseq_scratch1]\n\t" 521 LONG_L " %[src], %[rseq_scratch0]\n\t", 522 cmpfail) 523 #ifdef RSEQ_COMPARE_TWICE 524 RSEQ_ASM_DEFINE_CMPFAIL(6, 525 LONG_L " %[len], %[rseq_scratch2]\n\t" 526 LONG_L " %[dst], %[rseq_scratch1]\n\t" 527 LONG_L " %[src], %[rseq_scratch0]\n\t", 528 error1) 529 RSEQ_ASM_DEFINE_CMPFAIL(7, 530 LONG_L " %[len], %[rseq_scratch2]\n\t" 531 LONG_L " %[dst], %[rseq_scratch1]\n\t" 532 LONG_L " %[src], %[rseq_scratch0]\n\t", 533 error2) 534 #endif 535 : /* gcc asm goto does not allow outputs */ 536 : [cpu_id] "r" (cpu), 537 [current_cpu_id] "m" (__rseq_abi.cpu_id), 538 [rseq_cs] "m" (__rseq_abi.rseq_cs), 539 /* final store input */ 540 [v] "m" (*v), 541 [expect] "r" (expect), 542 [newv] "r" (newv), 543 /* try memcpy input */ 544 [dst] "r" (dst), 545 [src] "r" (src), 546 [len] "r" (len), 547 [rseq_scratch0] "m" (rseq_scratch[0]), 548 [rseq_scratch1] "m" (rseq_scratch[1]), 549 [rseq_scratch2] "m" (rseq_scratch[2]) 550 RSEQ_INJECT_INPUT 551 : "memory", "cc", "r0" 552 RSEQ_INJECT_CLOBBER 553 : abort, cmpfail 554 #ifdef RSEQ_COMPARE_TWICE 555 , error1, error2 556 #endif 557 ); 558 return 0; 559 abort: 560 RSEQ_INJECT_FAILED 561 return -1; 562 cmpfail: 563 return 1; 564 #ifdef RSEQ_COMPARE_TWICE 565 error1: 566 rseq_bug("cpu_id comparison failed"); 567 error2: 568 rseq_bug("expected value comparison failed"); 569 #endif 570 } 571 572 /* s390 is TSO. */ 573 static inline __attribute__((always_inline)) 574 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 575 void *dst, void *src, size_t len, 576 intptr_t newv, int cpu) 577 { 578 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, 579 newv, cpu); 580 } 581 #endif /* !RSEQ_SKIP_FASTPATH */ 582