1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 3 /* 4 * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the 5 * access-register mode nor the linkage stack this instruction will always 6 * cause a special-operation exception (the trap-enabled bit in the DUCT 7 * is and will stay 0). The instruction pattern is 8 * b2 ff 0f ff trap4 4095(%r0) 9 */ 10 #define RSEQ_SIG 0xB2FF0FFF 11 12 #define rseq_smp_mb() __asm__ __volatile__ ("bcr 15,0" ::: "memory") 13 #define rseq_smp_rmb() rseq_smp_mb() 14 #define rseq_smp_wmb() rseq_smp_mb() 15 16 #define rseq_smp_load_acquire(p) \ 17 __extension__ ({ \ 18 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 19 rseq_barrier(); \ 20 ____p1; \ 21 }) 22 23 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 24 25 #define rseq_smp_store_release(p, v) \ 26 do { \ 27 rseq_barrier(); \ 28 RSEQ_WRITE_ONCE(*p, v); \ 29 } while (0) 30 31 #ifdef RSEQ_SKIP_FASTPATH 32 #include "rseq-skip.h" 33 #else /* !RSEQ_SKIP_FASTPATH */ 34 35 #ifdef __s390x__ 36 37 #define LONG_L "lg" 38 #define LONG_S "stg" 39 #define LONG_LT_R "ltgr" 40 #define LONG_CMP "cg" 41 #define LONG_CMP_R "cgr" 42 #define LONG_ADDI "aghi" 43 #define LONG_ADD_R "agr" 44 45 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 46 start_ip, post_commit_offset, abort_ip) \ 47 ".pushsection __rseq_cs, \"aw\"\n\t" \ 48 ".balign 32\n\t" \ 49 __rseq_str(label) ":\n\t" \ 50 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 51 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 52 ".popsection\n\t" \ 53 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 54 ".quad " __rseq_str(label) "b\n\t" \ 55 ".popsection\n\t" 56 57 /* 58 * Exit points of a rseq critical section consist of all instructions outside 59 * of the critical section where a critical section can either branch to or 60 * reach through the normal course of its execution. The abort IP and the 61 * post-commit IP are already part of the __rseq_cs section and should not be 62 * explicitly defined as additional exit points. Knowing all exit points is 63 * useful to assist debuggers stepping over the critical section. 64 */ 65 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 66 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 67 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ 68 ".popsection\n\t" 69 70 #elif __s390__ 71 72 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 73 start_ip, post_commit_offset, abort_ip) \ 74 ".pushsection __rseq_cs, \"aw\"\n\t" \ 75 ".balign 32\n\t" \ 76 __rseq_str(label) ":\n\t" \ 77 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 78 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \ 79 ".popsection\n\t" \ 80 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 81 ".long 0x0, " __rseq_str(label) "b\n\t" \ 82 ".popsection\n\t" 83 84 /* 85 * Exit points of a rseq critical section consist of all instructions outside 86 * of the critical section where a critical section can either branch to or 87 * reach through the normal course of its execution. The abort IP and the 88 * post-commit IP are already part of the __rseq_cs section and should not be 89 * explicitly defined as additional exit points. Knowing all exit points is 90 * useful to assist debuggers stepping over the critical section. 91 */ 92 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 93 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 94 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \ 95 ".popsection\n\t" 96 97 #define LONG_L "l" 98 #define LONG_S "st" 99 #define LONG_LT_R "ltr" 100 #define LONG_CMP "c" 101 #define LONG_CMP_R "cr" 102 #define LONG_ADDI "ahi" 103 #define LONG_ADD_R "ar" 104 105 #endif 106 107 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 108 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 109 (post_commit_ip - start_ip), abort_ip) 110 111 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 112 RSEQ_INJECT_ASM(1) \ 113 "larl %%r0, " __rseq_str(cs_label) "\n\t" \ 114 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \ 115 __rseq_str(label) ":\n\t" 116 117 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 118 RSEQ_INJECT_ASM(2) \ 119 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \ 120 "jnz " __rseq_str(label) "\n\t" 121 122 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 123 ".pushsection __rseq_failure, \"ax\"\n\t" \ 124 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 125 __rseq_str(label) ":\n\t" \ 126 teardown \ 127 "jg %l[" __rseq_str(abort_label) "]\n\t" \ 128 ".popsection\n\t" 129 130 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 131 ".pushsection __rseq_failure, \"ax\"\n\t" \ 132 __rseq_str(label) ":\n\t" \ 133 teardown \ 134 "jg %l[" __rseq_str(cmpfail_label) "]\n\t" \ 135 ".popsection\n\t" 136 137 static inline __attribute__((always_inline)) 138 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 139 { 140 RSEQ_INJECT_C(9) 141 142 __asm__ __volatile__ goto ( 143 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 144 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 145 #ifdef RSEQ_COMPARE_TWICE 146 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 147 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 148 #endif 149 /* Start rseq by storing table entry pointer into rseq_cs. */ 150 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 151 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 152 RSEQ_INJECT_ASM(3) 153 LONG_CMP " %[expect], %[v]\n\t" 154 "jnz %l[cmpfail]\n\t" 155 RSEQ_INJECT_ASM(4) 156 #ifdef RSEQ_COMPARE_TWICE 157 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 158 LONG_CMP " %[expect], %[v]\n\t" 159 "jnz %l[error2]\n\t" 160 #endif 161 /* final store */ 162 LONG_S " %[newv], %[v]\n\t" 163 "2:\n\t" 164 RSEQ_INJECT_ASM(5) 165 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 166 : /* gcc asm goto does not allow outputs */ 167 : [cpu_id] "r" (cpu), 168 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 169 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 170 [v] "m" (*v), 171 [expect] "r" (expect), 172 [newv] "r" (newv) 173 RSEQ_INJECT_INPUT 174 : "memory", "cc", "r0" 175 RSEQ_INJECT_CLOBBER 176 : abort, cmpfail 177 #ifdef RSEQ_COMPARE_TWICE 178 , error1, error2 179 #endif 180 ); 181 rseq_after_asm_goto(); 182 return 0; 183 abort: 184 rseq_after_asm_goto(); 185 RSEQ_INJECT_FAILED 186 return -1; 187 cmpfail: 188 rseq_after_asm_goto(); 189 return 1; 190 #ifdef RSEQ_COMPARE_TWICE 191 error1: 192 rseq_after_asm_goto(); 193 rseq_bug("cpu_id comparison failed"); 194 error2: 195 rseq_after_asm_goto(); 196 rseq_bug("expected value comparison failed"); 197 #endif 198 } 199 200 /* 201 * Compare @v against @expectnot. When it does _not_ match, load @v 202 * into @load, and store the content of *@v + voffp into @v. 203 */ 204 static inline __attribute__((always_inline)) 205 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 206 long voffp, intptr_t *load, int cpu) 207 { 208 RSEQ_INJECT_C(9) 209 210 __asm__ __volatile__ goto ( 211 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 212 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 213 #ifdef RSEQ_COMPARE_TWICE 214 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 215 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 216 #endif 217 /* Start rseq by storing table entry pointer into rseq_cs. */ 218 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 219 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 220 RSEQ_INJECT_ASM(3) 221 LONG_L " %%r1, %[v]\n\t" 222 LONG_CMP_R " %%r1, %[expectnot]\n\t" 223 "je %l[cmpfail]\n\t" 224 RSEQ_INJECT_ASM(4) 225 #ifdef RSEQ_COMPARE_TWICE 226 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 227 LONG_L " %%r1, %[v]\n\t" 228 LONG_CMP_R " %%r1, %[expectnot]\n\t" 229 "je %l[error2]\n\t" 230 #endif 231 LONG_S " %%r1, %[load]\n\t" 232 LONG_ADD_R " %%r1, %[voffp]\n\t" 233 LONG_L " %%r1, 0(%%r1)\n\t" 234 /* final store */ 235 LONG_S " %%r1, %[v]\n\t" 236 "2:\n\t" 237 RSEQ_INJECT_ASM(5) 238 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 239 : /* gcc asm goto does not allow outputs */ 240 : [cpu_id] "r" (cpu), 241 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 242 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 243 /* final store input */ 244 [v] "m" (*v), 245 [expectnot] "r" (expectnot), 246 [voffp] "r" (voffp), 247 [load] "m" (*load) 248 RSEQ_INJECT_INPUT 249 : "memory", "cc", "r0", "r1" 250 RSEQ_INJECT_CLOBBER 251 : abort, cmpfail 252 #ifdef RSEQ_COMPARE_TWICE 253 , error1, error2 254 #endif 255 ); 256 rseq_after_asm_goto(); 257 return 0; 258 abort: 259 rseq_after_asm_goto(); 260 RSEQ_INJECT_FAILED 261 return -1; 262 cmpfail: 263 rseq_after_asm_goto(); 264 return 1; 265 #ifdef RSEQ_COMPARE_TWICE 266 error1: 267 rseq_after_asm_goto(); 268 rseq_bug("cpu_id comparison failed"); 269 error2: 270 rseq_after_asm_goto(); 271 rseq_bug("expected value comparison failed"); 272 #endif 273 } 274 275 static inline __attribute__((always_inline)) 276 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 277 { 278 RSEQ_INJECT_C(9) 279 280 __asm__ __volatile__ goto ( 281 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 282 #ifdef RSEQ_COMPARE_TWICE 283 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 284 #endif 285 /* Start rseq by storing table entry pointer into rseq_cs. */ 286 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 287 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 288 RSEQ_INJECT_ASM(3) 289 #ifdef RSEQ_COMPARE_TWICE 290 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 291 #endif 292 LONG_L " %%r0, %[v]\n\t" 293 LONG_ADD_R " %%r0, %[count]\n\t" 294 /* final store */ 295 LONG_S " %%r0, %[v]\n\t" 296 "2:\n\t" 297 RSEQ_INJECT_ASM(4) 298 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 299 : /* gcc asm goto does not allow outputs */ 300 : [cpu_id] "r" (cpu), 301 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 302 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 303 /* final store input */ 304 [v] "m" (*v), 305 [count] "r" (count) 306 RSEQ_INJECT_INPUT 307 : "memory", "cc", "r0" 308 RSEQ_INJECT_CLOBBER 309 : abort 310 #ifdef RSEQ_COMPARE_TWICE 311 , error1 312 #endif 313 ); 314 rseq_after_asm_goto(); 315 return 0; 316 abort: 317 rseq_after_asm_goto(); 318 RSEQ_INJECT_FAILED 319 return -1; 320 #ifdef RSEQ_COMPARE_TWICE 321 error1: 322 rseq_after_asm_goto(); 323 rseq_bug("cpu_id comparison failed"); 324 #endif 325 } 326 327 static inline __attribute__((always_inline)) 328 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 329 intptr_t *v2, intptr_t newv2, 330 intptr_t newv, int cpu) 331 { 332 RSEQ_INJECT_C(9) 333 334 __asm__ __volatile__ goto ( 335 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 336 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 337 #ifdef RSEQ_COMPARE_TWICE 338 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 339 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 340 #endif 341 /* Start rseq by storing table entry pointer into rseq_cs. */ 342 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 343 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 344 RSEQ_INJECT_ASM(3) 345 LONG_CMP " %[expect], %[v]\n\t" 346 "jnz %l[cmpfail]\n\t" 347 RSEQ_INJECT_ASM(4) 348 #ifdef RSEQ_COMPARE_TWICE 349 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 350 LONG_CMP " %[expect], %[v]\n\t" 351 "jnz %l[error2]\n\t" 352 #endif 353 /* try store */ 354 LONG_S " %[newv2], %[v2]\n\t" 355 RSEQ_INJECT_ASM(5) 356 /* final store */ 357 LONG_S " %[newv], %[v]\n\t" 358 "2:\n\t" 359 RSEQ_INJECT_ASM(6) 360 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 361 : /* gcc asm goto does not allow outputs */ 362 : [cpu_id] "r" (cpu), 363 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 364 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 365 /* try store input */ 366 [v2] "m" (*v2), 367 [newv2] "r" (newv2), 368 /* final store input */ 369 [v] "m" (*v), 370 [expect] "r" (expect), 371 [newv] "r" (newv) 372 RSEQ_INJECT_INPUT 373 : "memory", "cc", "r0" 374 RSEQ_INJECT_CLOBBER 375 : abort, cmpfail 376 #ifdef RSEQ_COMPARE_TWICE 377 , error1, error2 378 #endif 379 ); 380 rseq_after_asm_goto(); 381 return 0; 382 abort: 383 rseq_after_asm_goto(); 384 RSEQ_INJECT_FAILED 385 return -1; 386 cmpfail: 387 rseq_after_asm_goto(); 388 return 1; 389 #ifdef RSEQ_COMPARE_TWICE 390 error1: 391 rseq_after_asm_goto(); 392 rseq_bug("cpu_id comparison failed"); 393 error2: 394 rseq_after_asm_goto(); 395 rseq_bug("expected value comparison failed"); 396 #endif 397 } 398 399 /* s390 is TSO. */ 400 static inline __attribute__((always_inline)) 401 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 402 intptr_t *v2, intptr_t newv2, 403 intptr_t newv, int cpu) 404 { 405 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu); 406 } 407 408 static inline __attribute__((always_inline)) 409 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 410 intptr_t *v2, intptr_t expect2, 411 intptr_t newv, int cpu) 412 { 413 RSEQ_INJECT_C(9) 414 415 __asm__ __volatile__ goto ( 416 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 417 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 418 #ifdef RSEQ_COMPARE_TWICE 419 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 420 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 421 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 422 #endif 423 /* Start rseq by storing table entry pointer into rseq_cs. */ 424 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 425 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 426 RSEQ_INJECT_ASM(3) 427 LONG_CMP " %[expect], %[v]\n\t" 428 "jnz %l[cmpfail]\n\t" 429 RSEQ_INJECT_ASM(4) 430 LONG_CMP " %[expect2], %[v2]\n\t" 431 "jnz %l[cmpfail]\n\t" 432 RSEQ_INJECT_ASM(5) 433 #ifdef RSEQ_COMPARE_TWICE 434 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 435 LONG_CMP " %[expect], %[v]\n\t" 436 "jnz %l[error2]\n\t" 437 LONG_CMP " %[expect2], %[v2]\n\t" 438 "jnz %l[error3]\n\t" 439 #endif 440 /* final store */ 441 LONG_S " %[newv], %[v]\n\t" 442 "2:\n\t" 443 RSEQ_INJECT_ASM(6) 444 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 445 : /* gcc asm goto does not allow outputs */ 446 : [cpu_id] "r" (cpu), 447 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 448 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 449 /* cmp2 input */ 450 [v2] "m" (*v2), 451 [expect2] "r" (expect2), 452 /* final store input */ 453 [v] "m" (*v), 454 [expect] "r" (expect), 455 [newv] "r" (newv) 456 RSEQ_INJECT_INPUT 457 : "memory", "cc", "r0" 458 RSEQ_INJECT_CLOBBER 459 : abort, cmpfail 460 #ifdef RSEQ_COMPARE_TWICE 461 , error1, error2, error3 462 #endif 463 ); 464 rseq_after_asm_goto(); 465 return 0; 466 abort: 467 rseq_after_asm_goto(); 468 RSEQ_INJECT_FAILED 469 return -1; 470 cmpfail: 471 rseq_after_asm_goto(); 472 return 1; 473 #ifdef RSEQ_COMPARE_TWICE 474 error1: 475 rseq_after_asm_goto(); 476 rseq_bug("cpu_id comparison failed"); 477 error2: 478 rseq_after_asm_goto(); 479 rseq_bug("1st expected value comparison failed"); 480 error3: 481 rseq_after_asm_goto(); 482 rseq_bug("2nd expected value comparison failed"); 483 #endif 484 } 485 486 static inline __attribute__((always_inline)) 487 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 488 void *dst, void *src, size_t len, 489 intptr_t newv, int cpu) 490 { 491 uint64_t rseq_scratch[3]; 492 493 RSEQ_INJECT_C(9) 494 495 __asm__ __volatile__ goto ( 496 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 497 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 498 #ifdef RSEQ_COMPARE_TWICE 499 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 500 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 501 #endif 502 LONG_S " %[src], %[rseq_scratch0]\n\t" 503 LONG_S " %[dst], %[rseq_scratch1]\n\t" 504 LONG_S " %[len], %[rseq_scratch2]\n\t" 505 /* Start rseq by storing table entry pointer into rseq_cs. */ 506 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 507 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 508 RSEQ_INJECT_ASM(3) 509 LONG_CMP " %[expect], %[v]\n\t" 510 "jnz 5f\n\t" 511 RSEQ_INJECT_ASM(4) 512 #ifdef RSEQ_COMPARE_TWICE 513 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 514 LONG_CMP " %[expect], %[v]\n\t" 515 "jnz 7f\n\t" 516 #endif 517 /* try memcpy */ 518 LONG_LT_R " %[len], %[len]\n\t" 519 "jz 333f\n\t" 520 "222:\n\t" 521 "ic %%r0,0(%[src])\n\t" 522 "stc %%r0,0(%[dst])\n\t" 523 LONG_ADDI " %[src], 1\n\t" 524 LONG_ADDI " %[dst], 1\n\t" 525 LONG_ADDI " %[len], -1\n\t" 526 "jnz 222b\n\t" 527 "333:\n\t" 528 RSEQ_INJECT_ASM(5) 529 /* final store */ 530 LONG_S " %[newv], %[v]\n\t" 531 "2:\n\t" 532 RSEQ_INJECT_ASM(6) 533 /* teardown */ 534 LONG_L " %[len], %[rseq_scratch2]\n\t" 535 LONG_L " %[dst], %[rseq_scratch1]\n\t" 536 LONG_L " %[src], %[rseq_scratch0]\n\t" 537 RSEQ_ASM_DEFINE_ABORT(4, 538 LONG_L " %[len], %[rseq_scratch2]\n\t" 539 LONG_L " %[dst], %[rseq_scratch1]\n\t" 540 LONG_L " %[src], %[rseq_scratch0]\n\t", 541 abort) 542 RSEQ_ASM_DEFINE_CMPFAIL(5, 543 LONG_L " %[len], %[rseq_scratch2]\n\t" 544 LONG_L " %[dst], %[rseq_scratch1]\n\t" 545 LONG_L " %[src], %[rseq_scratch0]\n\t", 546 cmpfail) 547 #ifdef RSEQ_COMPARE_TWICE 548 RSEQ_ASM_DEFINE_CMPFAIL(6, 549 LONG_L " %[len], %[rseq_scratch2]\n\t" 550 LONG_L " %[dst], %[rseq_scratch1]\n\t" 551 LONG_L " %[src], %[rseq_scratch0]\n\t", 552 error1) 553 RSEQ_ASM_DEFINE_CMPFAIL(7, 554 LONG_L " %[len], %[rseq_scratch2]\n\t" 555 LONG_L " %[dst], %[rseq_scratch1]\n\t" 556 LONG_L " %[src], %[rseq_scratch0]\n\t", 557 error2) 558 #endif 559 : /* gcc asm goto does not allow outputs */ 560 : [cpu_id] "r" (cpu), 561 [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 562 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 563 /* final store input */ 564 [v] "m" (*v), 565 [expect] "r" (expect), 566 [newv] "r" (newv), 567 /* try memcpy input */ 568 [dst] "r" (dst), 569 [src] "r" (src), 570 [len] "r" (len), 571 [rseq_scratch0] "m" (rseq_scratch[0]), 572 [rseq_scratch1] "m" (rseq_scratch[1]), 573 [rseq_scratch2] "m" (rseq_scratch[2]) 574 RSEQ_INJECT_INPUT 575 : "memory", "cc", "r0" 576 RSEQ_INJECT_CLOBBER 577 : abort, cmpfail 578 #ifdef RSEQ_COMPARE_TWICE 579 , error1, error2 580 #endif 581 ); 582 rseq_after_asm_goto(); 583 return 0; 584 abort: 585 rseq_after_asm_goto(); 586 RSEQ_INJECT_FAILED 587 return -1; 588 cmpfail: 589 rseq_after_asm_goto(); 590 return 1; 591 #ifdef RSEQ_COMPARE_TWICE 592 error1: 593 rseq_after_asm_goto(); 594 rseq_bug("cpu_id comparison failed"); 595 error2: 596 rseq_after_asm_goto(); 597 rseq_bug("expected value comparison failed"); 598 #endif 599 } 600 601 /* s390 is TSO. */ 602 static inline __attribute__((always_inline)) 603 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 604 void *dst, void *src, size_t len, 605 intptr_t newv, int cpu) 606 { 607 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, 608 newv, cpu); 609 } 610 #endif /* !RSEQ_SKIP_FASTPATH */ 611