1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 3 /* 4 * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the 5 * access-register mode nor the linkage stack this instruction will always 6 * cause a special-operation exception (the trap-enabled bit in the DUCT 7 * is and will stay 0). The instruction pattern is 8 * b2 ff 0f ff trap4 4095(%r0) 9 */ 10 #define RSEQ_SIG 0xB2FF0FFF 11 12 #define rseq_smp_mb() __asm__ __volatile__ ("bcr 15,0" ::: "memory") 13 #define rseq_smp_rmb() rseq_smp_mb() 14 #define rseq_smp_wmb() rseq_smp_mb() 15 16 #define rseq_smp_load_acquire(p) \ 17 __extension__ ({ \ 18 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 19 rseq_barrier(); \ 20 ____p1; \ 21 }) 22 23 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 24 25 #define rseq_smp_store_release(p, v) \ 26 do { \ 27 rseq_barrier(); \ 28 RSEQ_WRITE_ONCE(*p, v); \ 29 } while (0) 30 31 #ifdef RSEQ_SKIP_FASTPATH 32 #include "rseq-skip.h" 33 #else /* !RSEQ_SKIP_FASTPATH */ 34 35 #ifdef __s390x__ 36 37 #define LONG_L "lg" 38 #define LONG_S "stg" 39 #define LONG_LT_R "ltgr" 40 #define LONG_CMP "cg" 41 #define LONG_CMP_R "cgr" 42 #define LONG_ADDI "aghi" 43 #define LONG_ADD_R "agr" 44 45 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 46 start_ip, post_commit_offset, abort_ip) \ 47 ".pushsection __rseq_table, \"aw\"\n\t" \ 48 ".balign 32\n\t" \ 49 __rseq_str(label) ":\n\t" \ 50 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 51 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 52 ".popsection\n\t" 53 54 #elif __s390__ 55 56 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 57 start_ip, post_commit_offset, abort_ip) \ 58 ".pushsection __rseq_table, \"aw\"\n\t" \ 59 ".balign 32\n\t" \ 60 __rseq_str(label) ":\n\t" \ 61 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 62 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \ 63 ".popsection\n\t" 64 65 #define LONG_L "l" 66 #define LONG_S "st" 67 #define LONG_LT_R "ltr" 68 #define LONG_CMP "c" 69 #define LONG_CMP_R "cr" 70 #define LONG_ADDI "ahi" 71 #define LONG_ADD_R "ar" 72 73 #endif 74 75 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 76 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 77 (post_commit_ip - start_ip), abort_ip) 78 79 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 80 RSEQ_INJECT_ASM(1) \ 81 "larl %%r0, " __rseq_str(cs_label) "\n\t" \ 82 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \ 83 __rseq_str(label) ":\n\t" 84 85 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 86 RSEQ_INJECT_ASM(2) \ 87 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \ 88 "jnz " __rseq_str(label) "\n\t" 89 90 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 91 ".pushsection __rseq_failure, \"ax\"\n\t" \ 92 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 93 __rseq_str(label) ":\n\t" \ 94 teardown \ 95 "j %l[" __rseq_str(abort_label) "]\n\t" \ 96 ".popsection\n\t" 97 98 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 99 ".pushsection __rseq_failure, \"ax\"\n\t" \ 100 __rseq_str(label) ":\n\t" \ 101 teardown \ 102 "j %l[" __rseq_str(cmpfail_label) "]\n\t" \ 103 ".popsection\n\t" 104 105 static inline __attribute__((always_inline)) 106 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 107 { 108 RSEQ_INJECT_C(9) 109 110 __asm__ __volatile__ goto ( 111 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 112 /* Start rseq by storing table entry pointer into rseq_cs. */ 113 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 114 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 115 RSEQ_INJECT_ASM(3) 116 LONG_CMP " %[expect], %[v]\n\t" 117 "jnz %l[cmpfail]\n\t" 118 RSEQ_INJECT_ASM(4) 119 #ifdef RSEQ_COMPARE_TWICE 120 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 121 LONG_CMP " %[expect], %[v]\n\t" 122 "jnz %l[error2]\n\t" 123 #endif 124 /* final store */ 125 LONG_S " %[newv], %[v]\n\t" 126 "2:\n\t" 127 RSEQ_INJECT_ASM(5) 128 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 129 : /* gcc asm goto does not allow outputs */ 130 : [cpu_id] "r" (cpu), 131 [current_cpu_id] "m" (__rseq_abi.cpu_id), 132 [rseq_cs] "m" (__rseq_abi.rseq_cs), 133 [v] "m" (*v), 134 [expect] "r" (expect), 135 [newv] "r" (newv) 136 RSEQ_INJECT_INPUT 137 : "memory", "cc", "r0" 138 RSEQ_INJECT_CLOBBER 139 : abort, cmpfail 140 #ifdef RSEQ_COMPARE_TWICE 141 , error1, error2 142 #endif 143 ); 144 return 0; 145 abort: 146 RSEQ_INJECT_FAILED 147 return -1; 148 cmpfail: 149 return 1; 150 #ifdef RSEQ_COMPARE_TWICE 151 error1: 152 rseq_bug("cpu_id comparison failed"); 153 error2: 154 rseq_bug("expected value comparison failed"); 155 #endif 156 } 157 158 /* 159 * Compare @v against @expectnot. When it does _not_ match, load @v 160 * into @load, and store the content of *@v + voffp into @v. 161 */ 162 static inline __attribute__((always_inline)) 163 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 164 off_t voffp, intptr_t *load, int cpu) 165 { 166 RSEQ_INJECT_C(9) 167 168 __asm__ __volatile__ goto ( 169 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 170 /* Start rseq by storing table entry pointer into rseq_cs. */ 171 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 172 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 173 RSEQ_INJECT_ASM(3) 174 LONG_L " %%r1, %[v]\n\t" 175 LONG_CMP_R " %%r1, %[expectnot]\n\t" 176 "je %l[cmpfail]\n\t" 177 RSEQ_INJECT_ASM(4) 178 #ifdef RSEQ_COMPARE_TWICE 179 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 180 LONG_L " %%r1, %[v]\n\t" 181 LONG_CMP_R " %%r1, %[expectnot]\n\t" 182 "je %l[error2]\n\t" 183 #endif 184 LONG_S " %%r1, %[load]\n\t" 185 LONG_ADD_R " %%r1, %[voffp]\n\t" 186 LONG_L " %%r1, 0(%%r1)\n\t" 187 /* final store */ 188 LONG_S " %%r1, %[v]\n\t" 189 "2:\n\t" 190 RSEQ_INJECT_ASM(5) 191 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 192 : /* gcc asm goto does not allow outputs */ 193 : [cpu_id] "r" (cpu), 194 [current_cpu_id] "m" (__rseq_abi.cpu_id), 195 [rseq_cs] "m" (__rseq_abi.rseq_cs), 196 /* final store input */ 197 [v] "m" (*v), 198 [expectnot] "r" (expectnot), 199 [voffp] "r" (voffp), 200 [load] "m" (*load) 201 RSEQ_INJECT_INPUT 202 : "memory", "cc", "r0", "r1" 203 RSEQ_INJECT_CLOBBER 204 : abort, cmpfail 205 #ifdef RSEQ_COMPARE_TWICE 206 , error1, error2 207 #endif 208 ); 209 return 0; 210 abort: 211 RSEQ_INJECT_FAILED 212 return -1; 213 cmpfail: 214 return 1; 215 #ifdef RSEQ_COMPARE_TWICE 216 error1: 217 rseq_bug("cpu_id comparison failed"); 218 error2: 219 rseq_bug("expected value comparison failed"); 220 #endif 221 } 222 223 static inline __attribute__((always_inline)) 224 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 225 { 226 RSEQ_INJECT_C(9) 227 228 __asm__ __volatile__ goto ( 229 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 230 /* Start rseq by storing table entry pointer into rseq_cs. */ 231 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 232 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 233 RSEQ_INJECT_ASM(3) 234 #ifdef RSEQ_COMPARE_TWICE 235 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 236 #endif 237 LONG_L " %%r0, %[v]\n\t" 238 LONG_ADD_R " %%r0, %[count]\n\t" 239 /* final store */ 240 LONG_S " %%r0, %[v]\n\t" 241 "2:\n\t" 242 RSEQ_INJECT_ASM(4) 243 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 244 : /* gcc asm goto does not allow outputs */ 245 : [cpu_id] "r" (cpu), 246 [current_cpu_id] "m" (__rseq_abi.cpu_id), 247 [rseq_cs] "m" (__rseq_abi.rseq_cs), 248 /* final store input */ 249 [v] "m" (*v), 250 [count] "r" (count) 251 RSEQ_INJECT_INPUT 252 : "memory", "cc", "r0" 253 RSEQ_INJECT_CLOBBER 254 : abort 255 #ifdef RSEQ_COMPARE_TWICE 256 , error1 257 #endif 258 ); 259 return 0; 260 abort: 261 RSEQ_INJECT_FAILED 262 return -1; 263 #ifdef RSEQ_COMPARE_TWICE 264 error1: 265 rseq_bug("cpu_id comparison failed"); 266 #endif 267 } 268 269 static inline __attribute__((always_inline)) 270 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 271 intptr_t *v2, intptr_t newv2, 272 intptr_t newv, int cpu) 273 { 274 RSEQ_INJECT_C(9) 275 276 __asm__ __volatile__ goto ( 277 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 278 /* Start rseq by storing table entry pointer into rseq_cs. */ 279 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 280 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 281 RSEQ_INJECT_ASM(3) 282 LONG_CMP " %[expect], %[v]\n\t" 283 "jnz %l[cmpfail]\n\t" 284 RSEQ_INJECT_ASM(4) 285 #ifdef RSEQ_COMPARE_TWICE 286 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 287 LONG_CMP " %[expect], %[v]\n\t" 288 "jnz %l[error2]\n\t" 289 #endif 290 /* try store */ 291 LONG_S " %[newv2], %[v2]\n\t" 292 RSEQ_INJECT_ASM(5) 293 /* final store */ 294 LONG_S " %[newv], %[v]\n\t" 295 "2:\n\t" 296 RSEQ_INJECT_ASM(6) 297 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 298 : /* gcc asm goto does not allow outputs */ 299 : [cpu_id] "r" (cpu), 300 [current_cpu_id] "m" (__rseq_abi.cpu_id), 301 [rseq_cs] "m" (__rseq_abi.rseq_cs), 302 /* try store input */ 303 [v2] "m" (*v2), 304 [newv2] "r" (newv2), 305 /* final store input */ 306 [v] "m" (*v), 307 [expect] "r" (expect), 308 [newv] "r" (newv) 309 RSEQ_INJECT_INPUT 310 : "memory", "cc", "r0" 311 RSEQ_INJECT_CLOBBER 312 : abort, cmpfail 313 #ifdef RSEQ_COMPARE_TWICE 314 , error1, error2 315 #endif 316 ); 317 return 0; 318 abort: 319 RSEQ_INJECT_FAILED 320 return -1; 321 cmpfail: 322 return 1; 323 #ifdef RSEQ_COMPARE_TWICE 324 error1: 325 rseq_bug("cpu_id comparison failed"); 326 error2: 327 rseq_bug("expected value comparison failed"); 328 #endif 329 } 330 331 /* s390 is TSO. */ 332 static inline __attribute__((always_inline)) 333 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 334 intptr_t *v2, intptr_t newv2, 335 intptr_t newv, int cpu) 336 { 337 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu); 338 } 339 340 static inline __attribute__((always_inline)) 341 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 342 intptr_t *v2, intptr_t expect2, 343 intptr_t newv, int cpu) 344 { 345 RSEQ_INJECT_C(9) 346 347 __asm__ __volatile__ goto ( 348 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 349 /* Start rseq by storing table entry pointer into rseq_cs. */ 350 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 351 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 352 RSEQ_INJECT_ASM(3) 353 LONG_CMP " %[expect], %[v]\n\t" 354 "jnz %l[cmpfail]\n\t" 355 RSEQ_INJECT_ASM(4) 356 LONG_CMP " %[expect2], %[v2]\n\t" 357 "jnz %l[cmpfail]\n\t" 358 RSEQ_INJECT_ASM(5) 359 #ifdef RSEQ_COMPARE_TWICE 360 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 361 LONG_CMP " %[expect], %[v]\n\t" 362 "jnz %l[error2]\n\t" 363 LONG_CMP " %[expect2], %[v2]\n\t" 364 "jnz %l[error3]\n\t" 365 #endif 366 /* final store */ 367 LONG_S " %[newv], %[v]\n\t" 368 "2:\n\t" 369 RSEQ_INJECT_ASM(6) 370 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 371 : /* gcc asm goto does not allow outputs */ 372 : [cpu_id] "r" (cpu), 373 [current_cpu_id] "m" (__rseq_abi.cpu_id), 374 [rseq_cs] "m" (__rseq_abi.rseq_cs), 375 /* cmp2 input */ 376 [v2] "m" (*v2), 377 [expect2] "r" (expect2), 378 /* final store input */ 379 [v] "m" (*v), 380 [expect] "r" (expect), 381 [newv] "r" (newv) 382 RSEQ_INJECT_INPUT 383 : "memory", "cc", "r0" 384 RSEQ_INJECT_CLOBBER 385 : abort, cmpfail 386 #ifdef RSEQ_COMPARE_TWICE 387 , error1, error2, error3 388 #endif 389 ); 390 return 0; 391 abort: 392 RSEQ_INJECT_FAILED 393 return -1; 394 cmpfail: 395 return 1; 396 #ifdef RSEQ_COMPARE_TWICE 397 error1: 398 rseq_bug("cpu_id comparison failed"); 399 error2: 400 rseq_bug("1st expected value comparison failed"); 401 error3: 402 rseq_bug("2nd expected value comparison failed"); 403 #endif 404 } 405 406 static inline __attribute__((always_inline)) 407 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 408 void *dst, void *src, size_t len, 409 intptr_t newv, int cpu) 410 { 411 uint64_t rseq_scratch[3]; 412 413 RSEQ_INJECT_C(9) 414 415 __asm__ __volatile__ goto ( 416 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 417 LONG_S " %[src], %[rseq_scratch0]\n\t" 418 LONG_S " %[dst], %[rseq_scratch1]\n\t" 419 LONG_S " %[len], %[rseq_scratch2]\n\t" 420 /* Start rseq by storing table entry pointer into rseq_cs. */ 421 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 422 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 423 RSEQ_INJECT_ASM(3) 424 LONG_CMP " %[expect], %[v]\n\t" 425 "jnz 5f\n\t" 426 RSEQ_INJECT_ASM(4) 427 #ifdef RSEQ_COMPARE_TWICE 428 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 429 LONG_CMP " %[expect], %[v]\n\t" 430 "jnz 7f\n\t" 431 #endif 432 /* try memcpy */ 433 LONG_LT_R " %[len], %[len]\n\t" 434 "jz 333f\n\t" 435 "222:\n\t" 436 "ic %%r0,0(%[src])\n\t" 437 "stc %%r0,0(%[dst])\n\t" 438 LONG_ADDI " %[src], 1\n\t" 439 LONG_ADDI " %[dst], 1\n\t" 440 LONG_ADDI " %[len], -1\n\t" 441 "jnz 222b\n\t" 442 "333:\n\t" 443 RSEQ_INJECT_ASM(5) 444 /* final store */ 445 LONG_S " %[newv], %[v]\n\t" 446 "2:\n\t" 447 RSEQ_INJECT_ASM(6) 448 /* teardown */ 449 LONG_L " %[len], %[rseq_scratch2]\n\t" 450 LONG_L " %[dst], %[rseq_scratch1]\n\t" 451 LONG_L " %[src], %[rseq_scratch0]\n\t" 452 RSEQ_ASM_DEFINE_ABORT(4, 453 LONG_L " %[len], %[rseq_scratch2]\n\t" 454 LONG_L " %[dst], %[rseq_scratch1]\n\t" 455 LONG_L " %[src], %[rseq_scratch0]\n\t", 456 abort) 457 RSEQ_ASM_DEFINE_CMPFAIL(5, 458 LONG_L " %[len], %[rseq_scratch2]\n\t" 459 LONG_L " %[dst], %[rseq_scratch1]\n\t" 460 LONG_L " %[src], %[rseq_scratch0]\n\t", 461 cmpfail) 462 #ifdef RSEQ_COMPARE_TWICE 463 RSEQ_ASM_DEFINE_CMPFAIL(6, 464 LONG_L " %[len], %[rseq_scratch2]\n\t" 465 LONG_L " %[dst], %[rseq_scratch1]\n\t" 466 LONG_L " %[src], %[rseq_scratch0]\n\t", 467 error1) 468 RSEQ_ASM_DEFINE_CMPFAIL(7, 469 LONG_L " %[len], %[rseq_scratch2]\n\t" 470 LONG_L " %[dst], %[rseq_scratch1]\n\t" 471 LONG_L " %[src], %[rseq_scratch0]\n\t", 472 error2) 473 #endif 474 : /* gcc asm goto does not allow outputs */ 475 : [cpu_id] "r" (cpu), 476 [current_cpu_id] "m" (__rseq_abi.cpu_id), 477 [rseq_cs] "m" (__rseq_abi.rseq_cs), 478 /* final store input */ 479 [v] "m" (*v), 480 [expect] "r" (expect), 481 [newv] "r" (newv), 482 /* try memcpy input */ 483 [dst] "r" (dst), 484 [src] "r" (src), 485 [len] "r" (len), 486 [rseq_scratch0] "m" (rseq_scratch[0]), 487 [rseq_scratch1] "m" (rseq_scratch[1]), 488 [rseq_scratch2] "m" (rseq_scratch[2]) 489 RSEQ_INJECT_INPUT 490 : "memory", "cc", "r0" 491 RSEQ_INJECT_CLOBBER 492 : abort, cmpfail 493 #ifdef RSEQ_COMPARE_TWICE 494 , error1, error2 495 #endif 496 ); 497 return 0; 498 abort: 499 RSEQ_INJECT_FAILED 500 return -1; 501 cmpfail: 502 return 1; 503 #ifdef RSEQ_COMPARE_TWICE 504 error1: 505 rseq_bug("cpu_id comparison failed"); 506 error2: 507 rseq_bug("expected value comparison failed"); 508 #endif 509 } 510 511 /* s390 is TSO. */ 512 static inline __attribute__((always_inline)) 513 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 514 void *dst, void *src, size_t len, 515 intptr_t newv, int cpu) 516 { 517 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, 518 newv, cpu); 519 } 520 #endif /* !RSEQ_SKIP_FASTPATH */ 521