1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * Author: Paul Burton <paul.burton@mips.com> 4 * (C) Copyright 2018 MIPS Tech LLC 5 * 6 * Based on rseq-arm.h: 7 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 8 */ 9 10 #define RSEQ_SIG 0x53053053 11 12 #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory") 13 #define rseq_smp_rmb() rseq_smp_mb() 14 #define rseq_smp_wmb() rseq_smp_mb() 15 16 #define rseq_smp_load_acquire(p) \ 17 __extension__ ({ \ 18 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 19 rseq_smp_mb(); \ 20 ____p1; \ 21 }) 22 23 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 24 25 #define rseq_smp_store_release(p, v) \ 26 do { \ 27 rseq_smp_mb(); \ 28 RSEQ_WRITE_ONCE(*p, v); \ 29 } while (0) 30 31 #ifdef RSEQ_SKIP_FASTPATH 32 #include "rseq-skip.h" 33 #else /* !RSEQ_SKIP_FASTPATH */ 34 35 #if _MIPS_SZLONG == 64 36 # define LONG ".dword" 37 # define LONG_LA "dla" 38 # define LONG_L "ld" 39 # define LONG_S "sd" 40 # define LONG_ADDI "daddiu" 41 # define U32_U64_PAD(x) x 42 #elif _MIPS_SZLONG == 32 43 # define LONG ".word" 44 # define LONG_LA "la" 45 # define LONG_L "lw" 46 # define LONG_S "sw" 47 # define LONG_ADDI "addiu" 48 # ifdef __BIG_ENDIAN 49 # define U32_U64_PAD(x) "0x0, " x 50 # else 51 # define U32_U64_PAD(x) x ", 0x0" 52 # endif 53 #else 54 # error unsupported _MIPS_SZLONG 55 #endif 56 57 #define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip, \ 58 post_commit_offset, abort_ip) \ 59 ".pushsection __rseq_table, \"aw\"\n\t" \ 60 ".balign 32\n\t" \ 61 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 62 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \ 63 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \ 64 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \ 65 ".popsection\n\t" 66 67 #define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \ 68 __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \ 69 (post_commit_ip - start_ip), abort_ip) 70 71 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 72 RSEQ_INJECT_ASM(1) \ 73 LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \ 74 LONG_S " $4, %[" __rseq_str(rseq_cs) "]\n\t" \ 75 __rseq_str(label) ":\n\t" 76 77 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 78 RSEQ_INJECT_ASM(2) \ 79 "lw $4, %[" __rseq_str(current_cpu_id) "]\n\t" \ 80 "bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t" 81 82 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 83 abort_label, version, flags, \ 84 start_ip, post_commit_offset, abort_ip) \ 85 ".balign 32\n\t" \ 86 __rseq_str(table_label) ":\n\t" \ 87 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 88 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \ 89 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \ 90 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \ 91 ".word " __rseq_str(RSEQ_SIG) "\n\t" \ 92 __rseq_str(label) ":\n\t" \ 93 teardown \ 94 "b %l[" __rseq_str(abort_label) "]\n\t" 95 96 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \ 97 start_ip, post_commit_ip, abort_ip) \ 98 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 99 abort_label, 0x0, 0x0, start_ip, \ 100 (post_commit_ip - start_ip), abort_ip) 101 102 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 103 __rseq_str(label) ":\n\t" \ 104 teardown \ 105 "b %l[" __rseq_str(cmpfail_label) "]\n\t" 106 107 #define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("") 108 109 static inline __attribute__((always_inline)) 110 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 111 { 112 RSEQ_INJECT_C(9) 113 114 rseq_workaround_gcc_asm_size_guess(); 115 __asm__ __volatile__ goto ( 116 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 117 /* Start rseq by storing table entry pointer into rseq_cs. */ 118 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 119 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 120 RSEQ_INJECT_ASM(3) 121 LONG_L " $4, %[v]\n\t" 122 "bne $4, %[expect], %l[cmpfail]\n\t" 123 RSEQ_INJECT_ASM(4) 124 #ifdef RSEQ_COMPARE_TWICE 125 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 126 LONG_L " $4, %[v]\n\t" 127 "bne $4, %[expect], %l[error2]\n\t" 128 #endif 129 /* final store */ 130 LONG_S " %[newv], %[v]\n\t" 131 "2:\n\t" 132 RSEQ_INJECT_ASM(5) 133 "b 5f\n\t" 134 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 135 "5:\n\t" 136 : /* gcc asm goto does not allow outputs */ 137 : [cpu_id] "r" (cpu), 138 [current_cpu_id] "m" (__rseq_abi.cpu_id), 139 [rseq_cs] "m" (__rseq_abi.rseq_cs), 140 [v] "m" (*v), 141 [expect] "r" (expect), 142 [newv] "r" (newv) 143 RSEQ_INJECT_INPUT 144 : "$4", "memory" 145 RSEQ_INJECT_CLOBBER 146 : abort, cmpfail 147 #ifdef RSEQ_COMPARE_TWICE 148 , error1, error2 149 #endif 150 ); 151 rseq_workaround_gcc_asm_size_guess(); 152 return 0; 153 abort: 154 rseq_workaround_gcc_asm_size_guess(); 155 RSEQ_INJECT_FAILED 156 return -1; 157 cmpfail: 158 rseq_workaround_gcc_asm_size_guess(); 159 return 1; 160 #ifdef RSEQ_COMPARE_TWICE 161 error1: 162 rseq_bug("cpu_id comparison failed"); 163 error2: 164 rseq_bug("expected value comparison failed"); 165 #endif 166 } 167 168 static inline __attribute__((always_inline)) 169 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 170 off_t voffp, intptr_t *load, int cpu) 171 { 172 RSEQ_INJECT_C(9) 173 174 rseq_workaround_gcc_asm_size_guess(); 175 __asm__ __volatile__ goto ( 176 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 177 /* Start rseq by storing table entry pointer into rseq_cs. */ 178 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 179 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 180 RSEQ_INJECT_ASM(3) 181 LONG_L " $4, %[v]\n\t" 182 "beq $4, %[expectnot], %l[cmpfail]\n\t" 183 RSEQ_INJECT_ASM(4) 184 #ifdef RSEQ_COMPARE_TWICE 185 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 186 LONG_L " $4, %[v]\n\t" 187 "beq $4, %[expectnot], %l[error2]\n\t" 188 #endif 189 LONG_S " $4, %[load]\n\t" 190 LONG_ADDI " $4, %[voffp]\n\t" 191 LONG_L " $4, 0($4)\n\t" 192 /* final store */ 193 LONG_S " $4, %[v]\n\t" 194 "2:\n\t" 195 RSEQ_INJECT_ASM(5) 196 "b 5f\n\t" 197 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 198 "5:\n\t" 199 : /* gcc asm goto does not allow outputs */ 200 : [cpu_id] "r" (cpu), 201 [current_cpu_id] "m" (__rseq_abi.cpu_id), 202 [rseq_cs] "m" (__rseq_abi.rseq_cs), 203 /* final store input */ 204 [v] "m" (*v), 205 [expectnot] "r" (expectnot), 206 [voffp] "Ir" (voffp), 207 [load] "m" (*load) 208 RSEQ_INJECT_INPUT 209 : "$4", "memory" 210 RSEQ_INJECT_CLOBBER 211 : abort, cmpfail 212 #ifdef RSEQ_COMPARE_TWICE 213 , error1, error2 214 #endif 215 ); 216 rseq_workaround_gcc_asm_size_guess(); 217 return 0; 218 abort: 219 rseq_workaround_gcc_asm_size_guess(); 220 RSEQ_INJECT_FAILED 221 return -1; 222 cmpfail: 223 rseq_workaround_gcc_asm_size_guess(); 224 return 1; 225 #ifdef RSEQ_COMPARE_TWICE 226 error1: 227 rseq_bug("cpu_id comparison failed"); 228 error2: 229 rseq_bug("expected value comparison failed"); 230 #endif 231 } 232 233 static inline __attribute__((always_inline)) 234 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 235 { 236 RSEQ_INJECT_C(9) 237 238 rseq_workaround_gcc_asm_size_guess(); 239 __asm__ __volatile__ goto ( 240 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 241 /* Start rseq by storing table entry pointer into rseq_cs. */ 242 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 243 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 244 RSEQ_INJECT_ASM(3) 245 #ifdef RSEQ_COMPARE_TWICE 246 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 247 #endif 248 LONG_L " $4, %[v]\n\t" 249 LONG_ADDI " $4, %[count]\n\t" 250 /* final store */ 251 LONG_S " $4, %[v]\n\t" 252 "2:\n\t" 253 RSEQ_INJECT_ASM(4) 254 "b 5f\n\t" 255 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 256 "5:\n\t" 257 : /* gcc asm goto does not allow outputs */ 258 : [cpu_id] "r" (cpu), 259 [current_cpu_id] "m" (__rseq_abi.cpu_id), 260 [rseq_cs] "m" (__rseq_abi.rseq_cs), 261 [v] "m" (*v), 262 [count] "Ir" (count) 263 RSEQ_INJECT_INPUT 264 : "$4", "memory" 265 RSEQ_INJECT_CLOBBER 266 : abort 267 #ifdef RSEQ_COMPARE_TWICE 268 , error1 269 #endif 270 ); 271 rseq_workaround_gcc_asm_size_guess(); 272 return 0; 273 abort: 274 rseq_workaround_gcc_asm_size_guess(); 275 RSEQ_INJECT_FAILED 276 return -1; 277 #ifdef RSEQ_COMPARE_TWICE 278 error1: 279 rseq_bug("cpu_id comparison failed"); 280 #endif 281 } 282 283 static inline __attribute__((always_inline)) 284 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 285 intptr_t *v2, intptr_t newv2, 286 intptr_t newv, int cpu) 287 { 288 RSEQ_INJECT_C(9) 289 290 rseq_workaround_gcc_asm_size_guess(); 291 __asm__ __volatile__ goto ( 292 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 293 /* Start rseq by storing table entry pointer into rseq_cs. */ 294 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 295 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 296 RSEQ_INJECT_ASM(3) 297 LONG_L " $4, %[v]\n\t" 298 "bne $4, %[expect], %l[cmpfail]\n\t" 299 RSEQ_INJECT_ASM(4) 300 #ifdef RSEQ_COMPARE_TWICE 301 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 302 LONG_L " $4, %[v]\n\t" 303 "bne $4, %[expect], %l[error2]\n\t" 304 #endif 305 /* try store */ 306 LONG_S " %[newv2], %[v2]\n\t" 307 RSEQ_INJECT_ASM(5) 308 /* final store */ 309 LONG_S " %[newv], %[v]\n\t" 310 "2:\n\t" 311 RSEQ_INJECT_ASM(6) 312 "b 5f\n\t" 313 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 314 "5:\n\t" 315 : /* gcc asm goto does not allow outputs */ 316 : [cpu_id] "r" (cpu), 317 [current_cpu_id] "m" (__rseq_abi.cpu_id), 318 [rseq_cs] "m" (__rseq_abi.rseq_cs), 319 /* try store input */ 320 [v2] "m" (*v2), 321 [newv2] "r" (newv2), 322 /* final store input */ 323 [v] "m" (*v), 324 [expect] "r" (expect), 325 [newv] "r" (newv) 326 RSEQ_INJECT_INPUT 327 : "$4", "memory" 328 RSEQ_INJECT_CLOBBER 329 : abort, cmpfail 330 #ifdef RSEQ_COMPARE_TWICE 331 , error1, error2 332 #endif 333 ); 334 rseq_workaround_gcc_asm_size_guess(); 335 return 0; 336 abort: 337 rseq_workaround_gcc_asm_size_guess(); 338 RSEQ_INJECT_FAILED 339 return -1; 340 cmpfail: 341 rseq_workaround_gcc_asm_size_guess(); 342 return 1; 343 #ifdef RSEQ_COMPARE_TWICE 344 error1: 345 rseq_bug("cpu_id comparison failed"); 346 error2: 347 rseq_bug("expected value comparison failed"); 348 #endif 349 } 350 351 static inline __attribute__((always_inline)) 352 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 353 intptr_t *v2, intptr_t newv2, 354 intptr_t newv, int cpu) 355 { 356 RSEQ_INJECT_C(9) 357 358 rseq_workaround_gcc_asm_size_guess(); 359 __asm__ __volatile__ goto ( 360 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 361 /* Start rseq by storing table entry pointer into rseq_cs. */ 362 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 363 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 364 RSEQ_INJECT_ASM(3) 365 LONG_L " $4, %[v]\n\t" 366 "bne $4, %[expect], %l[cmpfail]\n\t" 367 RSEQ_INJECT_ASM(4) 368 #ifdef RSEQ_COMPARE_TWICE 369 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 370 LONG_L " $4, %[v]\n\t" 371 "bne $4, %[expect], %l[error2]\n\t" 372 #endif 373 /* try store */ 374 LONG_S " %[newv2], %[v2]\n\t" 375 RSEQ_INJECT_ASM(5) 376 "sync\n\t" /* full sync provides store-release */ 377 /* final store */ 378 LONG_S " %[newv], %[v]\n\t" 379 "2:\n\t" 380 RSEQ_INJECT_ASM(6) 381 "b 5f\n\t" 382 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 383 "5:\n\t" 384 : /* gcc asm goto does not allow outputs */ 385 : [cpu_id] "r" (cpu), 386 [current_cpu_id] "m" (__rseq_abi.cpu_id), 387 [rseq_cs] "m" (__rseq_abi.rseq_cs), 388 /* try store input */ 389 [v2] "m" (*v2), 390 [newv2] "r" (newv2), 391 /* final store input */ 392 [v] "m" (*v), 393 [expect] "r" (expect), 394 [newv] "r" (newv) 395 RSEQ_INJECT_INPUT 396 : "$4", "memory" 397 RSEQ_INJECT_CLOBBER 398 : abort, cmpfail 399 #ifdef RSEQ_COMPARE_TWICE 400 , error1, error2 401 #endif 402 ); 403 rseq_workaround_gcc_asm_size_guess(); 404 return 0; 405 abort: 406 rseq_workaround_gcc_asm_size_guess(); 407 RSEQ_INJECT_FAILED 408 return -1; 409 cmpfail: 410 rseq_workaround_gcc_asm_size_guess(); 411 return 1; 412 #ifdef RSEQ_COMPARE_TWICE 413 error1: 414 rseq_bug("cpu_id comparison failed"); 415 error2: 416 rseq_bug("expected value comparison failed"); 417 #endif 418 } 419 420 static inline __attribute__((always_inline)) 421 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 422 intptr_t *v2, intptr_t expect2, 423 intptr_t newv, int cpu) 424 { 425 RSEQ_INJECT_C(9) 426 427 rseq_workaround_gcc_asm_size_guess(); 428 __asm__ __volatile__ goto ( 429 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 430 /* Start rseq by storing table entry pointer into rseq_cs. */ 431 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 432 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 433 RSEQ_INJECT_ASM(3) 434 LONG_L " $4, %[v]\n\t" 435 "bne $4, %[expect], %l[cmpfail]\n\t" 436 RSEQ_INJECT_ASM(4) 437 LONG_L " $4, %[v2]\n\t" 438 "bne $4, %[expect2], %l[cmpfail]\n\t" 439 RSEQ_INJECT_ASM(5) 440 #ifdef RSEQ_COMPARE_TWICE 441 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 442 LONG_L " $4, %[v]\n\t" 443 "bne $4, %[expect], %l[error2]\n\t" 444 LONG_L " $4, %[v2]\n\t" 445 "bne $4, %[expect2], %l[error3]\n\t" 446 #endif 447 /* final store */ 448 LONG_S " %[newv], %[v]\n\t" 449 "2:\n\t" 450 RSEQ_INJECT_ASM(6) 451 "b 5f\n\t" 452 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 453 "5:\n\t" 454 : /* gcc asm goto does not allow outputs */ 455 : [cpu_id] "r" (cpu), 456 [current_cpu_id] "m" (__rseq_abi.cpu_id), 457 [rseq_cs] "m" (__rseq_abi.rseq_cs), 458 /* cmp2 input */ 459 [v2] "m" (*v2), 460 [expect2] "r" (expect2), 461 /* final store input */ 462 [v] "m" (*v), 463 [expect] "r" (expect), 464 [newv] "r" (newv) 465 RSEQ_INJECT_INPUT 466 : "$4", "memory" 467 RSEQ_INJECT_CLOBBER 468 : abort, cmpfail 469 #ifdef RSEQ_COMPARE_TWICE 470 , error1, error2, error3 471 #endif 472 ); 473 rseq_workaround_gcc_asm_size_guess(); 474 return 0; 475 abort: 476 rseq_workaround_gcc_asm_size_guess(); 477 RSEQ_INJECT_FAILED 478 return -1; 479 cmpfail: 480 rseq_workaround_gcc_asm_size_guess(); 481 return 1; 482 #ifdef RSEQ_COMPARE_TWICE 483 error1: 484 rseq_bug("cpu_id comparison failed"); 485 error2: 486 rseq_bug("1st expected value comparison failed"); 487 error3: 488 rseq_bug("2nd expected value comparison failed"); 489 #endif 490 } 491 492 static inline __attribute__((always_inline)) 493 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 494 void *dst, void *src, size_t len, 495 intptr_t newv, int cpu) 496 { 497 uintptr_t rseq_scratch[3]; 498 499 RSEQ_INJECT_C(9) 500 501 rseq_workaround_gcc_asm_size_guess(); 502 __asm__ __volatile__ goto ( 503 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 504 LONG_S " %[src], %[rseq_scratch0]\n\t" 505 LONG_S " %[dst], %[rseq_scratch1]\n\t" 506 LONG_S " %[len], %[rseq_scratch2]\n\t" 507 /* Start rseq by storing table entry pointer into rseq_cs. */ 508 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 509 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 510 RSEQ_INJECT_ASM(3) 511 LONG_L " $4, %[v]\n\t" 512 "bne $4, %[expect], 5f\n\t" 513 RSEQ_INJECT_ASM(4) 514 #ifdef RSEQ_COMPARE_TWICE 515 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 516 LONG_L " $4, %[v]\n\t" 517 "bne $4, %[expect], 7f\n\t" 518 #endif 519 /* try memcpy */ 520 "beqz %[len], 333f\n\t" \ 521 "222:\n\t" \ 522 "lb $4, 0(%[src])\n\t" \ 523 "sb $4, 0(%[dst])\n\t" \ 524 LONG_ADDI " %[src], 1\n\t" \ 525 LONG_ADDI " %[dst], 1\n\t" \ 526 LONG_ADDI " %[len], -1\n\t" \ 527 "bnez %[len], 222b\n\t" \ 528 "333:\n\t" \ 529 RSEQ_INJECT_ASM(5) 530 /* final store */ 531 LONG_S " %[newv], %[v]\n\t" 532 "2:\n\t" 533 RSEQ_INJECT_ASM(6) 534 /* teardown */ 535 LONG_L " %[len], %[rseq_scratch2]\n\t" 536 LONG_L " %[dst], %[rseq_scratch1]\n\t" 537 LONG_L " %[src], %[rseq_scratch0]\n\t" 538 "b 8f\n\t" 539 RSEQ_ASM_DEFINE_ABORT(3, 4, 540 /* teardown */ 541 LONG_L " %[len], %[rseq_scratch2]\n\t" 542 LONG_L " %[dst], %[rseq_scratch1]\n\t" 543 LONG_L " %[src], %[rseq_scratch0]\n\t", 544 abort, 1b, 2b, 4f) 545 RSEQ_ASM_DEFINE_CMPFAIL(5, 546 /* teardown */ 547 LONG_L " %[len], %[rseq_scratch2]\n\t" 548 LONG_L " %[dst], %[rseq_scratch1]\n\t" 549 LONG_L " %[src], %[rseq_scratch0]\n\t", 550 cmpfail) 551 #ifdef RSEQ_COMPARE_TWICE 552 RSEQ_ASM_DEFINE_CMPFAIL(6, 553 /* teardown */ 554 LONG_L " %[len], %[rseq_scratch2]\n\t" 555 LONG_L " %[dst], %[rseq_scratch1]\n\t" 556 LONG_L " %[src], %[rseq_scratch0]\n\t", 557 error1) 558 RSEQ_ASM_DEFINE_CMPFAIL(7, 559 /* teardown */ 560 LONG_L " %[len], %[rseq_scratch2]\n\t" 561 LONG_L " %[dst], %[rseq_scratch1]\n\t" 562 LONG_L " %[src], %[rseq_scratch0]\n\t", 563 error2) 564 #endif 565 "8:\n\t" 566 : /* gcc asm goto does not allow outputs */ 567 : [cpu_id] "r" (cpu), 568 [current_cpu_id] "m" (__rseq_abi.cpu_id), 569 [rseq_cs] "m" (__rseq_abi.rseq_cs), 570 /* final store input */ 571 [v] "m" (*v), 572 [expect] "r" (expect), 573 [newv] "r" (newv), 574 /* try memcpy input */ 575 [dst] "r" (dst), 576 [src] "r" (src), 577 [len] "r" (len), 578 [rseq_scratch0] "m" (rseq_scratch[0]), 579 [rseq_scratch1] "m" (rseq_scratch[1]), 580 [rseq_scratch2] "m" (rseq_scratch[2]) 581 RSEQ_INJECT_INPUT 582 : "$4", "memory" 583 RSEQ_INJECT_CLOBBER 584 : abort, cmpfail 585 #ifdef RSEQ_COMPARE_TWICE 586 , error1, error2 587 #endif 588 ); 589 rseq_workaround_gcc_asm_size_guess(); 590 return 0; 591 abort: 592 rseq_workaround_gcc_asm_size_guess(); 593 RSEQ_INJECT_FAILED 594 return -1; 595 cmpfail: 596 rseq_workaround_gcc_asm_size_guess(); 597 return 1; 598 #ifdef RSEQ_COMPARE_TWICE 599 error1: 600 rseq_workaround_gcc_asm_size_guess(); 601 rseq_bug("cpu_id comparison failed"); 602 error2: 603 rseq_workaround_gcc_asm_size_guess(); 604 rseq_bug("expected value comparison failed"); 605 #endif 606 } 607 608 static inline __attribute__((always_inline)) 609 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 610 void *dst, void *src, size_t len, 611 intptr_t newv, int cpu) 612 { 613 uintptr_t rseq_scratch[3]; 614 615 RSEQ_INJECT_C(9) 616 617 rseq_workaround_gcc_asm_size_guess(); 618 __asm__ __volatile__ goto ( 619 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 620 LONG_S " %[src], %[rseq_scratch0]\n\t" 621 LONG_S " %[dst], %[rseq_scratch1]\n\t" 622 LONG_S " %[len], %[rseq_scratch2]\n\t" 623 /* Start rseq by storing table entry pointer into rseq_cs. */ 624 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 625 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 626 RSEQ_INJECT_ASM(3) 627 LONG_L " $4, %[v]\n\t" 628 "bne $4, %[expect], 5f\n\t" 629 RSEQ_INJECT_ASM(4) 630 #ifdef RSEQ_COMPARE_TWICE 631 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 632 LONG_L " $4, %[v]\n\t" 633 "bne $4, %[expect], 7f\n\t" 634 #endif 635 /* try memcpy */ 636 "beqz %[len], 333f\n\t" \ 637 "222:\n\t" \ 638 "lb $4, 0(%[src])\n\t" \ 639 "sb $4, 0(%[dst])\n\t" \ 640 LONG_ADDI " %[src], 1\n\t" \ 641 LONG_ADDI " %[dst], 1\n\t" \ 642 LONG_ADDI " %[len], -1\n\t" \ 643 "bnez %[len], 222b\n\t" \ 644 "333:\n\t" \ 645 RSEQ_INJECT_ASM(5) 646 "sync\n\t" /* full sync provides store-release */ 647 /* final store */ 648 LONG_S " %[newv], %[v]\n\t" 649 "2:\n\t" 650 RSEQ_INJECT_ASM(6) 651 /* teardown */ 652 LONG_L " %[len], %[rseq_scratch2]\n\t" 653 LONG_L " %[dst], %[rseq_scratch1]\n\t" 654 LONG_L " %[src], %[rseq_scratch0]\n\t" 655 "b 8f\n\t" 656 RSEQ_ASM_DEFINE_ABORT(3, 4, 657 /* teardown */ 658 LONG_L " %[len], %[rseq_scratch2]\n\t" 659 LONG_L " %[dst], %[rseq_scratch1]\n\t" 660 LONG_L " %[src], %[rseq_scratch0]\n\t", 661 abort, 1b, 2b, 4f) 662 RSEQ_ASM_DEFINE_CMPFAIL(5, 663 /* teardown */ 664 LONG_L " %[len], %[rseq_scratch2]\n\t" 665 LONG_L " %[dst], %[rseq_scratch1]\n\t" 666 LONG_L " %[src], %[rseq_scratch0]\n\t", 667 cmpfail) 668 #ifdef RSEQ_COMPARE_TWICE 669 RSEQ_ASM_DEFINE_CMPFAIL(6, 670 /* teardown */ 671 LONG_L " %[len], %[rseq_scratch2]\n\t" 672 LONG_L " %[dst], %[rseq_scratch1]\n\t" 673 LONG_L " %[src], %[rseq_scratch0]\n\t", 674 error1) 675 RSEQ_ASM_DEFINE_CMPFAIL(7, 676 /* teardown */ 677 LONG_L " %[len], %[rseq_scratch2]\n\t" 678 LONG_L " %[dst], %[rseq_scratch1]\n\t" 679 LONG_L " %[src], %[rseq_scratch0]\n\t", 680 error2) 681 #endif 682 "8:\n\t" 683 : /* gcc asm goto does not allow outputs */ 684 : [cpu_id] "r" (cpu), 685 [current_cpu_id] "m" (__rseq_abi.cpu_id), 686 [rseq_cs] "m" (__rseq_abi.rseq_cs), 687 /* final store input */ 688 [v] "m" (*v), 689 [expect] "r" (expect), 690 [newv] "r" (newv), 691 /* try memcpy input */ 692 [dst] "r" (dst), 693 [src] "r" (src), 694 [len] "r" (len), 695 [rseq_scratch0] "m" (rseq_scratch[0]), 696 [rseq_scratch1] "m" (rseq_scratch[1]), 697 [rseq_scratch2] "m" (rseq_scratch[2]) 698 RSEQ_INJECT_INPUT 699 : "$4", "memory" 700 RSEQ_INJECT_CLOBBER 701 : abort, cmpfail 702 #ifdef RSEQ_COMPARE_TWICE 703 , error1, error2 704 #endif 705 ); 706 rseq_workaround_gcc_asm_size_guess(); 707 return 0; 708 abort: 709 rseq_workaround_gcc_asm_size_guess(); 710 RSEQ_INJECT_FAILED 711 return -1; 712 cmpfail: 713 rseq_workaround_gcc_asm_size_guess(); 714 return 1; 715 #ifdef RSEQ_COMPARE_TWICE 716 error1: 717 rseq_workaround_gcc_asm_size_guess(); 718 rseq_bug("cpu_id comparison failed"); 719 error2: 720 rseq_workaround_gcc_asm_size_guess(); 721 rseq_bug("expected value comparison failed"); 722 #endif 723 } 724 725 #endif /* !RSEQ_SKIP_FASTPATH */ 726