1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-arm.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8 #define RSEQ_SIG 0x53053053 9 10 #define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 11 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 12 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 13 14 #define rseq_smp_load_acquire(p) \ 15 __extension__ ({ \ 16 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 17 rseq_smp_mb(); \ 18 ____p1; \ 19 }) 20 21 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 22 23 #define rseq_smp_store_release(p, v) \ 24 do { \ 25 rseq_smp_mb(); \ 26 RSEQ_WRITE_ONCE(*p, v); \ 27 } while (0) 28 29 #ifdef RSEQ_SKIP_FASTPATH 30 #include "rseq-skip.h" 31 #else /* !RSEQ_SKIP_FASTPATH */ 32 33 #define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip, \ 34 post_commit_offset, abort_ip) \ 35 ".pushsection __rseq_table, \"aw\"\n\t" \ 36 ".balign 32\n\t" \ 37 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 38 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 39 ".popsection\n\t" 40 41 #define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \ 42 __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \ 43 (post_commit_ip - start_ip), abort_ip) 44 45 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 46 RSEQ_INJECT_ASM(1) \ 47 "adr r0, " __rseq_str(cs_label) "\n\t" \ 48 "str r0, %[" __rseq_str(rseq_cs) "]\n\t" \ 49 __rseq_str(label) ":\n\t" 50 51 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 52 RSEQ_INJECT_ASM(2) \ 53 "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \ 54 "cmp %[" __rseq_str(cpu_id) "], r0\n\t" \ 55 "bne " __rseq_str(label) "\n\t" 56 57 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 58 abort_label, version, flags, \ 59 start_ip, post_commit_offset, abort_ip) \ 60 ".balign 32\n\t" \ 61 __rseq_str(table_label) ":\n\t" \ 62 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 63 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 64 ".word " __rseq_str(RSEQ_SIG) "\n\t" \ 65 __rseq_str(label) ":\n\t" \ 66 teardown \ 67 "b %l[" __rseq_str(abort_label) "]\n\t" 68 69 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \ 70 start_ip, post_commit_ip, abort_ip) \ 71 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 72 abort_label, 0x0, 0x0, start_ip, \ 73 (post_commit_ip - start_ip), abort_ip) 74 75 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 76 __rseq_str(label) ":\n\t" \ 77 teardown \ 78 "b %l[" __rseq_str(cmpfail_label) "]\n\t" 79 80 #define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("") 81 82 static inline __attribute__((always_inline)) 83 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 84 { 85 RSEQ_INJECT_C(9) 86 87 rseq_workaround_gcc_asm_size_guess(); 88 __asm__ __volatile__ goto ( 89 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 90 /* Start rseq by storing table entry pointer into rseq_cs. */ 91 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 92 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 93 RSEQ_INJECT_ASM(3) 94 "ldr r0, %[v]\n\t" 95 "cmp %[expect], r0\n\t" 96 "bne %l[cmpfail]\n\t" 97 RSEQ_INJECT_ASM(4) 98 #ifdef RSEQ_COMPARE_TWICE 99 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 100 "ldr r0, %[v]\n\t" 101 "cmp %[expect], r0\n\t" 102 "bne %l[error2]\n\t" 103 #endif 104 /* final store */ 105 "str %[newv], %[v]\n\t" 106 "2:\n\t" 107 RSEQ_INJECT_ASM(5) 108 "b 5f\n\t" 109 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 110 "5:\n\t" 111 : /* gcc asm goto does not allow outputs */ 112 : [cpu_id] "r" (cpu), 113 [current_cpu_id] "m" (__rseq_abi.cpu_id), 114 [rseq_cs] "m" (__rseq_abi.rseq_cs), 115 [v] "m" (*v), 116 [expect] "r" (expect), 117 [newv] "r" (newv) 118 RSEQ_INJECT_INPUT 119 : "r0", "memory", "cc" 120 RSEQ_INJECT_CLOBBER 121 : abort, cmpfail 122 #ifdef RSEQ_COMPARE_TWICE 123 , error1, error2 124 #endif 125 ); 126 rseq_workaround_gcc_asm_size_guess(); 127 return 0; 128 abort: 129 rseq_workaround_gcc_asm_size_guess(); 130 RSEQ_INJECT_FAILED 131 return -1; 132 cmpfail: 133 rseq_workaround_gcc_asm_size_guess(); 134 return 1; 135 #ifdef RSEQ_COMPARE_TWICE 136 error1: 137 rseq_bug("cpu_id comparison failed"); 138 error2: 139 rseq_bug("expected value comparison failed"); 140 #endif 141 } 142 143 static inline __attribute__((always_inline)) 144 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 145 off_t voffp, intptr_t *load, int cpu) 146 { 147 RSEQ_INJECT_C(9) 148 149 rseq_workaround_gcc_asm_size_guess(); 150 __asm__ __volatile__ goto ( 151 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 152 /* Start rseq by storing table entry pointer into rseq_cs. */ 153 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 154 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 155 RSEQ_INJECT_ASM(3) 156 "ldr r0, %[v]\n\t" 157 "cmp %[expectnot], r0\n\t" 158 "beq %l[cmpfail]\n\t" 159 RSEQ_INJECT_ASM(4) 160 #ifdef RSEQ_COMPARE_TWICE 161 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 162 "ldr r0, %[v]\n\t" 163 "cmp %[expectnot], r0\n\t" 164 "beq %l[error2]\n\t" 165 #endif 166 "str r0, %[load]\n\t" 167 "add r0, %[voffp]\n\t" 168 "ldr r0, [r0]\n\t" 169 /* final store */ 170 "str r0, %[v]\n\t" 171 "2:\n\t" 172 RSEQ_INJECT_ASM(5) 173 "b 5f\n\t" 174 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 175 "5:\n\t" 176 : /* gcc asm goto does not allow outputs */ 177 : [cpu_id] "r" (cpu), 178 [current_cpu_id] "m" (__rseq_abi.cpu_id), 179 [rseq_cs] "m" (__rseq_abi.rseq_cs), 180 /* final store input */ 181 [v] "m" (*v), 182 [expectnot] "r" (expectnot), 183 [voffp] "Ir" (voffp), 184 [load] "m" (*load) 185 RSEQ_INJECT_INPUT 186 : "r0", "memory", "cc" 187 RSEQ_INJECT_CLOBBER 188 : abort, cmpfail 189 #ifdef RSEQ_COMPARE_TWICE 190 , error1, error2 191 #endif 192 ); 193 rseq_workaround_gcc_asm_size_guess(); 194 return 0; 195 abort: 196 rseq_workaround_gcc_asm_size_guess(); 197 RSEQ_INJECT_FAILED 198 return -1; 199 cmpfail: 200 rseq_workaround_gcc_asm_size_guess(); 201 return 1; 202 #ifdef RSEQ_COMPARE_TWICE 203 error1: 204 rseq_bug("cpu_id comparison failed"); 205 error2: 206 rseq_bug("expected value comparison failed"); 207 #endif 208 } 209 210 static inline __attribute__((always_inline)) 211 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 212 { 213 RSEQ_INJECT_C(9) 214 215 rseq_workaround_gcc_asm_size_guess(); 216 __asm__ __volatile__ goto ( 217 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 218 /* Start rseq by storing table entry pointer into rseq_cs. */ 219 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 220 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 221 RSEQ_INJECT_ASM(3) 222 #ifdef RSEQ_COMPARE_TWICE 223 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 224 #endif 225 "ldr r0, %[v]\n\t" 226 "add r0, %[count]\n\t" 227 /* final store */ 228 "str r0, %[v]\n\t" 229 "2:\n\t" 230 RSEQ_INJECT_ASM(4) 231 "b 5f\n\t" 232 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 233 "5:\n\t" 234 : /* gcc asm goto does not allow outputs */ 235 : [cpu_id] "r" (cpu), 236 [current_cpu_id] "m" (__rseq_abi.cpu_id), 237 [rseq_cs] "m" (__rseq_abi.rseq_cs), 238 [v] "m" (*v), 239 [count] "Ir" (count) 240 RSEQ_INJECT_INPUT 241 : "r0", "memory", "cc" 242 RSEQ_INJECT_CLOBBER 243 : abort 244 #ifdef RSEQ_COMPARE_TWICE 245 , error1 246 #endif 247 ); 248 rseq_workaround_gcc_asm_size_guess(); 249 return 0; 250 abort: 251 rseq_workaround_gcc_asm_size_guess(); 252 RSEQ_INJECT_FAILED 253 return -1; 254 #ifdef RSEQ_COMPARE_TWICE 255 error1: 256 rseq_bug("cpu_id comparison failed"); 257 #endif 258 } 259 260 static inline __attribute__((always_inline)) 261 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 262 intptr_t *v2, intptr_t newv2, 263 intptr_t newv, int cpu) 264 { 265 RSEQ_INJECT_C(9) 266 267 rseq_workaround_gcc_asm_size_guess(); 268 __asm__ __volatile__ goto ( 269 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 270 /* Start rseq by storing table entry pointer into rseq_cs. */ 271 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 272 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 273 RSEQ_INJECT_ASM(3) 274 "ldr r0, %[v]\n\t" 275 "cmp %[expect], r0\n\t" 276 "bne %l[cmpfail]\n\t" 277 RSEQ_INJECT_ASM(4) 278 #ifdef RSEQ_COMPARE_TWICE 279 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 280 "ldr r0, %[v]\n\t" 281 "cmp %[expect], r0\n\t" 282 "bne %l[error2]\n\t" 283 #endif 284 /* try store */ 285 "str %[newv2], %[v2]\n\t" 286 RSEQ_INJECT_ASM(5) 287 /* final store */ 288 "str %[newv], %[v]\n\t" 289 "2:\n\t" 290 RSEQ_INJECT_ASM(6) 291 "b 5f\n\t" 292 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 293 "5:\n\t" 294 : /* gcc asm goto does not allow outputs */ 295 : [cpu_id] "r" (cpu), 296 [current_cpu_id] "m" (__rseq_abi.cpu_id), 297 [rseq_cs] "m" (__rseq_abi.rseq_cs), 298 /* try store input */ 299 [v2] "m" (*v2), 300 [newv2] "r" (newv2), 301 /* final store input */ 302 [v] "m" (*v), 303 [expect] "r" (expect), 304 [newv] "r" (newv) 305 RSEQ_INJECT_INPUT 306 : "r0", "memory", "cc" 307 RSEQ_INJECT_CLOBBER 308 : abort, cmpfail 309 #ifdef RSEQ_COMPARE_TWICE 310 , error1, error2 311 #endif 312 ); 313 rseq_workaround_gcc_asm_size_guess(); 314 return 0; 315 abort: 316 rseq_workaround_gcc_asm_size_guess(); 317 RSEQ_INJECT_FAILED 318 return -1; 319 cmpfail: 320 rseq_workaround_gcc_asm_size_guess(); 321 return 1; 322 #ifdef RSEQ_COMPARE_TWICE 323 error1: 324 rseq_bug("cpu_id comparison failed"); 325 error2: 326 rseq_bug("expected value comparison failed"); 327 #endif 328 } 329 330 static inline __attribute__((always_inline)) 331 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 332 intptr_t *v2, intptr_t newv2, 333 intptr_t newv, int cpu) 334 { 335 RSEQ_INJECT_C(9) 336 337 rseq_workaround_gcc_asm_size_guess(); 338 __asm__ __volatile__ goto ( 339 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 340 /* Start rseq by storing table entry pointer into rseq_cs. */ 341 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 342 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 343 RSEQ_INJECT_ASM(3) 344 "ldr r0, %[v]\n\t" 345 "cmp %[expect], r0\n\t" 346 "bne %l[cmpfail]\n\t" 347 RSEQ_INJECT_ASM(4) 348 #ifdef RSEQ_COMPARE_TWICE 349 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 350 "ldr r0, %[v]\n\t" 351 "cmp %[expect], r0\n\t" 352 "bne %l[error2]\n\t" 353 #endif 354 /* try store */ 355 "str %[newv2], %[v2]\n\t" 356 RSEQ_INJECT_ASM(5) 357 "dmb\n\t" /* full mb provides store-release */ 358 /* final store */ 359 "str %[newv], %[v]\n\t" 360 "2:\n\t" 361 RSEQ_INJECT_ASM(6) 362 "b 5f\n\t" 363 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 364 "5:\n\t" 365 : /* gcc asm goto does not allow outputs */ 366 : [cpu_id] "r" (cpu), 367 [current_cpu_id] "m" (__rseq_abi.cpu_id), 368 [rseq_cs] "m" (__rseq_abi.rseq_cs), 369 /* try store input */ 370 [v2] "m" (*v2), 371 [newv2] "r" (newv2), 372 /* final store input */ 373 [v] "m" (*v), 374 [expect] "r" (expect), 375 [newv] "r" (newv) 376 RSEQ_INJECT_INPUT 377 : "r0", "memory", "cc" 378 RSEQ_INJECT_CLOBBER 379 : abort, cmpfail 380 #ifdef RSEQ_COMPARE_TWICE 381 , error1, error2 382 #endif 383 ); 384 rseq_workaround_gcc_asm_size_guess(); 385 return 0; 386 abort: 387 rseq_workaround_gcc_asm_size_guess(); 388 RSEQ_INJECT_FAILED 389 return -1; 390 cmpfail: 391 rseq_workaround_gcc_asm_size_guess(); 392 return 1; 393 #ifdef RSEQ_COMPARE_TWICE 394 error1: 395 rseq_bug("cpu_id comparison failed"); 396 error2: 397 rseq_bug("expected value comparison failed"); 398 #endif 399 } 400 401 static inline __attribute__((always_inline)) 402 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 403 intptr_t *v2, intptr_t expect2, 404 intptr_t newv, int cpu) 405 { 406 RSEQ_INJECT_C(9) 407 408 rseq_workaround_gcc_asm_size_guess(); 409 __asm__ __volatile__ goto ( 410 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 411 /* Start rseq by storing table entry pointer into rseq_cs. */ 412 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 413 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 414 RSEQ_INJECT_ASM(3) 415 "ldr r0, %[v]\n\t" 416 "cmp %[expect], r0\n\t" 417 "bne %l[cmpfail]\n\t" 418 RSEQ_INJECT_ASM(4) 419 "ldr r0, %[v2]\n\t" 420 "cmp %[expect2], r0\n\t" 421 "bne %l[cmpfail]\n\t" 422 RSEQ_INJECT_ASM(5) 423 #ifdef RSEQ_COMPARE_TWICE 424 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 425 "ldr r0, %[v]\n\t" 426 "cmp %[expect], r0\n\t" 427 "bne %l[error2]\n\t" 428 "ldr r0, %[v2]\n\t" 429 "cmp %[expect2], r0\n\t" 430 "bne %l[error3]\n\t" 431 #endif 432 /* final store */ 433 "str %[newv], %[v]\n\t" 434 "2:\n\t" 435 RSEQ_INJECT_ASM(6) 436 "b 5f\n\t" 437 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 438 "5:\n\t" 439 : /* gcc asm goto does not allow outputs */ 440 : [cpu_id] "r" (cpu), 441 [current_cpu_id] "m" (__rseq_abi.cpu_id), 442 [rseq_cs] "m" (__rseq_abi.rseq_cs), 443 /* cmp2 input */ 444 [v2] "m" (*v2), 445 [expect2] "r" (expect2), 446 /* final store input */ 447 [v] "m" (*v), 448 [expect] "r" (expect), 449 [newv] "r" (newv) 450 RSEQ_INJECT_INPUT 451 : "r0", "memory", "cc" 452 RSEQ_INJECT_CLOBBER 453 : abort, cmpfail 454 #ifdef RSEQ_COMPARE_TWICE 455 , error1, error2, error3 456 #endif 457 ); 458 rseq_workaround_gcc_asm_size_guess(); 459 return 0; 460 abort: 461 rseq_workaround_gcc_asm_size_guess(); 462 RSEQ_INJECT_FAILED 463 return -1; 464 cmpfail: 465 rseq_workaround_gcc_asm_size_guess(); 466 return 1; 467 #ifdef RSEQ_COMPARE_TWICE 468 error1: 469 rseq_bug("cpu_id comparison failed"); 470 error2: 471 rseq_bug("1st expected value comparison failed"); 472 error3: 473 rseq_bug("2nd expected value comparison failed"); 474 #endif 475 } 476 477 static inline __attribute__((always_inline)) 478 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 479 void *dst, void *src, size_t len, 480 intptr_t newv, int cpu) 481 { 482 uint32_t rseq_scratch[3]; 483 484 RSEQ_INJECT_C(9) 485 486 rseq_workaround_gcc_asm_size_guess(); 487 __asm__ __volatile__ goto ( 488 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 489 "str %[src], %[rseq_scratch0]\n\t" 490 "str %[dst], %[rseq_scratch1]\n\t" 491 "str %[len], %[rseq_scratch2]\n\t" 492 /* Start rseq by storing table entry pointer into rseq_cs. */ 493 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 494 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 495 RSEQ_INJECT_ASM(3) 496 "ldr r0, %[v]\n\t" 497 "cmp %[expect], r0\n\t" 498 "bne 5f\n\t" 499 RSEQ_INJECT_ASM(4) 500 #ifdef RSEQ_COMPARE_TWICE 501 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 502 "ldr r0, %[v]\n\t" 503 "cmp %[expect], r0\n\t" 504 "bne 7f\n\t" 505 #endif 506 /* try memcpy */ 507 "cmp %[len], #0\n\t" \ 508 "beq 333f\n\t" \ 509 "222:\n\t" \ 510 "ldrb %%r0, [%[src]]\n\t" \ 511 "strb %%r0, [%[dst]]\n\t" \ 512 "adds %[src], #1\n\t" \ 513 "adds %[dst], #1\n\t" \ 514 "subs %[len], #1\n\t" \ 515 "bne 222b\n\t" \ 516 "333:\n\t" \ 517 RSEQ_INJECT_ASM(5) 518 /* final store */ 519 "str %[newv], %[v]\n\t" 520 "2:\n\t" 521 RSEQ_INJECT_ASM(6) 522 /* teardown */ 523 "ldr %[len], %[rseq_scratch2]\n\t" 524 "ldr %[dst], %[rseq_scratch1]\n\t" 525 "ldr %[src], %[rseq_scratch0]\n\t" 526 "b 8f\n\t" 527 RSEQ_ASM_DEFINE_ABORT(3, 4, 528 /* teardown */ 529 "ldr %[len], %[rseq_scratch2]\n\t" 530 "ldr %[dst], %[rseq_scratch1]\n\t" 531 "ldr %[src], %[rseq_scratch0]\n\t", 532 abort, 1b, 2b, 4f) 533 RSEQ_ASM_DEFINE_CMPFAIL(5, 534 /* teardown */ 535 "ldr %[len], %[rseq_scratch2]\n\t" 536 "ldr %[dst], %[rseq_scratch1]\n\t" 537 "ldr %[src], %[rseq_scratch0]\n\t", 538 cmpfail) 539 #ifdef RSEQ_COMPARE_TWICE 540 RSEQ_ASM_DEFINE_CMPFAIL(6, 541 /* teardown */ 542 "ldr %[len], %[rseq_scratch2]\n\t" 543 "ldr %[dst], %[rseq_scratch1]\n\t" 544 "ldr %[src], %[rseq_scratch0]\n\t", 545 error1) 546 RSEQ_ASM_DEFINE_CMPFAIL(7, 547 /* teardown */ 548 "ldr %[len], %[rseq_scratch2]\n\t" 549 "ldr %[dst], %[rseq_scratch1]\n\t" 550 "ldr %[src], %[rseq_scratch0]\n\t", 551 error2) 552 #endif 553 "8:\n\t" 554 : /* gcc asm goto does not allow outputs */ 555 : [cpu_id] "r" (cpu), 556 [current_cpu_id] "m" (__rseq_abi.cpu_id), 557 [rseq_cs] "m" (__rseq_abi.rseq_cs), 558 /* final store input */ 559 [v] "m" (*v), 560 [expect] "r" (expect), 561 [newv] "r" (newv), 562 /* try memcpy input */ 563 [dst] "r" (dst), 564 [src] "r" (src), 565 [len] "r" (len), 566 [rseq_scratch0] "m" (rseq_scratch[0]), 567 [rseq_scratch1] "m" (rseq_scratch[1]), 568 [rseq_scratch2] "m" (rseq_scratch[2]) 569 RSEQ_INJECT_INPUT 570 : "r0", "memory", "cc" 571 RSEQ_INJECT_CLOBBER 572 : abort, cmpfail 573 #ifdef RSEQ_COMPARE_TWICE 574 , error1, error2 575 #endif 576 ); 577 rseq_workaround_gcc_asm_size_guess(); 578 return 0; 579 abort: 580 rseq_workaround_gcc_asm_size_guess(); 581 RSEQ_INJECT_FAILED 582 return -1; 583 cmpfail: 584 rseq_workaround_gcc_asm_size_guess(); 585 return 1; 586 #ifdef RSEQ_COMPARE_TWICE 587 error1: 588 rseq_workaround_gcc_asm_size_guess(); 589 rseq_bug("cpu_id comparison failed"); 590 error2: 591 rseq_workaround_gcc_asm_size_guess(); 592 rseq_bug("expected value comparison failed"); 593 #endif 594 } 595 596 static inline __attribute__((always_inline)) 597 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 598 void *dst, void *src, size_t len, 599 intptr_t newv, int cpu) 600 { 601 uint32_t rseq_scratch[3]; 602 603 RSEQ_INJECT_C(9) 604 605 rseq_workaround_gcc_asm_size_guess(); 606 __asm__ __volatile__ goto ( 607 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 608 "str %[src], %[rseq_scratch0]\n\t" 609 "str %[dst], %[rseq_scratch1]\n\t" 610 "str %[len], %[rseq_scratch2]\n\t" 611 /* Start rseq by storing table entry pointer into rseq_cs. */ 612 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 613 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 614 RSEQ_INJECT_ASM(3) 615 "ldr r0, %[v]\n\t" 616 "cmp %[expect], r0\n\t" 617 "bne 5f\n\t" 618 RSEQ_INJECT_ASM(4) 619 #ifdef RSEQ_COMPARE_TWICE 620 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 621 "ldr r0, %[v]\n\t" 622 "cmp %[expect], r0\n\t" 623 "bne 7f\n\t" 624 #endif 625 /* try memcpy */ 626 "cmp %[len], #0\n\t" \ 627 "beq 333f\n\t" \ 628 "222:\n\t" \ 629 "ldrb %%r0, [%[src]]\n\t" \ 630 "strb %%r0, [%[dst]]\n\t" \ 631 "adds %[src], #1\n\t" \ 632 "adds %[dst], #1\n\t" \ 633 "subs %[len], #1\n\t" \ 634 "bne 222b\n\t" \ 635 "333:\n\t" \ 636 RSEQ_INJECT_ASM(5) 637 "dmb\n\t" /* full mb provides store-release */ 638 /* final store */ 639 "str %[newv], %[v]\n\t" 640 "2:\n\t" 641 RSEQ_INJECT_ASM(6) 642 /* teardown */ 643 "ldr %[len], %[rseq_scratch2]\n\t" 644 "ldr %[dst], %[rseq_scratch1]\n\t" 645 "ldr %[src], %[rseq_scratch0]\n\t" 646 "b 8f\n\t" 647 RSEQ_ASM_DEFINE_ABORT(3, 4, 648 /* teardown */ 649 "ldr %[len], %[rseq_scratch2]\n\t" 650 "ldr %[dst], %[rseq_scratch1]\n\t" 651 "ldr %[src], %[rseq_scratch0]\n\t", 652 abort, 1b, 2b, 4f) 653 RSEQ_ASM_DEFINE_CMPFAIL(5, 654 /* teardown */ 655 "ldr %[len], %[rseq_scratch2]\n\t" 656 "ldr %[dst], %[rseq_scratch1]\n\t" 657 "ldr %[src], %[rseq_scratch0]\n\t", 658 cmpfail) 659 #ifdef RSEQ_COMPARE_TWICE 660 RSEQ_ASM_DEFINE_CMPFAIL(6, 661 /* teardown */ 662 "ldr %[len], %[rseq_scratch2]\n\t" 663 "ldr %[dst], %[rseq_scratch1]\n\t" 664 "ldr %[src], %[rseq_scratch0]\n\t", 665 error1) 666 RSEQ_ASM_DEFINE_CMPFAIL(7, 667 /* teardown */ 668 "ldr %[len], %[rseq_scratch2]\n\t" 669 "ldr %[dst], %[rseq_scratch1]\n\t" 670 "ldr %[src], %[rseq_scratch0]\n\t", 671 error2) 672 #endif 673 "8:\n\t" 674 : /* gcc asm goto does not allow outputs */ 675 : [cpu_id] "r" (cpu), 676 [current_cpu_id] "m" (__rseq_abi.cpu_id), 677 [rseq_cs] "m" (__rseq_abi.rseq_cs), 678 /* final store input */ 679 [v] "m" (*v), 680 [expect] "r" (expect), 681 [newv] "r" (newv), 682 /* try memcpy input */ 683 [dst] "r" (dst), 684 [src] "r" (src), 685 [len] "r" (len), 686 [rseq_scratch0] "m" (rseq_scratch[0]), 687 [rseq_scratch1] "m" (rseq_scratch[1]), 688 [rseq_scratch2] "m" (rseq_scratch[2]) 689 RSEQ_INJECT_INPUT 690 : "r0", "memory", "cc" 691 RSEQ_INJECT_CLOBBER 692 : abort, cmpfail 693 #ifdef RSEQ_COMPARE_TWICE 694 , error1, error2 695 #endif 696 ); 697 rseq_workaround_gcc_asm_size_guess(); 698 return 0; 699 abort: 700 rseq_workaround_gcc_asm_size_guess(); 701 RSEQ_INJECT_FAILED 702 return -1; 703 cmpfail: 704 rseq_workaround_gcc_asm_size_guess(); 705 return 1; 706 #ifdef RSEQ_COMPARE_TWICE 707 error1: 708 rseq_workaround_gcc_asm_size_guess(); 709 rseq_bug("cpu_id comparison failed"); 710 error2: 711 rseq_workaround_gcc_asm_size_guess(); 712 rseq_bug("expected value comparison failed"); 713 #endif 714 } 715 716 #endif /* !RSEQ_SKIP_FASTPATH */ 717