1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-arm.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8 #define RSEQ_SIG 0x53053053 9 10 #define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 11 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 12 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 13 14 #define rseq_smp_load_acquire(p) \ 15 __extension__ ({ \ 16 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 17 rseq_smp_mb(); \ 18 ____p1; \ 19 }) 20 21 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 22 23 #define rseq_smp_store_release(p, v) \ 24 do { \ 25 rseq_smp_mb(); \ 26 RSEQ_WRITE_ONCE(*p, v); \ 27 } while (0) 28 29 #ifdef RSEQ_SKIP_FASTPATH 30 #include "rseq-skip.h" 31 #else /* !RSEQ_SKIP_FASTPATH */ 32 33 #define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip, \ 34 post_commit_offset, abort_ip) \ 35 ".pushsection __rseq_table, \"aw\"\n\t" \ 36 ".balign 32\n\t" \ 37 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 38 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 39 ".popsection\n\t" 40 41 #define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \ 42 __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \ 43 (post_commit_ip - start_ip), abort_ip) 44 45 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 46 RSEQ_INJECT_ASM(1) \ 47 "adr r0, " __rseq_str(cs_label) "\n\t" \ 48 "str r0, %[" __rseq_str(rseq_cs) "]\n\t" \ 49 __rseq_str(label) ":\n\t" 50 51 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 52 RSEQ_INJECT_ASM(2) \ 53 "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \ 54 "cmp %[" __rseq_str(cpu_id) "], r0\n\t" \ 55 "bne " __rseq_str(label) "\n\t" 56 57 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 58 abort_label, version, flags, \ 59 start_ip, post_commit_offset, abort_ip) \ 60 __rseq_str(table_label) ":\n\t" \ 61 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 62 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 63 ".word " __rseq_str(RSEQ_SIG) "\n\t" \ 64 __rseq_str(label) ":\n\t" \ 65 teardown \ 66 "b %l[" __rseq_str(abort_label) "]\n\t" 67 68 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \ 69 start_ip, post_commit_ip, abort_ip) \ 70 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 71 abort_label, 0x0, 0x0, start_ip, \ 72 (post_commit_ip - start_ip), abort_ip) 73 74 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 75 __rseq_str(label) ":\n\t" \ 76 teardown \ 77 "b %l[" __rseq_str(cmpfail_label) "]\n\t" 78 79 #define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("") 80 81 static inline __attribute__((always_inline)) 82 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 83 { 84 RSEQ_INJECT_C(9) 85 86 rseq_workaround_gcc_asm_size_guess(); 87 __asm__ __volatile__ goto ( 88 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 89 /* Start rseq by storing table entry pointer into rseq_cs. */ 90 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 91 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 92 RSEQ_INJECT_ASM(3) 93 "ldr r0, %[v]\n\t" 94 "cmp %[expect], r0\n\t" 95 "bne %l[cmpfail]\n\t" 96 RSEQ_INJECT_ASM(4) 97 #ifdef RSEQ_COMPARE_TWICE 98 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 99 "ldr r0, %[v]\n\t" 100 "cmp %[expect], r0\n\t" 101 "bne %l[error2]\n\t" 102 #endif 103 /* final store */ 104 "str %[newv], %[v]\n\t" 105 "2:\n\t" 106 RSEQ_INJECT_ASM(5) 107 "b 5f\n\t" 108 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 109 "5:\n\t" 110 : /* gcc asm goto does not allow outputs */ 111 : [cpu_id] "r" (cpu), 112 [current_cpu_id] "m" (__rseq_abi.cpu_id), 113 [rseq_cs] "m" (__rseq_abi.rseq_cs), 114 [v] "m" (*v), 115 [expect] "r" (expect), 116 [newv] "r" (newv) 117 RSEQ_INJECT_INPUT 118 : "r0", "memory", "cc" 119 RSEQ_INJECT_CLOBBER 120 : abort, cmpfail 121 #ifdef RSEQ_COMPARE_TWICE 122 , error1, error2 123 #endif 124 ); 125 rseq_workaround_gcc_asm_size_guess(); 126 return 0; 127 abort: 128 rseq_workaround_gcc_asm_size_guess(); 129 RSEQ_INJECT_FAILED 130 return -1; 131 cmpfail: 132 rseq_workaround_gcc_asm_size_guess(); 133 return 1; 134 #ifdef RSEQ_COMPARE_TWICE 135 error1: 136 rseq_bug("cpu_id comparison failed"); 137 error2: 138 rseq_bug("expected value comparison failed"); 139 #endif 140 } 141 142 static inline __attribute__((always_inline)) 143 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 144 off_t voffp, intptr_t *load, int cpu) 145 { 146 RSEQ_INJECT_C(9) 147 148 rseq_workaround_gcc_asm_size_guess(); 149 __asm__ __volatile__ goto ( 150 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 151 /* Start rseq by storing table entry pointer into rseq_cs. */ 152 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 153 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 154 RSEQ_INJECT_ASM(3) 155 "ldr r0, %[v]\n\t" 156 "cmp %[expectnot], r0\n\t" 157 "beq %l[cmpfail]\n\t" 158 RSEQ_INJECT_ASM(4) 159 #ifdef RSEQ_COMPARE_TWICE 160 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 161 "ldr r0, %[v]\n\t" 162 "cmp %[expectnot], r0\n\t" 163 "beq %l[error2]\n\t" 164 #endif 165 "str r0, %[load]\n\t" 166 "add r0, %[voffp]\n\t" 167 "ldr r0, [r0]\n\t" 168 /* final store */ 169 "str r0, %[v]\n\t" 170 "2:\n\t" 171 RSEQ_INJECT_ASM(5) 172 "b 5f\n\t" 173 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 174 "5:\n\t" 175 : /* gcc asm goto does not allow outputs */ 176 : [cpu_id] "r" (cpu), 177 [current_cpu_id] "m" (__rseq_abi.cpu_id), 178 [rseq_cs] "m" (__rseq_abi.rseq_cs), 179 /* final store input */ 180 [v] "m" (*v), 181 [expectnot] "r" (expectnot), 182 [voffp] "Ir" (voffp), 183 [load] "m" (*load) 184 RSEQ_INJECT_INPUT 185 : "r0", "memory", "cc" 186 RSEQ_INJECT_CLOBBER 187 : abort, cmpfail 188 #ifdef RSEQ_COMPARE_TWICE 189 , error1, error2 190 #endif 191 ); 192 rseq_workaround_gcc_asm_size_guess(); 193 return 0; 194 abort: 195 rseq_workaround_gcc_asm_size_guess(); 196 RSEQ_INJECT_FAILED 197 return -1; 198 cmpfail: 199 rseq_workaround_gcc_asm_size_guess(); 200 return 1; 201 #ifdef RSEQ_COMPARE_TWICE 202 error1: 203 rseq_bug("cpu_id comparison failed"); 204 error2: 205 rseq_bug("expected value comparison failed"); 206 #endif 207 } 208 209 static inline __attribute__((always_inline)) 210 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 211 { 212 RSEQ_INJECT_C(9) 213 214 rseq_workaround_gcc_asm_size_guess(); 215 __asm__ __volatile__ goto ( 216 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 217 /* Start rseq by storing table entry pointer into rseq_cs. */ 218 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 219 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 220 RSEQ_INJECT_ASM(3) 221 #ifdef RSEQ_COMPARE_TWICE 222 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 223 #endif 224 "ldr r0, %[v]\n\t" 225 "add r0, %[count]\n\t" 226 /* final store */ 227 "str r0, %[v]\n\t" 228 "2:\n\t" 229 RSEQ_INJECT_ASM(4) 230 "b 5f\n\t" 231 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 232 "5:\n\t" 233 : /* gcc asm goto does not allow outputs */ 234 : [cpu_id] "r" (cpu), 235 [current_cpu_id] "m" (__rseq_abi.cpu_id), 236 [rseq_cs] "m" (__rseq_abi.rseq_cs), 237 [v] "m" (*v), 238 [count] "Ir" (count) 239 RSEQ_INJECT_INPUT 240 : "r0", "memory", "cc" 241 RSEQ_INJECT_CLOBBER 242 : abort 243 #ifdef RSEQ_COMPARE_TWICE 244 , error1 245 #endif 246 ); 247 rseq_workaround_gcc_asm_size_guess(); 248 return 0; 249 abort: 250 rseq_workaround_gcc_asm_size_guess(); 251 RSEQ_INJECT_FAILED 252 return -1; 253 #ifdef RSEQ_COMPARE_TWICE 254 error1: 255 rseq_bug("cpu_id comparison failed"); 256 #endif 257 } 258 259 static inline __attribute__((always_inline)) 260 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 261 intptr_t *v2, intptr_t newv2, 262 intptr_t newv, int cpu) 263 { 264 RSEQ_INJECT_C(9) 265 266 rseq_workaround_gcc_asm_size_guess(); 267 __asm__ __volatile__ goto ( 268 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 269 /* Start rseq by storing table entry pointer into rseq_cs. */ 270 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 271 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 272 RSEQ_INJECT_ASM(3) 273 "ldr r0, %[v]\n\t" 274 "cmp %[expect], r0\n\t" 275 "bne %l[cmpfail]\n\t" 276 RSEQ_INJECT_ASM(4) 277 #ifdef RSEQ_COMPARE_TWICE 278 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 279 "ldr r0, %[v]\n\t" 280 "cmp %[expect], r0\n\t" 281 "bne %l[error2]\n\t" 282 #endif 283 /* try store */ 284 "str %[newv2], %[v2]\n\t" 285 RSEQ_INJECT_ASM(5) 286 /* final store */ 287 "str %[newv], %[v]\n\t" 288 "2:\n\t" 289 RSEQ_INJECT_ASM(6) 290 "b 5f\n\t" 291 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 292 "5:\n\t" 293 : /* gcc asm goto does not allow outputs */ 294 : [cpu_id] "r" (cpu), 295 [current_cpu_id] "m" (__rseq_abi.cpu_id), 296 [rseq_cs] "m" (__rseq_abi.rseq_cs), 297 /* try store input */ 298 [v2] "m" (*v2), 299 [newv2] "r" (newv2), 300 /* final store input */ 301 [v] "m" (*v), 302 [expect] "r" (expect), 303 [newv] "r" (newv) 304 RSEQ_INJECT_INPUT 305 : "r0", "memory", "cc" 306 RSEQ_INJECT_CLOBBER 307 : abort, cmpfail 308 #ifdef RSEQ_COMPARE_TWICE 309 , error1, error2 310 #endif 311 ); 312 rseq_workaround_gcc_asm_size_guess(); 313 return 0; 314 abort: 315 rseq_workaround_gcc_asm_size_guess(); 316 RSEQ_INJECT_FAILED 317 return -1; 318 cmpfail: 319 rseq_workaround_gcc_asm_size_guess(); 320 return 1; 321 #ifdef RSEQ_COMPARE_TWICE 322 error1: 323 rseq_bug("cpu_id comparison failed"); 324 error2: 325 rseq_bug("expected value comparison failed"); 326 #endif 327 } 328 329 static inline __attribute__((always_inline)) 330 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 331 intptr_t *v2, intptr_t newv2, 332 intptr_t newv, int cpu) 333 { 334 RSEQ_INJECT_C(9) 335 336 rseq_workaround_gcc_asm_size_guess(); 337 __asm__ __volatile__ goto ( 338 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 339 /* Start rseq by storing table entry pointer into rseq_cs. */ 340 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 341 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 342 RSEQ_INJECT_ASM(3) 343 "ldr r0, %[v]\n\t" 344 "cmp %[expect], r0\n\t" 345 "bne %l[cmpfail]\n\t" 346 RSEQ_INJECT_ASM(4) 347 #ifdef RSEQ_COMPARE_TWICE 348 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 349 "ldr r0, %[v]\n\t" 350 "cmp %[expect], r0\n\t" 351 "bne %l[error2]\n\t" 352 #endif 353 /* try store */ 354 "str %[newv2], %[v2]\n\t" 355 RSEQ_INJECT_ASM(5) 356 "dmb\n\t" /* full mb provides store-release */ 357 /* final store */ 358 "str %[newv], %[v]\n\t" 359 "2:\n\t" 360 RSEQ_INJECT_ASM(6) 361 "b 5f\n\t" 362 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 363 "5:\n\t" 364 : /* gcc asm goto does not allow outputs */ 365 : [cpu_id] "r" (cpu), 366 [current_cpu_id] "m" (__rseq_abi.cpu_id), 367 [rseq_cs] "m" (__rseq_abi.rseq_cs), 368 /* try store input */ 369 [v2] "m" (*v2), 370 [newv2] "r" (newv2), 371 /* final store input */ 372 [v] "m" (*v), 373 [expect] "r" (expect), 374 [newv] "r" (newv) 375 RSEQ_INJECT_INPUT 376 : "r0", "memory", "cc" 377 RSEQ_INJECT_CLOBBER 378 : abort, cmpfail 379 #ifdef RSEQ_COMPARE_TWICE 380 , error1, error2 381 #endif 382 ); 383 rseq_workaround_gcc_asm_size_guess(); 384 return 0; 385 abort: 386 rseq_workaround_gcc_asm_size_guess(); 387 RSEQ_INJECT_FAILED 388 return -1; 389 cmpfail: 390 rseq_workaround_gcc_asm_size_guess(); 391 return 1; 392 #ifdef RSEQ_COMPARE_TWICE 393 error1: 394 rseq_bug("cpu_id comparison failed"); 395 error2: 396 rseq_bug("expected value comparison failed"); 397 #endif 398 } 399 400 static inline __attribute__((always_inline)) 401 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 402 intptr_t *v2, intptr_t expect2, 403 intptr_t newv, int cpu) 404 { 405 RSEQ_INJECT_C(9) 406 407 rseq_workaround_gcc_asm_size_guess(); 408 __asm__ __volatile__ goto ( 409 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 410 /* Start rseq by storing table entry pointer into rseq_cs. */ 411 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 412 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 413 RSEQ_INJECT_ASM(3) 414 "ldr r0, %[v]\n\t" 415 "cmp %[expect], r0\n\t" 416 "bne %l[cmpfail]\n\t" 417 RSEQ_INJECT_ASM(4) 418 "ldr r0, %[v2]\n\t" 419 "cmp %[expect2], r0\n\t" 420 "bne %l[cmpfail]\n\t" 421 RSEQ_INJECT_ASM(5) 422 #ifdef RSEQ_COMPARE_TWICE 423 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 424 "ldr r0, %[v]\n\t" 425 "cmp %[expect], r0\n\t" 426 "bne %l[error2]\n\t" 427 "ldr r0, %[v2]\n\t" 428 "cmp %[expect2], r0\n\t" 429 "bne %l[error3]\n\t" 430 #endif 431 /* final store */ 432 "str %[newv], %[v]\n\t" 433 "2:\n\t" 434 RSEQ_INJECT_ASM(6) 435 "b 5f\n\t" 436 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 437 "5:\n\t" 438 : /* gcc asm goto does not allow outputs */ 439 : [cpu_id] "r" (cpu), 440 [current_cpu_id] "m" (__rseq_abi.cpu_id), 441 [rseq_cs] "m" (__rseq_abi.rseq_cs), 442 /* cmp2 input */ 443 [v2] "m" (*v2), 444 [expect2] "r" (expect2), 445 /* final store input */ 446 [v] "m" (*v), 447 [expect] "r" (expect), 448 [newv] "r" (newv) 449 RSEQ_INJECT_INPUT 450 : "r0", "memory", "cc" 451 RSEQ_INJECT_CLOBBER 452 : abort, cmpfail 453 #ifdef RSEQ_COMPARE_TWICE 454 , error1, error2, error3 455 #endif 456 ); 457 rseq_workaround_gcc_asm_size_guess(); 458 return 0; 459 abort: 460 rseq_workaround_gcc_asm_size_guess(); 461 RSEQ_INJECT_FAILED 462 return -1; 463 cmpfail: 464 rseq_workaround_gcc_asm_size_guess(); 465 return 1; 466 #ifdef RSEQ_COMPARE_TWICE 467 error1: 468 rseq_bug("cpu_id comparison failed"); 469 error2: 470 rseq_bug("1st expected value comparison failed"); 471 error3: 472 rseq_bug("2nd expected value comparison failed"); 473 #endif 474 } 475 476 static inline __attribute__((always_inline)) 477 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 478 void *dst, void *src, size_t len, 479 intptr_t newv, int cpu) 480 { 481 uint32_t rseq_scratch[3]; 482 483 RSEQ_INJECT_C(9) 484 485 rseq_workaround_gcc_asm_size_guess(); 486 __asm__ __volatile__ goto ( 487 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 488 "str %[src], %[rseq_scratch0]\n\t" 489 "str %[dst], %[rseq_scratch1]\n\t" 490 "str %[len], %[rseq_scratch2]\n\t" 491 /* Start rseq by storing table entry pointer into rseq_cs. */ 492 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 493 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 494 RSEQ_INJECT_ASM(3) 495 "ldr r0, %[v]\n\t" 496 "cmp %[expect], r0\n\t" 497 "bne 5f\n\t" 498 RSEQ_INJECT_ASM(4) 499 #ifdef RSEQ_COMPARE_TWICE 500 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 501 "ldr r0, %[v]\n\t" 502 "cmp %[expect], r0\n\t" 503 "bne 7f\n\t" 504 #endif 505 /* try memcpy */ 506 "cmp %[len], #0\n\t" \ 507 "beq 333f\n\t" \ 508 "222:\n\t" \ 509 "ldrb %%r0, [%[src]]\n\t" \ 510 "strb %%r0, [%[dst]]\n\t" \ 511 "adds %[src], #1\n\t" \ 512 "adds %[dst], #1\n\t" \ 513 "subs %[len], #1\n\t" \ 514 "bne 222b\n\t" \ 515 "333:\n\t" \ 516 RSEQ_INJECT_ASM(5) 517 /* final store */ 518 "str %[newv], %[v]\n\t" 519 "2:\n\t" 520 RSEQ_INJECT_ASM(6) 521 /* teardown */ 522 "ldr %[len], %[rseq_scratch2]\n\t" 523 "ldr %[dst], %[rseq_scratch1]\n\t" 524 "ldr %[src], %[rseq_scratch0]\n\t" 525 "b 8f\n\t" 526 RSEQ_ASM_DEFINE_ABORT(3, 4, 527 /* teardown */ 528 "ldr %[len], %[rseq_scratch2]\n\t" 529 "ldr %[dst], %[rseq_scratch1]\n\t" 530 "ldr %[src], %[rseq_scratch0]\n\t", 531 abort, 1b, 2b, 4f) 532 RSEQ_ASM_DEFINE_CMPFAIL(5, 533 /* teardown */ 534 "ldr %[len], %[rseq_scratch2]\n\t" 535 "ldr %[dst], %[rseq_scratch1]\n\t" 536 "ldr %[src], %[rseq_scratch0]\n\t", 537 cmpfail) 538 #ifdef RSEQ_COMPARE_TWICE 539 RSEQ_ASM_DEFINE_CMPFAIL(6, 540 /* teardown */ 541 "ldr %[len], %[rseq_scratch2]\n\t" 542 "ldr %[dst], %[rseq_scratch1]\n\t" 543 "ldr %[src], %[rseq_scratch0]\n\t", 544 error1) 545 RSEQ_ASM_DEFINE_CMPFAIL(7, 546 /* teardown */ 547 "ldr %[len], %[rseq_scratch2]\n\t" 548 "ldr %[dst], %[rseq_scratch1]\n\t" 549 "ldr %[src], %[rseq_scratch0]\n\t", 550 error2) 551 #endif 552 "8:\n\t" 553 : /* gcc asm goto does not allow outputs */ 554 : [cpu_id] "r" (cpu), 555 [current_cpu_id] "m" (__rseq_abi.cpu_id), 556 [rseq_cs] "m" (__rseq_abi.rseq_cs), 557 /* final store input */ 558 [v] "m" (*v), 559 [expect] "r" (expect), 560 [newv] "r" (newv), 561 /* try memcpy input */ 562 [dst] "r" (dst), 563 [src] "r" (src), 564 [len] "r" (len), 565 [rseq_scratch0] "m" (rseq_scratch[0]), 566 [rseq_scratch1] "m" (rseq_scratch[1]), 567 [rseq_scratch2] "m" (rseq_scratch[2]) 568 RSEQ_INJECT_INPUT 569 : "r0", "memory", "cc" 570 RSEQ_INJECT_CLOBBER 571 : abort, cmpfail 572 #ifdef RSEQ_COMPARE_TWICE 573 , error1, error2 574 #endif 575 ); 576 rseq_workaround_gcc_asm_size_guess(); 577 return 0; 578 abort: 579 rseq_workaround_gcc_asm_size_guess(); 580 RSEQ_INJECT_FAILED 581 return -1; 582 cmpfail: 583 rseq_workaround_gcc_asm_size_guess(); 584 return 1; 585 #ifdef RSEQ_COMPARE_TWICE 586 error1: 587 rseq_workaround_gcc_asm_size_guess(); 588 rseq_bug("cpu_id comparison failed"); 589 error2: 590 rseq_workaround_gcc_asm_size_guess(); 591 rseq_bug("expected value comparison failed"); 592 #endif 593 } 594 595 static inline __attribute__((always_inline)) 596 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 597 void *dst, void *src, size_t len, 598 intptr_t newv, int cpu) 599 { 600 uint32_t rseq_scratch[3]; 601 602 RSEQ_INJECT_C(9) 603 604 rseq_workaround_gcc_asm_size_guess(); 605 __asm__ __volatile__ goto ( 606 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 607 "str %[src], %[rseq_scratch0]\n\t" 608 "str %[dst], %[rseq_scratch1]\n\t" 609 "str %[len], %[rseq_scratch2]\n\t" 610 /* Start rseq by storing table entry pointer into rseq_cs. */ 611 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 612 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 613 RSEQ_INJECT_ASM(3) 614 "ldr r0, %[v]\n\t" 615 "cmp %[expect], r0\n\t" 616 "bne 5f\n\t" 617 RSEQ_INJECT_ASM(4) 618 #ifdef RSEQ_COMPARE_TWICE 619 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 620 "ldr r0, %[v]\n\t" 621 "cmp %[expect], r0\n\t" 622 "bne 7f\n\t" 623 #endif 624 /* try memcpy */ 625 "cmp %[len], #0\n\t" \ 626 "beq 333f\n\t" \ 627 "222:\n\t" \ 628 "ldrb %%r0, [%[src]]\n\t" \ 629 "strb %%r0, [%[dst]]\n\t" \ 630 "adds %[src], #1\n\t" \ 631 "adds %[dst], #1\n\t" \ 632 "subs %[len], #1\n\t" \ 633 "bne 222b\n\t" \ 634 "333:\n\t" \ 635 RSEQ_INJECT_ASM(5) 636 "dmb\n\t" /* full mb provides store-release */ 637 /* final store */ 638 "str %[newv], %[v]\n\t" 639 "2:\n\t" 640 RSEQ_INJECT_ASM(6) 641 /* teardown */ 642 "ldr %[len], %[rseq_scratch2]\n\t" 643 "ldr %[dst], %[rseq_scratch1]\n\t" 644 "ldr %[src], %[rseq_scratch0]\n\t" 645 "b 8f\n\t" 646 RSEQ_ASM_DEFINE_ABORT(3, 4, 647 /* teardown */ 648 "ldr %[len], %[rseq_scratch2]\n\t" 649 "ldr %[dst], %[rseq_scratch1]\n\t" 650 "ldr %[src], %[rseq_scratch0]\n\t", 651 abort, 1b, 2b, 4f) 652 RSEQ_ASM_DEFINE_CMPFAIL(5, 653 /* teardown */ 654 "ldr %[len], %[rseq_scratch2]\n\t" 655 "ldr %[dst], %[rseq_scratch1]\n\t" 656 "ldr %[src], %[rseq_scratch0]\n\t", 657 cmpfail) 658 #ifdef RSEQ_COMPARE_TWICE 659 RSEQ_ASM_DEFINE_CMPFAIL(6, 660 /* teardown */ 661 "ldr %[len], %[rseq_scratch2]\n\t" 662 "ldr %[dst], %[rseq_scratch1]\n\t" 663 "ldr %[src], %[rseq_scratch0]\n\t", 664 error1) 665 RSEQ_ASM_DEFINE_CMPFAIL(7, 666 /* teardown */ 667 "ldr %[len], %[rseq_scratch2]\n\t" 668 "ldr %[dst], %[rseq_scratch1]\n\t" 669 "ldr %[src], %[rseq_scratch0]\n\t", 670 error2) 671 #endif 672 "8:\n\t" 673 : /* gcc asm goto does not allow outputs */ 674 : [cpu_id] "r" (cpu), 675 [current_cpu_id] "m" (__rseq_abi.cpu_id), 676 [rseq_cs] "m" (__rseq_abi.rseq_cs), 677 /* final store input */ 678 [v] "m" (*v), 679 [expect] "r" (expect), 680 [newv] "r" (newv), 681 /* try memcpy input */ 682 [dst] "r" (dst), 683 [src] "r" (src), 684 [len] "r" (len), 685 [rseq_scratch0] "m" (rseq_scratch[0]), 686 [rseq_scratch1] "m" (rseq_scratch[1]), 687 [rseq_scratch2] "m" (rseq_scratch[2]) 688 RSEQ_INJECT_INPUT 689 : "r0", "memory", "cc" 690 RSEQ_INJECT_CLOBBER 691 : abort, cmpfail 692 #ifdef RSEQ_COMPARE_TWICE 693 , error1, error2 694 #endif 695 ); 696 rseq_workaround_gcc_asm_size_guess(); 697 return 0; 698 abort: 699 rseq_workaround_gcc_asm_size_guess(); 700 RSEQ_INJECT_FAILED 701 return -1; 702 cmpfail: 703 rseq_workaround_gcc_asm_size_guess(); 704 return 1; 705 #ifdef RSEQ_COMPARE_TWICE 706 error1: 707 rseq_workaround_gcc_asm_size_guess(); 708 rseq_bug("cpu_id comparison failed"); 709 error2: 710 rseq_workaround_gcc_asm_size_guess(); 711 rseq_bug("expected value comparison failed"); 712 #endif 713 } 714 715 #endif /* !RSEQ_SKIP_FASTPATH */ 716