1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-arm.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8 /* 9 * - ARM little endian 10 * 11 * RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand 12 * value 0x5de3. This traps if user-space reaches this instruction by mistake, 13 * and the uncommon operand ensures the kernel does not move the instruction 14 * pointer to attacker-controlled code on rseq abort. 15 * 16 * The instruction pattern in the A32 instruction set is: 17 * 18 * e7f5def3 udf #24035 ; 0x5de3 19 * 20 * This translates to the following instruction pattern in the T16 instruction 21 * set: 22 * 23 * little endian: 24 * def3 udf #243 ; 0xf3 25 * e7f5 b.n <7f5> 26 * 27 * - ARMv6+ big endian (BE8): 28 * 29 * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian 30 * code and big-endian data. The data value of the signature needs to have its 31 * byte order reversed to generate the trap instruction: 32 * 33 * Data: 0xf3def5e7 34 * 35 * Translates to this A32 instruction pattern: 36 * 37 * e7f5def3 udf #24035 ; 0x5de3 38 * 39 * Translates to this T16 instruction pattern: 40 * 41 * def3 udf #243 ; 0xf3 42 * e7f5 b.n <7f5> 43 * 44 * - Prior to ARMv6 big endian (BE32): 45 * 46 * Prior to ARMv6, -mbig-endian generates big-endian code and data 47 * (which match), so the endianness of the data representation of the 48 * signature should not be reversed. However, the choice between BE32 49 * and BE8 is done by the linker, so we cannot know whether code and 50 * data endianness will be mixed before the linker is invoked. So rather 51 * than try to play tricks with the linker, the rseq signature is simply 52 * data (not a trap instruction) prior to ARMv6 on big endian. This is 53 * why the signature is expressed as data (.word) rather than as 54 * instruction (.inst) in assembler. 55 */ 56 57 #ifdef __ARMEB__ 58 #define RSEQ_SIG 0xf3def5e7 /* udf #24035 ; 0x5de3 (ARMv6+) */ 59 #else 60 #define RSEQ_SIG 0xe7f5def3 /* udf #24035 ; 0x5de3 */ 61 #endif 62 63 #define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 64 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 65 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 66 67 #define rseq_smp_load_acquire(p) \ 68 __extension__ ({ \ 69 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 70 rseq_smp_mb(); \ 71 ____p1; \ 72 }) 73 74 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 75 76 #define rseq_smp_store_release(p, v) \ 77 do { \ 78 rseq_smp_mb(); \ 79 RSEQ_WRITE_ONCE(*p, v); \ 80 } while (0) 81 82 #ifdef RSEQ_SKIP_FASTPATH 83 #include "rseq-skip.h" 84 #else /* !RSEQ_SKIP_FASTPATH */ 85 86 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ 87 post_commit_offset, abort_ip) \ 88 ".pushsection __rseq_cs, \"aw\"\n\t" \ 89 ".balign 32\n\t" \ 90 __rseq_str(label) ":\n\t" \ 91 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 92 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 93 ".popsection\n\t" \ 94 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 95 ".word " __rseq_str(label) "b, 0x0\n\t" \ 96 ".popsection\n\t" 97 98 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 99 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 100 (post_commit_ip - start_ip), abort_ip) 101 102 /* 103 * Exit points of a rseq critical section consist of all instructions outside 104 * of the critical section where a critical section can either branch to or 105 * reach through the normal course of its execution. The abort IP and the 106 * post-commit IP are already part of the __rseq_cs section and should not be 107 * explicitly defined as additional exit points. Knowing all exit points is 108 * useful to assist debuggers stepping over the critical section. 109 */ 110 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 111 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 112 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ 113 ".popsection\n\t" 114 115 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 116 RSEQ_INJECT_ASM(1) \ 117 "adr r0, " __rseq_str(cs_label) "\n\t" \ 118 "str r0, %[" __rseq_str(rseq_cs) "]\n\t" \ 119 __rseq_str(label) ":\n\t" 120 121 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 122 RSEQ_INJECT_ASM(2) \ 123 "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \ 124 "cmp %[" __rseq_str(cpu_id) "], r0\n\t" \ 125 "bne " __rseq_str(label) "\n\t" 126 127 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 128 abort_label, version, flags, \ 129 start_ip, post_commit_offset, abort_ip) \ 130 ".balign 32\n\t" \ 131 __rseq_str(table_label) ":\n\t" \ 132 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 133 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 134 ".word " __rseq_str(RSEQ_SIG) "\n\t" \ 135 __rseq_str(label) ":\n\t" \ 136 teardown \ 137 "b %l[" __rseq_str(abort_label) "]\n\t" 138 139 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \ 140 start_ip, post_commit_ip, abort_ip) \ 141 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 142 abort_label, 0x0, 0x0, start_ip, \ 143 (post_commit_ip - start_ip), abort_ip) 144 145 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 146 __rseq_str(label) ":\n\t" \ 147 teardown \ 148 "b %l[" __rseq_str(cmpfail_label) "]\n\t" 149 150 #define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("") 151 152 static inline __attribute__((always_inline)) 153 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 154 { 155 RSEQ_INJECT_C(9) 156 157 rseq_workaround_gcc_asm_size_guess(); 158 __asm__ __volatile__ goto ( 159 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 160 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 161 #ifdef RSEQ_COMPARE_TWICE 162 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 163 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 164 #endif 165 /* Start rseq by storing table entry pointer into rseq_cs. */ 166 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 167 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 168 RSEQ_INJECT_ASM(3) 169 "ldr r0, %[v]\n\t" 170 "cmp %[expect], r0\n\t" 171 "bne %l[cmpfail]\n\t" 172 RSEQ_INJECT_ASM(4) 173 #ifdef RSEQ_COMPARE_TWICE 174 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 175 "ldr r0, %[v]\n\t" 176 "cmp %[expect], r0\n\t" 177 "bne %l[error2]\n\t" 178 #endif 179 /* final store */ 180 "str %[newv], %[v]\n\t" 181 "2:\n\t" 182 RSEQ_INJECT_ASM(5) 183 "b 5f\n\t" 184 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 185 "5:\n\t" 186 : /* gcc asm goto does not allow outputs */ 187 : [cpu_id] "r" (cpu), 188 [current_cpu_id] "m" (__rseq_abi.cpu_id), 189 [rseq_cs] "m" (__rseq_abi.rseq_cs), 190 [v] "m" (*v), 191 [expect] "r" (expect), 192 [newv] "r" (newv) 193 RSEQ_INJECT_INPUT 194 : "r0", "memory", "cc" 195 RSEQ_INJECT_CLOBBER 196 : abort, cmpfail 197 #ifdef RSEQ_COMPARE_TWICE 198 , error1, error2 199 #endif 200 ); 201 rseq_workaround_gcc_asm_size_guess(); 202 return 0; 203 abort: 204 rseq_workaround_gcc_asm_size_guess(); 205 RSEQ_INJECT_FAILED 206 return -1; 207 cmpfail: 208 rseq_workaround_gcc_asm_size_guess(); 209 return 1; 210 #ifdef RSEQ_COMPARE_TWICE 211 error1: 212 rseq_bug("cpu_id comparison failed"); 213 error2: 214 rseq_bug("expected value comparison failed"); 215 #endif 216 } 217 218 static inline __attribute__((always_inline)) 219 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 220 off_t voffp, intptr_t *load, int cpu) 221 { 222 RSEQ_INJECT_C(9) 223 224 rseq_workaround_gcc_asm_size_guess(); 225 __asm__ __volatile__ goto ( 226 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 227 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 228 #ifdef RSEQ_COMPARE_TWICE 229 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 230 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 231 #endif 232 /* Start rseq by storing table entry pointer into rseq_cs. */ 233 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 234 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 235 RSEQ_INJECT_ASM(3) 236 "ldr r0, %[v]\n\t" 237 "cmp %[expectnot], r0\n\t" 238 "beq %l[cmpfail]\n\t" 239 RSEQ_INJECT_ASM(4) 240 #ifdef RSEQ_COMPARE_TWICE 241 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 242 "ldr r0, %[v]\n\t" 243 "cmp %[expectnot], r0\n\t" 244 "beq %l[error2]\n\t" 245 #endif 246 "str r0, %[load]\n\t" 247 "add r0, %[voffp]\n\t" 248 "ldr r0, [r0]\n\t" 249 /* final store */ 250 "str r0, %[v]\n\t" 251 "2:\n\t" 252 RSEQ_INJECT_ASM(5) 253 "b 5f\n\t" 254 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 255 "5:\n\t" 256 : /* gcc asm goto does not allow outputs */ 257 : [cpu_id] "r" (cpu), 258 [current_cpu_id] "m" (__rseq_abi.cpu_id), 259 [rseq_cs] "m" (__rseq_abi.rseq_cs), 260 /* final store input */ 261 [v] "m" (*v), 262 [expectnot] "r" (expectnot), 263 [voffp] "Ir" (voffp), 264 [load] "m" (*load) 265 RSEQ_INJECT_INPUT 266 : "r0", "memory", "cc" 267 RSEQ_INJECT_CLOBBER 268 : abort, cmpfail 269 #ifdef RSEQ_COMPARE_TWICE 270 , error1, error2 271 #endif 272 ); 273 rseq_workaround_gcc_asm_size_guess(); 274 return 0; 275 abort: 276 rseq_workaround_gcc_asm_size_guess(); 277 RSEQ_INJECT_FAILED 278 return -1; 279 cmpfail: 280 rseq_workaround_gcc_asm_size_guess(); 281 return 1; 282 #ifdef RSEQ_COMPARE_TWICE 283 error1: 284 rseq_bug("cpu_id comparison failed"); 285 error2: 286 rseq_bug("expected value comparison failed"); 287 #endif 288 } 289 290 static inline __attribute__((always_inline)) 291 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 292 { 293 RSEQ_INJECT_C(9) 294 295 rseq_workaround_gcc_asm_size_guess(); 296 __asm__ __volatile__ goto ( 297 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 298 #ifdef RSEQ_COMPARE_TWICE 299 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 300 #endif 301 /* Start rseq by storing table entry pointer into rseq_cs. */ 302 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 303 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 304 RSEQ_INJECT_ASM(3) 305 #ifdef RSEQ_COMPARE_TWICE 306 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 307 #endif 308 "ldr r0, %[v]\n\t" 309 "add r0, %[count]\n\t" 310 /* final store */ 311 "str r0, %[v]\n\t" 312 "2:\n\t" 313 RSEQ_INJECT_ASM(4) 314 "b 5f\n\t" 315 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 316 "5:\n\t" 317 : /* gcc asm goto does not allow outputs */ 318 : [cpu_id] "r" (cpu), 319 [current_cpu_id] "m" (__rseq_abi.cpu_id), 320 [rseq_cs] "m" (__rseq_abi.rseq_cs), 321 [v] "m" (*v), 322 [count] "Ir" (count) 323 RSEQ_INJECT_INPUT 324 : "r0", "memory", "cc" 325 RSEQ_INJECT_CLOBBER 326 : abort 327 #ifdef RSEQ_COMPARE_TWICE 328 , error1 329 #endif 330 ); 331 rseq_workaround_gcc_asm_size_guess(); 332 return 0; 333 abort: 334 rseq_workaround_gcc_asm_size_guess(); 335 RSEQ_INJECT_FAILED 336 return -1; 337 #ifdef RSEQ_COMPARE_TWICE 338 error1: 339 rseq_bug("cpu_id comparison failed"); 340 #endif 341 } 342 343 static inline __attribute__((always_inline)) 344 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 345 intptr_t *v2, intptr_t newv2, 346 intptr_t newv, int cpu) 347 { 348 RSEQ_INJECT_C(9) 349 350 rseq_workaround_gcc_asm_size_guess(); 351 __asm__ __volatile__ goto ( 352 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 353 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 354 #ifdef RSEQ_COMPARE_TWICE 355 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 356 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 357 #endif 358 /* Start rseq by storing table entry pointer into rseq_cs. */ 359 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 360 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 361 RSEQ_INJECT_ASM(3) 362 "ldr r0, %[v]\n\t" 363 "cmp %[expect], r0\n\t" 364 "bne %l[cmpfail]\n\t" 365 RSEQ_INJECT_ASM(4) 366 #ifdef RSEQ_COMPARE_TWICE 367 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 368 "ldr r0, %[v]\n\t" 369 "cmp %[expect], r0\n\t" 370 "bne %l[error2]\n\t" 371 #endif 372 /* try store */ 373 "str %[newv2], %[v2]\n\t" 374 RSEQ_INJECT_ASM(5) 375 /* final store */ 376 "str %[newv], %[v]\n\t" 377 "2:\n\t" 378 RSEQ_INJECT_ASM(6) 379 "b 5f\n\t" 380 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 381 "5:\n\t" 382 : /* gcc asm goto does not allow outputs */ 383 : [cpu_id] "r" (cpu), 384 [current_cpu_id] "m" (__rseq_abi.cpu_id), 385 [rseq_cs] "m" (__rseq_abi.rseq_cs), 386 /* try store input */ 387 [v2] "m" (*v2), 388 [newv2] "r" (newv2), 389 /* final store input */ 390 [v] "m" (*v), 391 [expect] "r" (expect), 392 [newv] "r" (newv) 393 RSEQ_INJECT_INPUT 394 : "r0", "memory", "cc" 395 RSEQ_INJECT_CLOBBER 396 : abort, cmpfail 397 #ifdef RSEQ_COMPARE_TWICE 398 , error1, error2 399 #endif 400 ); 401 rseq_workaround_gcc_asm_size_guess(); 402 return 0; 403 abort: 404 rseq_workaround_gcc_asm_size_guess(); 405 RSEQ_INJECT_FAILED 406 return -1; 407 cmpfail: 408 rseq_workaround_gcc_asm_size_guess(); 409 return 1; 410 #ifdef RSEQ_COMPARE_TWICE 411 error1: 412 rseq_bug("cpu_id comparison failed"); 413 error2: 414 rseq_bug("expected value comparison failed"); 415 #endif 416 } 417 418 static inline __attribute__((always_inline)) 419 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 420 intptr_t *v2, intptr_t newv2, 421 intptr_t newv, int cpu) 422 { 423 RSEQ_INJECT_C(9) 424 425 rseq_workaround_gcc_asm_size_guess(); 426 __asm__ __volatile__ goto ( 427 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 428 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 429 #ifdef RSEQ_COMPARE_TWICE 430 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 431 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 432 #endif 433 /* Start rseq by storing table entry pointer into rseq_cs. */ 434 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 435 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 436 RSEQ_INJECT_ASM(3) 437 "ldr r0, %[v]\n\t" 438 "cmp %[expect], r0\n\t" 439 "bne %l[cmpfail]\n\t" 440 RSEQ_INJECT_ASM(4) 441 #ifdef RSEQ_COMPARE_TWICE 442 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 443 "ldr r0, %[v]\n\t" 444 "cmp %[expect], r0\n\t" 445 "bne %l[error2]\n\t" 446 #endif 447 /* try store */ 448 "str %[newv2], %[v2]\n\t" 449 RSEQ_INJECT_ASM(5) 450 "dmb\n\t" /* full mb provides store-release */ 451 /* final store */ 452 "str %[newv], %[v]\n\t" 453 "2:\n\t" 454 RSEQ_INJECT_ASM(6) 455 "b 5f\n\t" 456 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 457 "5:\n\t" 458 : /* gcc asm goto does not allow outputs */ 459 : [cpu_id] "r" (cpu), 460 [current_cpu_id] "m" (__rseq_abi.cpu_id), 461 [rseq_cs] "m" (__rseq_abi.rseq_cs), 462 /* try store input */ 463 [v2] "m" (*v2), 464 [newv2] "r" (newv2), 465 /* final store input */ 466 [v] "m" (*v), 467 [expect] "r" (expect), 468 [newv] "r" (newv) 469 RSEQ_INJECT_INPUT 470 : "r0", "memory", "cc" 471 RSEQ_INJECT_CLOBBER 472 : abort, cmpfail 473 #ifdef RSEQ_COMPARE_TWICE 474 , error1, error2 475 #endif 476 ); 477 rseq_workaround_gcc_asm_size_guess(); 478 return 0; 479 abort: 480 rseq_workaround_gcc_asm_size_guess(); 481 RSEQ_INJECT_FAILED 482 return -1; 483 cmpfail: 484 rseq_workaround_gcc_asm_size_guess(); 485 return 1; 486 #ifdef RSEQ_COMPARE_TWICE 487 error1: 488 rseq_bug("cpu_id comparison failed"); 489 error2: 490 rseq_bug("expected value comparison failed"); 491 #endif 492 } 493 494 static inline __attribute__((always_inline)) 495 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 496 intptr_t *v2, intptr_t expect2, 497 intptr_t newv, int cpu) 498 { 499 RSEQ_INJECT_C(9) 500 501 rseq_workaround_gcc_asm_size_guess(); 502 __asm__ __volatile__ goto ( 503 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 504 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 505 #ifdef RSEQ_COMPARE_TWICE 506 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 507 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 508 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 509 #endif 510 /* Start rseq by storing table entry pointer into rseq_cs. */ 511 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 512 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 513 RSEQ_INJECT_ASM(3) 514 "ldr r0, %[v]\n\t" 515 "cmp %[expect], r0\n\t" 516 "bne %l[cmpfail]\n\t" 517 RSEQ_INJECT_ASM(4) 518 "ldr r0, %[v2]\n\t" 519 "cmp %[expect2], r0\n\t" 520 "bne %l[cmpfail]\n\t" 521 RSEQ_INJECT_ASM(5) 522 #ifdef RSEQ_COMPARE_TWICE 523 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 524 "ldr r0, %[v]\n\t" 525 "cmp %[expect], r0\n\t" 526 "bne %l[error2]\n\t" 527 "ldr r0, %[v2]\n\t" 528 "cmp %[expect2], r0\n\t" 529 "bne %l[error3]\n\t" 530 #endif 531 /* final store */ 532 "str %[newv], %[v]\n\t" 533 "2:\n\t" 534 RSEQ_INJECT_ASM(6) 535 "b 5f\n\t" 536 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 537 "5:\n\t" 538 : /* gcc asm goto does not allow outputs */ 539 : [cpu_id] "r" (cpu), 540 [current_cpu_id] "m" (__rseq_abi.cpu_id), 541 [rseq_cs] "m" (__rseq_abi.rseq_cs), 542 /* cmp2 input */ 543 [v2] "m" (*v2), 544 [expect2] "r" (expect2), 545 /* final store input */ 546 [v] "m" (*v), 547 [expect] "r" (expect), 548 [newv] "r" (newv) 549 RSEQ_INJECT_INPUT 550 : "r0", "memory", "cc" 551 RSEQ_INJECT_CLOBBER 552 : abort, cmpfail 553 #ifdef RSEQ_COMPARE_TWICE 554 , error1, error2, error3 555 #endif 556 ); 557 rseq_workaround_gcc_asm_size_guess(); 558 return 0; 559 abort: 560 rseq_workaround_gcc_asm_size_guess(); 561 RSEQ_INJECT_FAILED 562 return -1; 563 cmpfail: 564 rseq_workaround_gcc_asm_size_guess(); 565 return 1; 566 #ifdef RSEQ_COMPARE_TWICE 567 error1: 568 rseq_bug("cpu_id comparison failed"); 569 error2: 570 rseq_bug("1st expected value comparison failed"); 571 error3: 572 rseq_bug("2nd expected value comparison failed"); 573 #endif 574 } 575 576 static inline __attribute__((always_inline)) 577 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 578 void *dst, void *src, size_t len, 579 intptr_t newv, int cpu) 580 { 581 uint32_t rseq_scratch[3]; 582 583 RSEQ_INJECT_C(9) 584 585 rseq_workaround_gcc_asm_size_guess(); 586 __asm__ __volatile__ goto ( 587 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 588 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 589 #ifdef RSEQ_COMPARE_TWICE 590 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 591 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 592 #endif 593 "str %[src], %[rseq_scratch0]\n\t" 594 "str %[dst], %[rseq_scratch1]\n\t" 595 "str %[len], %[rseq_scratch2]\n\t" 596 /* Start rseq by storing table entry pointer into rseq_cs. */ 597 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 598 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 599 RSEQ_INJECT_ASM(3) 600 "ldr r0, %[v]\n\t" 601 "cmp %[expect], r0\n\t" 602 "bne 5f\n\t" 603 RSEQ_INJECT_ASM(4) 604 #ifdef RSEQ_COMPARE_TWICE 605 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 606 "ldr r0, %[v]\n\t" 607 "cmp %[expect], r0\n\t" 608 "bne 7f\n\t" 609 #endif 610 /* try memcpy */ 611 "cmp %[len], #0\n\t" \ 612 "beq 333f\n\t" \ 613 "222:\n\t" \ 614 "ldrb %%r0, [%[src]]\n\t" \ 615 "strb %%r0, [%[dst]]\n\t" \ 616 "adds %[src], #1\n\t" \ 617 "adds %[dst], #1\n\t" \ 618 "subs %[len], #1\n\t" \ 619 "bne 222b\n\t" \ 620 "333:\n\t" \ 621 RSEQ_INJECT_ASM(5) 622 /* final store */ 623 "str %[newv], %[v]\n\t" 624 "2:\n\t" 625 RSEQ_INJECT_ASM(6) 626 /* teardown */ 627 "ldr %[len], %[rseq_scratch2]\n\t" 628 "ldr %[dst], %[rseq_scratch1]\n\t" 629 "ldr %[src], %[rseq_scratch0]\n\t" 630 "b 8f\n\t" 631 RSEQ_ASM_DEFINE_ABORT(3, 4, 632 /* teardown */ 633 "ldr %[len], %[rseq_scratch2]\n\t" 634 "ldr %[dst], %[rseq_scratch1]\n\t" 635 "ldr %[src], %[rseq_scratch0]\n\t", 636 abort, 1b, 2b, 4f) 637 RSEQ_ASM_DEFINE_CMPFAIL(5, 638 /* teardown */ 639 "ldr %[len], %[rseq_scratch2]\n\t" 640 "ldr %[dst], %[rseq_scratch1]\n\t" 641 "ldr %[src], %[rseq_scratch0]\n\t", 642 cmpfail) 643 #ifdef RSEQ_COMPARE_TWICE 644 RSEQ_ASM_DEFINE_CMPFAIL(6, 645 /* teardown */ 646 "ldr %[len], %[rseq_scratch2]\n\t" 647 "ldr %[dst], %[rseq_scratch1]\n\t" 648 "ldr %[src], %[rseq_scratch0]\n\t", 649 error1) 650 RSEQ_ASM_DEFINE_CMPFAIL(7, 651 /* teardown */ 652 "ldr %[len], %[rseq_scratch2]\n\t" 653 "ldr %[dst], %[rseq_scratch1]\n\t" 654 "ldr %[src], %[rseq_scratch0]\n\t", 655 error2) 656 #endif 657 "8:\n\t" 658 : /* gcc asm goto does not allow outputs */ 659 : [cpu_id] "r" (cpu), 660 [current_cpu_id] "m" (__rseq_abi.cpu_id), 661 [rseq_cs] "m" (__rseq_abi.rseq_cs), 662 /* final store input */ 663 [v] "m" (*v), 664 [expect] "r" (expect), 665 [newv] "r" (newv), 666 /* try memcpy input */ 667 [dst] "r" (dst), 668 [src] "r" (src), 669 [len] "r" (len), 670 [rseq_scratch0] "m" (rseq_scratch[0]), 671 [rseq_scratch1] "m" (rseq_scratch[1]), 672 [rseq_scratch2] "m" (rseq_scratch[2]) 673 RSEQ_INJECT_INPUT 674 : "r0", "memory", "cc" 675 RSEQ_INJECT_CLOBBER 676 : abort, cmpfail 677 #ifdef RSEQ_COMPARE_TWICE 678 , error1, error2 679 #endif 680 ); 681 rseq_workaround_gcc_asm_size_guess(); 682 return 0; 683 abort: 684 rseq_workaround_gcc_asm_size_guess(); 685 RSEQ_INJECT_FAILED 686 return -1; 687 cmpfail: 688 rseq_workaround_gcc_asm_size_guess(); 689 return 1; 690 #ifdef RSEQ_COMPARE_TWICE 691 error1: 692 rseq_workaround_gcc_asm_size_guess(); 693 rseq_bug("cpu_id comparison failed"); 694 error2: 695 rseq_workaround_gcc_asm_size_guess(); 696 rseq_bug("expected value comparison failed"); 697 #endif 698 } 699 700 static inline __attribute__((always_inline)) 701 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 702 void *dst, void *src, size_t len, 703 intptr_t newv, int cpu) 704 { 705 uint32_t rseq_scratch[3]; 706 707 RSEQ_INJECT_C(9) 708 709 rseq_workaround_gcc_asm_size_guess(); 710 __asm__ __volatile__ goto ( 711 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 712 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 713 #ifdef RSEQ_COMPARE_TWICE 714 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 715 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 716 #endif 717 "str %[src], %[rseq_scratch0]\n\t" 718 "str %[dst], %[rseq_scratch1]\n\t" 719 "str %[len], %[rseq_scratch2]\n\t" 720 /* Start rseq by storing table entry pointer into rseq_cs. */ 721 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 722 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 723 RSEQ_INJECT_ASM(3) 724 "ldr r0, %[v]\n\t" 725 "cmp %[expect], r0\n\t" 726 "bne 5f\n\t" 727 RSEQ_INJECT_ASM(4) 728 #ifdef RSEQ_COMPARE_TWICE 729 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 730 "ldr r0, %[v]\n\t" 731 "cmp %[expect], r0\n\t" 732 "bne 7f\n\t" 733 #endif 734 /* try memcpy */ 735 "cmp %[len], #0\n\t" \ 736 "beq 333f\n\t" \ 737 "222:\n\t" \ 738 "ldrb %%r0, [%[src]]\n\t" \ 739 "strb %%r0, [%[dst]]\n\t" \ 740 "adds %[src], #1\n\t" \ 741 "adds %[dst], #1\n\t" \ 742 "subs %[len], #1\n\t" \ 743 "bne 222b\n\t" \ 744 "333:\n\t" \ 745 RSEQ_INJECT_ASM(5) 746 "dmb\n\t" /* full mb provides store-release */ 747 /* final store */ 748 "str %[newv], %[v]\n\t" 749 "2:\n\t" 750 RSEQ_INJECT_ASM(6) 751 /* teardown */ 752 "ldr %[len], %[rseq_scratch2]\n\t" 753 "ldr %[dst], %[rseq_scratch1]\n\t" 754 "ldr %[src], %[rseq_scratch0]\n\t" 755 "b 8f\n\t" 756 RSEQ_ASM_DEFINE_ABORT(3, 4, 757 /* teardown */ 758 "ldr %[len], %[rseq_scratch2]\n\t" 759 "ldr %[dst], %[rseq_scratch1]\n\t" 760 "ldr %[src], %[rseq_scratch0]\n\t", 761 abort, 1b, 2b, 4f) 762 RSEQ_ASM_DEFINE_CMPFAIL(5, 763 /* teardown */ 764 "ldr %[len], %[rseq_scratch2]\n\t" 765 "ldr %[dst], %[rseq_scratch1]\n\t" 766 "ldr %[src], %[rseq_scratch0]\n\t", 767 cmpfail) 768 #ifdef RSEQ_COMPARE_TWICE 769 RSEQ_ASM_DEFINE_CMPFAIL(6, 770 /* teardown */ 771 "ldr %[len], %[rseq_scratch2]\n\t" 772 "ldr %[dst], %[rseq_scratch1]\n\t" 773 "ldr %[src], %[rseq_scratch0]\n\t", 774 error1) 775 RSEQ_ASM_DEFINE_CMPFAIL(7, 776 /* teardown */ 777 "ldr %[len], %[rseq_scratch2]\n\t" 778 "ldr %[dst], %[rseq_scratch1]\n\t" 779 "ldr %[src], %[rseq_scratch0]\n\t", 780 error2) 781 #endif 782 "8:\n\t" 783 : /* gcc asm goto does not allow outputs */ 784 : [cpu_id] "r" (cpu), 785 [current_cpu_id] "m" (__rseq_abi.cpu_id), 786 [rseq_cs] "m" (__rseq_abi.rseq_cs), 787 /* final store input */ 788 [v] "m" (*v), 789 [expect] "r" (expect), 790 [newv] "r" (newv), 791 /* try memcpy input */ 792 [dst] "r" (dst), 793 [src] "r" (src), 794 [len] "r" (len), 795 [rseq_scratch0] "m" (rseq_scratch[0]), 796 [rseq_scratch1] "m" (rseq_scratch[1]), 797 [rseq_scratch2] "m" (rseq_scratch[2]) 798 RSEQ_INJECT_INPUT 799 : "r0", "memory", "cc" 800 RSEQ_INJECT_CLOBBER 801 : abort, cmpfail 802 #ifdef RSEQ_COMPARE_TWICE 803 , error1, error2 804 #endif 805 ); 806 rseq_workaround_gcc_asm_size_guess(); 807 return 0; 808 abort: 809 rseq_workaround_gcc_asm_size_guess(); 810 RSEQ_INJECT_FAILED 811 return -1; 812 cmpfail: 813 rseq_workaround_gcc_asm_size_guess(); 814 return 1; 815 #ifdef RSEQ_COMPARE_TWICE 816 error1: 817 rseq_workaround_gcc_asm_size_guess(); 818 rseq_bug("cpu_id comparison failed"); 819 error2: 820 rseq_workaround_gcc_asm_size_guess(); 821 rseq_bug("expected value comparison failed"); 822 #endif 823 } 824 825 #endif /* !RSEQ_SKIP_FASTPATH */ 826