1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-arm.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8 /* 9 * RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand 10 * value 0x5de3. This traps if user-space reaches this instruction by mistake, 11 * and the uncommon operand ensures the kernel does not move the instruction 12 * pointer to attacker-controlled code on rseq abort. 13 * 14 * The instruction pattern in the A32 instruction set is: 15 * 16 * e7f5def3 udf #24035 ; 0x5de3 17 * 18 * This translates to the following instruction pattern in the T16 instruction 19 * set: 20 * 21 * little endian: 22 * def3 udf #243 ; 0xf3 23 * e7f5 b.n <7f5> 24 * 25 * pre-ARMv6 big endian code: 26 * e7f5 b.n <7f5> 27 * def3 udf #243 ; 0xf3 28 * 29 * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian 30 * code and big-endian data. Ensure the RSEQ_SIG data signature matches code 31 * endianness. Prior to ARMv6, -mbig-endian generates big-endian code and data 32 * (which match), so there is no need to reverse the endianness of the data 33 * representation of the signature. However, the choice between BE32 and BE8 34 * is done by the linker, so we cannot know whether code and data endianness 35 * will be mixed before the linker is invoked. 36 */ 37 38 #define RSEQ_SIG_CODE 0xe7f5def3 39 40 #ifndef __ASSEMBLER__ 41 42 #define RSEQ_SIG_DATA \ 43 ({ \ 44 int sig; \ 45 asm volatile ("b 2f\n\t" \ 46 "1: .inst " __rseq_str(RSEQ_SIG_CODE) "\n\t" \ 47 "2:\n\t" \ 48 "ldr %[sig], 1b\n\t" \ 49 : [sig] "=r" (sig)); \ 50 sig; \ 51 }) 52 53 #define RSEQ_SIG RSEQ_SIG_DATA 54 55 #endif 56 57 #define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 58 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 59 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 60 61 #define rseq_smp_load_acquire(p) \ 62 __extension__ ({ \ 63 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 64 rseq_smp_mb(); \ 65 ____p1; \ 66 }) 67 68 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 69 70 #define rseq_smp_store_release(p, v) \ 71 do { \ 72 rseq_smp_mb(); \ 73 RSEQ_WRITE_ONCE(*p, v); \ 74 } while (0) 75 76 #ifdef RSEQ_SKIP_FASTPATH 77 #include "rseq-skip.h" 78 #else /* !RSEQ_SKIP_FASTPATH */ 79 80 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ 81 post_commit_offset, abort_ip) \ 82 ".pushsection __rseq_cs, \"aw\"\n\t" \ 83 ".balign 32\n\t" \ 84 __rseq_str(label) ":\n\t" \ 85 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 86 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 87 ".popsection\n\t" \ 88 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 89 ".word " __rseq_str(label) "b, 0x0\n\t" \ 90 ".popsection\n\t" 91 92 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 93 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 94 (post_commit_ip - start_ip), abort_ip) 95 96 /* 97 * Exit points of a rseq critical section consist of all instructions outside 98 * of the critical section where a critical section can either branch to or 99 * reach through the normal course of its execution. The abort IP and the 100 * post-commit IP are already part of the __rseq_cs section and should not be 101 * explicitly defined as additional exit points. Knowing all exit points is 102 * useful to assist debuggers stepping over the critical section. 103 */ 104 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 105 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 106 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ 107 ".popsection\n\t" 108 109 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 110 RSEQ_INJECT_ASM(1) \ 111 "adr r0, " __rseq_str(cs_label) "\n\t" \ 112 "str r0, %[" __rseq_str(rseq_cs) "]\n\t" \ 113 __rseq_str(label) ":\n\t" 114 115 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 116 RSEQ_INJECT_ASM(2) \ 117 "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \ 118 "cmp %[" __rseq_str(cpu_id) "], r0\n\t" \ 119 "bne " __rseq_str(label) "\n\t" 120 121 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 122 abort_label, version, flags, \ 123 start_ip, post_commit_offset, abort_ip) \ 124 ".balign 32\n\t" \ 125 __rseq_str(table_label) ":\n\t" \ 126 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 127 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 128 ".arm\n\t" \ 129 ".inst " __rseq_str(RSEQ_SIG_CODE) "\n\t" \ 130 __rseq_str(label) ":\n\t" \ 131 teardown \ 132 "b %l[" __rseq_str(abort_label) "]\n\t" 133 134 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \ 135 start_ip, post_commit_ip, abort_ip) \ 136 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ 137 abort_label, 0x0, 0x0, start_ip, \ 138 (post_commit_ip - start_ip), abort_ip) 139 140 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 141 __rseq_str(label) ":\n\t" \ 142 teardown \ 143 "b %l[" __rseq_str(cmpfail_label) "]\n\t" 144 145 #define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("") 146 147 static inline __attribute__((always_inline)) 148 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 149 { 150 RSEQ_INJECT_C(9) 151 152 rseq_workaround_gcc_asm_size_guess(); 153 __asm__ __volatile__ goto ( 154 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 155 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 156 #ifdef RSEQ_COMPARE_TWICE 157 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 158 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 159 #endif 160 /* Start rseq by storing table entry pointer into rseq_cs. */ 161 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 162 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 163 RSEQ_INJECT_ASM(3) 164 "ldr r0, %[v]\n\t" 165 "cmp %[expect], r0\n\t" 166 "bne %l[cmpfail]\n\t" 167 RSEQ_INJECT_ASM(4) 168 #ifdef RSEQ_COMPARE_TWICE 169 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 170 "ldr r0, %[v]\n\t" 171 "cmp %[expect], r0\n\t" 172 "bne %l[error2]\n\t" 173 #endif 174 /* final store */ 175 "str %[newv], %[v]\n\t" 176 "2:\n\t" 177 RSEQ_INJECT_ASM(5) 178 "b 5f\n\t" 179 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 180 "5:\n\t" 181 : /* gcc asm goto does not allow outputs */ 182 : [cpu_id] "r" (cpu), 183 [current_cpu_id] "m" (__rseq_abi.cpu_id), 184 [rseq_cs] "m" (__rseq_abi.rseq_cs), 185 [v] "m" (*v), 186 [expect] "r" (expect), 187 [newv] "r" (newv) 188 RSEQ_INJECT_INPUT 189 : "r0", "memory", "cc" 190 RSEQ_INJECT_CLOBBER 191 : abort, cmpfail 192 #ifdef RSEQ_COMPARE_TWICE 193 , error1, error2 194 #endif 195 ); 196 rseq_workaround_gcc_asm_size_guess(); 197 return 0; 198 abort: 199 rseq_workaround_gcc_asm_size_guess(); 200 RSEQ_INJECT_FAILED 201 return -1; 202 cmpfail: 203 rseq_workaround_gcc_asm_size_guess(); 204 return 1; 205 #ifdef RSEQ_COMPARE_TWICE 206 error1: 207 rseq_bug("cpu_id comparison failed"); 208 error2: 209 rseq_bug("expected value comparison failed"); 210 #endif 211 } 212 213 static inline __attribute__((always_inline)) 214 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 215 off_t voffp, intptr_t *load, int cpu) 216 { 217 RSEQ_INJECT_C(9) 218 219 rseq_workaround_gcc_asm_size_guess(); 220 __asm__ __volatile__ goto ( 221 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 222 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 223 #ifdef RSEQ_COMPARE_TWICE 224 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 225 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 226 #endif 227 /* Start rseq by storing table entry pointer into rseq_cs. */ 228 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 229 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 230 RSEQ_INJECT_ASM(3) 231 "ldr r0, %[v]\n\t" 232 "cmp %[expectnot], r0\n\t" 233 "beq %l[cmpfail]\n\t" 234 RSEQ_INJECT_ASM(4) 235 #ifdef RSEQ_COMPARE_TWICE 236 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 237 "ldr r0, %[v]\n\t" 238 "cmp %[expectnot], r0\n\t" 239 "beq %l[error2]\n\t" 240 #endif 241 "str r0, %[load]\n\t" 242 "add r0, %[voffp]\n\t" 243 "ldr r0, [r0]\n\t" 244 /* final store */ 245 "str r0, %[v]\n\t" 246 "2:\n\t" 247 RSEQ_INJECT_ASM(5) 248 "b 5f\n\t" 249 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 250 "5:\n\t" 251 : /* gcc asm goto does not allow outputs */ 252 : [cpu_id] "r" (cpu), 253 [current_cpu_id] "m" (__rseq_abi.cpu_id), 254 [rseq_cs] "m" (__rseq_abi.rseq_cs), 255 /* final store input */ 256 [v] "m" (*v), 257 [expectnot] "r" (expectnot), 258 [voffp] "Ir" (voffp), 259 [load] "m" (*load) 260 RSEQ_INJECT_INPUT 261 : "r0", "memory", "cc" 262 RSEQ_INJECT_CLOBBER 263 : abort, cmpfail 264 #ifdef RSEQ_COMPARE_TWICE 265 , error1, error2 266 #endif 267 ); 268 rseq_workaround_gcc_asm_size_guess(); 269 return 0; 270 abort: 271 rseq_workaround_gcc_asm_size_guess(); 272 RSEQ_INJECT_FAILED 273 return -1; 274 cmpfail: 275 rseq_workaround_gcc_asm_size_guess(); 276 return 1; 277 #ifdef RSEQ_COMPARE_TWICE 278 error1: 279 rseq_bug("cpu_id comparison failed"); 280 error2: 281 rseq_bug("expected value comparison failed"); 282 #endif 283 } 284 285 static inline __attribute__((always_inline)) 286 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 287 { 288 RSEQ_INJECT_C(9) 289 290 rseq_workaround_gcc_asm_size_guess(); 291 __asm__ __volatile__ goto ( 292 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 293 #ifdef RSEQ_COMPARE_TWICE 294 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 295 #endif 296 /* Start rseq by storing table entry pointer into rseq_cs. */ 297 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 298 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 299 RSEQ_INJECT_ASM(3) 300 #ifdef RSEQ_COMPARE_TWICE 301 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 302 #endif 303 "ldr r0, %[v]\n\t" 304 "add r0, %[count]\n\t" 305 /* final store */ 306 "str r0, %[v]\n\t" 307 "2:\n\t" 308 RSEQ_INJECT_ASM(4) 309 "b 5f\n\t" 310 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 311 "5:\n\t" 312 : /* gcc asm goto does not allow outputs */ 313 : [cpu_id] "r" (cpu), 314 [current_cpu_id] "m" (__rseq_abi.cpu_id), 315 [rseq_cs] "m" (__rseq_abi.rseq_cs), 316 [v] "m" (*v), 317 [count] "Ir" (count) 318 RSEQ_INJECT_INPUT 319 : "r0", "memory", "cc" 320 RSEQ_INJECT_CLOBBER 321 : abort 322 #ifdef RSEQ_COMPARE_TWICE 323 , error1 324 #endif 325 ); 326 rseq_workaround_gcc_asm_size_guess(); 327 return 0; 328 abort: 329 rseq_workaround_gcc_asm_size_guess(); 330 RSEQ_INJECT_FAILED 331 return -1; 332 #ifdef RSEQ_COMPARE_TWICE 333 error1: 334 rseq_bug("cpu_id comparison failed"); 335 #endif 336 } 337 338 static inline __attribute__((always_inline)) 339 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 340 intptr_t *v2, intptr_t newv2, 341 intptr_t newv, int cpu) 342 { 343 RSEQ_INJECT_C(9) 344 345 rseq_workaround_gcc_asm_size_guess(); 346 __asm__ __volatile__ goto ( 347 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 348 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 349 #ifdef RSEQ_COMPARE_TWICE 350 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 351 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 352 #endif 353 /* Start rseq by storing table entry pointer into rseq_cs. */ 354 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 355 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 356 RSEQ_INJECT_ASM(3) 357 "ldr r0, %[v]\n\t" 358 "cmp %[expect], r0\n\t" 359 "bne %l[cmpfail]\n\t" 360 RSEQ_INJECT_ASM(4) 361 #ifdef RSEQ_COMPARE_TWICE 362 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 363 "ldr r0, %[v]\n\t" 364 "cmp %[expect], r0\n\t" 365 "bne %l[error2]\n\t" 366 #endif 367 /* try store */ 368 "str %[newv2], %[v2]\n\t" 369 RSEQ_INJECT_ASM(5) 370 /* final store */ 371 "str %[newv], %[v]\n\t" 372 "2:\n\t" 373 RSEQ_INJECT_ASM(6) 374 "b 5f\n\t" 375 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 376 "5:\n\t" 377 : /* gcc asm goto does not allow outputs */ 378 : [cpu_id] "r" (cpu), 379 [current_cpu_id] "m" (__rseq_abi.cpu_id), 380 [rseq_cs] "m" (__rseq_abi.rseq_cs), 381 /* try store input */ 382 [v2] "m" (*v2), 383 [newv2] "r" (newv2), 384 /* final store input */ 385 [v] "m" (*v), 386 [expect] "r" (expect), 387 [newv] "r" (newv) 388 RSEQ_INJECT_INPUT 389 : "r0", "memory", "cc" 390 RSEQ_INJECT_CLOBBER 391 : abort, cmpfail 392 #ifdef RSEQ_COMPARE_TWICE 393 , error1, error2 394 #endif 395 ); 396 rseq_workaround_gcc_asm_size_guess(); 397 return 0; 398 abort: 399 rseq_workaround_gcc_asm_size_guess(); 400 RSEQ_INJECT_FAILED 401 return -1; 402 cmpfail: 403 rseq_workaround_gcc_asm_size_guess(); 404 return 1; 405 #ifdef RSEQ_COMPARE_TWICE 406 error1: 407 rseq_bug("cpu_id comparison failed"); 408 error2: 409 rseq_bug("expected value comparison failed"); 410 #endif 411 } 412 413 static inline __attribute__((always_inline)) 414 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 415 intptr_t *v2, intptr_t newv2, 416 intptr_t newv, int cpu) 417 { 418 RSEQ_INJECT_C(9) 419 420 rseq_workaround_gcc_asm_size_guess(); 421 __asm__ __volatile__ goto ( 422 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 423 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 424 #ifdef RSEQ_COMPARE_TWICE 425 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 426 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 427 #endif 428 /* Start rseq by storing table entry pointer into rseq_cs. */ 429 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 430 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 431 RSEQ_INJECT_ASM(3) 432 "ldr r0, %[v]\n\t" 433 "cmp %[expect], r0\n\t" 434 "bne %l[cmpfail]\n\t" 435 RSEQ_INJECT_ASM(4) 436 #ifdef RSEQ_COMPARE_TWICE 437 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 438 "ldr r0, %[v]\n\t" 439 "cmp %[expect], r0\n\t" 440 "bne %l[error2]\n\t" 441 #endif 442 /* try store */ 443 "str %[newv2], %[v2]\n\t" 444 RSEQ_INJECT_ASM(5) 445 "dmb\n\t" /* full mb provides store-release */ 446 /* final store */ 447 "str %[newv], %[v]\n\t" 448 "2:\n\t" 449 RSEQ_INJECT_ASM(6) 450 "b 5f\n\t" 451 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 452 "5:\n\t" 453 : /* gcc asm goto does not allow outputs */ 454 : [cpu_id] "r" (cpu), 455 [current_cpu_id] "m" (__rseq_abi.cpu_id), 456 [rseq_cs] "m" (__rseq_abi.rseq_cs), 457 /* try store input */ 458 [v2] "m" (*v2), 459 [newv2] "r" (newv2), 460 /* final store input */ 461 [v] "m" (*v), 462 [expect] "r" (expect), 463 [newv] "r" (newv) 464 RSEQ_INJECT_INPUT 465 : "r0", "memory", "cc" 466 RSEQ_INJECT_CLOBBER 467 : abort, cmpfail 468 #ifdef RSEQ_COMPARE_TWICE 469 , error1, error2 470 #endif 471 ); 472 rseq_workaround_gcc_asm_size_guess(); 473 return 0; 474 abort: 475 rseq_workaround_gcc_asm_size_guess(); 476 RSEQ_INJECT_FAILED 477 return -1; 478 cmpfail: 479 rseq_workaround_gcc_asm_size_guess(); 480 return 1; 481 #ifdef RSEQ_COMPARE_TWICE 482 error1: 483 rseq_bug("cpu_id comparison failed"); 484 error2: 485 rseq_bug("expected value comparison failed"); 486 #endif 487 } 488 489 static inline __attribute__((always_inline)) 490 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 491 intptr_t *v2, intptr_t expect2, 492 intptr_t newv, int cpu) 493 { 494 RSEQ_INJECT_C(9) 495 496 rseq_workaround_gcc_asm_size_guess(); 497 __asm__ __volatile__ goto ( 498 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 499 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 500 #ifdef RSEQ_COMPARE_TWICE 501 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 502 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 503 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 504 #endif 505 /* Start rseq by storing table entry pointer into rseq_cs. */ 506 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 507 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 508 RSEQ_INJECT_ASM(3) 509 "ldr r0, %[v]\n\t" 510 "cmp %[expect], r0\n\t" 511 "bne %l[cmpfail]\n\t" 512 RSEQ_INJECT_ASM(4) 513 "ldr r0, %[v2]\n\t" 514 "cmp %[expect2], r0\n\t" 515 "bne %l[cmpfail]\n\t" 516 RSEQ_INJECT_ASM(5) 517 #ifdef RSEQ_COMPARE_TWICE 518 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 519 "ldr r0, %[v]\n\t" 520 "cmp %[expect], r0\n\t" 521 "bne %l[error2]\n\t" 522 "ldr r0, %[v2]\n\t" 523 "cmp %[expect2], r0\n\t" 524 "bne %l[error3]\n\t" 525 #endif 526 /* final store */ 527 "str %[newv], %[v]\n\t" 528 "2:\n\t" 529 RSEQ_INJECT_ASM(6) 530 "b 5f\n\t" 531 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 532 "5:\n\t" 533 : /* gcc asm goto does not allow outputs */ 534 : [cpu_id] "r" (cpu), 535 [current_cpu_id] "m" (__rseq_abi.cpu_id), 536 [rseq_cs] "m" (__rseq_abi.rseq_cs), 537 /* cmp2 input */ 538 [v2] "m" (*v2), 539 [expect2] "r" (expect2), 540 /* final store input */ 541 [v] "m" (*v), 542 [expect] "r" (expect), 543 [newv] "r" (newv) 544 RSEQ_INJECT_INPUT 545 : "r0", "memory", "cc" 546 RSEQ_INJECT_CLOBBER 547 : abort, cmpfail 548 #ifdef RSEQ_COMPARE_TWICE 549 , error1, error2, error3 550 #endif 551 ); 552 rseq_workaround_gcc_asm_size_guess(); 553 return 0; 554 abort: 555 rseq_workaround_gcc_asm_size_guess(); 556 RSEQ_INJECT_FAILED 557 return -1; 558 cmpfail: 559 rseq_workaround_gcc_asm_size_guess(); 560 return 1; 561 #ifdef RSEQ_COMPARE_TWICE 562 error1: 563 rseq_bug("cpu_id comparison failed"); 564 error2: 565 rseq_bug("1st expected value comparison failed"); 566 error3: 567 rseq_bug("2nd expected value comparison failed"); 568 #endif 569 } 570 571 static inline __attribute__((always_inline)) 572 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 573 void *dst, void *src, size_t len, 574 intptr_t newv, int cpu) 575 { 576 uint32_t rseq_scratch[3]; 577 578 RSEQ_INJECT_C(9) 579 580 rseq_workaround_gcc_asm_size_guess(); 581 __asm__ __volatile__ goto ( 582 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 583 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 584 #ifdef RSEQ_COMPARE_TWICE 585 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 586 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 587 #endif 588 "str %[src], %[rseq_scratch0]\n\t" 589 "str %[dst], %[rseq_scratch1]\n\t" 590 "str %[len], %[rseq_scratch2]\n\t" 591 /* Start rseq by storing table entry pointer into rseq_cs. */ 592 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 593 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 594 RSEQ_INJECT_ASM(3) 595 "ldr r0, %[v]\n\t" 596 "cmp %[expect], r0\n\t" 597 "bne 5f\n\t" 598 RSEQ_INJECT_ASM(4) 599 #ifdef RSEQ_COMPARE_TWICE 600 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 601 "ldr r0, %[v]\n\t" 602 "cmp %[expect], r0\n\t" 603 "bne 7f\n\t" 604 #endif 605 /* try memcpy */ 606 "cmp %[len], #0\n\t" \ 607 "beq 333f\n\t" \ 608 "222:\n\t" \ 609 "ldrb %%r0, [%[src]]\n\t" \ 610 "strb %%r0, [%[dst]]\n\t" \ 611 "adds %[src], #1\n\t" \ 612 "adds %[dst], #1\n\t" \ 613 "subs %[len], #1\n\t" \ 614 "bne 222b\n\t" \ 615 "333:\n\t" \ 616 RSEQ_INJECT_ASM(5) 617 /* final store */ 618 "str %[newv], %[v]\n\t" 619 "2:\n\t" 620 RSEQ_INJECT_ASM(6) 621 /* teardown */ 622 "ldr %[len], %[rseq_scratch2]\n\t" 623 "ldr %[dst], %[rseq_scratch1]\n\t" 624 "ldr %[src], %[rseq_scratch0]\n\t" 625 "b 8f\n\t" 626 RSEQ_ASM_DEFINE_ABORT(3, 4, 627 /* teardown */ 628 "ldr %[len], %[rseq_scratch2]\n\t" 629 "ldr %[dst], %[rseq_scratch1]\n\t" 630 "ldr %[src], %[rseq_scratch0]\n\t", 631 abort, 1b, 2b, 4f) 632 RSEQ_ASM_DEFINE_CMPFAIL(5, 633 /* teardown */ 634 "ldr %[len], %[rseq_scratch2]\n\t" 635 "ldr %[dst], %[rseq_scratch1]\n\t" 636 "ldr %[src], %[rseq_scratch0]\n\t", 637 cmpfail) 638 #ifdef RSEQ_COMPARE_TWICE 639 RSEQ_ASM_DEFINE_CMPFAIL(6, 640 /* teardown */ 641 "ldr %[len], %[rseq_scratch2]\n\t" 642 "ldr %[dst], %[rseq_scratch1]\n\t" 643 "ldr %[src], %[rseq_scratch0]\n\t", 644 error1) 645 RSEQ_ASM_DEFINE_CMPFAIL(7, 646 /* teardown */ 647 "ldr %[len], %[rseq_scratch2]\n\t" 648 "ldr %[dst], %[rseq_scratch1]\n\t" 649 "ldr %[src], %[rseq_scratch0]\n\t", 650 error2) 651 #endif 652 "8:\n\t" 653 : /* gcc asm goto does not allow outputs */ 654 : [cpu_id] "r" (cpu), 655 [current_cpu_id] "m" (__rseq_abi.cpu_id), 656 [rseq_cs] "m" (__rseq_abi.rseq_cs), 657 /* final store input */ 658 [v] "m" (*v), 659 [expect] "r" (expect), 660 [newv] "r" (newv), 661 /* try memcpy input */ 662 [dst] "r" (dst), 663 [src] "r" (src), 664 [len] "r" (len), 665 [rseq_scratch0] "m" (rseq_scratch[0]), 666 [rseq_scratch1] "m" (rseq_scratch[1]), 667 [rseq_scratch2] "m" (rseq_scratch[2]) 668 RSEQ_INJECT_INPUT 669 : "r0", "memory", "cc" 670 RSEQ_INJECT_CLOBBER 671 : abort, cmpfail 672 #ifdef RSEQ_COMPARE_TWICE 673 , error1, error2 674 #endif 675 ); 676 rseq_workaround_gcc_asm_size_guess(); 677 return 0; 678 abort: 679 rseq_workaround_gcc_asm_size_guess(); 680 RSEQ_INJECT_FAILED 681 return -1; 682 cmpfail: 683 rseq_workaround_gcc_asm_size_guess(); 684 return 1; 685 #ifdef RSEQ_COMPARE_TWICE 686 error1: 687 rseq_workaround_gcc_asm_size_guess(); 688 rseq_bug("cpu_id comparison failed"); 689 error2: 690 rseq_workaround_gcc_asm_size_guess(); 691 rseq_bug("expected value comparison failed"); 692 #endif 693 } 694 695 static inline __attribute__((always_inline)) 696 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 697 void *dst, void *src, size_t len, 698 intptr_t newv, int cpu) 699 { 700 uint32_t rseq_scratch[3]; 701 702 RSEQ_INJECT_C(9) 703 704 rseq_workaround_gcc_asm_size_guess(); 705 __asm__ __volatile__ goto ( 706 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 707 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 708 #ifdef RSEQ_COMPARE_TWICE 709 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 710 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 711 #endif 712 "str %[src], %[rseq_scratch0]\n\t" 713 "str %[dst], %[rseq_scratch1]\n\t" 714 "str %[len], %[rseq_scratch2]\n\t" 715 /* Start rseq by storing table entry pointer into rseq_cs. */ 716 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 717 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 718 RSEQ_INJECT_ASM(3) 719 "ldr r0, %[v]\n\t" 720 "cmp %[expect], r0\n\t" 721 "bne 5f\n\t" 722 RSEQ_INJECT_ASM(4) 723 #ifdef RSEQ_COMPARE_TWICE 724 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 725 "ldr r0, %[v]\n\t" 726 "cmp %[expect], r0\n\t" 727 "bne 7f\n\t" 728 #endif 729 /* try memcpy */ 730 "cmp %[len], #0\n\t" \ 731 "beq 333f\n\t" \ 732 "222:\n\t" \ 733 "ldrb %%r0, [%[src]]\n\t" \ 734 "strb %%r0, [%[dst]]\n\t" \ 735 "adds %[src], #1\n\t" \ 736 "adds %[dst], #1\n\t" \ 737 "subs %[len], #1\n\t" \ 738 "bne 222b\n\t" \ 739 "333:\n\t" \ 740 RSEQ_INJECT_ASM(5) 741 "dmb\n\t" /* full mb provides store-release */ 742 /* final store */ 743 "str %[newv], %[v]\n\t" 744 "2:\n\t" 745 RSEQ_INJECT_ASM(6) 746 /* teardown */ 747 "ldr %[len], %[rseq_scratch2]\n\t" 748 "ldr %[dst], %[rseq_scratch1]\n\t" 749 "ldr %[src], %[rseq_scratch0]\n\t" 750 "b 8f\n\t" 751 RSEQ_ASM_DEFINE_ABORT(3, 4, 752 /* teardown */ 753 "ldr %[len], %[rseq_scratch2]\n\t" 754 "ldr %[dst], %[rseq_scratch1]\n\t" 755 "ldr %[src], %[rseq_scratch0]\n\t", 756 abort, 1b, 2b, 4f) 757 RSEQ_ASM_DEFINE_CMPFAIL(5, 758 /* teardown */ 759 "ldr %[len], %[rseq_scratch2]\n\t" 760 "ldr %[dst], %[rseq_scratch1]\n\t" 761 "ldr %[src], %[rseq_scratch0]\n\t", 762 cmpfail) 763 #ifdef RSEQ_COMPARE_TWICE 764 RSEQ_ASM_DEFINE_CMPFAIL(6, 765 /* teardown */ 766 "ldr %[len], %[rseq_scratch2]\n\t" 767 "ldr %[dst], %[rseq_scratch1]\n\t" 768 "ldr %[src], %[rseq_scratch0]\n\t", 769 error1) 770 RSEQ_ASM_DEFINE_CMPFAIL(7, 771 /* teardown */ 772 "ldr %[len], %[rseq_scratch2]\n\t" 773 "ldr %[dst], %[rseq_scratch1]\n\t" 774 "ldr %[src], %[rseq_scratch0]\n\t", 775 error2) 776 #endif 777 "8:\n\t" 778 : /* gcc asm goto does not allow outputs */ 779 : [cpu_id] "r" (cpu), 780 [current_cpu_id] "m" (__rseq_abi.cpu_id), 781 [rseq_cs] "m" (__rseq_abi.rseq_cs), 782 /* final store input */ 783 [v] "m" (*v), 784 [expect] "r" (expect), 785 [newv] "r" (newv), 786 /* try memcpy input */ 787 [dst] "r" (dst), 788 [src] "r" (src), 789 [len] "r" (len), 790 [rseq_scratch0] "m" (rseq_scratch[0]), 791 [rseq_scratch1] "m" (rseq_scratch[1]), 792 [rseq_scratch2] "m" (rseq_scratch[2]) 793 RSEQ_INJECT_INPUT 794 : "r0", "memory", "cc" 795 RSEQ_INJECT_CLOBBER 796 : abort, cmpfail 797 #ifdef RSEQ_COMPARE_TWICE 798 , error1, error2 799 #endif 800 ); 801 rseq_workaround_gcc_asm_size_guess(); 802 return 0; 803 abort: 804 rseq_workaround_gcc_asm_size_guess(); 805 RSEQ_INJECT_FAILED 806 return -1; 807 cmpfail: 808 rseq_workaround_gcc_asm_size_guess(); 809 return 1; 810 #ifdef RSEQ_COMPARE_TWICE 811 error1: 812 rseq_workaround_gcc_asm_size_guess(); 813 rseq_bug("cpu_id comparison failed"); 814 error2: 815 rseq_workaround_gcc_asm_size_guess(); 816 rseq_bug("expected value comparison failed"); 817 #endif 818 } 819 820 #endif /* !RSEQ_SKIP_FASTPATH */ 821