1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-arm64.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com> 7 */ 8 9 /* 10 * aarch64 -mbig-endian generates mixed endianness code vs data: 11 * little-endian code and big-endian data. Ensure the RSEQ_SIG signature 12 * matches code endianness. 13 */ 14 #define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */ 15 16 #ifdef __AARCH64EB__ 17 #define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */ 18 #else 19 #define RSEQ_SIG_DATA RSEQ_SIG_CODE 20 #endif 21 22 #define RSEQ_SIG RSEQ_SIG_DATA 23 24 #define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory") 25 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory") 26 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory") 27 28 #define rseq_smp_load_acquire(p) \ 29 __extension__ ({ \ 30 __typeof(*p) ____p1; \ 31 switch (sizeof(*p)) { \ 32 case 1: \ 33 asm volatile ("ldarb %w0, %1" \ 34 : "=r" (*(__u8 *)p) \ 35 : "Q" (*p) : "memory"); \ 36 break; \ 37 case 2: \ 38 asm volatile ("ldarh %w0, %1" \ 39 : "=r" (*(__u16 *)p) \ 40 : "Q" (*p) : "memory"); \ 41 break; \ 42 case 4: \ 43 asm volatile ("ldar %w0, %1" \ 44 : "=r" (*(__u32 *)p) \ 45 : "Q" (*p) : "memory"); \ 46 break; \ 47 case 8: \ 48 asm volatile ("ldar %0, %1" \ 49 : "=r" (*(__u64 *)p) \ 50 : "Q" (*p) : "memory"); \ 51 break; \ 52 } \ 53 ____p1; \ 54 }) 55 56 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 57 58 #define rseq_smp_store_release(p, v) \ 59 do { \ 60 switch (sizeof(*p)) { \ 61 case 1: \ 62 asm volatile ("stlrb %w1, %0" \ 63 : "=Q" (*p) \ 64 : "r" ((__u8)v) \ 65 : "memory"); \ 66 break; \ 67 case 2: \ 68 asm volatile ("stlrh %w1, %0" \ 69 : "=Q" (*p) \ 70 : "r" ((__u16)v) \ 71 : "memory"); \ 72 break; \ 73 case 4: \ 74 asm volatile ("stlr %w1, %0" \ 75 : "=Q" (*p) \ 76 : "r" ((__u32)v) \ 77 : "memory"); \ 78 break; \ 79 case 8: \ 80 asm volatile ("stlr %1, %0" \ 81 : "=Q" (*p) \ 82 : "r" ((__u64)v) \ 83 : "memory"); \ 84 break; \ 85 } \ 86 } while (0) 87 88 #ifdef RSEQ_SKIP_FASTPATH 89 #include "rseq-skip.h" 90 #else /* !RSEQ_SKIP_FASTPATH */ 91 92 #define RSEQ_ASM_TMP_REG32 "w15" 93 #define RSEQ_ASM_TMP_REG "x15" 94 #define RSEQ_ASM_TMP_REG_2 "x14" 95 96 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ 97 post_commit_offset, abort_ip) \ 98 " .pushsection __rseq_cs, \"aw\"\n" \ 99 " .balign 32\n" \ 100 __rseq_str(label) ":\n" \ 101 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \ 102 " .quad " __rseq_str(start_ip) ", " \ 103 __rseq_str(post_commit_offset) ", " \ 104 __rseq_str(abort_ip) "\n" \ 105 " .popsection\n\t" \ 106 " .pushsection __rseq_cs_ptr_array, \"aw\"\n" \ 107 " .quad " __rseq_str(label) "b\n" \ 108 " .popsection\n" 109 110 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 111 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 112 (post_commit_ip - start_ip), abort_ip) 113 114 /* 115 * Exit points of a rseq critical section consist of all instructions outside 116 * of the critical section where a critical section can either branch to or 117 * reach through the normal course of its execution. The abort IP and the 118 * post-commit IP are already part of the __rseq_cs section and should not be 119 * explicitly defined as additional exit points. Knowing all exit points is 120 * useful to assist debuggers stepping over the critical section. 121 */ 122 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 123 " .pushsection __rseq_exit_point_array, \"aw\"\n" \ 124 " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \ 125 " .popsection\n" 126 127 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 128 RSEQ_INJECT_ASM(1) \ 129 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \ 130 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 131 ", :lo12:" __rseq_str(cs_label) "\n" \ 132 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \ 133 __rseq_str(label) ":\n" 134 135 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \ 136 " b 222f\n" \ 137 " .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \ 138 __rseq_str(label) ":\n" \ 139 " b %l[" __rseq_str(abort_label) "]\n" \ 140 "222:\n" 141 142 #define RSEQ_ASM_OP_STORE(value, var) \ 143 " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n" 144 145 #define RSEQ_ASM_OP_STORE_RELEASE(value, var) \ 146 " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n" 147 148 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ 149 RSEQ_ASM_OP_STORE(value, var) \ 150 __rseq_str(post_commit_label) ":\n" 151 152 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \ 153 RSEQ_ASM_OP_STORE_RELEASE(value, var) \ 154 __rseq_str(post_commit_label) ":\n" 155 156 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ 157 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \ 158 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 159 ", %[" __rseq_str(expect) "]\n" \ 160 " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n" 161 162 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \ 163 " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \ 164 " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \ 165 ", %w[" __rseq_str(expect) "]\n" \ 166 " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n" 167 168 #define RSEQ_ASM_OP_CMPNE(var, expect, label) \ 169 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \ 170 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 171 ", %[" __rseq_str(expect) "]\n" \ 172 " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n" 173 174 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 175 RSEQ_INJECT_ASM(2) \ 176 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label) 177 178 #define RSEQ_ASM_OP_R_LOAD(var) \ 179 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" 180 181 #define RSEQ_ASM_OP_R_STORE(var) \ 182 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" 183 184 #define RSEQ_ASM_OP_R_LOAD_OFF(offset) \ 185 " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \ 186 ", %[" __rseq_str(offset) "]]\n" 187 188 #define RSEQ_ASM_OP_R_ADD(count) \ 189 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 190 ", %[" __rseq_str(count) "]\n" 191 192 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ 193 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \ 194 __rseq_str(post_commit_label) ":\n" 195 196 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \ 197 " cbz %[" __rseq_str(len) "], 333f\n" \ 198 " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \ 199 "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \ 200 " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \ 201 ", " RSEQ_ASM_TMP_REG_2 "]\n" \ 202 " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \ 203 ", " RSEQ_ASM_TMP_REG_2 "]\n" \ 204 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \ 205 "333:\n" 206 207 static inline __attribute__((always_inline)) 208 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 209 { 210 RSEQ_INJECT_C(9) 211 212 __asm__ __volatile__ goto ( 213 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 214 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 215 #ifdef RSEQ_COMPARE_TWICE 216 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 217 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 218 #endif 219 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 220 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 221 RSEQ_INJECT_ASM(3) 222 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 223 RSEQ_INJECT_ASM(4) 224 #ifdef RSEQ_COMPARE_TWICE 225 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 226 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 227 #endif 228 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 229 RSEQ_INJECT_ASM(5) 230 RSEQ_ASM_DEFINE_ABORT(4, abort) 231 : /* gcc asm goto does not allow outputs */ 232 : [cpu_id] "r" (cpu), 233 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 234 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 235 [v] "Qo" (*v), 236 [expect] "r" (expect), 237 [newv] "r" (newv) 238 RSEQ_INJECT_INPUT 239 : "memory", RSEQ_ASM_TMP_REG 240 : abort, cmpfail 241 #ifdef RSEQ_COMPARE_TWICE 242 , error1, error2 243 #endif 244 ); 245 rseq_after_asm_goto(); 246 return 0; 247 abort: 248 rseq_after_asm_goto(); 249 RSEQ_INJECT_FAILED 250 return -1; 251 cmpfail: 252 rseq_after_asm_goto(); 253 return 1; 254 #ifdef RSEQ_COMPARE_TWICE 255 error1: 256 rseq_after_asm_goto(); 257 rseq_bug("cpu_id comparison failed"); 258 error2: 259 rseq_after_asm_goto(); 260 rseq_bug("expected value comparison failed"); 261 #endif 262 } 263 264 static inline __attribute__((always_inline)) 265 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 266 long voffp, intptr_t *load, int cpu) 267 { 268 RSEQ_INJECT_C(9) 269 270 __asm__ __volatile__ goto ( 271 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 272 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 273 #ifdef RSEQ_COMPARE_TWICE 274 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 275 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 276 #endif 277 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 278 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 279 RSEQ_INJECT_ASM(3) 280 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) 281 RSEQ_INJECT_ASM(4) 282 #ifdef RSEQ_COMPARE_TWICE 283 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 284 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) 285 #endif 286 RSEQ_ASM_OP_R_LOAD(v) 287 RSEQ_ASM_OP_R_STORE(load) 288 RSEQ_ASM_OP_R_LOAD_OFF(voffp) 289 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 290 RSEQ_INJECT_ASM(5) 291 RSEQ_ASM_DEFINE_ABORT(4, abort) 292 : /* gcc asm goto does not allow outputs */ 293 : [cpu_id] "r" (cpu), 294 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 295 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 296 [v] "Qo" (*v), 297 [expectnot] "r" (expectnot), 298 [load] "Qo" (*load), 299 [voffp] "r" (voffp) 300 RSEQ_INJECT_INPUT 301 : "memory", RSEQ_ASM_TMP_REG 302 : abort, cmpfail 303 #ifdef RSEQ_COMPARE_TWICE 304 , error1, error2 305 #endif 306 ); 307 rseq_after_asm_goto(); 308 return 0; 309 abort: 310 rseq_after_asm_goto(); 311 RSEQ_INJECT_FAILED 312 return -1; 313 cmpfail: 314 rseq_after_asm_goto(); 315 return 1; 316 #ifdef RSEQ_COMPARE_TWICE 317 error1: 318 rseq_after_asm_goto(); 319 rseq_bug("cpu_id comparison failed"); 320 error2: 321 rseq_after_asm_goto(); 322 rseq_bug("expected value comparison failed"); 323 #endif 324 } 325 326 static inline __attribute__((always_inline)) 327 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 328 { 329 RSEQ_INJECT_C(9) 330 331 __asm__ __volatile__ goto ( 332 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 333 #ifdef RSEQ_COMPARE_TWICE 334 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 335 #endif 336 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 337 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 338 RSEQ_INJECT_ASM(3) 339 #ifdef RSEQ_COMPARE_TWICE 340 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 341 #endif 342 RSEQ_ASM_OP_R_LOAD(v) 343 RSEQ_ASM_OP_R_ADD(count) 344 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 345 RSEQ_INJECT_ASM(4) 346 RSEQ_ASM_DEFINE_ABORT(4, abort) 347 : /* gcc asm goto does not allow outputs */ 348 : [cpu_id] "r" (cpu), 349 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 350 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 351 [v] "Qo" (*v), 352 [count] "r" (count) 353 RSEQ_INJECT_INPUT 354 : "memory", RSEQ_ASM_TMP_REG 355 : abort 356 #ifdef RSEQ_COMPARE_TWICE 357 , error1 358 #endif 359 ); 360 rseq_after_asm_goto(); 361 return 0; 362 abort: 363 rseq_after_asm_goto(); 364 RSEQ_INJECT_FAILED 365 return -1; 366 #ifdef RSEQ_COMPARE_TWICE 367 error1: 368 rseq_after_asm_goto(); 369 rseq_bug("cpu_id comparison failed"); 370 #endif 371 } 372 373 static inline __attribute__((always_inline)) 374 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 375 intptr_t *v2, intptr_t newv2, 376 intptr_t newv, int cpu) 377 { 378 RSEQ_INJECT_C(9) 379 380 __asm__ __volatile__ goto ( 381 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 382 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 383 #ifdef RSEQ_COMPARE_TWICE 384 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 385 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 386 #endif 387 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 388 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 389 RSEQ_INJECT_ASM(3) 390 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 391 RSEQ_INJECT_ASM(4) 392 #ifdef RSEQ_COMPARE_TWICE 393 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 394 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 395 #endif 396 RSEQ_ASM_OP_STORE(newv2, v2) 397 RSEQ_INJECT_ASM(5) 398 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 399 RSEQ_INJECT_ASM(6) 400 RSEQ_ASM_DEFINE_ABORT(4, abort) 401 : /* gcc asm goto does not allow outputs */ 402 : [cpu_id] "r" (cpu), 403 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 404 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 405 [expect] "r" (expect), 406 [v] "Qo" (*v), 407 [newv] "r" (newv), 408 [v2] "Qo" (*v2), 409 [newv2] "r" (newv2) 410 RSEQ_INJECT_INPUT 411 : "memory", RSEQ_ASM_TMP_REG 412 : abort, cmpfail 413 #ifdef RSEQ_COMPARE_TWICE 414 , error1, error2 415 #endif 416 ); 417 rseq_after_asm_goto(); 418 return 0; 419 abort: 420 rseq_after_asm_goto(); 421 RSEQ_INJECT_FAILED 422 return -1; 423 cmpfail: 424 rseq_after_asm_goto(); 425 return 1; 426 #ifdef RSEQ_COMPARE_TWICE 427 error1: 428 rseq_after_asm_goto(); 429 rseq_bug("cpu_id comparison failed"); 430 error2: 431 rseq_after_asm_goto(); 432 rseq_bug("expected value comparison failed"); 433 #endif 434 } 435 436 static inline __attribute__((always_inline)) 437 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 438 intptr_t *v2, intptr_t newv2, 439 intptr_t newv, int cpu) 440 { 441 RSEQ_INJECT_C(9) 442 443 __asm__ __volatile__ goto ( 444 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 445 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 446 #ifdef RSEQ_COMPARE_TWICE 447 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 448 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 449 #endif 450 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 451 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 452 RSEQ_INJECT_ASM(3) 453 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 454 RSEQ_INJECT_ASM(4) 455 #ifdef RSEQ_COMPARE_TWICE 456 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 457 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 458 #endif 459 RSEQ_ASM_OP_STORE(newv2, v2) 460 RSEQ_INJECT_ASM(5) 461 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 462 RSEQ_INJECT_ASM(6) 463 RSEQ_ASM_DEFINE_ABORT(4, abort) 464 : /* gcc asm goto does not allow outputs */ 465 : [cpu_id] "r" (cpu), 466 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 467 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 468 [expect] "r" (expect), 469 [v] "Qo" (*v), 470 [newv] "r" (newv), 471 [v2] "Qo" (*v2), 472 [newv2] "r" (newv2) 473 RSEQ_INJECT_INPUT 474 : "memory", RSEQ_ASM_TMP_REG 475 : abort, cmpfail 476 #ifdef RSEQ_COMPARE_TWICE 477 , error1, error2 478 #endif 479 ); 480 rseq_after_asm_goto(); 481 return 0; 482 abort: 483 rseq_after_asm_goto(); 484 RSEQ_INJECT_FAILED 485 return -1; 486 cmpfail: 487 rseq_after_asm_goto(); 488 return 1; 489 #ifdef RSEQ_COMPARE_TWICE 490 error1: 491 rseq_after_asm_goto(); 492 rseq_bug("cpu_id comparison failed"); 493 error2: 494 rseq_after_asm_goto(); 495 rseq_bug("expected value comparison failed"); 496 #endif 497 } 498 499 static inline __attribute__((always_inline)) 500 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 501 intptr_t *v2, intptr_t expect2, 502 intptr_t newv, int cpu) 503 { 504 RSEQ_INJECT_C(9) 505 506 __asm__ __volatile__ goto ( 507 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 508 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 509 #ifdef RSEQ_COMPARE_TWICE 510 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 511 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 512 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3]) 513 #endif 514 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 515 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 516 RSEQ_INJECT_ASM(3) 517 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 518 RSEQ_INJECT_ASM(4) 519 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) 520 RSEQ_INJECT_ASM(5) 521 #ifdef RSEQ_COMPARE_TWICE 522 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 523 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 524 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) 525 #endif 526 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 527 RSEQ_INJECT_ASM(6) 528 RSEQ_ASM_DEFINE_ABORT(4, abort) 529 : /* gcc asm goto does not allow outputs */ 530 : [cpu_id] "r" (cpu), 531 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 532 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 533 [v] "Qo" (*v), 534 [expect] "r" (expect), 535 [v2] "Qo" (*v2), 536 [expect2] "r" (expect2), 537 [newv] "r" (newv) 538 RSEQ_INJECT_INPUT 539 : "memory", RSEQ_ASM_TMP_REG 540 : abort, cmpfail 541 #ifdef RSEQ_COMPARE_TWICE 542 , error1, error2, error3 543 #endif 544 ); 545 rseq_after_asm_goto(); 546 return 0; 547 abort: 548 rseq_after_asm_goto(); 549 RSEQ_INJECT_FAILED 550 return -1; 551 cmpfail: 552 rseq_after_asm_goto(); 553 return 1; 554 #ifdef RSEQ_COMPARE_TWICE 555 error1: 556 rseq_after_asm_goto(); 557 rseq_bug("cpu_id comparison failed"); 558 error2: 559 rseq_after_asm_goto(); 560 rseq_bug("expected value comparison failed"); 561 error3: 562 rseq_after_asm_goto(); 563 rseq_bug("2nd expected value comparison failed"); 564 #endif 565 } 566 567 static inline __attribute__((always_inline)) 568 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 569 void *dst, void *src, size_t len, 570 intptr_t newv, int cpu) 571 { 572 RSEQ_INJECT_C(9) 573 574 __asm__ __volatile__ goto ( 575 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 576 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 577 #ifdef RSEQ_COMPARE_TWICE 578 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 579 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 580 #endif 581 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 582 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 583 RSEQ_INJECT_ASM(3) 584 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 585 RSEQ_INJECT_ASM(4) 586 #ifdef RSEQ_COMPARE_TWICE 587 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 588 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 589 #endif 590 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 591 RSEQ_INJECT_ASM(5) 592 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 593 RSEQ_INJECT_ASM(6) 594 RSEQ_ASM_DEFINE_ABORT(4, abort) 595 : /* gcc asm goto does not allow outputs */ 596 : [cpu_id] "r" (cpu), 597 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 598 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 599 [expect] "r" (expect), 600 [v] "Qo" (*v), 601 [newv] "r" (newv), 602 [dst] "r" (dst), 603 [src] "r" (src), 604 [len] "r" (len) 605 RSEQ_INJECT_INPUT 606 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 607 : abort, cmpfail 608 #ifdef RSEQ_COMPARE_TWICE 609 , error1, error2 610 #endif 611 ); 612 rseq_after_asm_goto(); 613 return 0; 614 abort: 615 rseq_after_asm_goto(); 616 RSEQ_INJECT_FAILED 617 return -1; 618 cmpfail: 619 rseq_after_asm_goto(); 620 return 1; 621 #ifdef RSEQ_COMPARE_TWICE 622 error1: 623 rseq_after_asm_goto(); 624 rseq_bug("cpu_id comparison failed"); 625 error2: 626 rseq_after_asm_goto(); 627 rseq_bug("expected value comparison failed"); 628 #endif 629 } 630 631 static inline __attribute__((always_inline)) 632 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 633 void *dst, void *src, size_t len, 634 intptr_t newv, int cpu) 635 { 636 RSEQ_INJECT_C(9) 637 638 __asm__ __volatile__ goto ( 639 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 640 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 641 #ifdef RSEQ_COMPARE_TWICE 642 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 643 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 644 #endif 645 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 646 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 647 RSEQ_INJECT_ASM(3) 648 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 649 RSEQ_INJECT_ASM(4) 650 #ifdef RSEQ_COMPARE_TWICE 651 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 652 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 653 #endif 654 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 655 RSEQ_INJECT_ASM(5) 656 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 657 RSEQ_INJECT_ASM(6) 658 RSEQ_ASM_DEFINE_ABORT(4, abort) 659 : /* gcc asm goto does not allow outputs */ 660 : [cpu_id] "r" (cpu), 661 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 662 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 663 [expect] "r" (expect), 664 [v] "Qo" (*v), 665 [newv] "r" (newv), 666 [dst] "r" (dst), 667 [src] "r" (src), 668 [len] "r" (len) 669 RSEQ_INJECT_INPUT 670 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 671 : abort, cmpfail 672 #ifdef RSEQ_COMPARE_TWICE 673 , error1, error2 674 #endif 675 ); 676 rseq_after_asm_goto(); 677 return 0; 678 abort: 679 rseq_after_asm_goto(); 680 RSEQ_INJECT_FAILED 681 return -1; 682 cmpfail: 683 rseq_after_asm_goto(); 684 return 1; 685 #ifdef RSEQ_COMPARE_TWICE 686 error1: 687 rseq_after_asm_goto(); 688 rseq_bug("cpu_id comparison failed"); 689 error2: 690 rseq_after_asm_goto(); 691 rseq_bug("expected value comparison failed"); 692 #endif 693 } 694 695 #endif /* !RSEQ_SKIP_FASTPATH */ 696