1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-arm64.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com> 7 */ 8 9 #define RSEQ_SIG 0xd428bc00 /* BRK #0x45E0 */ 10 11 #define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory") 12 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory") 13 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory") 14 15 #define rseq_smp_load_acquire(p) \ 16 __extension__ ({ \ 17 __typeof(*p) ____p1; \ 18 switch (sizeof(*p)) { \ 19 case 1: \ 20 asm volatile ("ldarb %w0, %1" \ 21 : "=r" (*(__u8 *)p) \ 22 : "Q" (*p) : "memory"); \ 23 break; \ 24 case 2: \ 25 asm volatile ("ldarh %w0, %1" \ 26 : "=r" (*(__u16 *)p) \ 27 : "Q" (*p) : "memory"); \ 28 break; \ 29 case 4: \ 30 asm volatile ("ldar %w0, %1" \ 31 : "=r" (*(__u32 *)p) \ 32 : "Q" (*p) : "memory"); \ 33 break; \ 34 case 8: \ 35 asm volatile ("ldar %0, %1" \ 36 : "=r" (*(__u64 *)p) \ 37 : "Q" (*p) : "memory"); \ 38 break; \ 39 } \ 40 ____p1; \ 41 }) 42 43 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 44 45 #define rseq_smp_store_release(p, v) \ 46 do { \ 47 switch (sizeof(*p)) { \ 48 case 1: \ 49 asm volatile ("stlrb %w1, %0" \ 50 : "=Q" (*p) \ 51 : "r" ((__u8)v) \ 52 : "memory"); \ 53 break; \ 54 case 2: \ 55 asm volatile ("stlrh %w1, %0" \ 56 : "=Q" (*p) \ 57 : "r" ((__u16)v) \ 58 : "memory"); \ 59 break; \ 60 case 4: \ 61 asm volatile ("stlr %w1, %0" \ 62 : "=Q" (*p) \ 63 : "r" ((__u32)v) \ 64 : "memory"); \ 65 break; \ 66 case 8: \ 67 asm volatile ("stlr %1, %0" \ 68 : "=Q" (*p) \ 69 : "r" ((__u64)v) \ 70 : "memory"); \ 71 break; \ 72 } \ 73 } while (0) 74 75 #ifdef RSEQ_SKIP_FASTPATH 76 #include "rseq-skip.h" 77 #else /* !RSEQ_SKIP_FASTPATH */ 78 79 #define RSEQ_ASM_TMP_REG32 "w15" 80 #define RSEQ_ASM_TMP_REG "x15" 81 #define RSEQ_ASM_TMP_REG_2 "x14" 82 83 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ 84 post_commit_offset, abort_ip) \ 85 " .pushsection __rseq_table, \"aw\"\n" \ 86 " .balign 32\n" \ 87 __rseq_str(label) ":\n" \ 88 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \ 89 " .quad " __rseq_str(start_ip) ", " \ 90 __rseq_str(post_commit_offset) ", " \ 91 __rseq_str(abort_ip) "\n" \ 92 " .popsection\n" 93 94 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 95 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 96 (post_commit_ip - start_ip), abort_ip) 97 98 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 99 RSEQ_INJECT_ASM(1) \ 100 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \ 101 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 102 ", :lo12:" __rseq_str(cs_label) "\n" \ 103 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \ 104 __rseq_str(label) ":\n" 105 106 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \ 107 " b 222f\n" \ 108 " .inst " __rseq_str(RSEQ_SIG) "\n" \ 109 __rseq_str(label) ":\n" \ 110 " b %l[" __rseq_str(abort_label) "]\n" \ 111 "222:\n" 112 113 #define RSEQ_ASM_OP_STORE(value, var) \ 114 " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n" 115 116 #define RSEQ_ASM_OP_STORE_RELEASE(value, var) \ 117 " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n" 118 119 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ 120 RSEQ_ASM_OP_STORE(value, var) \ 121 __rseq_str(post_commit_label) ":\n" 122 123 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \ 124 RSEQ_ASM_OP_STORE_RELEASE(value, var) \ 125 __rseq_str(post_commit_label) ":\n" 126 127 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ 128 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \ 129 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 130 ", %[" __rseq_str(expect) "]\n" \ 131 " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n" 132 133 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \ 134 " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \ 135 " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \ 136 ", %w[" __rseq_str(expect) "]\n" \ 137 " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n" 138 139 #define RSEQ_ASM_OP_CMPNE(var, expect, label) \ 140 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \ 141 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 142 ", %[" __rseq_str(expect) "]\n" \ 143 " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n" 144 145 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 146 RSEQ_INJECT_ASM(2) \ 147 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label) 148 149 #define RSEQ_ASM_OP_R_LOAD(var) \ 150 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" 151 152 #define RSEQ_ASM_OP_R_STORE(var) \ 153 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" 154 155 #define RSEQ_ASM_OP_R_LOAD_OFF(offset) \ 156 " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \ 157 ", %[" __rseq_str(offset) "]]\n" 158 159 #define RSEQ_ASM_OP_R_ADD(count) \ 160 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 161 ", %[" __rseq_str(count) "]\n" 162 163 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ 164 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \ 165 __rseq_str(post_commit_label) ":\n" 166 167 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \ 168 " cbz %[" __rseq_str(len) "], 333f\n" \ 169 " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \ 170 "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \ 171 " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \ 172 ", " RSEQ_ASM_TMP_REG_2 "]\n" \ 173 " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \ 174 ", " RSEQ_ASM_TMP_REG_2 "]\n" \ 175 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \ 176 "333:\n" 177 178 static inline __attribute__((always_inline)) 179 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 180 { 181 RSEQ_INJECT_C(9) 182 183 __asm__ __volatile__ goto ( 184 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 185 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 186 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 187 RSEQ_INJECT_ASM(3) 188 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 189 RSEQ_INJECT_ASM(4) 190 #ifdef RSEQ_COMPARE_TWICE 191 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 192 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 193 #endif 194 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 195 RSEQ_INJECT_ASM(5) 196 RSEQ_ASM_DEFINE_ABORT(4, abort) 197 : /* gcc asm goto does not allow outputs */ 198 : [cpu_id] "r" (cpu), 199 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 200 [rseq_cs] "m" (__rseq_abi.rseq_cs), 201 [v] "Qo" (*v), 202 [expect] "r" (expect), 203 [newv] "r" (newv) 204 RSEQ_INJECT_INPUT 205 : "memory", RSEQ_ASM_TMP_REG 206 : abort, cmpfail 207 #ifdef RSEQ_COMPARE_TWICE 208 , error1, error2 209 #endif 210 ); 211 212 return 0; 213 abort: 214 RSEQ_INJECT_FAILED 215 return -1; 216 cmpfail: 217 return 1; 218 #ifdef RSEQ_COMPARE_TWICE 219 error1: 220 rseq_bug("cpu_id comparison failed"); 221 error2: 222 rseq_bug("expected value comparison failed"); 223 #endif 224 } 225 226 static inline __attribute__((always_inline)) 227 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 228 off_t voffp, intptr_t *load, int cpu) 229 { 230 RSEQ_INJECT_C(9) 231 232 __asm__ __volatile__ goto ( 233 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 234 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 235 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 236 RSEQ_INJECT_ASM(3) 237 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) 238 RSEQ_INJECT_ASM(4) 239 #ifdef RSEQ_COMPARE_TWICE 240 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 241 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) 242 #endif 243 RSEQ_ASM_OP_R_LOAD(v) 244 RSEQ_ASM_OP_R_STORE(load) 245 RSEQ_ASM_OP_R_LOAD_OFF(voffp) 246 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 247 RSEQ_INJECT_ASM(5) 248 RSEQ_ASM_DEFINE_ABORT(4, abort) 249 : /* gcc asm goto does not allow outputs */ 250 : [cpu_id] "r" (cpu), 251 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 252 [rseq_cs] "m" (__rseq_abi.rseq_cs), 253 [v] "Qo" (*v), 254 [expectnot] "r" (expectnot), 255 [load] "Qo" (*load), 256 [voffp] "r" (voffp) 257 RSEQ_INJECT_INPUT 258 : "memory", RSEQ_ASM_TMP_REG 259 : abort, cmpfail 260 #ifdef RSEQ_COMPARE_TWICE 261 , error1, error2 262 #endif 263 ); 264 return 0; 265 abort: 266 RSEQ_INJECT_FAILED 267 return -1; 268 cmpfail: 269 return 1; 270 #ifdef RSEQ_COMPARE_TWICE 271 error1: 272 rseq_bug("cpu_id comparison failed"); 273 error2: 274 rseq_bug("expected value comparison failed"); 275 #endif 276 } 277 278 static inline __attribute__((always_inline)) 279 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 280 { 281 RSEQ_INJECT_C(9) 282 283 __asm__ __volatile__ goto ( 284 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 285 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 286 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 287 RSEQ_INJECT_ASM(3) 288 #ifdef RSEQ_COMPARE_TWICE 289 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 290 #endif 291 RSEQ_ASM_OP_R_LOAD(v) 292 RSEQ_ASM_OP_R_ADD(count) 293 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 294 RSEQ_INJECT_ASM(4) 295 RSEQ_ASM_DEFINE_ABORT(4, abort) 296 : /* gcc asm goto does not allow outputs */ 297 : [cpu_id] "r" (cpu), 298 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 299 [rseq_cs] "m" (__rseq_abi.rseq_cs), 300 [v] "Qo" (*v), 301 [count] "r" (count) 302 RSEQ_INJECT_INPUT 303 : "memory", RSEQ_ASM_TMP_REG 304 : abort 305 #ifdef RSEQ_COMPARE_TWICE 306 , error1 307 #endif 308 ); 309 return 0; 310 abort: 311 RSEQ_INJECT_FAILED 312 return -1; 313 #ifdef RSEQ_COMPARE_TWICE 314 error1: 315 rseq_bug("cpu_id comparison failed"); 316 #endif 317 } 318 319 static inline __attribute__((always_inline)) 320 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 321 intptr_t *v2, intptr_t newv2, 322 intptr_t newv, int cpu) 323 { 324 RSEQ_INJECT_C(9) 325 326 __asm__ __volatile__ goto ( 327 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 328 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 329 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 330 RSEQ_INJECT_ASM(3) 331 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 332 RSEQ_INJECT_ASM(4) 333 #ifdef RSEQ_COMPARE_TWICE 334 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 335 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 336 #endif 337 RSEQ_ASM_OP_STORE(newv2, v2) 338 RSEQ_INJECT_ASM(5) 339 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 340 RSEQ_INJECT_ASM(6) 341 RSEQ_ASM_DEFINE_ABORT(4, abort) 342 : /* gcc asm goto does not allow outputs */ 343 : [cpu_id] "r" (cpu), 344 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 345 [rseq_cs] "m" (__rseq_abi.rseq_cs), 346 [expect] "r" (expect), 347 [v] "Qo" (*v), 348 [newv] "r" (newv), 349 [v2] "Qo" (*v2), 350 [newv2] "r" (newv2) 351 RSEQ_INJECT_INPUT 352 : "memory", RSEQ_ASM_TMP_REG 353 : abort, cmpfail 354 #ifdef RSEQ_COMPARE_TWICE 355 , error1, error2 356 #endif 357 ); 358 359 return 0; 360 abort: 361 RSEQ_INJECT_FAILED 362 return -1; 363 cmpfail: 364 return 1; 365 #ifdef RSEQ_COMPARE_TWICE 366 error1: 367 rseq_bug("cpu_id comparison failed"); 368 error2: 369 rseq_bug("expected value comparison failed"); 370 #endif 371 } 372 373 static inline __attribute__((always_inline)) 374 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 375 intptr_t *v2, intptr_t newv2, 376 intptr_t newv, int cpu) 377 { 378 RSEQ_INJECT_C(9) 379 380 __asm__ __volatile__ goto ( 381 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 382 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 383 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 384 RSEQ_INJECT_ASM(3) 385 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 386 RSEQ_INJECT_ASM(4) 387 #ifdef RSEQ_COMPARE_TWICE 388 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 389 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 390 #endif 391 RSEQ_ASM_OP_STORE(newv2, v2) 392 RSEQ_INJECT_ASM(5) 393 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 394 RSEQ_INJECT_ASM(6) 395 RSEQ_ASM_DEFINE_ABORT(4, abort) 396 : /* gcc asm goto does not allow outputs */ 397 : [cpu_id] "r" (cpu), 398 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 399 [rseq_cs] "m" (__rseq_abi.rseq_cs), 400 [expect] "r" (expect), 401 [v] "Qo" (*v), 402 [newv] "r" (newv), 403 [v2] "Qo" (*v2), 404 [newv2] "r" (newv2) 405 RSEQ_INJECT_INPUT 406 : "memory", RSEQ_ASM_TMP_REG 407 : abort, cmpfail 408 #ifdef RSEQ_COMPARE_TWICE 409 , error1, error2 410 #endif 411 ); 412 413 return 0; 414 abort: 415 RSEQ_INJECT_FAILED 416 return -1; 417 cmpfail: 418 return 1; 419 #ifdef RSEQ_COMPARE_TWICE 420 error1: 421 rseq_bug("cpu_id comparison failed"); 422 error2: 423 rseq_bug("expected value comparison failed"); 424 #endif 425 } 426 427 static inline __attribute__((always_inline)) 428 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 429 intptr_t *v2, intptr_t expect2, 430 intptr_t newv, int cpu) 431 { 432 RSEQ_INJECT_C(9) 433 434 __asm__ __volatile__ goto ( 435 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 436 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 437 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 438 RSEQ_INJECT_ASM(3) 439 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 440 RSEQ_INJECT_ASM(4) 441 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) 442 RSEQ_INJECT_ASM(5) 443 #ifdef RSEQ_COMPARE_TWICE 444 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 445 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 446 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) 447 #endif 448 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 449 RSEQ_INJECT_ASM(6) 450 RSEQ_ASM_DEFINE_ABORT(4, abort) 451 : /* gcc asm goto does not allow outputs */ 452 : [cpu_id] "r" (cpu), 453 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 454 [rseq_cs] "m" (__rseq_abi.rseq_cs), 455 [v] "Qo" (*v), 456 [expect] "r" (expect), 457 [v2] "Qo" (*v2), 458 [expect2] "r" (expect2), 459 [newv] "r" (newv) 460 RSEQ_INJECT_INPUT 461 : "memory", RSEQ_ASM_TMP_REG 462 : abort, cmpfail 463 #ifdef RSEQ_COMPARE_TWICE 464 , error1, error2, error3 465 #endif 466 ); 467 468 return 0; 469 abort: 470 RSEQ_INJECT_FAILED 471 return -1; 472 cmpfail: 473 return 1; 474 #ifdef RSEQ_COMPARE_TWICE 475 error1: 476 rseq_bug("cpu_id comparison failed"); 477 error2: 478 rseq_bug("expected value comparison failed"); 479 error3: 480 rseq_bug("2nd expected value comparison failed"); 481 #endif 482 } 483 484 static inline __attribute__((always_inline)) 485 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 486 void *dst, void *src, size_t len, 487 intptr_t newv, int cpu) 488 { 489 RSEQ_INJECT_C(9) 490 491 __asm__ __volatile__ goto ( 492 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 493 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 494 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 495 RSEQ_INJECT_ASM(3) 496 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 497 RSEQ_INJECT_ASM(4) 498 #ifdef RSEQ_COMPARE_TWICE 499 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 500 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 501 #endif 502 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 503 RSEQ_INJECT_ASM(5) 504 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 505 RSEQ_INJECT_ASM(6) 506 RSEQ_ASM_DEFINE_ABORT(4, abort) 507 : /* gcc asm goto does not allow outputs */ 508 : [cpu_id] "r" (cpu), 509 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 510 [rseq_cs] "m" (__rseq_abi.rseq_cs), 511 [expect] "r" (expect), 512 [v] "Qo" (*v), 513 [newv] "r" (newv), 514 [dst] "r" (dst), 515 [src] "r" (src), 516 [len] "r" (len) 517 RSEQ_INJECT_INPUT 518 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 519 : abort, cmpfail 520 #ifdef RSEQ_COMPARE_TWICE 521 , error1, error2 522 #endif 523 ); 524 525 return 0; 526 abort: 527 RSEQ_INJECT_FAILED 528 return -1; 529 cmpfail: 530 return 1; 531 #ifdef RSEQ_COMPARE_TWICE 532 error1: 533 rseq_bug("cpu_id comparison failed"); 534 error2: 535 rseq_bug("expected value comparison failed"); 536 #endif 537 } 538 539 static inline __attribute__((always_inline)) 540 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 541 void *dst, void *src, size_t len, 542 intptr_t newv, int cpu) 543 { 544 RSEQ_INJECT_C(9) 545 546 __asm__ __volatile__ goto ( 547 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 548 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 549 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 550 RSEQ_INJECT_ASM(3) 551 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 552 RSEQ_INJECT_ASM(4) 553 #ifdef RSEQ_COMPARE_TWICE 554 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 555 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 556 #endif 557 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 558 RSEQ_INJECT_ASM(5) 559 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 560 RSEQ_INJECT_ASM(6) 561 RSEQ_ASM_DEFINE_ABORT(4, abort) 562 : /* gcc asm goto does not allow outputs */ 563 : [cpu_id] "r" (cpu), 564 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 565 [rseq_cs] "m" (__rseq_abi.rseq_cs), 566 [expect] "r" (expect), 567 [v] "Qo" (*v), 568 [newv] "r" (newv), 569 [dst] "r" (dst), 570 [src] "r" (src), 571 [len] "r" (len) 572 RSEQ_INJECT_INPUT 573 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 574 : abort, cmpfail 575 #ifdef RSEQ_COMPARE_TWICE 576 , error1, error2 577 #endif 578 ); 579 580 return 0; 581 abort: 582 RSEQ_INJECT_FAILED 583 return -1; 584 cmpfail: 585 return 1; 586 #ifdef RSEQ_COMPARE_TWICE 587 error1: 588 rseq_bug("cpu_id comparison failed"); 589 error2: 590 rseq_bug("expected value comparison failed"); 591 #endif 592 } 593 594 #endif /* !RSEQ_SKIP_FASTPATH */ 595