1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 3 #include "rseq-bits-template.h" 4 5 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 6 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 7 8 static inline __always_inline 9 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 10 { 11 RSEQ_INJECT_C(9) 12 13 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 14 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 15 #ifdef RSEQ_COMPARE_TWICE 16 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 17 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 18 #endif 19 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 20 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 21 RSEQ_INJECT_ASM(3) 22 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 23 RSEQ_INJECT_ASM(4) 24 #ifdef RSEQ_COMPARE_TWICE 25 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 26 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 27 #endif 28 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 29 RSEQ_INJECT_ASM(5) 30 RSEQ_ASM_DEFINE_ABORT(4, abort) 31 : /* gcc asm goto does not allow outputs */ 32 : [cpu_id] "r" (cpu), 33 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 34 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 35 [v] "m" (*v), 36 [expect] "r" (expect), 37 [newv] "r" (newv) 38 RSEQ_INJECT_INPUT 39 : "memory", RSEQ_ASM_TMP_REG_1 40 RSEQ_INJECT_CLOBBER 41 : abort, cmpfail 42 #ifdef RSEQ_COMPARE_TWICE 43 , error1, error2 44 #endif 45 ); 46 47 return 0; 48 abort: 49 RSEQ_INJECT_FAILED 50 return -1; 51 cmpfail: 52 return 1; 53 #ifdef RSEQ_COMPARE_TWICE 54 error1: 55 rseq_bug("cpu_id comparison failed"); 56 error2: 57 rseq_bug("expected value comparison failed"); 58 #endif 59 } 60 61 static inline __always_inline 62 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 63 off_t voffp, intptr_t *load, int cpu) 64 { 65 RSEQ_INJECT_C(9) 66 67 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 68 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 69 #ifdef RSEQ_COMPARE_TWICE 70 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 71 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 72 #endif 73 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 74 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 75 RSEQ_INJECT_ASM(3) 76 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]") 77 RSEQ_INJECT_ASM(4) 78 #ifdef RSEQ_COMPARE_TWICE 79 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 80 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]") 81 #endif 82 RSEQ_ASM_OP_R_LOAD(v) 83 RSEQ_ASM_OP_R_STORE(load) 84 RSEQ_ASM_OP_R_LOAD_OFF(voffp) 85 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 86 RSEQ_INJECT_ASM(5) 87 RSEQ_ASM_DEFINE_ABORT(4, abort) 88 : /* gcc asm goto does not allow outputs */ 89 : [cpu_id] "r" (cpu), 90 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 91 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 92 [v] "m" (*v), 93 [expectnot] "r" (expectnot), 94 [load] "m" (*load), 95 [voffp] "r" (voffp) 96 RSEQ_INJECT_INPUT 97 : "memory", RSEQ_ASM_TMP_REG_1 98 RSEQ_INJECT_CLOBBER 99 : abort, cmpfail 100 #ifdef RSEQ_COMPARE_TWICE 101 , error1, error2 102 #endif 103 ); 104 return 0; 105 abort: 106 RSEQ_INJECT_FAILED 107 return -1; 108 cmpfail: 109 return 1; 110 #ifdef RSEQ_COMPARE_TWICE 111 error1: 112 rseq_bug("cpu_id comparison failed"); 113 error2: 114 rseq_bug("expected value comparison failed"); 115 #endif 116 } 117 118 static inline __always_inline 119 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 120 { 121 RSEQ_INJECT_C(9) 122 123 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 124 #ifdef RSEQ_COMPARE_TWICE 125 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 126 #endif 127 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 128 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 129 RSEQ_INJECT_ASM(3) 130 #ifdef RSEQ_COMPARE_TWICE 131 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 132 #endif 133 RSEQ_ASM_OP_R_LOAD(v) 134 RSEQ_ASM_OP_R_ADD(count) 135 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 136 RSEQ_INJECT_ASM(4) 137 RSEQ_ASM_DEFINE_ABORT(4, abort) 138 : /* gcc asm goto does not allow outputs */ 139 : [cpu_id] "r" (cpu), 140 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 141 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 142 [v] "m" (*v), 143 [count] "r" (count) 144 RSEQ_INJECT_INPUT 145 : "memory", RSEQ_ASM_TMP_REG_1 146 RSEQ_INJECT_CLOBBER 147 : abort 148 #ifdef RSEQ_COMPARE_TWICE 149 , error1 150 #endif 151 ); 152 return 0; 153 abort: 154 RSEQ_INJECT_FAILED 155 return -1; 156 #ifdef RSEQ_COMPARE_TWICE 157 error1: 158 rseq_bug("cpu_id comparison failed"); 159 #endif 160 } 161 162 static inline __always_inline 163 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 164 intptr_t *v2, intptr_t expect2, 165 intptr_t newv, int cpu) 166 { 167 RSEQ_INJECT_C(9) 168 169 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 170 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 171 #ifdef RSEQ_COMPARE_TWICE 172 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 173 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 174 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]") 175 #endif 176 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 177 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 178 RSEQ_INJECT_ASM(3) 179 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 180 RSEQ_INJECT_ASM(4) 181 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]") 182 RSEQ_INJECT_ASM(5) 183 #ifdef RSEQ_COMPARE_TWICE 184 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 185 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 186 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]") 187 #endif 188 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 189 RSEQ_INJECT_ASM(6) 190 RSEQ_ASM_DEFINE_ABORT(4, abort) 191 : /* gcc asm goto does not allow outputs */ 192 : [cpu_id] "r" (cpu), 193 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 194 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 195 [v] "m" (*v), 196 [expect] "r" (expect), 197 [v2] "m" (*v2), 198 [expect2] "r" (expect2), 199 [newv] "r" (newv) 200 RSEQ_INJECT_INPUT 201 : "memory", RSEQ_ASM_TMP_REG_1 202 RSEQ_INJECT_CLOBBER 203 : abort, cmpfail 204 #ifdef RSEQ_COMPARE_TWICE 205 , error1, error2, error3 206 #endif 207 ); 208 209 return 0; 210 abort: 211 RSEQ_INJECT_FAILED 212 return -1; 213 cmpfail: 214 return 1; 215 #ifdef RSEQ_COMPARE_TWICE 216 error1: 217 rseq_bug("cpu_id comparison failed"); 218 error2: 219 rseq_bug("expected value comparison failed"); 220 error3: 221 rseq_bug("2nd expected value comparison failed"); 222 #endif 223 } 224 225 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV 226 227 /* 228 * pval = *(ptr+off) 229 * *pval += inc; 230 */ 231 static inline __always_inline 232 int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, intptr_t inc, int cpu) 233 { 234 RSEQ_INJECT_C(9) 235 236 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 237 #ifdef RSEQ_COMPARE_TWICE 238 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 239 #endif 240 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 241 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 242 RSEQ_INJECT_ASM(3) 243 #ifdef RSEQ_COMPARE_TWICE 244 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 245 #endif 246 RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3) 247 RSEQ_INJECT_ASM(4) 248 RSEQ_ASM_DEFINE_ABORT(4, abort) 249 : /* gcc asm goto does not allow outputs */ 250 : [cpu_id] "r" (cpu), 251 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 252 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 253 [ptr] "r" (ptr), 254 [off] "er" (off), 255 [inc] "er" (inc) 256 RSEQ_INJECT_INPUT 257 : "memory", RSEQ_ASM_TMP_REG_1 258 RSEQ_INJECT_CLOBBER 259 : abort 260 #ifdef RSEQ_COMPARE_TWICE 261 , error1 262 #endif 263 ); 264 return 0; 265 abort: 266 RSEQ_INJECT_FAILED 267 return -1; 268 #ifdef RSEQ_COMPARE_TWICE 269 error1: 270 rseq_bug("cpu_id comparison failed"); 271 #endif 272 } 273 274 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 275 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 276 277 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 278 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 279 280 static inline __always_inline 281 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 282 intptr_t *v2, intptr_t newv2, 283 intptr_t newv, int cpu) 284 { 285 RSEQ_INJECT_C(9) 286 287 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 288 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 289 #ifdef RSEQ_COMPARE_TWICE 290 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 291 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 292 #endif 293 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 294 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 295 RSEQ_INJECT_ASM(3) 296 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 297 RSEQ_INJECT_ASM(4) 298 #ifdef RSEQ_COMPARE_TWICE 299 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 300 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 301 #endif 302 RSEQ_ASM_OP_STORE(newv2, v2) 303 RSEQ_INJECT_ASM(5) 304 #ifdef RSEQ_TEMPLATE_MO_RELEASE 305 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 306 #else 307 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 308 #endif 309 RSEQ_INJECT_ASM(6) 310 RSEQ_ASM_DEFINE_ABORT(4, abort) 311 : /* gcc asm goto does not allow outputs */ 312 : [cpu_id] "r" (cpu), 313 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 314 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 315 [expect] "r" (expect), 316 [v] "m" (*v), 317 [newv] "r" (newv), 318 [v2] "m" (*v2), 319 [newv2] "r" (newv2) 320 RSEQ_INJECT_INPUT 321 : "memory", RSEQ_ASM_TMP_REG_1 322 RSEQ_INJECT_CLOBBER 323 : abort, cmpfail 324 #ifdef RSEQ_COMPARE_TWICE 325 , error1, error2 326 #endif 327 ); 328 329 return 0; 330 abort: 331 RSEQ_INJECT_FAILED 332 return -1; 333 cmpfail: 334 return 1; 335 #ifdef RSEQ_COMPARE_TWICE 336 error1: 337 rseq_bug("cpu_id comparison failed"); 338 error2: 339 rseq_bug("expected value comparison failed"); 340 #endif 341 } 342 343 static inline __always_inline 344 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 345 void *dst, void *src, size_t len, 346 intptr_t newv, int cpu) 347 { 348 RSEQ_INJECT_C(9) 349 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 350 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 351 #ifdef RSEQ_COMPARE_TWICE 352 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 353 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 354 #endif 355 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 356 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 357 RSEQ_INJECT_ASM(3) 358 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 359 RSEQ_INJECT_ASM(4) 360 #ifdef RSEQ_COMPARE_TWICE 361 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 362 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 363 #endif 364 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 365 RSEQ_INJECT_ASM(5) 366 #ifdef RSEQ_TEMPLATE_MO_RELEASE 367 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 368 #else 369 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 370 #endif 371 RSEQ_INJECT_ASM(6) 372 RSEQ_ASM_DEFINE_ABORT(4, abort) 373 : /* gcc asm goto does not allow outputs */ 374 : [cpu_id] "r" (cpu), 375 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 376 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 377 [expect] "r" (expect), 378 [v] "m" (*v), 379 [newv] "r" (newv), 380 [dst] "r" (dst), 381 [src] "r" (src), 382 [len] "r" (len) 383 RSEQ_INJECT_INPUT 384 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2, 385 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4 386 RSEQ_INJECT_CLOBBER 387 : abort, cmpfail 388 #ifdef RSEQ_COMPARE_TWICE 389 , error1, error2 390 #endif 391 ); 392 393 return 0; 394 abort: 395 RSEQ_INJECT_FAILED 396 return -1; 397 cmpfail: 398 return 1; 399 #ifdef RSEQ_COMPARE_TWICE 400 error1: 401 rseq_bug("cpu_id comparison failed"); 402 error2: 403 rseq_bug("expected value comparison failed"); 404 #endif 405 } 406 407 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 408 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 409 410 #include "rseq-bits-reset.h" 411