1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-arm64-bits.h 4 * 5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com> 7 */ 8 9 #include "rseq-bits-template.h" 10 11 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 12 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 13 14 static inline __attribute__((always_inline)) 15 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 16 { 17 RSEQ_INJECT_C(9) 18 19 __asm__ __volatile__ goto ( 20 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 21 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 22 #ifdef RSEQ_COMPARE_TWICE 23 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 24 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 25 #endif 26 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 27 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 28 RSEQ_INJECT_ASM(3) 29 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 30 RSEQ_INJECT_ASM(4) 31 #ifdef RSEQ_COMPARE_TWICE 32 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 33 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 34 #endif 35 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 36 RSEQ_INJECT_ASM(5) 37 RSEQ_ASM_DEFINE_ABORT(4, abort) 38 : /* gcc asm goto does not allow outputs */ 39 : [cpu_id] "r" (cpu), 40 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 41 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 42 [v] "Qo" (*v), 43 [expect] "r" (expect), 44 [newv] "r" (newv) 45 RSEQ_INJECT_INPUT 46 : "memory", RSEQ_ASM_TMP_REG 47 : abort, cmpfail 48 #ifdef RSEQ_COMPARE_TWICE 49 , error1, error2 50 #endif 51 ); 52 rseq_after_asm_goto(); 53 return 0; 54 abort: 55 rseq_after_asm_goto(); 56 RSEQ_INJECT_FAILED 57 return -1; 58 cmpfail: 59 rseq_after_asm_goto(); 60 return 1; 61 #ifdef RSEQ_COMPARE_TWICE 62 error1: 63 rseq_after_asm_goto(); 64 rseq_bug("cpu_id comparison failed"); 65 error2: 66 rseq_after_asm_goto(); 67 rseq_bug("expected value comparison failed"); 68 #endif 69 } 70 71 static inline __attribute__((always_inline)) 72 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 73 long voffp, intptr_t *load, int cpu) 74 { 75 RSEQ_INJECT_C(9) 76 77 __asm__ __volatile__ goto ( 78 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 79 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 80 #ifdef RSEQ_COMPARE_TWICE 81 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 82 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 83 #endif 84 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 85 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 86 RSEQ_INJECT_ASM(3) 87 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) 88 RSEQ_INJECT_ASM(4) 89 #ifdef RSEQ_COMPARE_TWICE 90 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 91 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) 92 #endif 93 RSEQ_ASM_OP_R_LOAD(v) 94 RSEQ_ASM_OP_R_STORE(load) 95 RSEQ_ASM_OP_R_LOAD_OFF(voffp) 96 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 97 RSEQ_INJECT_ASM(5) 98 RSEQ_ASM_DEFINE_ABORT(4, abort) 99 : /* gcc asm goto does not allow outputs */ 100 : [cpu_id] "r" (cpu), 101 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 102 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 103 [v] "Qo" (*v), 104 [expectnot] "r" (expectnot), 105 [load] "Qo" (*load), 106 [voffp] "r" (voffp) 107 RSEQ_INJECT_INPUT 108 : "memory", RSEQ_ASM_TMP_REG 109 : abort, cmpfail 110 #ifdef RSEQ_COMPARE_TWICE 111 , error1, error2 112 #endif 113 ); 114 rseq_after_asm_goto(); 115 return 0; 116 abort: 117 rseq_after_asm_goto(); 118 RSEQ_INJECT_FAILED 119 return -1; 120 cmpfail: 121 rseq_after_asm_goto(); 122 return 1; 123 #ifdef RSEQ_COMPARE_TWICE 124 error1: 125 rseq_after_asm_goto(); 126 rseq_bug("cpu_id comparison failed"); 127 error2: 128 rseq_after_asm_goto(); 129 rseq_bug("expected value comparison failed"); 130 #endif 131 } 132 133 static inline __attribute__((always_inline)) 134 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 135 { 136 RSEQ_INJECT_C(9) 137 138 __asm__ __volatile__ goto ( 139 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 140 #ifdef RSEQ_COMPARE_TWICE 141 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 142 #endif 143 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 144 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 145 RSEQ_INJECT_ASM(3) 146 #ifdef RSEQ_COMPARE_TWICE 147 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 148 #endif 149 RSEQ_ASM_OP_R_LOAD(v) 150 RSEQ_ASM_OP_R_ADD(count) 151 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 152 RSEQ_INJECT_ASM(4) 153 RSEQ_ASM_DEFINE_ABORT(4, abort) 154 : /* gcc asm goto does not allow outputs */ 155 : [cpu_id] "r" (cpu), 156 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 157 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 158 [v] "Qo" (*v), 159 [count] "r" (count) 160 RSEQ_INJECT_INPUT 161 : "memory", RSEQ_ASM_TMP_REG 162 : abort 163 #ifdef RSEQ_COMPARE_TWICE 164 , error1 165 #endif 166 ); 167 rseq_after_asm_goto(); 168 return 0; 169 abort: 170 rseq_after_asm_goto(); 171 RSEQ_INJECT_FAILED 172 return -1; 173 #ifdef RSEQ_COMPARE_TWICE 174 error1: 175 rseq_after_asm_goto(); 176 rseq_bug("cpu_id comparison failed"); 177 #endif 178 } 179 180 static inline __attribute__((always_inline)) 181 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 182 intptr_t *v2, intptr_t expect2, 183 intptr_t newv, int cpu) 184 { 185 RSEQ_INJECT_C(9) 186 187 __asm__ __volatile__ goto ( 188 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 189 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 190 #ifdef RSEQ_COMPARE_TWICE 191 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 192 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 193 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3]) 194 #endif 195 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 196 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 197 RSEQ_INJECT_ASM(3) 198 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 199 RSEQ_INJECT_ASM(4) 200 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) 201 RSEQ_INJECT_ASM(5) 202 #ifdef RSEQ_COMPARE_TWICE 203 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 204 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 205 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) 206 #endif 207 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 208 RSEQ_INJECT_ASM(6) 209 RSEQ_ASM_DEFINE_ABORT(4, abort) 210 : /* gcc asm goto does not allow outputs */ 211 : [cpu_id] "r" (cpu), 212 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 213 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 214 [v] "Qo" (*v), 215 [expect] "r" (expect), 216 [v2] "Qo" (*v2), 217 [expect2] "r" (expect2), 218 [newv] "r" (newv) 219 RSEQ_INJECT_INPUT 220 : "memory", RSEQ_ASM_TMP_REG 221 : abort, cmpfail 222 #ifdef RSEQ_COMPARE_TWICE 223 , error1, error2, error3 224 #endif 225 ); 226 rseq_after_asm_goto(); 227 return 0; 228 abort: 229 rseq_after_asm_goto(); 230 RSEQ_INJECT_FAILED 231 return -1; 232 cmpfail: 233 rseq_after_asm_goto(); 234 return 1; 235 #ifdef RSEQ_COMPARE_TWICE 236 error1: 237 rseq_after_asm_goto(); 238 rseq_bug("cpu_id comparison failed"); 239 error2: 240 rseq_after_asm_goto(); 241 rseq_bug("expected value comparison failed"); 242 error3: 243 rseq_after_asm_goto(); 244 rseq_bug("2nd expected value comparison failed"); 245 #endif 246 } 247 248 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 249 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 250 251 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 252 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 253 254 static inline __attribute__((always_inline)) 255 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 256 intptr_t *v2, intptr_t newv2, 257 intptr_t newv, int cpu) 258 { 259 RSEQ_INJECT_C(9) 260 261 __asm__ __volatile__ goto ( 262 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 263 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 264 #ifdef RSEQ_COMPARE_TWICE 265 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 266 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 267 #endif 268 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 269 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 270 RSEQ_INJECT_ASM(3) 271 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 272 RSEQ_INJECT_ASM(4) 273 #ifdef RSEQ_COMPARE_TWICE 274 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 275 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 276 #endif 277 RSEQ_ASM_OP_STORE(newv2, v2) 278 RSEQ_INJECT_ASM(5) 279 #ifdef RSEQ_TEMPLATE_MO_RELEASE 280 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 281 #else 282 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 283 #endif 284 RSEQ_INJECT_ASM(6) 285 RSEQ_ASM_DEFINE_ABORT(4, abort) 286 : /* gcc asm goto does not allow outputs */ 287 : [cpu_id] "r" (cpu), 288 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 289 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 290 [expect] "r" (expect), 291 [v] "Qo" (*v), 292 [newv] "r" (newv), 293 [v2] "Qo" (*v2), 294 [newv2] "r" (newv2) 295 RSEQ_INJECT_INPUT 296 : "memory", RSEQ_ASM_TMP_REG 297 : abort, cmpfail 298 #ifdef RSEQ_COMPARE_TWICE 299 , error1, error2 300 #endif 301 ); 302 rseq_after_asm_goto(); 303 return 0; 304 abort: 305 rseq_after_asm_goto(); 306 RSEQ_INJECT_FAILED 307 return -1; 308 cmpfail: 309 rseq_after_asm_goto(); 310 return 1; 311 #ifdef RSEQ_COMPARE_TWICE 312 error1: 313 rseq_after_asm_goto(); 314 rseq_bug("cpu_id comparison failed"); 315 error2: 316 rseq_after_asm_goto(); 317 rseq_bug("expected value comparison failed"); 318 #endif 319 } 320 321 static inline __attribute__((always_inline)) 322 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 323 void *dst, void *src, size_t len, 324 intptr_t newv, int cpu) 325 { 326 RSEQ_INJECT_C(9) 327 328 __asm__ __volatile__ goto ( 329 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 330 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 331 #ifdef RSEQ_COMPARE_TWICE 332 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 333 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 334 #endif 335 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 336 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 337 RSEQ_INJECT_ASM(3) 338 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 339 RSEQ_INJECT_ASM(4) 340 #ifdef RSEQ_COMPARE_TWICE 341 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 342 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 343 #endif 344 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 345 RSEQ_INJECT_ASM(5) 346 #ifdef RSEQ_TEMPLATE_MO_RELEASE 347 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 348 #else 349 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 350 #endif 351 RSEQ_INJECT_ASM(6) 352 RSEQ_ASM_DEFINE_ABORT(4, abort) 353 : /* gcc asm goto does not allow outputs */ 354 : [cpu_id] "r" (cpu), 355 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 356 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 357 [expect] "r" (expect), 358 [v] "Qo" (*v), 359 [newv] "r" (newv), 360 [dst] "r" (dst), 361 [src] "r" (src), 362 [len] "r" (len) 363 RSEQ_INJECT_INPUT 364 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 365 : abort, cmpfail 366 #ifdef RSEQ_COMPARE_TWICE 367 , error1, error2 368 #endif 369 ); 370 rseq_after_asm_goto(); 371 return 0; 372 abort: 373 rseq_after_asm_goto(); 374 RSEQ_INJECT_FAILED 375 return -1; 376 cmpfail: 377 rseq_after_asm_goto(); 378 return 1; 379 #ifdef RSEQ_COMPARE_TWICE 380 error1: 381 rseq_after_asm_goto(); 382 rseq_bug("cpu_id comparison failed"); 383 error2: 384 rseq_after_asm_goto(); 385 rseq_bug("expected value comparison failed"); 386 #endif 387 } 388 389 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 390 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 391 392 #include "rseq-bits-reset.h" 393