1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-x86-bits.h 4 * 5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8 #include "rseq-bits-template.h" 9 10 #ifdef __x86_64__ 11 12 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 13 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 14 15 static inline __attribute__((always_inline)) 16 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 17 { 18 RSEQ_INJECT_C(9) 19 20 __asm__ __volatile__ goto ( 21 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 22 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 23 #ifdef RSEQ_COMPARE_TWICE 24 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 25 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 26 #endif 27 /* Start rseq by storing table entry pointer into rseq_cs. */ 28 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 29 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 30 RSEQ_INJECT_ASM(3) 31 "cmpq %[v], %[expect]\n\t" 32 "jnz %l[cmpfail]\n\t" 33 RSEQ_INJECT_ASM(4) 34 #ifdef RSEQ_COMPARE_TWICE 35 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 36 "cmpq %[v], %[expect]\n\t" 37 "jnz %l[error2]\n\t" 38 #endif 39 /* final store */ 40 "movq %[newv], %[v]\n\t" 41 "2:\n\t" 42 RSEQ_INJECT_ASM(5) 43 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 44 : /* gcc asm goto does not allow outputs */ 45 : [cpu_id] "r" (cpu), 46 [rseq_offset] "r" (rseq_offset), 47 [v] "m" (*v), 48 [expect] "r" (expect), 49 [newv] "r" (newv) 50 : "memory", "cc", "rax" 51 RSEQ_INJECT_CLOBBER 52 : abort, cmpfail 53 #ifdef RSEQ_COMPARE_TWICE 54 , error1, error2 55 #endif 56 ); 57 rseq_after_asm_goto(); 58 return 0; 59 abort: 60 rseq_after_asm_goto(); 61 RSEQ_INJECT_FAILED 62 return -1; 63 cmpfail: 64 rseq_after_asm_goto(); 65 return 1; 66 #ifdef RSEQ_COMPARE_TWICE 67 error1: 68 rseq_after_asm_goto(); 69 rseq_bug("cpu_id comparison failed"); 70 error2: 71 rseq_after_asm_goto(); 72 rseq_bug("expected value comparison failed"); 73 #endif 74 } 75 76 /* 77 * Compare @v against @expectnot. When it does _not_ match, load @v 78 * into @load, and store the content of *@v + voffp into @v. 79 */ 80 static inline __attribute__((always_inline)) 81 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 82 long voffp, intptr_t *load, int cpu) 83 { 84 RSEQ_INJECT_C(9) 85 86 __asm__ __volatile__ goto ( 87 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 88 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 89 #ifdef RSEQ_COMPARE_TWICE 90 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 91 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 92 #endif 93 /* Start rseq by storing table entry pointer into rseq_cs. */ 94 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 95 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 96 RSEQ_INJECT_ASM(3) 97 "movq %[v], %%rbx\n\t" 98 "cmpq %%rbx, %[expectnot]\n\t" 99 "je %l[cmpfail]\n\t" 100 RSEQ_INJECT_ASM(4) 101 #ifdef RSEQ_COMPARE_TWICE 102 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 103 "movq %[v], %%rbx\n\t" 104 "cmpq %%rbx, %[expectnot]\n\t" 105 "je %l[error2]\n\t" 106 #endif 107 "movq %%rbx, %[load]\n\t" 108 "addq %[voffp], %%rbx\n\t" 109 "movq (%%rbx), %%rbx\n\t" 110 /* final store */ 111 "movq %%rbx, %[v]\n\t" 112 "2:\n\t" 113 RSEQ_INJECT_ASM(5) 114 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 115 : /* gcc asm goto does not allow outputs */ 116 : [cpu_id] "r" (cpu), 117 [rseq_offset] "r" (rseq_offset), 118 /* final store input */ 119 [v] "m" (*v), 120 [expectnot] "r" (expectnot), 121 [voffp] "er" (voffp), 122 [load] "m" (*load) 123 : "memory", "cc", "rax", "rbx" 124 RSEQ_INJECT_CLOBBER 125 : abort, cmpfail 126 #ifdef RSEQ_COMPARE_TWICE 127 , error1, error2 128 #endif 129 ); 130 rseq_after_asm_goto(); 131 return 0; 132 abort: 133 rseq_after_asm_goto(); 134 RSEQ_INJECT_FAILED 135 return -1; 136 cmpfail: 137 rseq_after_asm_goto(); 138 return 1; 139 #ifdef RSEQ_COMPARE_TWICE 140 error1: 141 rseq_after_asm_goto(); 142 rseq_bug("cpu_id comparison failed"); 143 error2: 144 rseq_after_asm_goto(); 145 rseq_bug("expected value comparison failed"); 146 #endif 147 } 148 149 static inline __attribute__((always_inline)) 150 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 151 { 152 RSEQ_INJECT_C(9) 153 154 __asm__ __volatile__ goto ( 155 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 156 #ifdef RSEQ_COMPARE_TWICE 157 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 158 #endif 159 /* Start rseq by storing table entry pointer into rseq_cs. */ 160 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 161 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 162 RSEQ_INJECT_ASM(3) 163 #ifdef RSEQ_COMPARE_TWICE 164 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 165 #endif 166 /* final store */ 167 "addq %[count], %[v]\n\t" 168 "2:\n\t" 169 RSEQ_INJECT_ASM(4) 170 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 171 : /* gcc asm goto does not allow outputs */ 172 : [cpu_id] "r" (cpu), 173 [rseq_offset] "r" (rseq_offset), 174 /* final store input */ 175 [v] "m" (*v), 176 [count] "er" (count) 177 : "memory", "cc", "rax" 178 RSEQ_INJECT_CLOBBER 179 : abort 180 #ifdef RSEQ_COMPARE_TWICE 181 , error1 182 #endif 183 ); 184 rseq_after_asm_goto(); 185 return 0; 186 abort: 187 rseq_after_asm_goto(); 188 RSEQ_INJECT_FAILED 189 return -1; 190 #ifdef RSEQ_COMPARE_TWICE 191 error1: 192 rseq_after_asm_goto(); 193 rseq_bug("cpu_id comparison failed"); 194 #endif 195 } 196 197 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV 198 199 /* 200 * pval = *(ptr+off) 201 * *pval += inc; 202 */ 203 static inline __attribute__((always_inline)) 204 int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, long off, intptr_t inc, int cpu) 205 { 206 RSEQ_INJECT_C(9) 207 208 __asm__ __volatile__ goto ( 209 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 210 #ifdef RSEQ_COMPARE_TWICE 211 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 212 #endif 213 /* Start rseq by storing table entry pointer into rseq_cs. */ 214 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 215 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 216 RSEQ_INJECT_ASM(3) 217 #ifdef RSEQ_COMPARE_TWICE 218 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 219 #endif 220 /* get p+v */ 221 "movq %[ptr], %%rbx\n\t" 222 "addq %[off], %%rbx\n\t" 223 /* get pv */ 224 "movq (%%rbx), %%rcx\n\t" 225 /* *pv += inc */ 226 "addq %[inc], (%%rcx)\n\t" 227 "2:\n\t" 228 RSEQ_INJECT_ASM(4) 229 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 230 : /* gcc asm goto does not allow outputs */ 231 : [cpu_id] "r" (cpu), 232 [rseq_offset] "r" (rseq_offset), 233 /* final store input */ 234 [ptr] "m" (*ptr), 235 [off] "er" (off), 236 [inc] "er" (inc) 237 : "memory", "cc", "rax", "rbx", "rcx" 238 RSEQ_INJECT_CLOBBER 239 : abort 240 #ifdef RSEQ_COMPARE_TWICE 241 , error1 242 #endif 243 ); 244 return 0; 245 abort: 246 RSEQ_INJECT_FAILED 247 return -1; 248 #ifdef RSEQ_COMPARE_TWICE 249 error1: 250 rseq_bug("cpu_id comparison failed"); 251 #endif 252 } 253 254 static inline __attribute__((always_inline)) 255 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 256 intptr_t *v2, intptr_t expect2, 257 intptr_t newv, int cpu) 258 { 259 RSEQ_INJECT_C(9) 260 261 __asm__ __volatile__ goto ( 262 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 263 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 264 #ifdef RSEQ_COMPARE_TWICE 265 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 266 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 267 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 268 #endif 269 /* Start rseq by storing table entry pointer into rseq_cs. */ 270 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 271 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 272 RSEQ_INJECT_ASM(3) 273 "cmpq %[v], %[expect]\n\t" 274 "jnz %l[cmpfail]\n\t" 275 RSEQ_INJECT_ASM(4) 276 "cmpq %[v2], %[expect2]\n\t" 277 "jnz %l[cmpfail]\n\t" 278 RSEQ_INJECT_ASM(5) 279 #ifdef RSEQ_COMPARE_TWICE 280 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 281 "cmpq %[v], %[expect]\n\t" 282 "jnz %l[error2]\n\t" 283 "cmpq %[v2], %[expect2]\n\t" 284 "jnz %l[error3]\n\t" 285 #endif 286 /* final store */ 287 "movq %[newv], %[v]\n\t" 288 "2:\n\t" 289 RSEQ_INJECT_ASM(6) 290 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 291 : /* gcc asm goto does not allow outputs */ 292 : [cpu_id] "r" (cpu), 293 [rseq_offset] "r" (rseq_offset), 294 /* cmp2 input */ 295 [v2] "m" (*v2), 296 [expect2] "r" (expect2), 297 /* final store input */ 298 [v] "m" (*v), 299 [expect] "r" (expect), 300 [newv] "r" (newv) 301 : "memory", "cc", "rax" 302 RSEQ_INJECT_CLOBBER 303 : abort, cmpfail 304 #ifdef RSEQ_COMPARE_TWICE 305 , error1, error2, error3 306 #endif 307 ); 308 rseq_after_asm_goto(); 309 return 0; 310 abort: 311 rseq_after_asm_goto(); 312 RSEQ_INJECT_FAILED 313 return -1; 314 cmpfail: 315 rseq_after_asm_goto(); 316 return 1; 317 #ifdef RSEQ_COMPARE_TWICE 318 error1: 319 rseq_after_asm_goto(); 320 rseq_bug("cpu_id comparison failed"); 321 error2: 322 rseq_after_asm_goto(); 323 rseq_bug("1st expected value comparison failed"); 324 error3: 325 rseq_after_asm_goto(); 326 rseq_bug("2nd expected value comparison failed"); 327 #endif 328 } 329 330 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 331 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 332 333 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 334 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 335 336 static inline __attribute__((always_inline)) 337 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 338 intptr_t *v2, intptr_t newv2, 339 intptr_t newv, int cpu) 340 { 341 RSEQ_INJECT_C(9) 342 343 __asm__ __volatile__ goto ( 344 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 345 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 346 #ifdef RSEQ_COMPARE_TWICE 347 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 348 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 349 #endif 350 /* Start rseq by storing table entry pointer into rseq_cs. */ 351 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 352 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 353 RSEQ_INJECT_ASM(3) 354 "cmpq %[v], %[expect]\n\t" 355 "jnz %l[cmpfail]\n\t" 356 RSEQ_INJECT_ASM(4) 357 #ifdef RSEQ_COMPARE_TWICE 358 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 359 "cmpq %[v], %[expect]\n\t" 360 "jnz %l[error2]\n\t" 361 #endif 362 /* try store */ 363 "movq %[newv2], %[v2]\n\t" 364 RSEQ_INJECT_ASM(5) 365 /* final store */ 366 "movq %[newv], %[v]\n\t" 367 "2:\n\t" 368 RSEQ_INJECT_ASM(6) 369 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 370 : /* gcc asm goto does not allow outputs */ 371 : [cpu_id] "r" (cpu), 372 [rseq_offset] "r" (rseq_offset), 373 /* try store input */ 374 [v2] "m" (*v2), 375 [newv2] "r" (newv2), 376 /* final store input */ 377 [v] "m" (*v), 378 [expect] "r" (expect), 379 [newv] "r" (newv) 380 : "memory", "cc", "rax" 381 RSEQ_INJECT_CLOBBER 382 : abort, cmpfail 383 #ifdef RSEQ_COMPARE_TWICE 384 , error1, error2 385 #endif 386 ); 387 rseq_after_asm_goto(); 388 return 0; 389 abort: 390 rseq_after_asm_goto(); 391 RSEQ_INJECT_FAILED 392 return -1; 393 cmpfail: 394 rseq_after_asm_goto(); 395 return 1; 396 #ifdef RSEQ_COMPARE_TWICE 397 error1: 398 rseq_after_asm_goto(); 399 rseq_bug("cpu_id comparison failed"); 400 error2: 401 rseq_after_asm_goto(); 402 rseq_bug("expected value comparison failed"); 403 #endif 404 } 405 406 static inline __attribute__((always_inline)) 407 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 408 void *dst, void *src, size_t len, 409 intptr_t newv, int cpu) 410 { 411 uint64_t rseq_scratch[3]; 412 413 RSEQ_INJECT_C(9) 414 415 __asm__ __volatile__ goto ( 416 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 417 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 418 #ifdef RSEQ_COMPARE_TWICE 419 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 420 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 421 #endif 422 "movq %[src], %[rseq_scratch0]\n\t" 423 "movq %[dst], %[rseq_scratch1]\n\t" 424 "movq %[len], %[rseq_scratch2]\n\t" 425 /* Start rseq by storing table entry pointer into rseq_cs. */ 426 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 427 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 428 RSEQ_INJECT_ASM(3) 429 "cmpq %[v], %[expect]\n\t" 430 "jnz 5f\n\t" 431 RSEQ_INJECT_ASM(4) 432 #ifdef RSEQ_COMPARE_TWICE 433 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) 434 "cmpq %[v], %[expect]\n\t" 435 "jnz 7f\n\t" 436 #endif 437 /* try memcpy */ 438 "test %[len], %[len]\n\t" \ 439 "jz 333f\n\t" \ 440 "222:\n\t" \ 441 "movb (%[src]), %%al\n\t" \ 442 "movb %%al, (%[dst])\n\t" \ 443 "inc %[src]\n\t" \ 444 "inc %[dst]\n\t" \ 445 "dec %[len]\n\t" \ 446 "jnz 222b\n\t" \ 447 "333:\n\t" \ 448 RSEQ_INJECT_ASM(5) 449 /* final store */ 450 "movq %[newv], %[v]\n\t" 451 "2:\n\t" 452 RSEQ_INJECT_ASM(6) 453 /* teardown */ 454 "movq %[rseq_scratch2], %[len]\n\t" 455 "movq %[rseq_scratch1], %[dst]\n\t" 456 "movq %[rseq_scratch0], %[src]\n\t" 457 RSEQ_ASM_DEFINE_ABORT(4, 458 "movq %[rseq_scratch2], %[len]\n\t" 459 "movq %[rseq_scratch1], %[dst]\n\t" 460 "movq %[rseq_scratch0], %[src]\n\t", 461 abort) 462 RSEQ_ASM_DEFINE_CMPFAIL(5, 463 "movq %[rseq_scratch2], %[len]\n\t" 464 "movq %[rseq_scratch1], %[dst]\n\t" 465 "movq %[rseq_scratch0], %[src]\n\t", 466 cmpfail) 467 #ifdef RSEQ_COMPARE_TWICE 468 RSEQ_ASM_DEFINE_CMPFAIL(6, 469 "movq %[rseq_scratch2], %[len]\n\t" 470 "movq %[rseq_scratch1], %[dst]\n\t" 471 "movq %[rseq_scratch0], %[src]\n\t", 472 error1) 473 RSEQ_ASM_DEFINE_CMPFAIL(7, 474 "movq %[rseq_scratch2], %[len]\n\t" 475 "movq %[rseq_scratch1], %[dst]\n\t" 476 "movq %[rseq_scratch0], %[src]\n\t", 477 error2) 478 #endif 479 : /* gcc asm goto does not allow outputs */ 480 : [cpu_id] "r" (cpu), 481 [rseq_offset] "r" (rseq_offset), 482 /* final store input */ 483 [v] "m" (*v), 484 [expect] "r" (expect), 485 [newv] "r" (newv), 486 /* try memcpy input */ 487 [dst] "r" (dst), 488 [src] "r" (src), 489 [len] "r" (len), 490 [rseq_scratch0] "m" (rseq_scratch[0]), 491 [rseq_scratch1] "m" (rseq_scratch[1]), 492 [rseq_scratch2] "m" (rseq_scratch[2]) 493 : "memory", "cc", "rax" 494 RSEQ_INJECT_CLOBBER 495 : abort, cmpfail 496 #ifdef RSEQ_COMPARE_TWICE 497 , error1, error2 498 #endif 499 ); 500 rseq_after_asm_goto(); 501 return 0; 502 abort: 503 rseq_after_asm_goto(); 504 RSEQ_INJECT_FAILED 505 return -1; 506 cmpfail: 507 rseq_after_asm_goto(); 508 return 1; 509 #ifdef RSEQ_COMPARE_TWICE 510 error1: 511 rseq_after_asm_goto(); 512 rseq_bug("cpu_id comparison failed"); 513 error2: 514 rseq_after_asm_goto(); 515 rseq_bug("expected value comparison failed"); 516 #endif 517 } 518 519 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 520 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 521 522 #elif defined(__i386__) 523 524 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 525 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 526 527 static inline __attribute__((always_inline)) 528 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 529 { 530 RSEQ_INJECT_C(9) 531 532 __asm__ __volatile__ goto ( 533 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 534 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 535 #ifdef RSEQ_COMPARE_TWICE 536 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 537 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 538 #endif 539 /* Start rseq by storing table entry pointer into rseq_cs. */ 540 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 541 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 542 RSEQ_INJECT_ASM(3) 543 "cmpl %[v], %[expect]\n\t" 544 "jnz %l[cmpfail]\n\t" 545 RSEQ_INJECT_ASM(4) 546 #ifdef RSEQ_COMPARE_TWICE 547 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 548 "cmpl %[v], %[expect]\n\t" 549 "jnz %l[error2]\n\t" 550 #endif 551 /* final store */ 552 "movl %[newv], %[v]\n\t" 553 "2:\n\t" 554 RSEQ_INJECT_ASM(5) 555 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 556 : /* gcc asm goto does not allow outputs */ 557 : [cpu_id] "r" (cpu), 558 [rseq_offset] "r" (rseq_offset), 559 [v] "m" (*v), 560 [expect] "r" (expect), 561 [newv] "r" (newv) 562 : "memory", "cc", "eax" 563 RSEQ_INJECT_CLOBBER 564 : abort, cmpfail 565 #ifdef RSEQ_COMPARE_TWICE 566 , error1, error2 567 #endif 568 ); 569 rseq_after_asm_goto(); 570 return 0; 571 abort: 572 rseq_after_asm_goto(); 573 RSEQ_INJECT_FAILED 574 return -1; 575 cmpfail: 576 rseq_after_asm_goto(); 577 return 1; 578 #ifdef RSEQ_COMPARE_TWICE 579 error1: 580 rseq_after_asm_goto(); 581 rseq_bug("cpu_id comparison failed"); 582 error2: 583 rseq_after_asm_goto(); 584 rseq_bug("expected value comparison failed"); 585 #endif 586 } 587 588 /* 589 * Compare @v against @expectnot. When it does _not_ match, load @v 590 * into @load, and store the content of *@v + voffp into @v. 591 */ 592 static inline __attribute__((always_inline)) 593 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 594 long voffp, intptr_t *load, int cpu) 595 { 596 RSEQ_INJECT_C(9) 597 598 __asm__ __volatile__ goto ( 599 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 600 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 601 #ifdef RSEQ_COMPARE_TWICE 602 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 603 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 604 #endif 605 /* Start rseq by storing table entry pointer into rseq_cs. */ 606 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 607 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 608 RSEQ_INJECT_ASM(3) 609 "movl %[v], %%ebx\n\t" 610 "cmpl %%ebx, %[expectnot]\n\t" 611 "je %l[cmpfail]\n\t" 612 RSEQ_INJECT_ASM(4) 613 #ifdef RSEQ_COMPARE_TWICE 614 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 615 "movl %[v], %%ebx\n\t" 616 "cmpl %%ebx, %[expectnot]\n\t" 617 "je %l[error2]\n\t" 618 #endif 619 "movl %%ebx, %[load]\n\t" 620 "addl %[voffp], %%ebx\n\t" 621 "movl (%%ebx), %%ebx\n\t" 622 /* final store */ 623 "movl %%ebx, %[v]\n\t" 624 "2:\n\t" 625 RSEQ_INJECT_ASM(5) 626 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 627 : /* gcc asm goto does not allow outputs */ 628 : [cpu_id] "r" (cpu), 629 [rseq_offset] "r" (rseq_offset), 630 /* final store input */ 631 [v] "m" (*v), 632 [expectnot] "r" (expectnot), 633 [voffp] "ir" (voffp), 634 [load] "m" (*load) 635 : "memory", "cc", "eax", "ebx" 636 RSEQ_INJECT_CLOBBER 637 : abort, cmpfail 638 #ifdef RSEQ_COMPARE_TWICE 639 , error1, error2 640 #endif 641 ); 642 rseq_after_asm_goto(); 643 return 0; 644 abort: 645 rseq_after_asm_goto(); 646 RSEQ_INJECT_FAILED 647 return -1; 648 cmpfail: 649 rseq_after_asm_goto(); 650 return 1; 651 #ifdef RSEQ_COMPARE_TWICE 652 error1: 653 rseq_after_asm_goto(); 654 rseq_bug("cpu_id comparison failed"); 655 error2: 656 rseq_after_asm_goto(); 657 rseq_bug("expected value comparison failed"); 658 #endif 659 } 660 661 static inline __attribute__((always_inline)) 662 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 663 { 664 RSEQ_INJECT_C(9) 665 666 __asm__ __volatile__ goto ( 667 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 668 #ifdef RSEQ_COMPARE_TWICE 669 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 670 #endif 671 /* Start rseq by storing table entry pointer into rseq_cs. */ 672 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 673 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 674 RSEQ_INJECT_ASM(3) 675 #ifdef RSEQ_COMPARE_TWICE 676 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 677 #endif 678 /* final store */ 679 "addl %[count], %[v]\n\t" 680 "2:\n\t" 681 RSEQ_INJECT_ASM(4) 682 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 683 : /* gcc asm goto does not allow outputs */ 684 : [cpu_id] "r" (cpu), 685 [rseq_offset] "r" (rseq_offset), 686 /* final store input */ 687 [v] "m" (*v), 688 [count] "ir" (count) 689 : "memory", "cc", "eax" 690 RSEQ_INJECT_CLOBBER 691 : abort 692 #ifdef RSEQ_COMPARE_TWICE 693 , error1 694 #endif 695 ); 696 rseq_after_asm_goto(); 697 return 0; 698 abort: 699 rseq_after_asm_goto(); 700 RSEQ_INJECT_FAILED 701 return -1; 702 #ifdef RSEQ_COMPARE_TWICE 703 error1: 704 rseq_after_asm_goto(); 705 rseq_bug("cpu_id comparison failed"); 706 #endif 707 } 708 709 static inline __attribute__((always_inline)) 710 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 711 intptr_t *v2, intptr_t expect2, 712 intptr_t newv, int cpu) 713 { 714 RSEQ_INJECT_C(9) 715 716 __asm__ __volatile__ goto ( 717 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 718 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 719 #ifdef RSEQ_COMPARE_TWICE 720 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 721 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 722 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 723 #endif 724 /* Start rseq by storing table entry pointer into rseq_cs. */ 725 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 726 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 727 RSEQ_INJECT_ASM(3) 728 "cmpl %[v], %[expect]\n\t" 729 "jnz %l[cmpfail]\n\t" 730 RSEQ_INJECT_ASM(4) 731 "cmpl %[expect2], %[v2]\n\t" 732 "jnz %l[cmpfail]\n\t" 733 RSEQ_INJECT_ASM(5) 734 #ifdef RSEQ_COMPARE_TWICE 735 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 736 "cmpl %[v], %[expect]\n\t" 737 "jnz %l[error2]\n\t" 738 "cmpl %[expect2], %[v2]\n\t" 739 "jnz %l[error3]\n\t" 740 #endif 741 "movl %[newv], %%eax\n\t" 742 /* final store */ 743 "movl %%eax, %[v]\n\t" 744 "2:\n\t" 745 RSEQ_INJECT_ASM(6) 746 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 747 : /* gcc asm goto does not allow outputs */ 748 : [cpu_id] "r" (cpu), 749 [rseq_offset] "r" (rseq_offset), 750 /* cmp2 input */ 751 [v2] "m" (*v2), 752 [expect2] "r" (expect2), 753 /* final store input */ 754 [v] "m" (*v), 755 [expect] "r" (expect), 756 [newv] "m" (newv) 757 : "memory", "cc", "eax" 758 RSEQ_INJECT_CLOBBER 759 : abort, cmpfail 760 #ifdef RSEQ_COMPARE_TWICE 761 , error1, error2, error3 762 #endif 763 ); 764 rseq_after_asm_goto(); 765 return 0; 766 abort: 767 rseq_after_asm_goto(); 768 RSEQ_INJECT_FAILED 769 return -1; 770 cmpfail: 771 rseq_after_asm_goto(); 772 return 1; 773 #ifdef RSEQ_COMPARE_TWICE 774 error1: 775 rseq_after_asm_goto(); 776 rseq_bug("cpu_id comparison failed"); 777 error2: 778 rseq_after_asm_goto(); 779 rseq_bug("1st expected value comparison failed"); 780 error3: 781 rseq_after_asm_goto(); 782 rseq_bug("2nd expected value comparison failed"); 783 #endif 784 } 785 786 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 787 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 788 789 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 790 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 791 792 static inline __attribute__((always_inline)) 793 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 794 intptr_t *v2, intptr_t newv2, 795 intptr_t newv, int cpu) 796 { 797 RSEQ_INJECT_C(9) 798 799 __asm__ __volatile__ goto ( 800 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 801 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 802 #ifdef RSEQ_COMPARE_TWICE 803 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 804 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 805 #endif 806 /* Start rseq by storing table entry pointer into rseq_cs. */ 807 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 808 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 809 RSEQ_INJECT_ASM(3) 810 "movl %[expect], %%eax\n\t" 811 "cmpl %[v], %%eax\n\t" 812 "jnz %l[cmpfail]\n\t" 813 RSEQ_INJECT_ASM(4) 814 #ifdef RSEQ_COMPARE_TWICE 815 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 816 "movl %[expect], %%eax\n\t" 817 "cmpl %[v], %%eax\n\t" 818 "jnz %l[error2]\n\t" 819 #endif 820 /* try store */ 821 "movl %[newv2], %[v2]\n\t" 822 RSEQ_INJECT_ASM(5) 823 #ifdef RSEQ_TEMPLATE_MO_RELEASE 824 "lock; addl $0,-128(%%esp)\n\t" 825 #endif 826 /* final store */ 827 "movl %[newv], %[v]\n\t" 828 "2:\n\t" 829 RSEQ_INJECT_ASM(6) 830 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 831 : /* gcc asm goto does not allow outputs */ 832 : [cpu_id] "r" (cpu), 833 [rseq_offset] "r" (rseq_offset), 834 /* try store input */ 835 [v2] "m" (*v2), 836 [newv2] "r" (newv2), 837 /* final store input */ 838 [v] "m" (*v), 839 [expect] "m" (expect), 840 [newv] "r" (newv) 841 : "memory", "cc", "eax" 842 RSEQ_INJECT_CLOBBER 843 : abort, cmpfail 844 #ifdef RSEQ_COMPARE_TWICE 845 , error1, error2 846 #endif 847 ); 848 rseq_after_asm_goto(); 849 return 0; 850 abort: 851 rseq_after_asm_goto(); 852 RSEQ_INJECT_FAILED 853 return -1; 854 cmpfail: 855 rseq_after_asm_goto(); 856 return 1; 857 #ifdef RSEQ_COMPARE_TWICE 858 error1: 859 rseq_after_asm_goto(); 860 rseq_bug("cpu_id comparison failed"); 861 error2: 862 rseq_after_asm_goto(); 863 rseq_bug("expected value comparison failed"); 864 #endif 865 866 } 867 868 /* TODO: implement a faster memcpy. */ 869 static inline __attribute__((always_inline)) 870 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 871 void *dst, void *src, size_t len, 872 intptr_t newv, int cpu) 873 { 874 uint32_t rseq_scratch[3]; 875 876 RSEQ_INJECT_C(9) 877 878 __asm__ __volatile__ goto ( 879 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 880 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 881 #ifdef RSEQ_COMPARE_TWICE 882 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 883 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 884 #endif 885 "movl %[src], %[rseq_scratch0]\n\t" 886 "movl %[dst], %[rseq_scratch1]\n\t" 887 "movl %[len], %[rseq_scratch2]\n\t" 888 /* Start rseq by storing table entry pointer into rseq_cs. */ 889 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 890 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 891 RSEQ_INJECT_ASM(3) 892 "movl %[expect], %%eax\n\t" 893 "cmpl %%eax, %[v]\n\t" 894 "jnz 5f\n\t" 895 RSEQ_INJECT_ASM(4) 896 #ifdef RSEQ_COMPARE_TWICE 897 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) 898 "movl %[expect], %%eax\n\t" 899 "cmpl %%eax, %[v]\n\t" 900 "jnz 7f\n\t" 901 #endif 902 /* try memcpy */ 903 "test %[len], %[len]\n\t" \ 904 "jz 333f\n\t" \ 905 "222:\n\t" \ 906 "movb (%[src]), %%al\n\t" \ 907 "movb %%al, (%[dst])\n\t" \ 908 "inc %[src]\n\t" \ 909 "inc %[dst]\n\t" \ 910 "dec %[len]\n\t" \ 911 "jnz 222b\n\t" \ 912 "333:\n\t" \ 913 RSEQ_INJECT_ASM(5) 914 #ifdef RSEQ_TEMPLATE_MO_RELEASE 915 "lock; addl $0,-128(%%esp)\n\t" 916 #endif 917 "movl %[newv], %%eax\n\t" 918 /* final store */ 919 "movl %%eax, %[v]\n\t" 920 "2:\n\t" 921 RSEQ_INJECT_ASM(6) 922 /* teardown */ 923 "movl %[rseq_scratch2], %[len]\n\t" 924 "movl %[rseq_scratch1], %[dst]\n\t" 925 "movl %[rseq_scratch0], %[src]\n\t" 926 RSEQ_ASM_DEFINE_ABORT(4, 927 "movl %[rseq_scratch2], %[len]\n\t" 928 "movl %[rseq_scratch1], %[dst]\n\t" 929 "movl %[rseq_scratch0], %[src]\n\t", 930 abort) 931 RSEQ_ASM_DEFINE_CMPFAIL(5, 932 "movl %[rseq_scratch2], %[len]\n\t" 933 "movl %[rseq_scratch1], %[dst]\n\t" 934 "movl %[rseq_scratch0], %[src]\n\t", 935 cmpfail) 936 #ifdef RSEQ_COMPARE_TWICE 937 RSEQ_ASM_DEFINE_CMPFAIL(6, 938 "movl %[rseq_scratch2], %[len]\n\t" 939 "movl %[rseq_scratch1], %[dst]\n\t" 940 "movl %[rseq_scratch0], %[src]\n\t", 941 error1) 942 RSEQ_ASM_DEFINE_CMPFAIL(7, 943 "movl %[rseq_scratch2], %[len]\n\t" 944 "movl %[rseq_scratch1], %[dst]\n\t" 945 "movl %[rseq_scratch0], %[src]\n\t", 946 error2) 947 #endif 948 : /* gcc asm goto does not allow outputs */ 949 : [cpu_id] "r" (cpu), 950 [rseq_offset] "r" (rseq_offset), 951 /* final store input */ 952 [v] "m" (*v), 953 [expect] "m" (expect), 954 [newv] "m" (newv), 955 /* try memcpy input */ 956 [dst] "r" (dst), 957 [src] "r" (src), 958 [len] "r" (len), 959 [rseq_scratch0] "m" (rseq_scratch[0]), 960 [rseq_scratch1] "m" (rseq_scratch[1]), 961 [rseq_scratch2] "m" (rseq_scratch[2]) 962 : "memory", "cc", "eax" 963 RSEQ_INJECT_CLOBBER 964 : abort, cmpfail 965 #ifdef RSEQ_COMPARE_TWICE 966 , error1, error2 967 #endif 968 ); 969 rseq_after_asm_goto(); 970 return 0; 971 abort: 972 rseq_after_asm_goto(); 973 RSEQ_INJECT_FAILED 974 return -1; 975 cmpfail: 976 rseq_after_asm_goto(); 977 return 1; 978 #ifdef RSEQ_COMPARE_TWICE 979 error1: 980 rseq_after_asm_goto(); 981 rseq_bug("cpu_id comparison failed"); 982 error2: 983 rseq_after_asm_goto(); 984 rseq_bug("expected value comparison failed"); 985 #endif 986 } 987 988 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 989 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 990 991 #endif 992 993 #include "rseq-bits-reset.h" 994