1 // SPDX-License-Identifier: LGPL-2.1 2 #define _GNU_SOURCE 3 #include <assert.h> 4 #include <pthread.h> 5 #include <sched.h> 6 #include <stdint.h> 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <string.h> 10 #include <syscall.h> 11 #include <unistd.h> 12 #include <poll.h> 13 #include <sys/types.h> 14 #include <signal.h> 15 #include <errno.h> 16 #include <stddef.h> 17 18 static inline pid_t rseq_gettid(void) 19 { 20 return syscall(__NR_gettid); 21 } 22 23 #define NR_INJECT 9 24 static int loop_cnt[NR_INJECT + 1]; 25 26 static int loop_cnt_1 asm("asm_loop_cnt_1") __attribute__((used)); 27 static int loop_cnt_2 asm("asm_loop_cnt_2") __attribute__((used)); 28 static int loop_cnt_3 asm("asm_loop_cnt_3") __attribute__((used)); 29 static int loop_cnt_4 asm("asm_loop_cnt_4") __attribute__((used)); 30 static int loop_cnt_5 asm("asm_loop_cnt_5") __attribute__((used)); 31 static int loop_cnt_6 asm("asm_loop_cnt_6") __attribute__((used)); 32 33 static int opt_modulo, verbose; 34 35 static int opt_yield, opt_signal, opt_sleep, 36 opt_disable_rseq, opt_threads = 200, 37 opt_disable_mod = 0, opt_test = 's', opt_mb = 0; 38 39 #ifndef RSEQ_SKIP_FASTPATH 40 static long long opt_reps = 5000; 41 #else 42 static long long opt_reps = 100; 43 #endif 44 45 static __thread __attribute__((tls_model("initial-exec"))) 46 unsigned int signals_delivered; 47 48 #ifndef BENCHMARK 49 50 static __thread __attribute__((tls_model("initial-exec"), unused)) 51 unsigned int yield_mod_cnt, nr_abort; 52 53 #define printf_verbose(fmt, ...) \ 54 do { \ 55 if (verbose) \ 56 printf(fmt, ## __VA_ARGS__); \ 57 } while (0) 58 59 #ifdef __i386__ 60 61 #define INJECT_ASM_REG "eax" 62 63 #define RSEQ_INJECT_CLOBBER \ 64 , INJECT_ASM_REG 65 66 #define RSEQ_INJECT_ASM(n) \ 67 "mov asm_loop_cnt_" #n ", %%" INJECT_ASM_REG "\n\t" \ 68 "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \ 69 "jz 333f\n\t" \ 70 "222:\n\t" \ 71 "dec %%" INJECT_ASM_REG "\n\t" \ 72 "jnz 222b\n\t" \ 73 "333:\n\t" 74 75 #elif defined(__x86_64__) 76 77 #define INJECT_ASM_REG_P "rax" 78 #define INJECT_ASM_REG "eax" 79 80 #define RSEQ_INJECT_CLOBBER \ 81 , INJECT_ASM_REG_P \ 82 , INJECT_ASM_REG 83 84 #define RSEQ_INJECT_ASM(n) \ 85 "lea asm_loop_cnt_" #n "(%%rip), %%" INJECT_ASM_REG_P "\n\t" \ 86 "mov (%%" INJECT_ASM_REG_P "), %%" INJECT_ASM_REG "\n\t" \ 87 "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \ 88 "jz 333f\n\t" \ 89 "222:\n\t" \ 90 "dec %%" INJECT_ASM_REG "\n\t" \ 91 "jnz 222b\n\t" \ 92 "333:\n\t" 93 94 #elif defined(__s390__) 95 96 #define RSEQ_INJECT_INPUT \ 97 , [loop_cnt_1]"m"(loop_cnt[1]) \ 98 , [loop_cnt_2]"m"(loop_cnt[2]) \ 99 , [loop_cnt_3]"m"(loop_cnt[3]) \ 100 , [loop_cnt_4]"m"(loop_cnt[4]) \ 101 , [loop_cnt_5]"m"(loop_cnt[5]) \ 102 , [loop_cnt_6]"m"(loop_cnt[6]) 103 104 #define INJECT_ASM_REG "r12" 105 106 #define RSEQ_INJECT_CLOBBER \ 107 , INJECT_ASM_REG 108 109 #define RSEQ_INJECT_ASM(n) \ 110 "l %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ 111 "ltr %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG "\n\t" \ 112 "je 333f\n\t" \ 113 "222:\n\t" \ 114 "ahi %%" INJECT_ASM_REG ", -1\n\t" \ 115 "jnz 222b\n\t" \ 116 "333:\n\t" 117 118 #elif defined(__ARMEL__) 119 120 #define RSEQ_INJECT_INPUT \ 121 , [loop_cnt_1]"m"(loop_cnt[1]) \ 122 , [loop_cnt_2]"m"(loop_cnt[2]) \ 123 , [loop_cnt_3]"m"(loop_cnt[3]) \ 124 , [loop_cnt_4]"m"(loop_cnt[4]) \ 125 , [loop_cnt_5]"m"(loop_cnt[5]) \ 126 , [loop_cnt_6]"m"(loop_cnt[6]) 127 128 #define INJECT_ASM_REG "r4" 129 130 #define RSEQ_INJECT_CLOBBER \ 131 , INJECT_ASM_REG 132 133 #define RSEQ_INJECT_ASM(n) \ 134 "ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ 135 "cmp " INJECT_ASM_REG ", #0\n\t" \ 136 "beq 333f\n\t" \ 137 "222:\n\t" \ 138 "subs " INJECT_ASM_REG ", #1\n\t" \ 139 "bne 222b\n\t" \ 140 "333:\n\t" 141 142 #elif defined(__AARCH64EL__) 143 144 #define RSEQ_INJECT_INPUT \ 145 , [loop_cnt_1] "Qo" (loop_cnt[1]) \ 146 , [loop_cnt_2] "Qo" (loop_cnt[2]) \ 147 , [loop_cnt_3] "Qo" (loop_cnt[3]) \ 148 , [loop_cnt_4] "Qo" (loop_cnt[4]) \ 149 , [loop_cnt_5] "Qo" (loop_cnt[5]) \ 150 , [loop_cnt_6] "Qo" (loop_cnt[6]) 151 152 #define INJECT_ASM_REG RSEQ_ASM_TMP_REG32 153 154 #define RSEQ_INJECT_ASM(n) \ 155 " ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n" \ 156 " cbz " INJECT_ASM_REG ", 333f\n" \ 157 "222:\n" \ 158 " sub " INJECT_ASM_REG ", " INJECT_ASM_REG ", #1\n" \ 159 " cbnz " INJECT_ASM_REG ", 222b\n" \ 160 "333:\n" 161 162 #elif __PPC__ 163 164 #define RSEQ_INJECT_INPUT \ 165 , [loop_cnt_1]"m"(loop_cnt[1]) \ 166 , [loop_cnt_2]"m"(loop_cnt[2]) \ 167 , [loop_cnt_3]"m"(loop_cnt[3]) \ 168 , [loop_cnt_4]"m"(loop_cnt[4]) \ 169 , [loop_cnt_5]"m"(loop_cnt[5]) \ 170 , [loop_cnt_6]"m"(loop_cnt[6]) 171 172 #define INJECT_ASM_REG "r18" 173 174 #define RSEQ_INJECT_CLOBBER \ 175 , INJECT_ASM_REG 176 177 #define RSEQ_INJECT_ASM(n) \ 178 "lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ 179 "cmpwi %%" INJECT_ASM_REG ", 0\n\t" \ 180 "beq 333f\n\t" \ 181 "222:\n\t" \ 182 "subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \ 183 "bne 222b\n\t" \ 184 "333:\n\t" 185 186 #elif defined(__mips__) 187 188 #define RSEQ_INJECT_INPUT \ 189 , [loop_cnt_1]"m"(loop_cnt[1]) \ 190 , [loop_cnt_2]"m"(loop_cnt[2]) \ 191 , [loop_cnt_3]"m"(loop_cnt[3]) \ 192 , [loop_cnt_4]"m"(loop_cnt[4]) \ 193 , [loop_cnt_5]"m"(loop_cnt[5]) \ 194 , [loop_cnt_6]"m"(loop_cnt[6]) 195 196 #define INJECT_ASM_REG "$5" 197 198 #define RSEQ_INJECT_CLOBBER \ 199 , INJECT_ASM_REG 200 201 #define RSEQ_INJECT_ASM(n) \ 202 "lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ 203 "beqz " INJECT_ASM_REG ", 333f\n\t" \ 204 "222:\n\t" \ 205 "addiu " INJECT_ASM_REG ", -1\n\t" \ 206 "bnez " INJECT_ASM_REG ", 222b\n\t" \ 207 "333:\n\t" 208 209 #else 210 #error unsupported target 211 #endif 212 213 #define RSEQ_INJECT_FAILED \ 214 nr_abort++; 215 216 #define RSEQ_INJECT_C(n) \ 217 { \ 218 int loc_i, loc_nr_loops = loop_cnt[n]; \ 219 \ 220 for (loc_i = 0; loc_i < loc_nr_loops; loc_i++) { \ 221 rseq_barrier(); \ 222 } \ 223 if (loc_nr_loops == -1 && opt_modulo) { \ 224 if (yield_mod_cnt == opt_modulo - 1) { \ 225 if (opt_sleep > 0) \ 226 poll(NULL, 0, opt_sleep); \ 227 if (opt_yield) \ 228 sched_yield(); \ 229 if (opt_signal) \ 230 raise(SIGUSR1); \ 231 yield_mod_cnt = 0; \ 232 } else { \ 233 yield_mod_cnt++; \ 234 } \ 235 } \ 236 } 237 238 #else 239 240 #define printf_verbose(fmt, ...) 241 242 #endif /* BENCHMARK */ 243 244 #include "rseq.h" 245 246 struct percpu_lock_entry { 247 intptr_t v; 248 } __attribute__((aligned(128))); 249 250 struct percpu_lock { 251 struct percpu_lock_entry c[CPU_SETSIZE]; 252 }; 253 254 struct test_data_entry { 255 intptr_t count; 256 } __attribute__((aligned(128))); 257 258 struct spinlock_test_data { 259 struct percpu_lock lock; 260 struct test_data_entry c[CPU_SETSIZE]; 261 }; 262 263 struct spinlock_thread_test_data { 264 struct spinlock_test_data *data; 265 long long reps; 266 int reg; 267 }; 268 269 struct inc_test_data { 270 struct test_data_entry c[CPU_SETSIZE]; 271 }; 272 273 struct inc_thread_test_data { 274 struct inc_test_data *data; 275 long long reps; 276 int reg; 277 }; 278 279 struct percpu_list_node { 280 intptr_t data; 281 struct percpu_list_node *next; 282 }; 283 284 struct percpu_list_entry { 285 struct percpu_list_node *head; 286 } __attribute__((aligned(128))); 287 288 struct percpu_list { 289 struct percpu_list_entry c[CPU_SETSIZE]; 290 }; 291 292 #define BUFFER_ITEM_PER_CPU 100 293 294 struct percpu_buffer_node { 295 intptr_t data; 296 }; 297 298 struct percpu_buffer_entry { 299 intptr_t offset; 300 intptr_t buflen; 301 struct percpu_buffer_node **array; 302 } __attribute__((aligned(128))); 303 304 struct percpu_buffer { 305 struct percpu_buffer_entry c[CPU_SETSIZE]; 306 }; 307 308 #define MEMCPY_BUFFER_ITEM_PER_CPU 100 309 310 struct percpu_memcpy_buffer_node { 311 intptr_t data1; 312 uint64_t data2; 313 }; 314 315 struct percpu_memcpy_buffer_entry { 316 intptr_t offset; 317 intptr_t buflen; 318 struct percpu_memcpy_buffer_node *array; 319 } __attribute__((aligned(128))); 320 321 struct percpu_memcpy_buffer { 322 struct percpu_memcpy_buffer_entry c[CPU_SETSIZE]; 323 }; 324 325 /* A simple percpu spinlock. Grabs lock on current cpu. */ 326 static int rseq_this_cpu_lock(struct percpu_lock *lock) 327 { 328 int cpu; 329 330 for (;;) { 331 int ret; 332 333 cpu = rseq_cpu_start(); 334 ret = rseq_cmpeqv_storev(&lock->c[cpu].v, 335 0, 1, cpu); 336 if (rseq_likely(!ret)) 337 break; 338 /* Retry if comparison fails or rseq aborts. */ 339 } 340 /* 341 * Acquire semantic when taking lock after control dependency. 342 * Matches rseq_smp_store_release(). 343 */ 344 rseq_smp_acquire__after_ctrl_dep(); 345 return cpu; 346 } 347 348 static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu) 349 { 350 assert(lock->c[cpu].v == 1); 351 /* 352 * Release lock, with release semantic. Matches 353 * rseq_smp_acquire__after_ctrl_dep(). 354 */ 355 rseq_smp_store_release(&lock->c[cpu].v, 0); 356 } 357 358 void *test_percpu_spinlock_thread(void *arg) 359 { 360 struct spinlock_thread_test_data *thread_data = arg; 361 struct spinlock_test_data *data = thread_data->data; 362 long long i, reps; 363 364 if (!opt_disable_rseq && thread_data->reg && 365 rseq_register_current_thread()) 366 abort(); 367 reps = thread_data->reps; 368 for (i = 0; i < reps; i++) { 369 int cpu = rseq_cpu_start(); 370 371 cpu = rseq_this_cpu_lock(&data->lock); 372 data->c[cpu].count++; 373 rseq_percpu_unlock(&data->lock, cpu); 374 #ifndef BENCHMARK 375 if (i != 0 && !(i % (reps / 10))) 376 printf_verbose("tid %d: count %lld\n", 377 (int) rseq_gettid(), i); 378 #endif 379 } 380 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", 381 (int) rseq_gettid(), nr_abort, signals_delivered); 382 if (!opt_disable_rseq && thread_data->reg && 383 rseq_unregister_current_thread()) 384 abort(); 385 return NULL; 386 } 387 388 /* 389 * A simple test which implements a sharded counter using a per-cpu 390 * lock. Obviously real applications might prefer to simply use a 391 * per-cpu increment; however, this is reasonable for a test and the 392 * lock can be extended to synchronize more complicated operations. 393 */ 394 void test_percpu_spinlock(void) 395 { 396 const int num_threads = opt_threads; 397 int i, ret; 398 uint64_t sum; 399 pthread_t test_threads[num_threads]; 400 struct spinlock_test_data data; 401 struct spinlock_thread_test_data thread_data[num_threads]; 402 403 memset(&data, 0, sizeof(data)); 404 for (i = 0; i < num_threads; i++) { 405 thread_data[i].reps = opt_reps; 406 if (opt_disable_mod <= 0 || (i % opt_disable_mod)) 407 thread_data[i].reg = 1; 408 else 409 thread_data[i].reg = 0; 410 thread_data[i].data = &data; 411 ret = pthread_create(&test_threads[i], NULL, 412 test_percpu_spinlock_thread, 413 &thread_data[i]); 414 if (ret) { 415 errno = ret; 416 perror("pthread_create"); 417 abort(); 418 } 419 } 420 421 for (i = 0; i < num_threads; i++) { 422 ret = pthread_join(test_threads[i], NULL); 423 if (ret) { 424 errno = ret; 425 perror("pthread_join"); 426 abort(); 427 } 428 } 429 430 sum = 0; 431 for (i = 0; i < CPU_SETSIZE; i++) 432 sum += data.c[i].count; 433 434 assert(sum == (uint64_t)opt_reps * num_threads); 435 } 436 437 void *test_percpu_inc_thread(void *arg) 438 { 439 struct inc_thread_test_data *thread_data = arg; 440 struct inc_test_data *data = thread_data->data; 441 long long i, reps; 442 443 if (!opt_disable_rseq && thread_data->reg && 444 rseq_register_current_thread()) 445 abort(); 446 reps = thread_data->reps; 447 for (i = 0; i < reps; i++) { 448 int ret; 449 450 do { 451 int cpu; 452 453 cpu = rseq_cpu_start(); 454 ret = rseq_addv(&data->c[cpu].count, 1, cpu); 455 } while (rseq_unlikely(ret)); 456 #ifndef BENCHMARK 457 if (i != 0 && !(i % (reps / 10))) 458 printf_verbose("tid %d: count %lld\n", 459 (int) rseq_gettid(), i); 460 #endif 461 } 462 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", 463 (int) rseq_gettid(), nr_abort, signals_delivered); 464 if (!opt_disable_rseq && thread_data->reg && 465 rseq_unregister_current_thread()) 466 abort(); 467 return NULL; 468 } 469 470 void test_percpu_inc(void) 471 { 472 const int num_threads = opt_threads; 473 int i, ret; 474 uint64_t sum; 475 pthread_t test_threads[num_threads]; 476 struct inc_test_data data; 477 struct inc_thread_test_data thread_data[num_threads]; 478 479 memset(&data, 0, sizeof(data)); 480 for (i = 0; i < num_threads; i++) { 481 thread_data[i].reps = opt_reps; 482 if (opt_disable_mod <= 0 || (i % opt_disable_mod)) 483 thread_data[i].reg = 1; 484 else 485 thread_data[i].reg = 0; 486 thread_data[i].data = &data; 487 ret = pthread_create(&test_threads[i], NULL, 488 test_percpu_inc_thread, 489 &thread_data[i]); 490 if (ret) { 491 errno = ret; 492 perror("pthread_create"); 493 abort(); 494 } 495 } 496 497 for (i = 0; i < num_threads; i++) { 498 ret = pthread_join(test_threads[i], NULL); 499 if (ret) { 500 errno = ret; 501 perror("pthread_join"); 502 abort(); 503 } 504 } 505 506 sum = 0; 507 for (i = 0; i < CPU_SETSIZE; i++) 508 sum += data.c[i].count; 509 510 assert(sum == (uint64_t)opt_reps * num_threads); 511 } 512 513 void this_cpu_list_push(struct percpu_list *list, 514 struct percpu_list_node *node, 515 int *_cpu) 516 { 517 int cpu; 518 519 for (;;) { 520 intptr_t *targetptr, newval, expect; 521 int ret; 522 523 cpu = rseq_cpu_start(); 524 /* Load list->c[cpu].head with single-copy atomicity. */ 525 expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head); 526 newval = (intptr_t)node; 527 targetptr = (intptr_t *)&list->c[cpu].head; 528 node->next = (struct percpu_list_node *)expect; 529 ret = rseq_cmpeqv_storev(targetptr, expect, newval, cpu); 530 if (rseq_likely(!ret)) 531 break; 532 /* Retry if comparison fails or rseq aborts. */ 533 } 534 if (_cpu) 535 *_cpu = cpu; 536 } 537 538 /* 539 * Unlike a traditional lock-less linked list; the availability of a 540 * rseq primitive allows us to implement pop without concerns over 541 * ABA-type races. 542 */ 543 struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list, 544 int *_cpu) 545 { 546 struct percpu_list_node *node = NULL; 547 int cpu; 548 549 for (;;) { 550 struct percpu_list_node *head; 551 intptr_t *targetptr, expectnot, *load; 552 off_t offset; 553 int ret; 554 555 cpu = rseq_cpu_start(); 556 targetptr = (intptr_t *)&list->c[cpu].head; 557 expectnot = (intptr_t)NULL; 558 offset = offsetof(struct percpu_list_node, next); 559 load = (intptr_t *)&head; 560 ret = rseq_cmpnev_storeoffp_load(targetptr, expectnot, 561 offset, load, cpu); 562 if (rseq_likely(!ret)) { 563 node = head; 564 break; 565 } 566 if (ret > 0) 567 break; 568 /* Retry if rseq aborts. */ 569 } 570 if (_cpu) 571 *_cpu = cpu; 572 return node; 573 } 574 575 /* 576 * __percpu_list_pop is not safe against concurrent accesses. Should 577 * only be used on lists that are not concurrently modified. 578 */ 579 struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu) 580 { 581 struct percpu_list_node *node; 582 583 node = list->c[cpu].head; 584 if (!node) 585 return NULL; 586 list->c[cpu].head = node->next; 587 return node; 588 } 589 590 void *test_percpu_list_thread(void *arg) 591 { 592 long long i, reps; 593 struct percpu_list *list = (struct percpu_list *)arg; 594 595 if (!opt_disable_rseq && rseq_register_current_thread()) 596 abort(); 597 598 reps = opt_reps; 599 for (i = 0; i < reps; i++) { 600 struct percpu_list_node *node; 601 602 node = this_cpu_list_pop(list, NULL); 603 if (opt_yield) 604 sched_yield(); /* encourage shuffling */ 605 if (node) 606 this_cpu_list_push(list, node, NULL); 607 } 608 609 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", 610 (int) rseq_gettid(), nr_abort, signals_delivered); 611 if (!opt_disable_rseq && rseq_unregister_current_thread()) 612 abort(); 613 614 return NULL; 615 } 616 617 /* Simultaneous modification to a per-cpu linked list from many threads. */ 618 void test_percpu_list(void) 619 { 620 const int num_threads = opt_threads; 621 int i, j, ret; 622 uint64_t sum = 0, expected_sum = 0; 623 struct percpu_list list; 624 pthread_t test_threads[num_threads]; 625 cpu_set_t allowed_cpus; 626 627 memset(&list, 0, sizeof(list)); 628 629 /* Generate list entries for every usable cpu. */ 630 sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); 631 for (i = 0; i < CPU_SETSIZE; i++) { 632 if (!CPU_ISSET(i, &allowed_cpus)) 633 continue; 634 for (j = 1; j <= 100; j++) { 635 struct percpu_list_node *node; 636 637 expected_sum += j; 638 639 node = malloc(sizeof(*node)); 640 assert(node); 641 node->data = j; 642 node->next = list.c[i].head; 643 list.c[i].head = node; 644 } 645 } 646 647 for (i = 0; i < num_threads; i++) { 648 ret = pthread_create(&test_threads[i], NULL, 649 test_percpu_list_thread, &list); 650 if (ret) { 651 errno = ret; 652 perror("pthread_create"); 653 abort(); 654 } 655 } 656 657 for (i = 0; i < num_threads; i++) { 658 ret = pthread_join(test_threads[i], NULL); 659 if (ret) { 660 errno = ret; 661 perror("pthread_join"); 662 abort(); 663 } 664 } 665 666 for (i = 0; i < CPU_SETSIZE; i++) { 667 struct percpu_list_node *node; 668 669 if (!CPU_ISSET(i, &allowed_cpus)) 670 continue; 671 672 while ((node = __percpu_list_pop(&list, i))) { 673 sum += node->data; 674 free(node); 675 } 676 } 677 678 /* 679 * All entries should now be accounted for (unless some external 680 * actor is interfering with our allowed affinity while this 681 * test is running). 682 */ 683 assert(sum == expected_sum); 684 } 685 686 bool this_cpu_buffer_push(struct percpu_buffer *buffer, 687 struct percpu_buffer_node *node, 688 int *_cpu) 689 { 690 bool result = false; 691 int cpu; 692 693 for (;;) { 694 intptr_t *targetptr_spec, newval_spec; 695 intptr_t *targetptr_final, newval_final; 696 intptr_t offset; 697 int ret; 698 699 cpu = rseq_cpu_start(); 700 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); 701 if (offset == buffer->c[cpu].buflen) 702 break; 703 newval_spec = (intptr_t)node; 704 targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset]; 705 newval_final = offset + 1; 706 targetptr_final = &buffer->c[cpu].offset; 707 if (opt_mb) 708 ret = rseq_cmpeqv_trystorev_storev_release( 709 targetptr_final, offset, targetptr_spec, 710 newval_spec, newval_final, cpu); 711 else 712 ret = rseq_cmpeqv_trystorev_storev(targetptr_final, 713 offset, targetptr_spec, newval_spec, 714 newval_final, cpu); 715 if (rseq_likely(!ret)) { 716 result = true; 717 break; 718 } 719 /* Retry if comparison fails or rseq aborts. */ 720 } 721 if (_cpu) 722 *_cpu = cpu; 723 return result; 724 } 725 726 struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer, 727 int *_cpu) 728 { 729 struct percpu_buffer_node *head; 730 int cpu; 731 732 for (;;) { 733 intptr_t *targetptr, newval; 734 intptr_t offset; 735 int ret; 736 737 cpu = rseq_cpu_start(); 738 /* Load offset with single-copy atomicity. */ 739 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); 740 if (offset == 0) { 741 head = NULL; 742 break; 743 } 744 head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]); 745 newval = offset - 1; 746 targetptr = (intptr_t *)&buffer->c[cpu].offset; 747 ret = rseq_cmpeqv_cmpeqv_storev(targetptr, offset, 748 (intptr_t *)&buffer->c[cpu].array[offset - 1], 749 (intptr_t)head, newval, cpu); 750 if (rseq_likely(!ret)) 751 break; 752 /* Retry if comparison fails or rseq aborts. */ 753 } 754 if (_cpu) 755 *_cpu = cpu; 756 return head; 757 } 758 759 /* 760 * __percpu_buffer_pop is not safe against concurrent accesses. Should 761 * only be used on buffers that are not concurrently modified. 762 */ 763 struct percpu_buffer_node *__percpu_buffer_pop(struct percpu_buffer *buffer, 764 int cpu) 765 { 766 struct percpu_buffer_node *head; 767 intptr_t offset; 768 769 offset = buffer->c[cpu].offset; 770 if (offset == 0) 771 return NULL; 772 head = buffer->c[cpu].array[offset - 1]; 773 buffer->c[cpu].offset = offset - 1; 774 return head; 775 } 776 777 void *test_percpu_buffer_thread(void *arg) 778 { 779 long long i, reps; 780 struct percpu_buffer *buffer = (struct percpu_buffer *)arg; 781 782 if (!opt_disable_rseq && rseq_register_current_thread()) 783 abort(); 784 785 reps = opt_reps; 786 for (i = 0; i < reps; i++) { 787 struct percpu_buffer_node *node; 788 789 node = this_cpu_buffer_pop(buffer, NULL); 790 if (opt_yield) 791 sched_yield(); /* encourage shuffling */ 792 if (node) { 793 if (!this_cpu_buffer_push(buffer, node, NULL)) { 794 /* Should increase buffer size. */ 795 abort(); 796 } 797 } 798 } 799 800 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", 801 (int) rseq_gettid(), nr_abort, signals_delivered); 802 if (!opt_disable_rseq && rseq_unregister_current_thread()) 803 abort(); 804 805 return NULL; 806 } 807 808 /* Simultaneous modification to a per-cpu buffer from many threads. */ 809 void test_percpu_buffer(void) 810 { 811 const int num_threads = opt_threads; 812 int i, j, ret; 813 uint64_t sum = 0, expected_sum = 0; 814 struct percpu_buffer buffer; 815 pthread_t test_threads[num_threads]; 816 cpu_set_t allowed_cpus; 817 818 memset(&buffer, 0, sizeof(buffer)); 819 820 /* Generate list entries for every usable cpu. */ 821 sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); 822 for (i = 0; i < CPU_SETSIZE; i++) { 823 if (!CPU_ISSET(i, &allowed_cpus)) 824 continue; 825 /* Worse-case is every item in same CPU. */ 826 buffer.c[i].array = 827 malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE * 828 BUFFER_ITEM_PER_CPU); 829 assert(buffer.c[i].array); 830 buffer.c[i].buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU; 831 for (j = 1; j <= BUFFER_ITEM_PER_CPU; j++) { 832 struct percpu_buffer_node *node; 833 834 expected_sum += j; 835 836 /* 837 * We could theoretically put the word-sized 838 * "data" directly in the buffer. However, we 839 * want to model objects that would not fit 840 * within a single word, so allocate an object 841 * for each node. 842 */ 843 node = malloc(sizeof(*node)); 844 assert(node); 845 node->data = j; 846 buffer.c[i].array[j - 1] = node; 847 buffer.c[i].offset++; 848 } 849 } 850 851 for (i = 0; i < num_threads; i++) { 852 ret = pthread_create(&test_threads[i], NULL, 853 test_percpu_buffer_thread, &buffer); 854 if (ret) { 855 errno = ret; 856 perror("pthread_create"); 857 abort(); 858 } 859 } 860 861 for (i = 0; i < num_threads; i++) { 862 ret = pthread_join(test_threads[i], NULL); 863 if (ret) { 864 errno = ret; 865 perror("pthread_join"); 866 abort(); 867 } 868 } 869 870 for (i = 0; i < CPU_SETSIZE; i++) { 871 struct percpu_buffer_node *node; 872 873 if (!CPU_ISSET(i, &allowed_cpus)) 874 continue; 875 876 while ((node = __percpu_buffer_pop(&buffer, i))) { 877 sum += node->data; 878 free(node); 879 } 880 free(buffer.c[i].array); 881 } 882 883 /* 884 * All entries should now be accounted for (unless some external 885 * actor is interfering with our allowed affinity while this 886 * test is running). 887 */ 888 assert(sum == expected_sum); 889 } 890 891 bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer, 892 struct percpu_memcpy_buffer_node item, 893 int *_cpu) 894 { 895 bool result = false; 896 int cpu; 897 898 for (;;) { 899 intptr_t *targetptr_final, newval_final, offset; 900 char *destptr, *srcptr; 901 size_t copylen; 902 int ret; 903 904 cpu = rseq_cpu_start(); 905 /* Load offset with single-copy atomicity. */ 906 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); 907 if (offset == buffer->c[cpu].buflen) 908 break; 909 destptr = (char *)&buffer->c[cpu].array[offset]; 910 srcptr = (char *)&item; 911 /* copylen must be <= 4kB. */ 912 copylen = sizeof(item); 913 newval_final = offset + 1; 914 targetptr_final = &buffer->c[cpu].offset; 915 if (opt_mb) 916 ret = rseq_cmpeqv_trymemcpy_storev_release( 917 targetptr_final, offset, 918 destptr, srcptr, copylen, 919 newval_final, cpu); 920 else 921 ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final, 922 offset, destptr, srcptr, copylen, 923 newval_final, cpu); 924 if (rseq_likely(!ret)) { 925 result = true; 926 break; 927 } 928 /* Retry if comparison fails or rseq aborts. */ 929 } 930 if (_cpu) 931 *_cpu = cpu; 932 return result; 933 } 934 935 bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, 936 struct percpu_memcpy_buffer_node *item, 937 int *_cpu) 938 { 939 bool result = false; 940 int cpu; 941 942 for (;;) { 943 intptr_t *targetptr_final, newval_final, offset; 944 char *destptr, *srcptr; 945 size_t copylen; 946 int ret; 947 948 cpu = rseq_cpu_start(); 949 /* Load offset with single-copy atomicity. */ 950 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); 951 if (offset == 0) 952 break; 953 destptr = (char *)item; 954 srcptr = (char *)&buffer->c[cpu].array[offset - 1]; 955 /* copylen must be <= 4kB. */ 956 copylen = sizeof(*item); 957 newval_final = offset - 1; 958 targetptr_final = &buffer->c[cpu].offset; 959 ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final, 960 offset, destptr, srcptr, copylen, 961 newval_final, cpu); 962 if (rseq_likely(!ret)) { 963 result = true; 964 break; 965 } 966 /* Retry if comparison fails or rseq aborts. */ 967 } 968 if (_cpu) 969 *_cpu = cpu; 970 return result; 971 } 972 973 /* 974 * __percpu_memcpy_buffer_pop is not safe against concurrent accesses. Should 975 * only be used on buffers that are not concurrently modified. 976 */ 977 bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, 978 struct percpu_memcpy_buffer_node *item, 979 int cpu) 980 { 981 intptr_t offset; 982 983 offset = buffer->c[cpu].offset; 984 if (offset == 0) 985 return false; 986 memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item)); 987 buffer->c[cpu].offset = offset - 1; 988 return true; 989 } 990 991 void *test_percpu_memcpy_buffer_thread(void *arg) 992 { 993 long long i, reps; 994 struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg; 995 996 if (!opt_disable_rseq && rseq_register_current_thread()) 997 abort(); 998 999 reps = opt_reps; 1000 for (i = 0; i < reps; i++) { 1001 struct percpu_memcpy_buffer_node item; 1002 bool result; 1003 1004 result = this_cpu_memcpy_buffer_pop(buffer, &item, NULL); 1005 if (opt_yield) 1006 sched_yield(); /* encourage shuffling */ 1007 if (result) { 1008 if (!this_cpu_memcpy_buffer_push(buffer, item, NULL)) { 1009 /* Should increase buffer size. */ 1010 abort(); 1011 } 1012 } 1013 } 1014 1015 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", 1016 (int) rseq_gettid(), nr_abort, signals_delivered); 1017 if (!opt_disable_rseq && rseq_unregister_current_thread()) 1018 abort(); 1019 1020 return NULL; 1021 } 1022 1023 /* Simultaneous modification to a per-cpu buffer from many threads. */ 1024 void test_percpu_memcpy_buffer(void) 1025 { 1026 const int num_threads = opt_threads; 1027 int i, j, ret; 1028 uint64_t sum = 0, expected_sum = 0; 1029 struct percpu_memcpy_buffer buffer; 1030 pthread_t test_threads[num_threads]; 1031 cpu_set_t allowed_cpus; 1032 1033 memset(&buffer, 0, sizeof(buffer)); 1034 1035 /* Generate list entries for every usable cpu. */ 1036 sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); 1037 for (i = 0; i < CPU_SETSIZE; i++) { 1038 if (!CPU_ISSET(i, &allowed_cpus)) 1039 continue; 1040 /* Worse-case is every item in same CPU. */ 1041 buffer.c[i].array = 1042 malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE * 1043 MEMCPY_BUFFER_ITEM_PER_CPU); 1044 assert(buffer.c[i].array); 1045 buffer.c[i].buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU; 1046 for (j = 1; j <= MEMCPY_BUFFER_ITEM_PER_CPU; j++) { 1047 expected_sum += 2 * j + 1; 1048 1049 /* 1050 * We could theoretically put the word-sized 1051 * "data" directly in the buffer. However, we 1052 * want to model objects that would not fit 1053 * within a single word, so allocate an object 1054 * for each node. 1055 */ 1056 buffer.c[i].array[j - 1].data1 = j; 1057 buffer.c[i].array[j - 1].data2 = j + 1; 1058 buffer.c[i].offset++; 1059 } 1060 } 1061 1062 for (i = 0; i < num_threads; i++) { 1063 ret = pthread_create(&test_threads[i], NULL, 1064 test_percpu_memcpy_buffer_thread, 1065 &buffer); 1066 if (ret) { 1067 errno = ret; 1068 perror("pthread_create"); 1069 abort(); 1070 } 1071 } 1072 1073 for (i = 0; i < num_threads; i++) { 1074 ret = pthread_join(test_threads[i], NULL); 1075 if (ret) { 1076 errno = ret; 1077 perror("pthread_join"); 1078 abort(); 1079 } 1080 } 1081 1082 for (i = 0; i < CPU_SETSIZE; i++) { 1083 struct percpu_memcpy_buffer_node item; 1084 1085 if (!CPU_ISSET(i, &allowed_cpus)) 1086 continue; 1087 1088 while (__percpu_memcpy_buffer_pop(&buffer, &item, i)) { 1089 sum += item.data1; 1090 sum += item.data2; 1091 } 1092 free(buffer.c[i].array); 1093 } 1094 1095 /* 1096 * All entries should now be accounted for (unless some external 1097 * actor is interfering with our allowed affinity while this 1098 * test is running). 1099 */ 1100 assert(sum == expected_sum); 1101 } 1102 1103 static void test_signal_interrupt_handler(int signo) 1104 { 1105 signals_delivered++; 1106 } 1107 1108 static int set_signal_handler(void) 1109 { 1110 int ret = 0; 1111 struct sigaction sa; 1112 sigset_t sigset; 1113 1114 ret = sigemptyset(&sigset); 1115 if (ret < 0) { 1116 perror("sigemptyset"); 1117 return ret; 1118 } 1119 1120 sa.sa_handler = test_signal_interrupt_handler; 1121 sa.sa_mask = sigset; 1122 sa.sa_flags = 0; 1123 ret = sigaction(SIGUSR1, &sa, NULL); 1124 if (ret < 0) { 1125 perror("sigaction"); 1126 return ret; 1127 } 1128 1129 printf_verbose("Signal handler set for SIGUSR1\n"); 1130 1131 return ret; 1132 } 1133 1134 static void show_usage(int argc, char **argv) 1135 { 1136 printf("Usage : %s <OPTIONS>\n", 1137 argv[0]); 1138 printf("OPTIONS:\n"); 1139 printf(" [-1 loops] Number of loops for delay injection 1\n"); 1140 printf(" [-2 loops] Number of loops for delay injection 2\n"); 1141 printf(" [-3 loops] Number of loops for delay injection 3\n"); 1142 printf(" [-4 loops] Number of loops for delay injection 4\n"); 1143 printf(" [-5 loops] Number of loops for delay injection 5\n"); 1144 printf(" [-6 loops] Number of loops for delay injection 6\n"); 1145 printf(" [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n"); 1146 printf(" [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n"); 1147 printf(" [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n"); 1148 printf(" [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n"); 1149 printf(" [-y] Yield\n"); 1150 printf(" [-k] Kill thread with signal\n"); 1151 printf(" [-s S] S: =0: disabled (default), >0: sleep time (ms)\n"); 1152 printf(" [-t N] Number of threads (default 200)\n"); 1153 printf(" [-r N] Number of repetitions per thread (default 5000)\n"); 1154 printf(" [-d] Disable rseq system call (no initialization)\n"); 1155 printf(" [-D M] Disable rseq for each M threads\n"); 1156 printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement\n"); 1157 printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n"); 1158 printf(" [-v] Verbose output.\n"); 1159 printf(" [-h] Show this help.\n"); 1160 printf("\n"); 1161 } 1162 1163 int main(int argc, char **argv) 1164 { 1165 int i; 1166 1167 for (i = 1; i < argc; i++) { 1168 if (argv[i][0] != '-') 1169 continue; 1170 switch (argv[i][1]) { 1171 case '1': 1172 case '2': 1173 case '3': 1174 case '4': 1175 case '5': 1176 case '6': 1177 case '7': 1178 case '8': 1179 case '9': 1180 if (argc < i + 2) { 1181 show_usage(argc, argv); 1182 goto error; 1183 } 1184 loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]); 1185 i++; 1186 break; 1187 case 'm': 1188 if (argc < i + 2) { 1189 show_usage(argc, argv); 1190 goto error; 1191 } 1192 opt_modulo = atol(argv[i + 1]); 1193 if (opt_modulo < 0) { 1194 show_usage(argc, argv); 1195 goto error; 1196 } 1197 i++; 1198 break; 1199 case 's': 1200 if (argc < i + 2) { 1201 show_usage(argc, argv); 1202 goto error; 1203 } 1204 opt_sleep = atol(argv[i + 1]); 1205 if (opt_sleep < 0) { 1206 show_usage(argc, argv); 1207 goto error; 1208 } 1209 i++; 1210 break; 1211 case 'y': 1212 opt_yield = 1; 1213 break; 1214 case 'k': 1215 opt_signal = 1; 1216 break; 1217 case 'd': 1218 opt_disable_rseq = 1; 1219 break; 1220 case 'D': 1221 if (argc < i + 2) { 1222 show_usage(argc, argv); 1223 goto error; 1224 } 1225 opt_disable_mod = atol(argv[i + 1]); 1226 if (opt_disable_mod < 0) { 1227 show_usage(argc, argv); 1228 goto error; 1229 } 1230 i++; 1231 break; 1232 case 't': 1233 if (argc < i + 2) { 1234 show_usage(argc, argv); 1235 goto error; 1236 } 1237 opt_threads = atol(argv[i + 1]); 1238 if (opt_threads < 0) { 1239 show_usage(argc, argv); 1240 goto error; 1241 } 1242 i++; 1243 break; 1244 case 'r': 1245 if (argc < i + 2) { 1246 show_usage(argc, argv); 1247 goto error; 1248 } 1249 opt_reps = atoll(argv[i + 1]); 1250 if (opt_reps < 0) { 1251 show_usage(argc, argv); 1252 goto error; 1253 } 1254 i++; 1255 break; 1256 case 'h': 1257 show_usage(argc, argv); 1258 goto end; 1259 case 'T': 1260 if (argc < i + 2) { 1261 show_usage(argc, argv); 1262 goto error; 1263 } 1264 opt_test = *argv[i + 1]; 1265 switch (opt_test) { 1266 case 's': 1267 case 'l': 1268 case 'i': 1269 case 'b': 1270 case 'm': 1271 break; 1272 default: 1273 show_usage(argc, argv); 1274 goto error; 1275 } 1276 i++; 1277 break; 1278 case 'v': 1279 verbose = 1; 1280 break; 1281 case 'M': 1282 opt_mb = 1; 1283 break; 1284 default: 1285 show_usage(argc, argv); 1286 goto error; 1287 } 1288 } 1289 1290 loop_cnt_1 = loop_cnt[1]; 1291 loop_cnt_2 = loop_cnt[2]; 1292 loop_cnt_3 = loop_cnt[3]; 1293 loop_cnt_4 = loop_cnt[4]; 1294 loop_cnt_5 = loop_cnt[5]; 1295 loop_cnt_6 = loop_cnt[6]; 1296 1297 if (set_signal_handler()) 1298 goto error; 1299 1300 if (!opt_disable_rseq && rseq_register_current_thread()) 1301 goto error; 1302 switch (opt_test) { 1303 case 's': 1304 printf_verbose("spinlock\n"); 1305 test_percpu_spinlock(); 1306 break; 1307 case 'l': 1308 printf_verbose("linked list\n"); 1309 test_percpu_list(); 1310 break; 1311 case 'b': 1312 printf_verbose("buffer\n"); 1313 test_percpu_buffer(); 1314 break; 1315 case 'm': 1316 printf_verbose("memcpy buffer\n"); 1317 test_percpu_memcpy_buffer(); 1318 break; 1319 case 'i': 1320 printf_verbose("counter increment\n"); 1321 test_percpu_inc(); 1322 break; 1323 } 1324 if (!opt_disable_rseq && rseq_unregister_current_thread()) 1325 abort(); 1326 end: 1327 return 0; 1328 1329 error: 1330 return -1; 1331 } 1332