1 // SPDX-License-Identifier: LGPL-2.1 2 #define _GNU_SOURCE 3 #include <assert.h> 4 #include <pthread.h> 5 #include <sched.h> 6 #include <stdint.h> 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <string.h> 10 #include <syscall.h> 11 #include <unistd.h> 12 #include <poll.h> 13 #include <sys/types.h> 14 #include <signal.h> 15 #include <errno.h> 16 #include <stddef.h> 17 18 static inline pid_t gettid(void) 19 { 20 return syscall(__NR_gettid); 21 } 22 23 #define NR_INJECT 9 24 static int loop_cnt[NR_INJECT + 1]; 25 26 static int loop_cnt_1 asm("asm_loop_cnt_1") __attribute__((used)); 27 static int loop_cnt_2 asm("asm_loop_cnt_2") __attribute__((used)); 28 static int loop_cnt_3 asm("asm_loop_cnt_3") __attribute__((used)); 29 static int loop_cnt_4 asm("asm_loop_cnt_4") __attribute__((used)); 30 static int loop_cnt_5 asm("asm_loop_cnt_5") __attribute__((used)); 31 static int loop_cnt_6 asm("asm_loop_cnt_6") __attribute__((used)); 32 33 static int opt_modulo, verbose; 34 35 static int opt_yield, opt_signal, opt_sleep, 36 opt_disable_rseq, opt_threads = 200, 37 opt_disable_mod = 0, opt_test = 's', opt_mb = 0; 38 39 #ifndef RSEQ_SKIP_FASTPATH 40 static long long opt_reps = 5000; 41 #else 42 static long long opt_reps = 100; 43 #endif 44 45 static __thread __attribute__((tls_model("initial-exec"))) 46 unsigned int signals_delivered; 47 48 #ifndef BENCHMARK 49 50 static __thread __attribute__((tls_model("initial-exec"), unused)) 51 unsigned int yield_mod_cnt, nr_abort; 52 53 #define printf_verbose(fmt, ...) \ 54 do { \ 55 if (verbose) \ 56 printf(fmt, ## __VA_ARGS__); \ 57 } while (0) 58 59 #if defined(__x86_64__) || defined(__i386__) 60 61 #define INJECT_ASM_REG "eax" 62 63 #define RSEQ_INJECT_CLOBBER \ 64 , INJECT_ASM_REG 65 66 #ifdef __i386__ 67 68 #define RSEQ_INJECT_ASM(n) \ 69 "mov asm_loop_cnt_" #n ", %%" INJECT_ASM_REG "\n\t" \ 70 "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \ 71 "jz 333f\n\t" \ 72 "222:\n\t" \ 73 "dec %%" INJECT_ASM_REG "\n\t" \ 74 "jnz 222b\n\t" \ 75 "333:\n\t" 76 77 #elif defined(__x86_64__) 78 79 #define RSEQ_INJECT_ASM(n) \ 80 "lea asm_loop_cnt_" #n "(%%rip), %%" INJECT_ASM_REG "\n\t" \ 81 "mov (%%" INJECT_ASM_REG "), %%" INJECT_ASM_REG "\n\t" \ 82 "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \ 83 "jz 333f\n\t" \ 84 "222:\n\t" \ 85 "dec %%" INJECT_ASM_REG "\n\t" \ 86 "jnz 222b\n\t" \ 87 "333:\n\t" 88 89 #else 90 #error "Unsupported architecture" 91 #endif 92 93 #elif defined(__ARMEL__) 94 95 #define RSEQ_INJECT_INPUT \ 96 , [loop_cnt_1]"m"(loop_cnt[1]) \ 97 , [loop_cnt_2]"m"(loop_cnt[2]) \ 98 , [loop_cnt_3]"m"(loop_cnt[3]) \ 99 , [loop_cnt_4]"m"(loop_cnt[4]) \ 100 , [loop_cnt_5]"m"(loop_cnt[5]) \ 101 , [loop_cnt_6]"m"(loop_cnt[6]) 102 103 #define INJECT_ASM_REG "r4" 104 105 #define RSEQ_INJECT_CLOBBER \ 106 , INJECT_ASM_REG 107 108 #define RSEQ_INJECT_ASM(n) \ 109 "ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ 110 "cmp " INJECT_ASM_REG ", #0\n\t" \ 111 "beq 333f\n\t" \ 112 "222:\n\t" \ 113 "subs " INJECT_ASM_REG ", #1\n\t" \ 114 "bne 222b\n\t" \ 115 "333:\n\t" 116 117 #elif __PPC__ 118 119 #define RSEQ_INJECT_INPUT \ 120 , [loop_cnt_1]"m"(loop_cnt[1]) \ 121 , [loop_cnt_2]"m"(loop_cnt[2]) \ 122 , [loop_cnt_3]"m"(loop_cnt[3]) \ 123 , [loop_cnt_4]"m"(loop_cnt[4]) \ 124 , [loop_cnt_5]"m"(loop_cnt[5]) \ 125 , [loop_cnt_6]"m"(loop_cnt[6]) 126 127 #define INJECT_ASM_REG "r18" 128 129 #define RSEQ_INJECT_CLOBBER \ 130 , INJECT_ASM_REG 131 132 #define RSEQ_INJECT_ASM(n) \ 133 "lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ 134 "cmpwi %%" INJECT_ASM_REG ", 0\n\t" \ 135 "beq 333f\n\t" \ 136 "222:\n\t" \ 137 "subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \ 138 "bne 222b\n\t" \ 139 "333:\n\t" 140 141 #elif defined(__mips__) 142 143 #define RSEQ_INJECT_INPUT \ 144 , [loop_cnt_1]"m"(loop_cnt[1]) \ 145 , [loop_cnt_2]"m"(loop_cnt[2]) \ 146 , [loop_cnt_3]"m"(loop_cnt[3]) \ 147 , [loop_cnt_4]"m"(loop_cnt[4]) \ 148 , [loop_cnt_5]"m"(loop_cnt[5]) \ 149 , [loop_cnt_6]"m"(loop_cnt[6]) 150 151 #define INJECT_ASM_REG "$5" 152 153 #define RSEQ_INJECT_CLOBBER \ 154 , INJECT_ASM_REG 155 156 #define RSEQ_INJECT_ASM(n) \ 157 "lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ 158 "beqz " INJECT_ASM_REG ", 333f\n\t" \ 159 "222:\n\t" \ 160 "addiu " INJECT_ASM_REG ", -1\n\t" \ 161 "bnez " INJECT_ASM_REG ", 222b\n\t" \ 162 "333:\n\t" 163 164 #else 165 #error unsupported target 166 #endif 167 168 #define RSEQ_INJECT_FAILED \ 169 nr_abort++; 170 171 #define RSEQ_INJECT_C(n) \ 172 { \ 173 int loc_i, loc_nr_loops = loop_cnt[n]; \ 174 \ 175 for (loc_i = 0; loc_i < loc_nr_loops; loc_i++) { \ 176 rseq_barrier(); \ 177 } \ 178 if (loc_nr_loops == -1 && opt_modulo) { \ 179 if (yield_mod_cnt == opt_modulo - 1) { \ 180 if (opt_sleep > 0) \ 181 poll(NULL, 0, opt_sleep); \ 182 if (opt_yield) \ 183 sched_yield(); \ 184 if (opt_signal) \ 185 raise(SIGUSR1); \ 186 yield_mod_cnt = 0; \ 187 } else { \ 188 yield_mod_cnt++; \ 189 } \ 190 } \ 191 } 192 193 #else 194 195 #define printf_verbose(fmt, ...) 196 197 #endif /* BENCHMARK */ 198 199 #include "rseq.h" 200 201 struct percpu_lock_entry { 202 intptr_t v; 203 } __attribute__((aligned(128))); 204 205 struct percpu_lock { 206 struct percpu_lock_entry c[CPU_SETSIZE]; 207 }; 208 209 struct test_data_entry { 210 intptr_t count; 211 } __attribute__((aligned(128))); 212 213 struct spinlock_test_data { 214 struct percpu_lock lock; 215 struct test_data_entry c[CPU_SETSIZE]; 216 }; 217 218 struct spinlock_thread_test_data { 219 struct spinlock_test_data *data; 220 long long reps; 221 int reg; 222 }; 223 224 struct inc_test_data { 225 struct test_data_entry c[CPU_SETSIZE]; 226 }; 227 228 struct inc_thread_test_data { 229 struct inc_test_data *data; 230 long long reps; 231 int reg; 232 }; 233 234 struct percpu_list_node { 235 intptr_t data; 236 struct percpu_list_node *next; 237 }; 238 239 struct percpu_list_entry { 240 struct percpu_list_node *head; 241 } __attribute__((aligned(128))); 242 243 struct percpu_list { 244 struct percpu_list_entry c[CPU_SETSIZE]; 245 }; 246 247 #define BUFFER_ITEM_PER_CPU 100 248 249 struct percpu_buffer_node { 250 intptr_t data; 251 }; 252 253 struct percpu_buffer_entry { 254 intptr_t offset; 255 intptr_t buflen; 256 struct percpu_buffer_node **array; 257 } __attribute__((aligned(128))); 258 259 struct percpu_buffer { 260 struct percpu_buffer_entry c[CPU_SETSIZE]; 261 }; 262 263 #define MEMCPY_BUFFER_ITEM_PER_CPU 100 264 265 struct percpu_memcpy_buffer_node { 266 intptr_t data1; 267 uint64_t data2; 268 }; 269 270 struct percpu_memcpy_buffer_entry { 271 intptr_t offset; 272 intptr_t buflen; 273 struct percpu_memcpy_buffer_node *array; 274 } __attribute__((aligned(128))); 275 276 struct percpu_memcpy_buffer { 277 struct percpu_memcpy_buffer_entry c[CPU_SETSIZE]; 278 }; 279 280 /* A simple percpu spinlock. Grabs lock on current cpu. */ 281 static int rseq_this_cpu_lock(struct percpu_lock *lock) 282 { 283 int cpu; 284 285 for (;;) { 286 int ret; 287 288 cpu = rseq_cpu_start(); 289 ret = rseq_cmpeqv_storev(&lock->c[cpu].v, 290 0, 1, cpu); 291 if (rseq_likely(!ret)) 292 break; 293 /* Retry if comparison fails or rseq aborts. */ 294 } 295 /* 296 * Acquire semantic when taking lock after control dependency. 297 * Matches rseq_smp_store_release(). 298 */ 299 rseq_smp_acquire__after_ctrl_dep(); 300 return cpu; 301 } 302 303 static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu) 304 { 305 assert(lock->c[cpu].v == 1); 306 /* 307 * Release lock, with release semantic. Matches 308 * rseq_smp_acquire__after_ctrl_dep(). 309 */ 310 rseq_smp_store_release(&lock->c[cpu].v, 0); 311 } 312 313 void *test_percpu_spinlock_thread(void *arg) 314 { 315 struct spinlock_thread_test_data *thread_data = arg; 316 struct spinlock_test_data *data = thread_data->data; 317 long long i, reps; 318 319 if (!opt_disable_rseq && thread_data->reg && 320 rseq_register_current_thread()) 321 abort(); 322 reps = thread_data->reps; 323 for (i = 0; i < reps; i++) { 324 int cpu = rseq_cpu_start(); 325 326 cpu = rseq_this_cpu_lock(&data->lock); 327 data->c[cpu].count++; 328 rseq_percpu_unlock(&data->lock, cpu); 329 #ifndef BENCHMARK 330 if (i != 0 && !(i % (reps / 10))) 331 printf_verbose("tid %d: count %lld\n", (int) gettid(), i); 332 #endif 333 } 334 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", 335 (int) gettid(), nr_abort, signals_delivered); 336 if (!opt_disable_rseq && thread_data->reg && 337 rseq_unregister_current_thread()) 338 abort(); 339 return NULL; 340 } 341 342 /* 343 * A simple test which implements a sharded counter using a per-cpu 344 * lock. Obviously real applications might prefer to simply use a 345 * per-cpu increment; however, this is reasonable for a test and the 346 * lock can be extended to synchronize more complicated operations. 347 */ 348 void test_percpu_spinlock(void) 349 { 350 const int num_threads = opt_threads; 351 int i, ret; 352 uint64_t sum; 353 pthread_t test_threads[num_threads]; 354 struct spinlock_test_data data; 355 struct spinlock_thread_test_data thread_data[num_threads]; 356 357 memset(&data, 0, sizeof(data)); 358 for (i = 0; i < num_threads; i++) { 359 thread_data[i].reps = opt_reps; 360 if (opt_disable_mod <= 0 || (i % opt_disable_mod)) 361 thread_data[i].reg = 1; 362 else 363 thread_data[i].reg = 0; 364 thread_data[i].data = &data; 365 ret = pthread_create(&test_threads[i], NULL, 366 test_percpu_spinlock_thread, 367 &thread_data[i]); 368 if (ret) { 369 errno = ret; 370 perror("pthread_create"); 371 abort(); 372 } 373 } 374 375 for (i = 0; i < num_threads; i++) { 376 ret = pthread_join(test_threads[i], NULL); 377 if (ret) { 378 errno = ret; 379 perror("pthread_join"); 380 abort(); 381 } 382 } 383 384 sum = 0; 385 for (i = 0; i < CPU_SETSIZE; i++) 386 sum += data.c[i].count; 387 388 assert(sum == (uint64_t)opt_reps * num_threads); 389 } 390 391 void *test_percpu_inc_thread(void *arg) 392 { 393 struct inc_thread_test_data *thread_data = arg; 394 struct inc_test_data *data = thread_data->data; 395 long long i, reps; 396 397 if (!opt_disable_rseq && thread_data->reg && 398 rseq_register_current_thread()) 399 abort(); 400 reps = thread_data->reps; 401 for (i = 0; i < reps; i++) { 402 int ret; 403 404 do { 405 int cpu; 406 407 cpu = rseq_cpu_start(); 408 ret = rseq_addv(&data->c[cpu].count, 1, cpu); 409 } while (rseq_unlikely(ret)); 410 #ifndef BENCHMARK 411 if (i != 0 && !(i % (reps / 10))) 412 printf_verbose("tid %d: count %lld\n", (int) gettid(), i); 413 #endif 414 } 415 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", 416 (int) gettid(), nr_abort, signals_delivered); 417 if (!opt_disable_rseq && thread_data->reg && 418 rseq_unregister_current_thread()) 419 abort(); 420 return NULL; 421 } 422 423 void test_percpu_inc(void) 424 { 425 const int num_threads = opt_threads; 426 int i, ret; 427 uint64_t sum; 428 pthread_t test_threads[num_threads]; 429 struct inc_test_data data; 430 struct inc_thread_test_data thread_data[num_threads]; 431 432 memset(&data, 0, sizeof(data)); 433 for (i = 0; i < num_threads; i++) { 434 thread_data[i].reps = opt_reps; 435 if (opt_disable_mod <= 0 || (i % opt_disable_mod)) 436 thread_data[i].reg = 1; 437 else 438 thread_data[i].reg = 0; 439 thread_data[i].data = &data; 440 ret = pthread_create(&test_threads[i], NULL, 441 test_percpu_inc_thread, 442 &thread_data[i]); 443 if (ret) { 444 errno = ret; 445 perror("pthread_create"); 446 abort(); 447 } 448 } 449 450 for (i = 0; i < num_threads; i++) { 451 ret = pthread_join(test_threads[i], NULL); 452 if (ret) { 453 errno = ret; 454 perror("pthread_join"); 455 abort(); 456 } 457 } 458 459 sum = 0; 460 for (i = 0; i < CPU_SETSIZE; i++) 461 sum += data.c[i].count; 462 463 assert(sum == (uint64_t)opt_reps * num_threads); 464 } 465 466 void this_cpu_list_push(struct percpu_list *list, 467 struct percpu_list_node *node, 468 int *_cpu) 469 { 470 int cpu; 471 472 for (;;) { 473 intptr_t *targetptr, newval, expect; 474 int ret; 475 476 cpu = rseq_cpu_start(); 477 /* Load list->c[cpu].head with single-copy atomicity. */ 478 expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head); 479 newval = (intptr_t)node; 480 targetptr = (intptr_t *)&list->c[cpu].head; 481 node->next = (struct percpu_list_node *)expect; 482 ret = rseq_cmpeqv_storev(targetptr, expect, newval, cpu); 483 if (rseq_likely(!ret)) 484 break; 485 /* Retry if comparison fails or rseq aborts. */ 486 } 487 if (_cpu) 488 *_cpu = cpu; 489 } 490 491 /* 492 * Unlike a traditional lock-less linked list; the availability of a 493 * rseq primitive allows us to implement pop without concerns over 494 * ABA-type races. 495 */ 496 struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list, 497 int *_cpu) 498 { 499 struct percpu_list_node *node = NULL; 500 int cpu; 501 502 for (;;) { 503 struct percpu_list_node *head; 504 intptr_t *targetptr, expectnot, *load; 505 off_t offset; 506 int ret; 507 508 cpu = rseq_cpu_start(); 509 targetptr = (intptr_t *)&list->c[cpu].head; 510 expectnot = (intptr_t)NULL; 511 offset = offsetof(struct percpu_list_node, next); 512 load = (intptr_t *)&head; 513 ret = rseq_cmpnev_storeoffp_load(targetptr, expectnot, 514 offset, load, cpu); 515 if (rseq_likely(!ret)) { 516 node = head; 517 break; 518 } 519 if (ret > 0) 520 break; 521 /* Retry if rseq aborts. */ 522 } 523 if (_cpu) 524 *_cpu = cpu; 525 return node; 526 } 527 528 /* 529 * __percpu_list_pop is not safe against concurrent accesses. Should 530 * only be used on lists that are not concurrently modified. 531 */ 532 struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu) 533 { 534 struct percpu_list_node *node; 535 536 node = list->c[cpu].head; 537 if (!node) 538 return NULL; 539 list->c[cpu].head = node->next; 540 return node; 541 } 542 543 void *test_percpu_list_thread(void *arg) 544 { 545 long long i, reps; 546 struct percpu_list *list = (struct percpu_list *)arg; 547 548 if (!opt_disable_rseq && rseq_register_current_thread()) 549 abort(); 550 551 reps = opt_reps; 552 for (i = 0; i < reps; i++) { 553 struct percpu_list_node *node; 554 555 node = this_cpu_list_pop(list, NULL); 556 if (opt_yield) 557 sched_yield(); /* encourage shuffling */ 558 if (node) 559 this_cpu_list_push(list, node, NULL); 560 } 561 562 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", 563 (int) gettid(), nr_abort, signals_delivered); 564 if (!opt_disable_rseq && rseq_unregister_current_thread()) 565 abort(); 566 567 return NULL; 568 } 569 570 /* Simultaneous modification to a per-cpu linked list from many threads. */ 571 void test_percpu_list(void) 572 { 573 const int num_threads = opt_threads; 574 int i, j, ret; 575 uint64_t sum = 0, expected_sum = 0; 576 struct percpu_list list; 577 pthread_t test_threads[num_threads]; 578 cpu_set_t allowed_cpus; 579 580 memset(&list, 0, sizeof(list)); 581 582 /* Generate list entries for every usable cpu. */ 583 sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); 584 for (i = 0; i < CPU_SETSIZE; i++) { 585 if (!CPU_ISSET(i, &allowed_cpus)) 586 continue; 587 for (j = 1; j <= 100; j++) { 588 struct percpu_list_node *node; 589 590 expected_sum += j; 591 592 node = malloc(sizeof(*node)); 593 assert(node); 594 node->data = j; 595 node->next = list.c[i].head; 596 list.c[i].head = node; 597 } 598 } 599 600 for (i = 0; i < num_threads; i++) { 601 ret = pthread_create(&test_threads[i], NULL, 602 test_percpu_list_thread, &list); 603 if (ret) { 604 errno = ret; 605 perror("pthread_create"); 606 abort(); 607 } 608 } 609 610 for (i = 0; i < num_threads; i++) { 611 ret = pthread_join(test_threads[i], NULL); 612 if (ret) { 613 errno = ret; 614 perror("pthread_join"); 615 abort(); 616 } 617 } 618 619 for (i = 0; i < CPU_SETSIZE; i++) { 620 struct percpu_list_node *node; 621 622 if (!CPU_ISSET(i, &allowed_cpus)) 623 continue; 624 625 while ((node = __percpu_list_pop(&list, i))) { 626 sum += node->data; 627 free(node); 628 } 629 } 630 631 /* 632 * All entries should now be accounted for (unless some external 633 * actor is interfering with our allowed affinity while this 634 * test is running). 635 */ 636 assert(sum == expected_sum); 637 } 638 639 bool this_cpu_buffer_push(struct percpu_buffer *buffer, 640 struct percpu_buffer_node *node, 641 int *_cpu) 642 { 643 bool result = false; 644 int cpu; 645 646 for (;;) { 647 intptr_t *targetptr_spec, newval_spec; 648 intptr_t *targetptr_final, newval_final; 649 intptr_t offset; 650 int ret; 651 652 cpu = rseq_cpu_start(); 653 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); 654 if (offset == buffer->c[cpu].buflen) 655 break; 656 newval_spec = (intptr_t)node; 657 targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset]; 658 newval_final = offset + 1; 659 targetptr_final = &buffer->c[cpu].offset; 660 if (opt_mb) 661 ret = rseq_cmpeqv_trystorev_storev_release( 662 targetptr_final, offset, targetptr_spec, 663 newval_spec, newval_final, cpu); 664 else 665 ret = rseq_cmpeqv_trystorev_storev(targetptr_final, 666 offset, targetptr_spec, newval_spec, 667 newval_final, cpu); 668 if (rseq_likely(!ret)) { 669 result = true; 670 break; 671 } 672 /* Retry if comparison fails or rseq aborts. */ 673 } 674 if (_cpu) 675 *_cpu = cpu; 676 return result; 677 } 678 679 struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer, 680 int *_cpu) 681 { 682 struct percpu_buffer_node *head; 683 int cpu; 684 685 for (;;) { 686 intptr_t *targetptr, newval; 687 intptr_t offset; 688 int ret; 689 690 cpu = rseq_cpu_start(); 691 /* Load offset with single-copy atomicity. */ 692 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); 693 if (offset == 0) { 694 head = NULL; 695 break; 696 } 697 head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]); 698 newval = offset - 1; 699 targetptr = (intptr_t *)&buffer->c[cpu].offset; 700 ret = rseq_cmpeqv_cmpeqv_storev(targetptr, offset, 701 (intptr_t *)&buffer->c[cpu].array[offset - 1], 702 (intptr_t)head, newval, cpu); 703 if (rseq_likely(!ret)) 704 break; 705 /* Retry if comparison fails or rseq aborts. */ 706 } 707 if (_cpu) 708 *_cpu = cpu; 709 return head; 710 } 711 712 /* 713 * __percpu_buffer_pop is not safe against concurrent accesses. Should 714 * only be used on buffers that are not concurrently modified. 715 */ 716 struct percpu_buffer_node *__percpu_buffer_pop(struct percpu_buffer *buffer, 717 int cpu) 718 { 719 struct percpu_buffer_node *head; 720 intptr_t offset; 721 722 offset = buffer->c[cpu].offset; 723 if (offset == 0) 724 return NULL; 725 head = buffer->c[cpu].array[offset - 1]; 726 buffer->c[cpu].offset = offset - 1; 727 return head; 728 } 729 730 void *test_percpu_buffer_thread(void *arg) 731 { 732 long long i, reps; 733 struct percpu_buffer *buffer = (struct percpu_buffer *)arg; 734 735 if (!opt_disable_rseq && rseq_register_current_thread()) 736 abort(); 737 738 reps = opt_reps; 739 for (i = 0; i < reps; i++) { 740 struct percpu_buffer_node *node; 741 742 node = this_cpu_buffer_pop(buffer, NULL); 743 if (opt_yield) 744 sched_yield(); /* encourage shuffling */ 745 if (node) { 746 if (!this_cpu_buffer_push(buffer, node, NULL)) { 747 /* Should increase buffer size. */ 748 abort(); 749 } 750 } 751 } 752 753 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", 754 (int) gettid(), nr_abort, signals_delivered); 755 if (!opt_disable_rseq && rseq_unregister_current_thread()) 756 abort(); 757 758 return NULL; 759 } 760 761 /* Simultaneous modification to a per-cpu buffer from many threads. */ 762 void test_percpu_buffer(void) 763 { 764 const int num_threads = opt_threads; 765 int i, j, ret; 766 uint64_t sum = 0, expected_sum = 0; 767 struct percpu_buffer buffer; 768 pthread_t test_threads[num_threads]; 769 cpu_set_t allowed_cpus; 770 771 memset(&buffer, 0, sizeof(buffer)); 772 773 /* Generate list entries for every usable cpu. */ 774 sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); 775 for (i = 0; i < CPU_SETSIZE; i++) { 776 if (!CPU_ISSET(i, &allowed_cpus)) 777 continue; 778 /* Worse-case is every item in same CPU. */ 779 buffer.c[i].array = 780 malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE * 781 BUFFER_ITEM_PER_CPU); 782 assert(buffer.c[i].array); 783 buffer.c[i].buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU; 784 for (j = 1; j <= BUFFER_ITEM_PER_CPU; j++) { 785 struct percpu_buffer_node *node; 786 787 expected_sum += j; 788 789 /* 790 * We could theoretically put the word-sized 791 * "data" directly in the buffer. However, we 792 * want to model objects that would not fit 793 * within a single word, so allocate an object 794 * for each node. 795 */ 796 node = malloc(sizeof(*node)); 797 assert(node); 798 node->data = j; 799 buffer.c[i].array[j - 1] = node; 800 buffer.c[i].offset++; 801 } 802 } 803 804 for (i = 0; i < num_threads; i++) { 805 ret = pthread_create(&test_threads[i], NULL, 806 test_percpu_buffer_thread, &buffer); 807 if (ret) { 808 errno = ret; 809 perror("pthread_create"); 810 abort(); 811 } 812 } 813 814 for (i = 0; i < num_threads; i++) { 815 ret = pthread_join(test_threads[i], NULL); 816 if (ret) { 817 errno = ret; 818 perror("pthread_join"); 819 abort(); 820 } 821 } 822 823 for (i = 0; i < CPU_SETSIZE; i++) { 824 struct percpu_buffer_node *node; 825 826 if (!CPU_ISSET(i, &allowed_cpus)) 827 continue; 828 829 while ((node = __percpu_buffer_pop(&buffer, i))) { 830 sum += node->data; 831 free(node); 832 } 833 free(buffer.c[i].array); 834 } 835 836 /* 837 * All entries should now be accounted for (unless some external 838 * actor is interfering with our allowed affinity while this 839 * test is running). 840 */ 841 assert(sum == expected_sum); 842 } 843 844 bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer, 845 struct percpu_memcpy_buffer_node item, 846 int *_cpu) 847 { 848 bool result = false; 849 int cpu; 850 851 for (;;) { 852 intptr_t *targetptr_final, newval_final, offset; 853 char *destptr, *srcptr; 854 size_t copylen; 855 int ret; 856 857 cpu = rseq_cpu_start(); 858 /* Load offset with single-copy atomicity. */ 859 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); 860 if (offset == buffer->c[cpu].buflen) 861 break; 862 destptr = (char *)&buffer->c[cpu].array[offset]; 863 srcptr = (char *)&item; 864 /* copylen must be <= 4kB. */ 865 copylen = sizeof(item); 866 newval_final = offset + 1; 867 targetptr_final = &buffer->c[cpu].offset; 868 if (opt_mb) 869 ret = rseq_cmpeqv_trymemcpy_storev_release( 870 targetptr_final, offset, 871 destptr, srcptr, copylen, 872 newval_final, cpu); 873 else 874 ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final, 875 offset, destptr, srcptr, copylen, 876 newval_final, cpu); 877 if (rseq_likely(!ret)) { 878 result = true; 879 break; 880 } 881 /* Retry if comparison fails or rseq aborts. */ 882 } 883 if (_cpu) 884 *_cpu = cpu; 885 return result; 886 } 887 888 bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, 889 struct percpu_memcpy_buffer_node *item, 890 int *_cpu) 891 { 892 bool result = false; 893 int cpu; 894 895 for (;;) { 896 intptr_t *targetptr_final, newval_final, offset; 897 char *destptr, *srcptr; 898 size_t copylen; 899 int ret; 900 901 cpu = rseq_cpu_start(); 902 /* Load offset with single-copy atomicity. */ 903 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); 904 if (offset == 0) 905 break; 906 destptr = (char *)item; 907 srcptr = (char *)&buffer->c[cpu].array[offset - 1]; 908 /* copylen must be <= 4kB. */ 909 copylen = sizeof(*item); 910 newval_final = offset - 1; 911 targetptr_final = &buffer->c[cpu].offset; 912 ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final, 913 offset, destptr, srcptr, copylen, 914 newval_final, cpu); 915 if (rseq_likely(!ret)) { 916 result = true; 917 break; 918 } 919 /* Retry if comparison fails or rseq aborts. */ 920 } 921 if (_cpu) 922 *_cpu = cpu; 923 return result; 924 } 925 926 /* 927 * __percpu_memcpy_buffer_pop is not safe against concurrent accesses. Should 928 * only be used on buffers that are not concurrently modified. 929 */ 930 bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, 931 struct percpu_memcpy_buffer_node *item, 932 int cpu) 933 { 934 intptr_t offset; 935 936 offset = buffer->c[cpu].offset; 937 if (offset == 0) 938 return false; 939 memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item)); 940 buffer->c[cpu].offset = offset - 1; 941 return true; 942 } 943 944 void *test_percpu_memcpy_buffer_thread(void *arg) 945 { 946 long long i, reps; 947 struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg; 948 949 if (!opt_disable_rseq && rseq_register_current_thread()) 950 abort(); 951 952 reps = opt_reps; 953 for (i = 0; i < reps; i++) { 954 struct percpu_memcpy_buffer_node item; 955 bool result; 956 957 result = this_cpu_memcpy_buffer_pop(buffer, &item, NULL); 958 if (opt_yield) 959 sched_yield(); /* encourage shuffling */ 960 if (result) { 961 if (!this_cpu_memcpy_buffer_push(buffer, item, NULL)) { 962 /* Should increase buffer size. */ 963 abort(); 964 } 965 } 966 } 967 968 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", 969 (int) gettid(), nr_abort, signals_delivered); 970 if (!opt_disable_rseq && rseq_unregister_current_thread()) 971 abort(); 972 973 return NULL; 974 } 975 976 /* Simultaneous modification to a per-cpu buffer from many threads. */ 977 void test_percpu_memcpy_buffer(void) 978 { 979 const int num_threads = opt_threads; 980 int i, j, ret; 981 uint64_t sum = 0, expected_sum = 0; 982 struct percpu_memcpy_buffer buffer; 983 pthread_t test_threads[num_threads]; 984 cpu_set_t allowed_cpus; 985 986 memset(&buffer, 0, sizeof(buffer)); 987 988 /* Generate list entries for every usable cpu. */ 989 sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); 990 for (i = 0; i < CPU_SETSIZE; i++) { 991 if (!CPU_ISSET(i, &allowed_cpus)) 992 continue; 993 /* Worse-case is every item in same CPU. */ 994 buffer.c[i].array = 995 malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE * 996 MEMCPY_BUFFER_ITEM_PER_CPU); 997 assert(buffer.c[i].array); 998 buffer.c[i].buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU; 999 for (j = 1; j <= MEMCPY_BUFFER_ITEM_PER_CPU; j++) { 1000 expected_sum += 2 * j + 1; 1001 1002 /* 1003 * We could theoretically put the word-sized 1004 * "data" directly in the buffer. However, we 1005 * want to model objects that would not fit 1006 * within a single word, so allocate an object 1007 * for each node. 1008 */ 1009 buffer.c[i].array[j - 1].data1 = j; 1010 buffer.c[i].array[j - 1].data2 = j + 1; 1011 buffer.c[i].offset++; 1012 } 1013 } 1014 1015 for (i = 0; i < num_threads; i++) { 1016 ret = pthread_create(&test_threads[i], NULL, 1017 test_percpu_memcpy_buffer_thread, 1018 &buffer); 1019 if (ret) { 1020 errno = ret; 1021 perror("pthread_create"); 1022 abort(); 1023 } 1024 } 1025 1026 for (i = 0; i < num_threads; i++) { 1027 ret = pthread_join(test_threads[i], NULL); 1028 if (ret) { 1029 errno = ret; 1030 perror("pthread_join"); 1031 abort(); 1032 } 1033 } 1034 1035 for (i = 0; i < CPU_SETSIZE; i++) { 1036 struct percpu_memcpy_buffer_node item; 1037 1038 if (!CPU_ISSET(i, &allowed_cpus)) 1039 continue; 1040 1041 while (__percpu_memcpy_buffer_pop(&buffer, &item, i)) { 1042 sum += item.data1; 1043 sum += item.data2; 1044 } 1045 free(buffer.c[i].array); 1046 } 1047 1048 /* 1049 * All entries should now be accounted for (unless some external 1050 * actor is interfering with our allowed affinity while this 1051 * test is running). 1052 */ 1053 assert(sum == expected_sum); 1054 } 1055 1056 static void test_signal_interrupt_handler(int signo) 1057 { 1058 signals_delivered++; 1059 } 1060 1061 static int set_signal_handler(void) 1062 { 1063 int ret = 0; 1064 struct sigaction sa; 1065 sigset_t sigset; 1066 1067 ret = sigemptyset(&sigset); 1068 if (ret < 0) { 1069 perror("sigemptyset"); 1070 return ret; 1071 } 1072 1073 sa.sa_handler = test_signal_interrupt_handler; 1074 sa.sa_mask = sigset; 1075 sa.sa_flags = 0; 1076 ret = sigaction(SIGUSR1, &sa, NULL); 1077 if (ret < 0) { 1078 perror("sigaction"); 1079 return ret; 1080 } 1081 1082 printf_verbose("Signal handler set for SIGUSR1\n"); 1083 1084 return ret; 1085 } 1086 1087 static void show_usage(int argc, char **argv) 1088 { 1089 printf("Usage : %s <OPTIONS>\n", 1090 argv[0]); 1091 printf("OPTIONS:\n"); 1092 printf(" [-1 loops] Number of loops for delay injection 1\n"); 1093 printf(" [-2 loops] Number of loops for delay injection 2\n"); 1094 printf(" [-3 loops] Number of loops for delay injection 3\n"); 1095 printf(" [-4 loops] Number of loops for delay injection 4\n"); 1096 printf(" [-5 loops] Number of loops for delay injection 5\n"); 1097 printf(" [-6 loops] Number of loops for delay injection 6\n"); 1098 printf(" [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n"); 1099 printf(" [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n"); 1100 printf(" [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n"); 1101 printf(" [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n"); 1102 printf(" [-y] Yield\n"); 1103 printf(" [-k] Kill thread with signal\n"); 1104 printf(" [-s S] S: =0: disabled (default), >0: sleep time (ms)\n"); 1105 printf(" [-t N] Number of threads (default 200)\n"); 1106 printf(" [-r N] Number of repetitions per thread (default 5000)\n"); 1107 printf(" [-d] Disable rseq system call (no initialization)\n"); 1108 printf(" [-D M] Disable rseq for each M threads\n"); 1109 printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement\n"); 1110 printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n"); 1111 printf(" [-v] Verbose output.\n"); 1112 printf(" [-h] Show this help.\n"); 1113 printf("\n"); 1114 } 1115 1116 int main(int argc, char **argv) 1117 { 1118 int i; 1119 1120 for (i = 1; i < argc; i++) { 1121 if (argv[i][0] != '-') 1122 continue; 1123 switch (argv[i][1]) { 1124 case '1': 1125 case '2': 1126 case '3': 1127 case '4': 1128 case '5': 1129 case '6': 1130 case '7': 1131 case '8': 1132 case '9': 1133 if (argc < i + 2) { 1134 show_usage(argc, argv); 1135 goto error; 1136 } 1137 loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]); 1138 i++; 1139 break; 1140 case 'm': 1141 if (argc < i + 2) { 1142 show_usage(argc, argv); 1143 goto error; 1144 } 1145 opt_modulo = atol(argv[i + 1]); 1146 if (opt_modulo < 0) { 1147 show_usage(argc, argv); 1148 goto error; 1149 } 1150 i++; 1151 break; 1152 case 's': 1153 if (argc < i + 2) { 1154 show_usage(argc, argv); 1155 goto error; 1156 } 1157 opt_sleep = atol(argv[i + 1]); 1158 if (opt_sleep < 0) { 1159 show_usage(argc, argv); 1160 goto error; 1161 } 1162 i++; 1163 break; 1164 case 'y': 1165 opt_yield = 1; 1166 break; 1167 case 'k': 1168 opt_signal = 1; 1169 break; 1170 case 'd': 1171 opt_disable_rseq = 1; 1172 break; 1173 case 'D': 1174 if (argc < i + 2) { 1175 show_usage(argc, argv); 1176 goto error; 1177 } 1178 opt_disable_mod = atol(argv[i + 1]); 1179 if (opt_disable_mod < 0) { 1180 show_usage(argc, argv); 1181 goto error; 1182 } 1183 i++; 1184 break; 1185 case 't': 1186 if (argc < i + 2) { 1187 show_usage(argc, argv); 1188 goto error; 1189 } 1190 opt_threads = atol(argv[i + 1]); 1191 if (opt_threads < 0) { 1192 show_usage(argc, argv); 1193 goto error; 1194 } 1195 i++; 1196 break; 1197 case 'r': 1198 if (argc < i + 2) { 1199 show_usage(argc, argv); 1200 goto error; 1201 } 1202 opt_reps = atoll(argv[i + 1]); 1203 if (opt_reps < 0) { 1204 show_usage(argc, argv); 1205 goto error; 1206 } 1207 i++; 1208 break; 1209 case 'h': 1210 show_usage(argc, argv); 1211 goto end; 1212 case 'T': 1213 if (argc < i + 2) { 1214 show_usage(argc, argv); 1215 goto error; 1216 } 1217 opt_test = *argv[i + 1]; 1218 switch (opt_test) { 1219 case 's': 1220 case 'l': 1221 case 'i': 1222 case 'b': 1223 case 'm': 1224 break; 1225 default: 1226 show_usage(argc, argv); 1227 goto error; 1228 } 1229 i++; 1230 break; 1231 case 'v': 1232 verbose = 1; 1233 break; 1234 case 'M': 1235 opt_mb = 1; 1236 break; 1237 default: 1238 show_usage(argc, argv); 1239 goto error; 1240 } 1241 } 1242 1243 loop_cnt_1 = loop_cnt[1]; 1244 loop_cnt_2 = loop_cnt[2]; 1245 loop_cnt_3 = loop_cnt[3]; 1246 loop_cnt_4 = loop_cnt[4]; 1247 loop_cnt_5 = loop_cnt[5]; 1248 loop_cnt_6 = loop_cnt[6]; 1249 1250 if (set_signal_handler()) 1251 goto error; 1252 1253 if (!opt_disable_rseq && rseq_register_current_thread()) 1254 goto error; 1255 switch (opt_test) { 1256 case 's': 1257 printf_verbose("spinlock\n"); 1258 test_percpu_spinlock(); 1259 break; 1260 case 'l': 1261 printf_verbose("linked list\n"); 1262 test_percpu_list(); 1263 break; 1264 case 'b': 1265 printf_verbose("buffer\n"); 1266 test_percpu_buffer(); 1267 break; 1268 case 'm': 1269 printf_verbose("memcpy buffer\n"); 1270 test_percpu_memcpy_buffer(); 1271 break; 1272 case 'i': 1273 printf_verbose("counter increment\n"); 1274 test_percpu_inc(); 1275 break; 1276 } 1277 if (!opt_disable_rseq && rseq_unregister_current_thread()) 1278 abort(); 1279 end: 1280 return 0; 1281 1282 error: 1283 return -1; 1284 } 1285