1 // SPDX-License-Identifier: LGPL-2.1
2 #define _GNU_SOURCE
3 #include <assert.h>
4 #include <linux/membarrier.h>
5 #include <pthread.h>
6 #include <sched.h>
7 #include <stdatomic.h>
8 #include <stdint.h>
9 #include <stdio.h>
10 #include <stdlib.h>
11 #include <string.h>
12 #include <syscall.h>
13 #include <unistd.h>
14 #include <poll.h>
15 #include <sys/types.h>
16 #include <signal.h>
17 #include <errno.h>
18 #include <stddef.h>
19 #include <stdbool.h>
20
rseq_gettid(void)21 static inline pid_t rseq_gettid(void)
22 {
23 return syscall(__NR_gettid);
24 }
25
26 #define NR_INJECT 9
27 static int loop_cnt[NR_INJECT + 1];
28
29 static int loop_cnt_1 asm("asm_loop_cnt_1") __attribute__((used));
30 static int loop_cnt_2 asm("asm_loop_cnt_2") __attribute__((used));
31 static int loop_cnt_3 asm("asm_loop_cnt_3") __attribute__((used));
32 static int loop_cnt_4 asm("asm_loop_cnt_4") __attribute__((used));
33 static int loop_cnt_5 asm("asm_loop_cnt_5") __attribute__((used));
34 static int loop_cnt_6 asm("asm_loop_cnt_6") __attribute__((used));
35
36 static int opt_modulo, verbose;
37
38 static int opt_yield, opt_signal, opt_sleep,
39 opt_disable_rseq, opt_threads = 200,
40 opt_disable_mod = 0, opt_test = 's';
41
42 static long long opt_reps = 5000;
43
44 static __thread __attribute__((tls_model("initial-exec")))
45 unsigned int signals_delivered;
46
47 #ifndef BENCHMARK
48
49 static __thread __attribute__((tls_model("initial-exec"), unused))
50 unsigned int yield_mod_cnt, nr_abort;
51
52 #define printf_verbose(fmt, ...) \
53 do { \
54 if (verbose) \
55 printf(fmt, ## __VA_ARGS__); \
56 } while (0)
57
58 #ifdef __i386__
59
60 #define INJECT_ASM_REG "eax"
61
62 #define RSEQ_INJECT_CLOBBER \
63 , INJECT_ASM_REG
64
65 #define RSEQ_INJECT_ASM(n) \
66 "mov asm_loop_cnt_" #n ", %%" INJECT_ASM_REG "\n\t" \
67 "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
68 "jz 333f\n\t" \
69 "222:\n\t" \
70 "dec %%" INJECT_ASM_REG "\n\t" \
71 "jnz 222b\n\t" \
72 "333:\n\t"
73
74 #elif defined(__x86_64__)
75
76 #define INJECT_ASM_REG_P "rax"
77 #define INJECT_ASM_REG "eax"
78
79 #define RSEQ_INJECT_CLOBBER \
80 , INJECT_ASM_REG_P \
81 , INJECT_ASM_REG
82
83 #define RSEQ_INJECT_ASM(n) \
84 "lea asm_loop_cnt_" #n "(%%rip), %%" INJECT_ASM_REG_P "\n\t" \
85 "mov (%%" INJECT_ASM_REG_P "), %%" INJECT_ASM_REG "\n\t" \
86 "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
87 "jz 333f\n\t" \
88 "222:\n\t" \
89 "dec %%" INJECT_ASM_REG "\n\t" \
90 "jnz 222b\n\t" \
91 "333:\n\t"
92
93 #elif defined(__s390__)
94
95 #define RSEQ_INJECT_INPUT \
96 , [loop_cnt_1]"m"(loop_cnt[1]) \
97 , [loop_cnt_2]"m"(loop_cnt[2]) \
98 , [loop_cnt_3]"m"(loop_cnt[3]) \
99 , [loop_cnt_4]"m"(loop_cnt[4]) \
100 , [loop_cnt_5]"m"(loop_cnt[5]) \
101 , [loop_cnt_6]"m"(loop_cnt[6])
102
103 #define INJECT_ASM_REG "r12"
104
105 #define RSEQ_INJECT_CLOBBER \
106 , INJECT_ASM_REG
107
108 #define RSEQ_INJECT_ASM(n) \
109 "l %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
110 "ltr %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG "\n\t" \
111 "je 333f\n\t" \
112 "222:\n\t" \
113 "ahi %%" INJECT_ASM_REG ", -1\n\t" \
114 "jnz 222b\n\t" \
115 "333:\n\t"
116
117 #elif defined(__ARMEL__)
118
119 #define RSEQ_INJECT_INPUT \
120 , [loop_cnt_1]"m"(loop_cnt[1]) \
121 , [loop_cnt_2]"m"(loop_cnt[2]) \
122 , [loop_cnt_3]"m"(loop_cnt[3]) \
123 , [loop_cnt_4]"m"(loop_cnt[4]) \
124 , [loop_cnt_5]"m"(loop_cnt[5]) \
125 , [loop_cnt_6]"m"(loop_cnt[6])
126
127 #define INJECT_ASM_REG "r4"
128
129 #define RSEQ_INJECT_CLOBBER \
130 , INJECT_ASM_REG
131
132 #define RSEQ_INJECT_ASM(n) \
133 "ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
134 "cmp " INJECT_ASM_REG ", #0\n\t" \
135 "beq 333f\n\t" \
136 "222:\n\t" \
137 "subs " INJECT_ASM_REG ", #1\n\t" \
138 "bne 222b\n\t" \
139 "333:\n\t"
140
141 #elif defined(__AARCH64EL__)
142
143 #define RSEQ_INJECT_INPUT \
144 , [loop_cnt_1] "Qo" (loop_cnt[1]) \
145 , [loop_cnt_2] "Qo" (loop_cnt[2]) \
146 , [loop_cnt_3] "Qo" (loop_cnt[3]) \
147 , [loop_cnt_4] "Qo" (loop_cnt[4]) \
148 , [loop_cnt_5] "Qo" (loop_cnt[5]) \
149 , [loop_cnt_6] "Qo" (loop_cnt[6])
150
151 #define INJECT_ASM_REG RSEQ_ASM_TMP_REG32
152
153 #define RSEQ_INJECT_ASM(n) \
154 " ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n" \
155 " cbz " INJECT_ASM_REG ", 333f\n" \
156 "222:\n" \
157 " sub " INJECT_ASM_REG ", " INJECT_ASM_REG ", #1\n" \
158 " cbnz " INJECT_ASM_REG ", 222b\n" \
159 "333:\n"
160
161 #elif defined(__PPC__)
162
163 #define RSEQ_INJECT_INPUT \
164 , [loop_cnt_1]"m"(loop_cnt[1]) \
165 , [loop_cnt_2]"m"(loop_cnt[2]) \
166 , [loop_cnt_3]"m"(loop_cnt[3]) \
167 , [loop_cnt_4]"m"(loop_cnt[4]) \
168 , [loop_cnt_5]"m"(loop_cnt[5]) \
169 , [loop_cnt_6]"m"(loop_cnt[6])
170
171 #define INJECT_ASM_REG "r18"
172
173 #define RSEQ_INJECT_CLOBBER \
174 , INJECT_ASM_REG
175
176 #define RSEQ_INJECT_ASM(n) \
177 "lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
178 "cmpwi %%" INJECT_ASM_REG ", 0\n\t" \
179 "beq 333f\n\t" \
180 "222:\n\t" \
181 "subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \
182 "bne 222b\n\t" \
183 "333:\n\t"
184
185 #elif defined(__mips__)
186
187 #define RSEQ_INJECT_INPUT \
188 , [loop_cnt_1]"m"(loop_cnt[1]) \
189 , [loop_cnt_2]"m"(loop_cnt[2]) \
190 , [loop_cnt_3]"m"(loop_cnt[3]) \
191 , [loop_cnt_4]"m"(loop_cnt[4]) \
192 , [loop_cnt_5]"m"(loop_cnt[5]) \
193 , [loop_cnt_6]"m"(loop_cnt[6])
194
195 #define INJECT_ASM_REG "$5"
196
197 #define RSEQ_INJECT_CLOBBER \
198 , INJECT_ASM_REG
199
200 #define RSEQ_INJECT_ASM(n) \
201 "lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
202 "beqz " INJECT_ASM_REG ", 333f\n\t" \
203 "222:\n\t" \
204 "addiu " INJECT_ASM_REG ", -1\n\t" \
205 "bnez " INJECT_ASM_REG ", 222b\n\t" \
206 "333:\n\t"
207 #elif defined(__riscv)
208
209 #define RSEQ_INJECT_INPUT \
210 , [loop_cnt_1]"m"(loop_cnt[1]) \
211 , [loop_cnt_2]"m"(loop_cnt[2]) \
212 , [loop_cnt_3]"m"(loop_cnt[3]) \
213 , [loop_cnt_4]"m"(loop_cnt[4]) \
214 , [loop_cnt_5]"m"(loop_cnt[5]) \
215 , [loop_cnt_6]"m"(loop_cnt[6])
216
217 #define INJECT_ASM_REG "t1"
218
219 #define RSEQ_INJECT_CLOBBER \
220 , INJECT_ASM_REG
221
222 #define RSEQ_INJECT_ASM(n) \
223 "lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
224 "beqz " INJECT_ASM_REG ", 333f\n\t" \
225 "222:\n\t" \
226 "addi " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t" \
227 "bnez " INJECT_ASM_REG ", 222b\n\t" \
228 "333:\n\t"
229
230
231 #else
232 #error unsupported target
233 #endif
234
235 #define RSEQ_INJECT_FAILED \
236 nr_abort++;
237
238 #define RSEQ_INJECT_C(n) \
239 { \
240 int loc_i, loc_nr_loops = loop_cnt[n]; \
241 \
242 for (loc_i = 0; loc_i < loc_nr_loops; loc_i++) { \
243 rseq_barrier(); \
244 } \
245 if (loc_nr_loops == -1 && opt_modulo) { \
246 if (yield_mod_cnt == opt_modulo - 1) { \
247 if (opt_sleep > 0) \
248 poll(NULL, 0, opt_sleep); \
249 if (opt_yield) \
250 sched_yield(); \
251 if (opt_signal) \
252 raise(SIGUSR1); \
253 yield_mod_cnt = 0; \
254 } else { \
255 yield_mod_cnt++; \
256 } \
257 } \
258 }
259
260 #else
261
262 #define printf_verbose(fmt, ...)
263
264 #endif /* BENCHMARK */
265
266 #include "rseq.h"
267
268 static enum rseq_mo opt_mo = RSEQ_MO_RELAXED;
269
270 #ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
271 #define TEST_MEMBARRIER
272
sys_membarrier(int cmd,int flags,int cpu_id)273 static int sys_membarrier(int cmd, int flags, int cpu_id)
274 {
275 return syscall(__NR_membarrier, cmd, flags, cpu_id);
276 }
277 #endif
278
279 #ifdef BUILDOPT_RSEQ_PERCPU_MM_CID
280 # define RSEQ_PERCPU RSEQ_PERCPU_MM_CID
281 static
get_current_cpu_id(void)282 int get_current_cpu_id(void)
283 {
284 return rseq_current_mm_cid();
285 }
286 static
rseq_validate_cpu_id(void)287 bool rseq_validate_cpu_id(void)
288 {
289 return rseq_mm_cid_available();
290 }
291 # ifdef TEST_MEMBARRIER
292 /*
293 * Membarrier does not currently support targeting a mm_cid, so
294 * issue the barrier on all cpus.
295 */
296 static
rseq_membarrier_expedited(int cpu)297 int rseq_membarrier_expedited(int cpu)
298 {
299 return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ,
300 0, 0);
301 }
302 # endif /* TEST_MEMBARRIER */
303 #else
304 # define RSEQ_PERCPU RSEQ_PERCPU_CPU_ID
305 static
get_current_cpu_id(void)306 int get_current_cpu_id(void)
307 {
308 return rseq_cpu_start();
309 }
310 static
rseq_validate_cpu_id(void)311 bool rseq_validate_cpu_id(void)
312 {
313 return rseq_current_cpu_raw() >= 0;
314 }
315 # ifdef TEST_MEMBARRIER
316 static
rseq_membarrier_expedited(int cpu)317 int rseq_membarrier_expedited(int cpu)
318 {
319 return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ,
320 MEMBARRIER_CMD_FLAG_CPU, cpu);
321 }
322 # endif /* TEST_MEMBARRIER */
323 #endif
324
325 struct percpu_lock_entry {
326 intptr_t v;
327 } __attribute__((aligned(128)));
328
329 struct percpu_lock {
330 struct percpu_lock_entry c[CPU_SETSIZE];
331 };
332
333 struct test_data_entry {
334 intptr_t count;
335 } __attribute__((aligned(128)));
336
337 struct spinlock_test_data {
338 struct percpu_lock lock;
339 struct test_data_entry c[CPU_SETSIZE];
340 };
341
342 struct spinlock_thread_test_data {
343 struct spinlock_test_data *data;
344 long long reps;
345 int reg;
346 };
347
348 struct inc_test_data {
349 struct test_data_entry c[CPU_SETSIZE];
350 };
351
352 struct inc_thread_test_data {
353 struct inc_test_data *data;
354 long long reps;
355 int reg;
356 };
357
358 struct percpu_list_node {
359 intptr_t data;
360 struct percpu_list_node *next;
361 };
362
363 struct percpu_list_entry {
364 struct percpu_list_node *head;
365 } __attribute__((aligned(128)));
366
367 struct percpu_list {
368 struct percpu_list_entry c[CPU_SETSIZE];
369 };
370
371 #define BUFFER_ITEM_PER_CPU 100
372
373 struct percpu_buffer_node {
374 intptr_t data;
375 };
376
377 struct percpu_buffer_entry {
378 intptr_t offset;
379 intptr_t buflen;
380 struct percpu_buffer_node **array;
381 } __attribute__((aligned(128)));
382
383 struct percpu_buffer {
384 struct percpu_buffer_entry c[CPU_SETSIZE];
385 };
386
387 #define MEMCPY_BUFFER_ITEM_PER_CPU 100
388
389 struct percpu_memcpy_buffer_node {
390 intptr_t data1;
391 uint64_t data2;
392 };
393
394 struct percpu_memcpy_buffer_entry {
395 intptr_t offset;
396 intptr_t buflen;
397 struct percpu_memcpy_buffer_node *array;
398 } __attribute__((aligned(128)));
399
400 struct percpu_memcpy_buffer {
401 struct percpu_memcpy_buffer_entry c[CPU_SETSIZE];
402 };
403
404 /* A simple percpu spinlock. Grabs lock on current cpu. */
rseq_this_cpu_lock(struct percpu_lock * lock)405 static int rseq_this_cpu_lock(struct percpu_lock *lock)
406 {
407 int cpu;
408
409 for (;;) {
410 int ret;
411
412 cpu = get_current_cpu_id();
413 if (cpu < 0) {
414 fprintf(stderr, "pid: %d: tid: %d, cpu: %d: cid: %d\n",
415 getpid(), (int) rseq_gettid(), rseq_current_cpu_raw(), cpu);
416 abort();
417 }
418 ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU,
419 &lock->c[cpu].v,
420 0, 1, cpu);
421 if (rseq_likely(!ret))
422 break;
423 /* Retry if comparison fails or rseq aborts. */
424 }
425 /*
426 * Acquire semantic when taking lock after control dependency.
427 * Matches rseq_smp_store_release().
428 */
429 rseq_smp_acquire__after_ctrl_dep();
430 return cpu;
431 }
432
rseq_percpu_unlock(struct percpu_lock * lock,int cpu)433 static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu)
434 {
435 assert(lock->c[cpu].v == 1);
436 /*
437 * Release lock, with release semantic. Matches
438 * rseq_smp_acquire__after_ctrl_dep().
439 */
440 rseq_smp_store_release(&lock->c[cpu].v, 0);
441 }
442
test_percpu_spinlock_thread(void * arg)443 void *test_percpu_spinlock_thread(void *arg)
444 {
445 struct spinlock_thread_test_data *thread_data = arg;
446 struct spinlock_test_data *data = thread_data->data;
447 long long i, reps;
448
449 if (!opt_disable_rseq && thread_data->reg &&
450 rseq_register_current_thread())
451 abort();
452 reps = thread_data->reps;
453 for (i = 0; i < reps; i++) {
454 int cpu = rseq_this_cpu_lock(&data->lock);
455 data->c[cpu].count++;
456 rseq_percpu_unlock(&data->lock, cpu);
457 #ifndef BENCHMARK
458 if (i != 0 && !(i % (reps / 10)))
459 printf_verbose("tid %d: count %lld\n",
460 (int) rseq_gettid(), i);
461 #endif
462 }
463 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
464 (int) rseq_gettid(), nr_abort, signals_delivered);
465 if (!opt_disable_rseq && thread_data->reg &&
466 rseq_unregister_current_thread())
467 abort();
468 return NULL;
469 }
470
471 /*
472 * A simple test which implements a sharded counter using a per-cpu
473 * lock. Obviously real applications might prefer to simply use a
474 * per-cpu increment; however, this is reasonable for a test and the
475 * lock can be extended to synchronize more complicated operations.
476 */
test_percpu_spinlock(void)477 void test_percpu_spinlock(void)
478 {
479 const int num_threads = opt_threads;
480 int i, ret;
481 uint64_t sum;
482 pthread_t test_threads[num_threads];
483 struct spinlock_test_data data;
484 struct spinlock_thread_test_data thread_data[num_threads];
485
486 memset(&data, 0, sizeof(data));
487 for (i = 0; i < num_threads; i++) {
488 thread_data[i].reps = opt_reps;
489 if (opt_disable_mod <= 0 || (i % opt_disable_mod))
490 thread_data[i].reg = 1;
491 else
492 thread_data[i].reg = 0;
493 thread_data[i].data = &data;
494 ret = pthread_create(&test_threads[i], NULL,
495 test_percpu_spinlock_thread,
496 &thread_data[i]);
497 if (ret) {
498 errno = ret;
499 perror("pthread_create");
500 abort();
501 }
502 }
503
504 for (i = 0; i < num_threads; i++) {
505 ret = pthread_join(test_threads[i], NULL);
506 if (ret) {
507 errno = ret;
508 perror("pthread_join");
509 abort();
510 }
511 }
512
513 sum = 0;
514 for (i = 0; i < CPU_SETSIZE; i++)
515 sum += data.c[i].count;
516
517 assert(sum == (uint64_t)opt_reps * num_threads);
518 }
519
test_percpu_inc_thread(void * arg)520 void *test_percpu_inc_thread(void *arg)
521 {
522 struct inc_thread_test_data *thread_data = arg;
523 struct inc_test_data *data = thread_data->data;
524 long long i, reps;
525
526 if (!opt_disable_rseq && thread_data->reg &&
527 rseq_register_current_thread())
528 abort();
529 reps = thread_data->reps;
530 for (i = 0; i < reps; i++) {
531 int ret;
532
533 do {
534 int cpu;
535
536 cpu = get_current_cpu_id();
537 ret = rseq_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU,
538 &data->c[cpu].count, 1, cpu);
539 } while (rseq_unlikely(ret));
540 #ifndef BENCHMARK
541 if (i != 0 && !(i % (reps / 10)))
542 printf_verbose("tid %d: count %lld\n",
543 (int) rseq_gettid(), i);
544 #endif
545 }
546 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
547 (int) rseq_gettid(), nr_abort, signals_delivered);
548 if (!opt_disable_rseq && thread_data->reg &&
549 rseq_unregister_current_thread())
550 abort();
551 return NULL;
552 }
553
test_percpu_inc(void)554 void test_percpu_inc(void)
555 {
556 const int num_threads = opt_threads;
557 int i, ret;
558 uint64_t sum;
559 pthread_t test_threads[num_threads];
560 struct inc_test_data data;
561 struct inc_thread_test_data thread_data[num_threads];
562
563 memset(&data, 0, sizeof(data));
564 for (i = 0; i < num_threads; i++) {
565 thread_data[i].reps = opt_reps;
566 if (opt_disable_mod <= 0 || (i % opt_disable_mod))
567 thread_data[i].reg = 1;
568 else
569 thread_data[i].reg = 0;
570 thread_data[i].data = &data;
571 ret = pthread_create(&test_threads[i], NULL,
572 test_percpu_inc_thread,
573 &thread_data[i]);
574 if (ret) {
575 errno = ret;
576 perror("pthread_create");
577 abort();
578 }
579 }
580
581 for (i = 0; i < num_threads; i++) {
582 ret = pthread_join(test_threads[i], NULL);
583 if (ret) {
584 errno = ret;
585 perror("pthread_join");
586 abort();
587 }
588 }
589
590 sum = 0;
591 for (i = 0; i < CPU_SETSIZE; i++)
592 sum += data.c[i].count;
593
594 assert(sum == (uint64_t)opt_reps * num_threads);
595 }
596
this_cpu_list_push(struct percpu_list * list,struct percpu_list_node * node,int * _cpu)597 void this_cpu_list_push(struct percpu_list *list,
598 struct percpu_list_node *node,
599 int *_cpu)
600 {
601 int cpu;
602
603 for (;;) {
604 intptr_t *targetptr, newval, expect;
605 int ret;
606
607 cpu = get_current_cpu_id();
608 /* Load list->c[cpu].head with single-copy atomicity. */
609 expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head);
610 newval = (intptr_t)node;
611 targetptr = (intptr_t *)&list->c[cpu].head;
612 node->next = (struct percpu_list_node *)expect;
613 ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU,
614 targetptr, expect, newval, cpu);
615 if (rseq_likely(!ret))
616 break;
617 /* Retry if comparison fails or rseq aborts. */
618 }
619 if (_cpu)
620 *_cpu = cpu;
621 }
622
623 /*
624 * Unlike a traditional lock-less linked list; the availability of a
625 * rseq primitive allows us to implement pop without concerns over
626 * ABA-type races.
627 */
this_cpu_list_pop(struct percpu_list * list,int * _cpu)628 struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list,
629 int *_cpu)
630 {
631 struct percpu_list_node *node = NULL;
632 int cpu;
633
634 for (;;) {
635 struct percpu_list_node *head;
636 intptr_t *targetptr, expectnot, *load;
637 long offset;
638 int ret;
639
640 cpu = get_current_cpu_id();
641 targetptr = (intptr_t *)&list->c[cpu].head;
642 expectnot = (intptr_t)NULL;
643 offset = offsetof(struct percpu_list_node, next);
644 load = (intptr_t *)&head;
645 ret = rseq_cmpnev_storeoffp_load(RSEQ_MO_RELAXED, RSEQ_PERCPU,
646 targetptr, expectnot,
647 offset, load, cpu);
648 if (rseq_likely(!ret)) {
649 node = head;
650 break;
651 }
652 if (ret > 0)
653 break;
654 /* Retry if rseq aborts. */
655 }
656 if (_cpu)
657 *_cpu = cpu;
658 return node;
659 }
660
661 /*
662 * __percpu_list_pop is not safe against concurrent accesses. Should
663 * only be used on lists that are not concurrently modified.
664 */
__percpu_list_pop(struct percpu_list * list,int cpu)665 struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu)
666 {
667 struct percpu_list_node *node;
668
669 node = list->c[cpu].head;
670 if (!node)
671 return NULL;
672 list->c[cpu].head = node->next;
673 return node;
674 }
675
test_percpu_list_thread(void * arg)676 void *test_percpu_list_thread(void *arg)
677 {
678 long long i, reps;
679 struct percpu_list *list = (struct percpu_list *)arg;
680
681 if (!opt_disable_rseq && rseq_register_current_thread())
682 abort();
683
684 reps = opt_reps;
685 for (i = 0; i < reps; i++) {
686 struct percpu_list_node *node;
687
688 node = this_cpu_list_pop(list, NULL);
689 if (opt_yield)
690 sched_yield(); /* encourage shuffling */
691 if (node)
692 this_cpu_list_push(list, node, NULL);
693 }
694
695 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
696 (int) rseq_gettid(), nr_abort, signals_delivered);
697 if (!opt_disable_rseq && rseq_unregister_current_thread())
698 abort();
699
700 return NULL;
701 }
702
703 /* Simultaneous modification to a per-cpu linked list from many threads. */
test_percpu_list(void)704 void test_percpu_list(void)
705 {
706 const int num_threads = opt_threads;
707 int i, j, ret;
708 uint64_t sum = 0, expected_sum = 0;
709 struct percpu_list list;
710 pthread_t test_threads[num_threads];
711 cpu_set_t allowed_cpus;
712
713 memset(&list, 0, sizeof(list));
714
715 /* Generate list entries for every usable cpu. */
716 sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
717 for (i = 0; i < CPU_SETSIZE; i++) {
718 if (!CPU_ISSET(i, &allowed_cpus))
719 continue;
720 for (j = 1; j <= 100; j++) {
721 struct percpu_list_node *node;
722
723 expected_sum += j;
724
725 node = malloc(sizeof(*node));
726 assert(node);
727 node->data = j;
728 node->next = list.c[i].head;
729 list.c[i].head = node;
730 }
731 }
732
733 for (i = 0; i < num_threads; i++) {
734 ret = pthread_create(&test_threads[i], NULL,
735 test_percpu_list_thread, &list);
736 if (ret) {
737 errno = ret;
738 perror("pthread_create");
739 abort();
740 }
741 }
742
743 for (i = 0; i < num_threads; i++) {
744 ret = pthread_join(test_threads[i], NULL);
745 if (ret) {
746 errno = ret;
747 perror("pthread_join");
748 abort();
749 }
750 }
751
752 for (i = 0; i < CPU_SETSIZE; i++) {
753 struct percpu_list_node *node;
754
755 if (!CPU_ISSET(i, &allowed_cpus))
756 continue;
757
758 while ((node = __percpu_list_pop(&list, i))) {
759 sum += node->data;
760 free(node);
761 }
762 }
763
764 /*
765 * All entries should now be accounted for (unless some external
766 * actor is interfering with our allowed affinity while this
767 * test is running).
768 */
769 assert(sum == expected_sum);
770 }
771
this_cpu_buffer_push(struct percpu_buffer * buffer,struct percpu_buffer_node * node,int * _cpu)772 bool this_cpu_buffer_push(struct percpu_buffer *buffer,
773 struct percpu_buffer_node *node,
774 int *_cpu)
775 {
776 bool result = false;
777 int cpu;
778
779 for (;;) {
780 intptr_t *targetptr_spec, newval_spec;
781 intptr_t *targetptr_final, newval_final;
782 intptr_t offset;
783 int ret;
784
785 cpu = get_current_cpu_id();
786 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
787 if (offset == buffer->c[cpu].buflen)
788 break;
789 newval_spec = (intptr_t)node;
790 targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset];
791 newval_final = offset + 1;
792 targetptr_final = &buffer->c[cpu].offset;
793 ret = rseq_cmpeqv_trystorev_storev(opt_mo, RSEQ_PERCPU,
794 targetptr_final, offset, targetptr_spec,
795 newval_spec, newval_final, cpu);
796 if (rseq_likely(!ret)) {
797 result = true;
798 break;
799 }
800 /* Retry if comparison fails or rseq aborts. */
801 }
802 if (_cpu)
803 *_cpu = cpu;
804 return result;
805 }
806
this_cpu_buffer_pop(struct percpu_buffer * buffer,int * _cpu)807 struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer,
808 int *_cpu)
809 {
810 struct percpu_buffer_node *head;
811 int cpu;
812
813 for (;;) {
814 intptr_t *targetptr, newval;
815 intptr_t offset;
816 int ret;
817
818 cpu = get_current_cpu_id();
819 /* Load offset with single-copy atomicity. */
820 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
821 if (offset == 0) {
822 head = NULL;
823 break;
824 }
825 head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]);
826 newval = offset - 1;
827 targetptr = (intptr_t *)&buffer->c[cpu].offset;
828 ret = rseq_cmpeqv_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU,
829 targetptr, offset,
830 (intptr_t *)&buffer->c[cpu].array[offset - 1],
831 (intptr_t)head, newval, cpu);
832 if (rseq_likely(!ret))
833 break;
834 /* Retry if comparison fails or rseq aborts. */
835 }
836 if (_cpu)
837 *_cpu = cpu;
838 return head;
839 }
840
841 /*
842 * __percpu_buffer_pop is not safe against concurrent accesses. Should
843 * only be used on buffers that are not concurrently modified.
844 */
__percpu_buffer_pop(struct percpu_buffer * buffer,int cpu)845 struct percpu_buffer_node *__percpu_buffer_pop(struct percpu_buffer *buffer,
846 int cpu)
847 {
848 struct percpu_buffer_node *head;
849 intptr_t offset;
850
851 offset = buffer->c[cpu].offset;
852 if (offset == 0)
853 return NULL;
854 head = buffer->c[cpu].array[offset - 1];
855 buffer->c[cpu].offset = offset - 1;
856 return head;
857 }
858
test_percpu_buffer_thread(void * arg)859 void *test_percpu_buffer_thread(void *arg)
860 {
861 long long i, reps;
862 struct percpu_buffer *buffer = (struct percpu_buffer *)arg;
863
864 if (!opt_disable_rseq && rseq_register_current_thread())
865 abort();
866
867 reps = opt_reps;
868 for (i = 0; i < reps; i++) {
869 struct percpu_buffer_node *node;
870
871 node = this_cpu_buffer_pop(buffer, NULL);
872 if (opt_yield)
873 sched_yield(); /* encourage shuffling */
874 if (node) {
875 if (!this_cpu_buffer_push(buffer, node, NULL)) {
876 /* Should increase buffer size. */
877 abort();
878 }
879 }
880 }
881
882 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
883 (int) rseq_gettid(), nr_abort, signals_delivered);
884 if (!opt_disable_rseq && rseq_unregister_current_thread())
885 abort();
886
887 return NULL;
888 }
889
890 /* Simultaneous modification to a per-cpu buffer from many threads. */
test_percpu_buffer(void)891 void test_percpu_buffer(void)
892 {
893 const int num_threads = opt_threads;
894 int i, j, ret;
895 uint64_t sum = 0, expected_sum = 0;
896 struct percpu_buffer buffer;
897 pthread_t test_threads[num_threads];
898 cpu_set_t allowed_cpus;
899
900 memset(&buffer, 0, sizeof(buffer));
901
902 /* Generate list entries for every usable cpu. */
903 sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
904 for (i = 0; i < CPU_SETSIZE; i++) {
905 if (!CPU_ISSET(i, &allowed_cpus))
906 continue;
907 /* Worse-case is every item in same CPU. */
908 buffer.c[i].array =
909 malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *
910 BUFFER_ITEM_PER_CPU);
911 assert(buffer.c[i].array);
912 buffer.c[i].buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU;
913 for (j = 1; j <= BUFFER_ITEM_PER_CPU; j++) {
914 struct percpu_buffer_node *node;
915
916 expected_sum += j;
917
918 /*
919 * We could theoretically put the word-sized
920 * "data" directly in the buffer. However, we
921 * want to model objects that would not fit
922 * within a single word, so allocate an object
923 * for each node.
924 */
925 node = malloc(sizeof(*node));
926 assert(node);
927 node->data = j;
928 buffer.c[i].array[j - 1] = node;
929 buffer.c[i].offset++;
930 }
931 }
932
933 for (i = 0; i < num_threads; i++) {
934 ret = pthread_create(&test_threads[i], NULL,
935 test_percpu_buffer_thread, &buffer);
936 if (ret) {
937 errno = ret;
938 perror("pthread_create");
939 abort();
940 }
941 }
942
943 for (i = 0; i < num_threads; i++) {
944 ret = pthread_join(test_threads[i], NULL);
945 if (ret) {
946 errno = ret;
947 perror("pthread_join");
948 abort();
949 }
950 }
951
952 for (i = 0; i < CPU_SETSIZE; i++) {
953 struct percpu_buffer_node *node;
954
955 if (!CPU_ISSET(i, &allowed_cpus))
956 continue;
957
958 while ((node = __percpu_buffer_pop(&buffer, i))) {
959 sum += node->data;
960 free(node);
961 }
962 free(buffer.c[i].array);
963 }
964
965 /*
966 * All entries should now be accounted for (unless some external
967 * actor is interfering with our allowed affinity while this
968 * test is running).
969 */
970 assert(sum == expected_sum);
971 }
972
this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer * buffer,struct percpu_memcpy_buffer_node item,int * _cpu)973 bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer,
974 struct percpu_memcpy_buffer_node item,
975 int *_cpu)
976 {
977 bool result = false;
978 int cpu;
979
980 for (;;) {
981 intptr_t *targetptr_final, newval_final, offset;
982 char *destptr, *srcptr;
983 size_t copylen;
984 int ret;
985
986 cpu = get_current_cpu_id();
987 /* Load offset with single-copy atomicity. */
988 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
989 if (offset == buffer->c[cpu].buflen)
990 break;
991 destptr = (char *)&buffer->c[cpu].array[offset];
992 srcptr = (char *)&item;
993 /* copylen must be <= 4kB. */
994 copylen = sizeof(item);
995 newval_final = offset + 1;
996 targetptr_final = &buffer->c[cpu].offset;
997 ret = rseq_cmpeqv_trymemcpy_storev(
998 opt_mo, RSEQ_PERCPU,
999 targetptr_final, offset,
1000 destptr, srcptr, copylen,
1001 newval_final, cpu);
1002 if (rseq_likely(!ret)) {
1003 result = true;
1004 break;
1005 }
1006 /* Retry if comparison fails or rseq aborts. */
1007 }
1008 if (_cpu)
1009 *_cpu = cpu;
1010 return result;
1011 }
1012
this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer * buffer,struct percpu_memcpy_buffer_node * item,int * _cpu)1013 bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,
1014 struct percpu_memcpy_buffer_node *item,
1015 int *_cpu)
1016 {
1017 bool result = false;
1018 int cpu;
1019
1020 for (;;) {
1021 intptr_t *targetptr_final, newval_final, offset;
1022 char *destptr, *srcptr;
1023 size_t copylen;
1024 int ret;
1025
1026 cpu = get_current_cpu_id();
1027 /* Load offset with single-copy atomicity. */
1028 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
1029 if (offset == 0)
1030 break;
1031 destptr = (char *)item;
1032 srcptr = (char *)&buffer->c[cpu].array[offset - 1];
1033 /* copylen must be <= 4kB. */
1034 copylen = sizeof(*item);
1035 newval_final = offset - 1;
1036 targetptr_final = &buffer->c[cpu].offset;
1037 ret = rseq_cmpeqv_trymemcpy_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU,
1038 targetptr_final, offset, destptr, srcptr, copylen,
1039 newval_final, cpu);
1040 if (rseq_likely(!ret)) {
1041 result = true;
1042 break;
1043 }
1044 /* Retry if comparison fails or rseq aborts. */
1045 }
1046 if (_cpu)
1047 *_cpu = cpu;
1048 return result;
1049 }
1050
1051 /*
1052 * __percpu_memcpy_buffer_pop is not safe against concurrent accesses. Should
1053 * only be used on buffers that are not concurrently modified.
1054 */
__percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer * buffer,struct percpu_memcpy_buffer_node * item,int cpu)1055 bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,
1056 struct percpu_memcpy_buffer_node *item,
1057 int cpu)
1058 {
1059 intptr_t offset;
1060
1061 offset = buffer->c[cpu].offset;
1062 if (offset == 0)
1063 return false;
1064 memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item));
1065 buffer->c[cpu].offset = offset - 1;
1066 return true;
1067 }
1068
test_percpu_memcpy_buffer_thread(void * arg)1069 void *test_percpu_memcpy_buffer_thread(void *arg)
1070 {
1071 long long i, reps;
1072 struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg;
1073
1074 if (!opt_disable_rseq && rseq_register_current_thread())
1075 abort();
1076
1077 reps = opt_reps;
1078 for (i = 0; i < reps; i++) {
1079 struct percpu_memcpy_buffer_node item;
1080 bool result;
1081
1082 result = this_cpu_memcpy_buffer_pop(buffer, &item, NULL);
1083 if (opt_yield)
1084 sched_yield(); /* encourage shuffling */
1085 if (result) {
1086 if (!this_cpu_memcpy_buffer_push(buffer, item, NULL)) {
1087 /* Should increase buffer size. */
1088 abort();
1089 }
1090 }
1091 }
1092
1093 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
1094 (int) rseq_gettid(), nr_abort, signals_delivered);
1095 if (!opt_disable_rseq && rseq_unregister_current_thread())
1096 abort();
1097
1098 return NULL;
1099 }
1100
1101 /* Simultaneous modification to a per-cpu buffer from many threads. */
test_percpu_memcpy_buffer(void)1102 void test_percpu_memcpy_buffer(void)
1103 {
1104 const int num_threads = opt_threads;
1105 int i, j, ret;
1106 uint64_t sum = 0, expected_sum = 0;
1107 struct percpu_memcpy_buffer buffer;
1108 pthread_t test_threads[num_threads];
1109 cpu_set_t allowed_cpus;
1110
1111 memset(&buffer, 0, sizeof(buffer));
1112
1113 /* Generate list entries for every usable cpu. */
1114 sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
1115 for (i = 0; i < CPU_SETSIZE; i++) {
1116 if (!CPU_ISSET(i, &allowed_cpus))
1117 continue;
1118 /* Worse-case is every item in same CPU. */
1119 buffer.c[i].array =
1120 malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *
1121 MEMCPY_BUFFER_ITEM_PER_CPU);
1122 assert(buffer.c[i].array);
1123 buffer.c[i].buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU;
1124 for (j = 1; j <= MEMCPY_BUFFER_ITEM_PER_CPU; j++) {
1125 expected_sum += 2 * j + 1;
1126
1127 /*
1128 * We could theoretically put the word-sized
1129 * "data" directly in the buffer. However, we
1130 * want to model objects that would not fit
1131 * within a single word, so allocate an object
1132 * for each node.
1133 */
1134 buffer.c[i].array[j - 1].data1 = j;
1135 buffer.c[i].array[j - 1].data2 = j + 1;
1136 buffer.c[i].offset++;
1137 }
1138 }
1139
1140 for (i = 0; i < num_threads; i++) {
1141 ret = pthread_create(&test_threads[i], NULL,
1142 test_percpu_memcpy_buffer_thread,
1143 &buffer);
1144 if (ret) {
1145 errno = ret;
1146 perror("pthread_create");
1147 abort();
1148 }
1149 }
1150
1151 for (i = 0; i < num_threads; i++) {
1152 ret = pthread_join(test_threads[i], NULL);
1153 if (ret) {
1154 errno = ret;
1155 perror("pthread_join");
1156 abort();
1157 }
1158 }
1159
1160 for (i = 0; i < CPU_SETSIZE; i++) {
1161 struct percpu_memcpy_buffer_node item;
1162
1163 if (!CPU_ISSET(i, &allowed_cpus))
1164 continue;
1165
1166 while (__percpu_memcpy_buffer_pop(&buffer, &item, i)) {
1167 sum += item.data1;
1168 sum += item.data2;
1169 }
1170 free(buffer.c[i].array);
1171 }
1172
1173 /*
1174 * All entries should now be accounted for (unless some external
1175 * actor is interfering with our allowed affinity while this
1176 * test is running).
1177 */
1178 assert(sum == expected_sum);
1179 }
1180
test_signal_interrupt_handler(int signo)1181 static void test_signal_interrupt_handler(int signo)
1182 {
1183 signals_delivered++;
1184 }
1185
set_signal_handler(void)1186 static int set_signal_handler(void)
1187 {
1188 int ret = 0;
1189 struct sigaction sa;
1190 sigset_t sigset;
1191
1192 ret = sigemptyset(&sigset);
1193 if (ret < 0) {
1194 perror("sigemptyset");
1195 return ret;
1196 }
1197
1198 sa.sa_handler = test_signal_interrupt_handler;
1199 sa.sa_mask = sigset;
1200 sa.sa_flags = 0;
1201 ret = sigaction(SIGUSR1, &sa, NULL);
1202 if (ret < 0) {
1203 perror("sigaction");
1204 return ret;
1205 }
1206
1207 printf_verbose("Signal handler set for SIGUSR1\n");
1208
1209 return ret;
1210 }
1211
1212 /* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */
1213 #ifdef TEST_MEMBARRIER
1214 struct test_membarrier_thread_args {
1215 int stop;
1216 intptr_t percpu_list_ptr;
1217 };
1218
1219 /* Worker threads modify data in their "active" percpu lists. */
test_membarrier_worker_thread(void * arg)1220 void *test_membarrier_worker_thread(void *arg)
1221 {
1222 struct test_membarrier_thread_args *args =
1223 (struct test_membarrier_thread_args *)arg;
1224 const int iters = opt_reps;
1225 int i;
1226
1227 if (rseq_register_current_thread()) {
1228 fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
1229 errno, strerror(errno));
1230 abort();
1231 }
1232
1233 /* Wait for initialization. */
1234 while (!atomic_load(&args->percpu_list_ptr)) {}
1235
1236 for (i = 0; i < iters; ++i) {
1237 int ret;
1238
1239 do {
1240 int cpu = get_current_cpu_id();
1241
1242 ret = rseq_offset_deref_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU,
1243 &args->percpu_list_ptr,
1244 sizeof(struct percpu_list_entry) * cpu, 1, cpu);
1245 } while (rseq_unlikely(ret));
1246 }
1247
1248 if (rseq_unregister_current_thread()) {
1249 fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
1250 errno, strerror(errno));
1251 abort();
1252 }
1253 return NULL;
1254 }
1255
test_membarrier_init_percpu_list(struct percpu_list * list)1256 void test_membarrier_init_percpu_list(struct percpu_list *list)
1257 {
1258 int i;
1259
1260 memset(list, 0, sizeof(*list));
1261 for (i = 0; i < CPU_SETSIZE; i++) {
1262 struct percpu_list_node *node;
1263
1264 node = malloc(sizeof(*node));
1265 assert(node);
1266 node->data = 0;
1267 node->next = NULL;
1268 list->c[i].head = node;
1269 }
1270 }
1271
test_membarrier_free_percpu_list(struct percpu_list * list)1272 void test_membarrier_free_percpu_list(struct percpu_list *list)
1273 {
1274 int i;
1275
1276 for (i = 0; i < CPU_SETSIZE; i++)
1277 free(list->c[i].head);
1278 }
1279
1280 /*
1281 * The manager thread swaps per-cpu lists that worker threads see,
1282 * and validates that there are no unexpected modifications.
1283 */
test_membarrier_manager_thread(void * arg)1284 void *test_membarrier_manager_thread(void *arg)
1285 {
1286 struct test_membarrier_thread_args *args =
1287 (struct test_membarrier_thread_args *)arg;
1288 struct percpu_list list_a, list_b;
1289 intptr_t expect_a = 0, expect_b = 0;
1290 int cpu_a = 0, cpu_b = 0;
1291
1292 if (rseq_register_current_thread()) {
1293 fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
1294 errno, strerror(errno));
1295 abort();
1296 }
1297
1298 /* Init lists. */
1299 test_membarrier_init_percpu_list(&list_a);
1300 test_membarrier_init_percpu_list(&list_b);
1301
1302 atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
1303
1304 while (!atomic_load(&args->stop)) {
1305 /* list_a is "active". */
1306 cpu_a = rand() % CPU_SETSIZE;
1307 /*
1308 * As list_b is "inactive", we should never see changes
1309 * to list_b.
1310 */
1311 if (expect_b != atomic_load(&list_b.c[cpu_b].head->data)) {
1312 fprintf(stderr, "Membarrier test failed\n");
1313 abort();
1314 }
1315
1316 /* Make list_b "active". */
1317 atomic_store(&args->percpu_list_ptr, (intptr_t)&list_b);
1318 if (rseq_membarrier_expedited(cpu_a) &&
1319 errno != ENXIO /* missing CPU */) {
1320 perror("sys_membarrier");
1321 abort();
1322 }
1323 /*
1324 * Cpu A should now only modify list_b, so the values
1325 * in list_a should be stable.
1326 */
1327 expect_a = atomic_load(&list_a.c[cpu_a].head->data);
1328
1329 cpu_b = rand() % CPU_SETSIZE;
1330 /*
1331 * As list_a is "inactive", we should never see changes
1332 * to list_a.
1333 */
1334 if (expect_a != atomic_load(&list_a.c[cpu_a].head->data)) {
1335 fprintf(stderr, "Membarrier test failed\n");
1336 abort();
1337 }
1338
1339 /* Make list_a "active". */
1340 atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
1341 if (rseq_membarrier_expedited(cpu_b) &&
1342 errno != ENXIO /* missing CPU*/) {
1343 perror("sys_membarrier");
1344 abort();
1345 }
1346 /* Remember a value from list_b. */
1347 expect_b = atomic_load(&list_b.c[cpu_b].head->data);
1348 }
1349
1350 test_membarrier_free_percpu_list(&list_a);
1351 test_membarrier_free_percpu_list(&list_b);
1352
1353 if (rseq_unregister_current_thread()) {
1354 fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
1355 errno, strerror(errno));
1356 abort();
1357 }
1358 return NULL;
1359 }
1360
test_membarrier(void)1361 void test_membarrier(void)
1362 {
1363 const int num_threads = opt_threads;
1364 struct test_membarrier_thread_args thread_args;
1365 pthread_t worker_threads[num_threads];
1366 pthread_t manager_thread;
1367 int i, ret;
1368
1369 if (sys_membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ, 0, 0)) {
1370 perror("sys_membarrier");
1371 abort();
1372 }
1373
1374 thread_args.stop = 0;
1375 thread_args.percpu_list_ptr = 0;
1376 ret = pthread_create(&manager_thread, NULL,
1377 test_membarrier_manager_thread, &thread_args);
1378 if (ret) {
1379 errno = ret;
1380 perror("pthread_create");
1381 abort();
1382 }
1383
1384 for (i = 0; i < num_threads; i++) {
1385 ret = pthread_create(&worker_threads[i], NULL,
1386 test_membarrier_worker_thread, &thread_args);
1387 if (ret) {
1388 errno = ret;
1389 perror("pthread_create");
1390 abort();
1391 }
1392 }
1393
1394
1395 for (i = 0; i < num_threads; i++) {
1396 ret = pthread_join(worker_threads[i], NULL);
1397 if (ret) {
1398 errno = ret;
1399 perror("pthread_join");
1400 abort();
1401 }
1402 }
1403
1404 atomic_store(&thread_args.stop, 1);
1405 ret = pthread_join(manager_thread, NULL);
1406 if (ret) {
1407 errno = ret;
1408 perror("pthread_join");
1409 abort();
1410 }
1411 }
1412 #else /* TEST_MEMBARRIER */
test_membarrier(void)1413 void test_membarrier(void)
1414 {
1415 fprintf(stderr, "rseq_offset_deref_addv is not implemented on this architecture. "
1416 "Skipping membarrier test.\n");
1417 }
1418 #endif
1419
show_usage(int argc,char ** argv)1420 static void show_usage(int argc, char **argv)
1421 {
1422 printf("Usage : %s <OPTIONS>\n",
1423 argv[0]);
1424 printf("OPTIONS:\n");
1425 printf(" [-1 loops] Number of loops for delay injection 1\n");
1426 printf(" [-2 loops] Number of loops for delay injection 2\n");
1427 printf(" [-3 loops] Number of loops for delay injection 3\n");
1428 printf(" [-4 loops] Number of loops for delay injection 4\n");
1429 printf(" [-5 loops] Number of loops for delay injection 5\n");
1430 printf(" [-6 loops] Number of loops for delay injection 6\n");
1431 printf(" [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n");
1432 printf(" [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n");
1433 printf(" [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n");
1434 printf(" [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n");
1435 printf(" [-y] Yield\n");
1436 printf(" [-k] Kill thread with signal\n");
1437 printf(" [-s S] S: =0: disabled (default), >0: sleep time (ms)\n");
1438 printf(" [-t N] Number of threads (default 200)\n");
1439 printf(" [-r N] Number of repetitions per thread (default 5000)\n");
1440 printf(" [-d] Disable rseq system call (no initialization)\n");
1441 printf(" [-D M] Disable rseq for each M threads\n");
1442 printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement, membarrie(r)\n");
1443 printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n");
1444 printf(" [-v] Verbose output.\n");
1445 printf(" [-h] Show this help.\n");
1446 printf("\n");
1447 }
1448
main(int argc,char ** argv)1449 int main(int argc, char **argv)
1450 {
1451 int i;
1452
1453 for (i = 1; i < argc; i++) {
1454 if (argv[i][0] != '-')
1455 continue;
1456 switch (argv[i][1]) {
1457 case '1':
1458 case '2':
1459 case '3':
1460 case '4':
1461 case '5':
1462 case '6':
1463 case '7':
1464 case '8':
1465 case '9':
1466 if (argc < i + 2) {
1467 show_usage(argc, argv);
1468 goto error;
1469 }
1470 loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]);
1471 i++;
1472 break;
1473 case 'm':
1474 if (argc < i + 2) {
1475 show_usage(argc, argv);
1476 goto error;
1477 }
1478 opt_modulo = atol(argv[i + 1]);
1479 if (opt_modulo < 0) {
1480 show_usage(argc, argv);
1481 goto error;
1482 }
1483 i++;
1484 break;
1485 case 's':
1486 if (argc < i + 2) {
1487 show_usage(argc, argv);
1488 goto error;
1489 }
1490 opt_sleep = atol(argv[i + 1]);
1491 if (opt_sleep < 0) {
1492 show_usage(argc, argv);
1493 goto error;
1494 }
1495 i++;
1496 break;
1497 case 'y':
1498 opt_yield = 1;
1499 break;
1500 case 'k':
1501 opt_signal = 1;
1502 break;
1503 case 'd':
1504 opt_disable_rseq = 1;
1505 break;
1506 case 'D':
1507 if (argc < i + 2) {
1508 show_usage(argc, argv);
1509 goto error;
1510 }
1511 opt_disable_mod = atol(argv[i + 1]);
1512 if (opt_disable_mod < 0) {
1513 show_usage(argc, argv);
1514 goto error;
1515 }
1516 i++;
1517 break;
1518 case 't':
1519 if (argc < i + 2) {
1520 show_usage(argc, argv);
1521 goto error;
1522 }
1523 opt_threads = atol(argv[i + 1]);
1524 if (opt_threads < 0) {
1525 show_usage(argc, argv);
1526 goto error;
1527 }
1528 i++;
1529 break;
1530 case 'r':
1531 if (argc < i + 2) {
1532 show_usage(argc, argv);
1533 goto error;
1534 }
1535 opt_reps = atoll(argv[i + 1]);
1536 if (opt_reps < 0) {
1537 show_usage(argc, argv);
1538 goto error;
1539 }
1540 i++;
1541 break;
1542 case 'h':
1543 show_usage(argc, argv);
1544 goto end;
1545 case 'T':
1546 if (argc < i + 2) {
1547 show_usage(argc, argv);
1548 goto error;
1549 }
1550 opt_test = *argv[i + 1];
1551 switch (opt_test) {
1552 case 's':
1553 case 'l':
1554 case 'i':
1555 case 'b':
1556 case 'm':
1557 case 'r':
1558 break;
1559 default:
1560 show_usage(argc, argv);
1561 goto error;
1562 }
1563 i++;
1564 break;
1565 case 'v':
1566 verbose = 1;
1567 break;
1568 case 'M':
1569 opt_mo = RSEQ_MO_RELEASE;
1570 break;
1571 default:
1572 show_usage(argc, argv);
1573 goto error;
1574 }
1575 }
1576
1577 loop_cnt_1 = loop_cnt[1];
1578 loop_cnt_2 = loop_cnt[2];
1579 loop_cnt_3 = loop_cnt[3];
1580 loop_cnt_4 = loop_cnt[4];
1581 loop_cnt_5 = loop_cnt[5];
1582 loop_cnt_6 = loop_cnt[6];
1583
1584 if (set_signal_handler())
1585 goto error;
1586
1587 if (!opt_disable_rseq && rseq_register_current_thread())
1588 goto error;
1589 if (!opt_disable_rseq && !rseq_validate_cpu_id()) {
1590 fprintf(stderr, "Error: cpu id getter unavailable\n");
1591 goto error;
1592 }
1593 switch (opt_test) {
1594 case 's':
1595 printf_verbose("spinlock\n");
1596 test_percpu_spinlock();
1597 break;
1598 case 'l':
1599 printf_verbose("linked list\n");
1600 test_percpu_list();
1601 break;
1602 case 'b':
1603 printf_verbose("buffer\n");
1604 test_percpu_buffer();
1605 break;
1606 case 'm':
1607 printf_verbose("memcpy buffer\n");
1608 test_percpu_memcpy_buffer();
1609 break;
1610 case 'i':
1611 printf_verbose("counter increment\n");
1612 test_percpu_inc();
1613 break;
1614 case 'r':
1615 printf_verbose("membarrier\n");
1616 test_membarrier();
1617 break;
1618 }
1619 if (!opt_disable_rseq && rseq_unregister_current_thread())
1620 abort();
1621 end:
1622 return 0;
1623
1624 error:
1625 return -1;
1626 }
1627