1 // SPDX-License-Identifier: LGPL-2.1
2 #define _GNU_SOURCE
3 #include <assert.h>
4 #include <pthread.h>
5 #include <sched.h>
6 #include <stdint.h>
7 #include <stdio.h>
8 #include <stdlib.h>
9 #include <string.h>
10 #include <syscall.h>
11 #include <unistd.h>
12 #include <poll.h>
13 #include <sys/types.h>
14 #include <signal.h>
15 #include <errno.h>
16 #include <stddef.h>
17 
18 static inline pid_t gettid(void)
19 {
20 	return syscall(__NR_gettid);
21 }
22 
23 #define NR_INJECT	9
24 static int loop_cnt[NR_INJECT + 1];
25 
26 static int loop_cnt_1 asm("asm_loop_cnt_1") __attribute__((used));
27 static int loop_cnt_2 asm("asm_loop_cnt_2") __attribute__((used));
28 static int loop_cnt_3 asm("asm_loop_cnt_3") __attribute__((used));
29 static int loop_cnt_4 asm("asm_loop_cnt_4") __attribute__((used));
30 static int loop_cnt_5 asm("asm_loop_cnt_5") __attribute__((used));
31 static int loop_cnt_6 asm("asm_loop_cnt_6") __attribute__((used));
32 
33 static int opt_modulo, verbose;
34 
35 static int opt_yield, opt_signal, opt_sleep,
36 		opt_disable_rseq, opt_threads = 200,
37 		opt_disable_mod = 0, opt_test = 's', opt_mb = 0;
38 
39 #ifndef RSEQ_SKIP_FASTPATH
40 static long long opt_reps = 5000;
41 #else
42 static long long opt_reps = 100;
43 #endif
44 
45 static __thread __attribute__((tls_model("initial-exec")))
46 unsigned int signals_delivered;
47 
48 #ifndef BENCHMARK
49 
50 static __thread __attribute__((tls_model("initial-exec"), unused))
51 unsigned int yield_mod_cnt, nr_abort;
52 
53 #define printf_verbose(fmt, ...)			\
54 	do {						\
55 		if (verbose)				\
56 			printf(fmt, ## __VA_ARGS__);	\
57 	} while (0)
58 
59 #if defined(__x86_64__) || defined(__i386__)
60 
61 #define INJECT_ASM_REG	"eax"
62 
63 #define RSEQ_INJECT_CLOBBER \
64 	, INJECT_ASM_REG
65 
66 #ifdef __i386__
67 
68 #define RSEQ_INJECT_ASM(n) \
69 	"mov asm_loop_cnt_" #n ", %%" INJECT_ASM_REG "\n\t" \
70 	"test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
71 	"jz 333f\n\t" \
72 	"222:\n\t" \
73 	"dec %%" INJECT_ASM_REG "\n\t" \
74 	"jnz 222b\n\t" \
75 	"333:\n\t"
76 
77 #elif defined(__x86_64__)
78 
79 #define RSEQ_INJECT_ASM(n) \
80 	"lea asm_loop_cnt_" #n "(%%rip), %%" INJECT_ASM_REG "\n\t" \
81 	"mov (%%" INJECT_ASM_REG "), %%" INJECT_ASM_REG "\n\t" \
82 	"test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
83 	"jz 333f\n\t" \
84 	"222:\n\t" \
85 	"dec %%" INJECT_ASM_REG "\n\t" \
86 	"jnz 222b\n\t" \
87 	"333:\n\t"
88 
89 #else
90 #error "Unsupported architecture"
91 #endif
92 
93 #elif defined(__ARMEL__)
94 
95 #define RSEQ_INJECT_INPUT \
96 	, [loop_cnt_1]"m"(loop_cnt[1]) \
97 	, [loop_cnt_2]"m"(loop_cnt[2]) \
98 	, [loop_cnt_3]"m"(loop_cnt[3]) \
99 	, [loop_cnt_4]"m"(loop_cnt[4]) \
100 	, [loop_cnt_5]"m"(loop_cnt[5]) \
101 	, [loop_cnt_6]"m"(loop_cnt[6])
102 
103 #define INJECT_ASM_REG	"r4"
104 
105 #define RSEQ_INJECT_CLOBBER \
106 	, INJECT_ASM_REG
107 
108 #define RSEQ_INJECT_ASM(n) \
109 	"ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
110 	"cmp " INJECT_ASM_REG ", #0\n\t" \
111 	"beq 333f\n\t" \
112 	"222:\n\t" \
113 	"subs " INJECT_ASM_REG ", #1\n\t" \
114 	"bne 222b\n\t" \
115 	"333:\n\t"
116 
117 #elif __PPC__
118 
119 #define RSEQ_INJECT_INPUT \
120 	, [loop_cnt_1]"m"(loop_cnt[1]) \
121 	, [loop_cnt_2]"m"(loop_cnt[2]) \
122 	, [loop_cnt_3]"m"(loop_cnt[3]) \
123 	, [loop_cnt_4]"m"(loop_cnt[4]) \
124 	, [loop_cnt_5]"m"(loop_cnt[5]) \
125 	, [loop_cnt_6]"m"(loop_cnt[6])
126 
127 #define INJECT_ASM_REG	"r18"
128 
129 #define RSEQ_INJECT_CLOBBER \
130 	, INJECT_ASM_REG
131 
132 #define RSEQ_INJECT_ASM(n) \
133 	"lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
134 	"cmpwi %%" INJECT_ASM_REG ", 0\n\t" \
135 	"beq 333f\n\t" \
136 	"222:\n\t" \
137 	"subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \
138 	"bne 222b\n\t" \
139 	"333:\n\t"
140 
141 #elif defined(__mips__)
142 
143 #define RSEQ_INJECT_INPUT \
144 	, [loop_cnt_1]"m"(loop_cnt[1]) \
145 	, [loop_cnt_2]"m"(loop_cnt[2]) \
146 	, [loop_cnt_3]"m"(loop_cnt[3]) \
147 	, [loop_cnt_4]"m"(loop_cnt[4]) \
148 	, [loop_cnt_5]"m"(loop_cnt[5]) \
149 	, [loop_cnt_6]"m"(loop_cnt[6])
150 
151 #define INJECT_ASM_REG	"$5"
152 
153 #define RSEQ_INJECT_CLOBBER \
154 	, INJECT_ASM_REG
155 
156 #define RSEQ_INJECT_ASM(n) \
157 	"lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
158 	"beqz " INJECT_ASM_REG ", 333f\n\t" \
159 	"222:\n\t" \
160 	"addiu " INJECT_ASM_REG ", -1\n\t" \
161 	"bnez " INJECT_ASM_REG ", 222b\n\t" \
162 	"333:\n\t"
163 
164 #else
165 #error unsupported target
166 #endif
167 
168 #define RSEQ_INJECT_FAILED \
169 	nr_abort++;
170 
171 #define RSEQ_INJECT_C(n) \
172 { \
173 	int loc_i, loc_nr_loops = loop_cnt[n]; \
174 	\
175 	for (loc_i = 0; loc_i < loc_nr_loops; loc_i++) { \
176 		rseq_barrier(); \
177 	} \
178 	if (loc_nr_loops == -1 && opt_modulo) { \
179 		if (yield_mod_cnt == opt_modulo - 1) { \
180 			if (opt_sleep > 0) \
181 				poll(NULL, 0, opt_sleep); \
182 			if (opt_yield) \
183 				sched_yield(); \
184 			if (opt_signal) \
185 				raise(SIGUSR1); \
186 			yield_mod_cnt = 0; \
187 		} else { \
188 			yield_mod_cnt++; \
189 		} \
190 	} \
191 }
192 
193 #else
194 
195 #define printf_verbose(fmt, ...)
196 
197 #endif /* BENCHMARK */
198 
199 #include "rseq.h"
200 
201 struct percpu_lock_entry {
202 	intptr_t v;
203 } __attribute__((aligned(128)));
204 
205 struct percpu_lock {
206 	struct percpu_lock_entry c[CPU_SETSIZE];
207 };
208 
209 struct test_data_entry {
210 	intptr_t count;
211 } __attribute__((aligned(128)));
212 
213 struct spinlock_test_data {
214 	struct percpu_lock lock;
215 	struct test_data_entry c[CPU_SETSIZE];
216 };
217 
218 struct spinlock_thread_test_data {
219 	struct spinlock_test_data *data;
220 	long long reps;
221 	int reg;
222 };
223 
224 struct inc_test_data {
225 	struct test_data_entry c[CPU_SETSIZE];
226 };
227 
228 struct inc_thread_test_data {
229 	struct inc_test_data *data;
230 	long long reps;
231 	int reg;
232 };
233 
234 struct percpu_list_node {
235 	intptr_t data;
236 	struct percpu_list_node *next;
237 };
238 
239 struct percpu_list_entry {
240 	struct percpu_list_node *head;
241 } __attribute__((aligned(128)));
242 
243 struct percpu_list {
244 	struct percpu_list_entry c[CPU_SETSIZE];
245 };
246 
247 #define BUFFER_ITEM_PER_CPU	100
248 
249 struct percpu_buffer_node {
250 	intptr_t data;
251 };
252 
253 struct percpu_buffer_entry {
254 	intptr_t offset;
255 	intptr_t buflen;
256 	struct percpu_buffer_node **array;
257 } __attribute__((aligned(128)));
258 
259 struct percpu_buffer {
260 	struct percpu_buffer_entry c[CPU_SETSIZE];
261 };
262 
263 #define MEMCPY_BUFFER_ITEM_PER_CPU	100
264 
265 struct percpu_memcpy_buffer_node {
266 	intptr_t data1;
267 	uint64_t data2;
268 };
269 
270 struct percpu_memcpy_buffer_entry {
271 	intptr_t offset;
272 	intptr_t buflen;
273 	struct percpu_memcpy_buffer_node *array;
274 } __attribute__((aligned(128)));
275 
276 struct percpu_memcpy_buffer {
277 	struct percpu_memcpy_buffer_entry c[CPU_SETSIZE];
278 };
279 
280 /* A simple percpu spinlock. Grabs lock on current cpu. */
281 static int rseq_this_cpu_lock(struct percpu_lock *lock)
282 {
283 	int cpu;
284 
285 	for (;;) {
286 		int ret;
287 
288 		cpu = rseq_cpu_start();
289 		ret = rseq_cmpeqv_storev(&lock->c[cpu].v,
290 					 0, 1, cpu);
291 		if (rseq_likely(!ret))
292 			break;
293 		/* Retry if comparison fails or rseq aborts. */
294 	}
295 	/*
296 	 * Acquire semantic when taking lock after control dependency.
297 	 * Matches rseq_smp_store_release().
298 	 */
299 	rseq_smp_acquire__after_ctrl_dep();
300 	return cpu;
301 }
302 
303 static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu)
304 {
305 	assert(lock->c[cpu].v == 1);
306 	/*
307 	 * Release lock, with release semantic. Matches
308 	 * rseq_smp_acquire__after_ctrl_dep().
309 	 */
310 	rseq_smp_store_release(&lock->c[cpu].v, 0);
311 }
312 
313 void *test_percpu_spinlock_thread(void *arg)
314 {
315 	struct spinlock_thread_test_data *thread_data = arg;
316 	struct spinlock_test_data *data = thread_data->data;
317 	long long i, reps;
318 
319 	if (!opt_disable_rseq && thread_data->reg &&
320 	    rseq_register_current_thread())
321 		abort();
322 	reps = thread_data->reps;
323 	for (i = 0; i < reps; i++) {
324 		int cpu = rseq_cpu_start();
325 
326 		cpu = rseq_this_cpu_lock(&data->lock);
327 		data->c[cpu].count++;
328 		rseq_percpu_unlock(&data->lock, cpu);
329 #ifndef BENCHMARK
330 		if (i != 0 && !(i % (reps / 10)))
331 			printf_verbose("tid %d: count %lld\n", (int) gettid(), i);
332 #endif
333 	}
334 	printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
335 		       (int) gettid(), nr_abort, signals_delivered);
336 	if (!opt_disable_rseq && thread_data->reg &&
337 	    rseq_unregister_current_thread())
338 		abort();
339 	return NULL;
340 }
341 
342 /*
343  * A simple test which implements a sharded counter using a per-cpu
344  * lock.  Obviously real applications might prefer to simply use a
345  * per-cpu increment; however, this is reasonable for a test and the
346  * lock can be extended to synchronize more complicated operations.
347  */
348 void test_percpu_spinlock(void)
349 {
350 	const int num_threads = opt_threads;
351 	int i, ret;
352 	uint64_t sum;
353 	pthread_t test_threads[num_threads];
354 	struct spinlock_test_data data;
355 	struct spinlock_thread_test_data thread_data[num_threads];
356 
357 	memset(&data, 0, sizeof(data));
358 	for (i = 0; i < num_threads; i++) {
359 		thread_data[i].reps = opt_reps;
360 		if (opt_disable_mod <= 0 || (i % opt_disable_mod))
361 			thread_data[i].reg = 1;
362 		else
363 			thread_data[i].reg = 0;
364 		thread_data[i].data = &data;
365 		ret = pthread_create(&test_threads[i], NULL,
366 				     test_percpu_spinlock_thread,
367 				     &thread_data[i]);
368 		if (ret) {
369 			errno = ret;
370 			perror("pthread_create");
371 			abort();
372 		}
373 	}
374 
375 	for (i = 0; i < num_threads; i++) {
376 		ret = pthread_join(test_threads[i], NULL);
377 		if (ret) {
378 			errno = ret;
379 			perror("pthread_join");
380 			abort();
381 		}
382 	}
383 
384 	sum = 0;
385 	for (i = 0; i < CPU_SETSIZE; i++)
386 		sum += data.c[i].count;
387 
388 	assert(sum == (uint64_t)opt_reps * num_threads);
389 }
390 
391 void *test_percpu_inc_thread(void *arg)
392 {
393 	struct inc_thread_test_data *thread_data = arg;
394 	struct inc_test_data *data = thread_data->data;
395 	long long i, reps;
396 
397 	if (!opt_disable_rseq && thread_data->reg &&
398 	    rseq_register_current_thread())
399 		abort();
400 	reps = thread_data->reps;
401 	for (i = 0; i < reps; i++) {
402 		int ret;
403 
404 		do {
405 			int cpu;
406 
407 			cpu = rseq_cpu_start();
408 			ret = rseq_addv(&data->c[cpu].count, 1, cpu);
409 		} while (rseq_unlikely(ret));
410 #ifndef BENCHMARK
411 		if (i != 0 && !(i % (reps / 10)))
412 			printf_verbose("tid %d: count %lld\n", (int) gettid(), i);
413 #endif
414 	}
415 	printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
416 		       (int) gettid(), nr_abort, signals_delivered);
417 	if (!opt_disable_rseq && thread_data->reg &&
418 	    rseq_unregister_current_thread())
419 		abort();
420 	return NULL;
421 }
422 
423 void test_percpu_inc(void)
424 {
425 	const int num_threads = opt_threads;
426 	int i, ret;
427 	uint64_t sum;
428 	pthread_t test_threads[num_threads];
429 	struct inc_test_data data;
430 	struct inc_thread_test_data thread_data[num_threads];
431 
432 	memset(&data, 0, sizeof(data));
433 	for (i = 0; i < num_threads; i++) {
434 		thread_data[i].reps = opt_reps;
435 		if (opt_disable_mod <= 0 || (i % opt_disable_mod))
436 			thread_data[i].reg = 1;
437 		else
438 			thread_data[i].reg = 0;
439 		thread_data[i].data = &data;
440 		ret = pthread_create(&test_threads[i], NULL,
441 				     test_percpu_inc_thread,
442 				     &thread_data[i]);
443 		if (ret) {
444 			errno = ret;
445 			perror("pthread_create");
446 			abort();
447 		}
448 	}
449 
450 	for (i = 0; i < num_threads; i++) {
451 		ret = pthread_join(test_threads[i], NULL);
452 		if (ret) {
453 			errno = ret;
454 			perror("pthread_join");
455 			abort();
456 		}
457 	}
458 
459 	sum = 0;
460 	for (i = 0; i < CPU_SETSIZE; i++)
461 		sum += data.c[i].count;
462 
463 	assert(sum == (uint64_t)opt_reps * num_threads);
464 }
465 
466 void this_cpu_list_push(struct percpu_list *list,
467 			struct percpu_list_node *node,
468 			int *_cpu)
469 {
470 	int cpu;
471 
472 	for (;;) {
473 		intptr_t *targetptr, newval, expect;
474 		int ret;
475 
476 		cpu = rseq_cpu_start();
477 		/* Load list->c[cpu].head with single-copy atomicity. */
478 		expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head);
479 		newval = (intptr_t)node;
480 		targetptr = (intptr_t *)&list->c[cpu].head;
481 		node->next = (struct percpu_list_node *)expect;
482 		ret = rseq_cmpeqv_storev(targetptr, expect, newval, cpu);
483 		if (rseq_likely(!ret))
484 			break;
485 		/* Retry if comparison fails or rseq aborts. */
486 	}
487 	if (_cpu)
488 		*_cpu = cpu;
489 }
490 
491 /*
492  * Unlike a traditional lock-less linked list; the availability of a
493  * rseq primitive allows us to implement pop without concerns over
494  * ABA-type races.
495  */
496 struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list,
497 					   int *_cpu)
498 {
499 	struct percpu_list_node *node = NULL;
500 	int cpu;
501 
502 	for (;;) {
503 		struct percpu_list_node *head;
504 		intptr_t *targetptr, expectnot, *load;
505 		off_t offset;
506 		int ret;
507 
508 		cpu = rseq_cpu_start();
509 		targetptr = (intptr_t *)&list->c[cpu].head;
510 		expectnot = (intptr_t)NULL;
511 		offset = offsetof(struct percpu_list_node, next);
512 		load = (intptr_t *)&head;
513 		ret = rseq_cmpnev_storeoffp_load(targetptr, expectnot,
514 						   offset, load, cpu);
515 		if (rseq_likely(!ret)) {
516 			node = head;
517 			break;
518 		}
519 		if (ret > 0)
520 			break;
521 		/* Retry if rseq aborts. */
522 	}
523 	if (_cpu)
524 		*_cpu = cpu;
525 	return node;
526 }
527 
528 /*
529  * __percpu_list_pop is not safe against concurrent accesses. Should
530  * only be used on lists that are not concurrently modified.
531  */
532 struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu)
533 {
534 	struct percpu_list_node *node;
535 
536 	node = list->c[cpu].head;
537 	if (!node)
538 		return NULL;
539 	list->c[cpu].head = node->next;
540 	return node;
541 }
542 
543 void *test_percpu_list_thread(void *arg)
544 {
545 	long long i, reps;
546 	struct percpu_list *list = (struct percpu_list *)arg;
547 
548 	if (!opt_disable_rseq && rseq_register_current_thread())
549 		abort();
550 
551 	reps = opt_reps;
552 	for (i = 0; i < reps; i++) {
553 		struct percpu_list_node *node;
554 
555 		node = this_cpu_list_pop(list, NULL);
556 		if (opt_yield)
557 			sched_yield();  /* encourage shuffling */
558 		if (node)
559 			this_cpu_list_push(list, node, NULL);
560 	}
561 
562 	printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
563 		       (int) gettid(), nr_abort, signals_delivered);
564 	if (!opt_disable_rseq && rseq_unregister_current_thread())
565 		abort();
566 
567 	return NULL;
568 }
569 
570 /* Simultaneous modification to a per-cpu linked list from many threads.  */
571 void test_percpu_list(void)
572 {
573 	const int num_threads = opt_threads;
574 	int i, j, ret;
575 	uint64_t sum = 0, expected_sum = 0;
576 	struct percpu_list list;
577 	pthread_t test_threads[num_threads];
578 	cpu_set_t allowed_cpus;
579 
580 	memset(&list, 0, sizeof(list));
581 
582 	/* Generate list entries for every usable cpu. */
583 	sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
584 	for (i = 0; i < CPU_SETSIZE; i++) {
585 		if (!CPU_ISSET(i, &allowed_cpus))
586 			continue;
587 		for (j = 1; j <= 100; j++) {
588 			struct percpu_list_node *node;
589 
590 			expected_sum += j;
591 
592 			node = malloc(sizeof(*node));
593 			assert(node);
594 			node->data = j;
595 			node->next = list.c[i].head;
596 			list.c[i].head = node;
597 		}
598 	}
599 
600 	for (i = 0; i < num_threads; i++) {
601 		ret = pthread_create(&test_threads[i], NULL,
602 				     test_percpu_list_thread, &list);
603 		if (ret) {
604 			errno = ret;
605 			perror("pthread_create");
606 			abort();
607 		}
608 	}
609 
610 	for (i = 0; i < num_threads; i++) {
611 		ret = pthread_join(test_threads[i], NULL);
612 		if (ret) {
613 			errno = ret;
614 			perror("pthread_join");
615 			abort();
616 		}
617 	}
618 
619 	for (i = 0; i < CPU_SETSIZE; i++) {
620 		struct percpu_list_node *node;
621 
622 		if (!CPU_ISSET(i, &allowed_cpus))
623 			continue;
624 
625 		while ((node = __percpu_list_pop(&list, i))) {
626 			sum += node->data;
627 			free(node);
628 		}
629 	}
630 
631 	/*
632 	 * All entries should now be accounted for (unless some external
633 	 * actor is interfering with our allowed affinity while this
634 	 * test is running).
635 	 */
636 	assert(sum == expected_sum);
637 }
638 
639 bool this_cpu_buffer_push(struct percpu_buffer *buffer,
640 			  struct percpu_buffer_node *node,
641 			  int *_cpu)
642 {
643 	bool result = false;
644 	int cpu;
645 
646 	for (;;) {
647 		intptr_t *targetptr_spec, newval_spec;
648 		intptr_t *targetptr_final, newval_final;
649 		intptr_t offset;
650 		int ret;
651 
652 		cpu = rseq_cpu_start();
653 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
654 		if (offset == buffer->c[cpu].buflen)
655 			break;
656 		newval_spec = (intptr_t)node;
657 		targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset];
658 		newval_final = offset + 1;
659 		targetptr_final = &buffer->c[cpu].offset;
660 		if (opt_mb)
661 			ret = rseq_cmpeqv_trystorev_storev_release(
662 				targetptr_final, offset, targetptr_spec,
663 				newval_spec, newval_final, cpu);
664 		else
665 			ret = rseq_cmpeqv_trystorev_storev(targetptr_final,
666 				offset, targetptr_spec, newval_spec,
667 				newval_final, cpu);
668 		if (rseq_likely(!ret)) {
669 			result = true;
670 			break;
671 		}
672 		/* Retry if comparison fails or rseq aborts. */
673 	}
674 	if (_cpu)
675 		*_cpu = cpu;
676 	return result;
677 }
678 
679 struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer,
680 					       int *_cpu)
681 {
682 	struct percpu_buffer_node *head;
683 	int cpu;
684 
685 	for (;;) {
686 		intptr_t *targetptr, newval;
687 		intptr_t offset;
688 		int ret;
689 
690 		cpu = rseq_cpu_start();
691 		/* Load offset with single-copy atomicity. */
692 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
693 		if (offset == 0) {
694 			head = NULL;
695 			break;
696 		}
697 		head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]);
698 		newval = offset - 1;
699 		targetptr = (intptr_t *)&buffer->c[cpu].offset;
700 		ret = rseq_cmpeqv_cmpeqv_storev(targetptr, offset,
701 			(intptr_t *)&buffer->c[cpu].array[offset - 1],
702 			(intptr_t)head, newval, cpu);
703 		if (rseq_likely(!ret))
704 			break;
705 		/* Retry if comparison fails or rseq aborts. */
706 	}
707 	if (_cpu)
708 		*_cpu = cpu;
709 	return head;
710 }
711 
712 /*
713  * __percpu_buffer_pop is not safe against concurrent accesses. Should
714  * only be used on buffers that are not concurrently modified.
715  */
716 struct percpu_buffer_node *__percpu_buffer_pop(struct percpu_buffer *buffer,
717 					       int cpu)
718 {
719 	struct percpu_buffer_node *head;
720 	intptr_t offset;
721 
722 	offset = buffer->c[cpu].offset;
723 	if (offset == 0)
724 		return NULL;
725 	head = buffer->c[cpu].array[offset - 1];
726 	buffer->c[cpu].offset = offset - 1;
727 	return head;
728 }
729 
730 void *test_percpu_buffer_thread(void *arg)
731 {
732 	long long i, reps;
733 	struct percpu_buffer *buffer = (struct percpu_buffer *)arg;
734 
735 	if (!opt_disable_rseq && rseq_register_current_thread())
736 		abort();
737 
738 	reps = opt_reps;
739 	for (i = 0; i < reps; i++) {
740 		struct percpu_buffer_node *node;
741 
742 		node = this_cpu_buffer_pop(buffer, NULL);
743 		if (opt_yield)
744 			sched_yield();  /* encourage shuffling */
745 		if (node) {
746 			if (!this_cpu_buffer_push(buffer, node, NULL)) {
747 				/* Should increase buffer size. */
748 				abort();
749 			}
750 		}
751 	}
752 
753 	printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
754 		       (int) gettid(), nr_abort, signals_delivered);
755 	if (!opt_disable_rseq && rseq_unregister_current_thread())
756 		abort();
757 
758 	return NULL;
759 }
760 
761 /* Simultaneous modification to a per-cpu buffer from many threads.  */
762 void test_percpu_buffer(void)
763 {
764 	const int num_threads = opt_threads;
765 	int i, j, ret;
766 	uint64_t sum = 0, expected_sum = 0;
767 	struct percpu_buffer buffer;
768 	pthread_t test_threads[num_threads];
769 	cpu_set_t allowed_cpus;
770 
771 	memset(&buffer, 0, sizeof(buffer));
772 
773 	/* Generate list entries for every usable cpu. */
774 	sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
775 	for (i = 0; i < CPU_SETSIZE; i++) {
776 		if (!CPU_ISSET(i, &allowed_cpus))
777 			continue;
778 		/* Worse-case is every item in same CPU. */
779 		buffer.c[i].array =
780 			malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *
781 			       BUFFER_ITEM_PER_CPU);
782 		assert(buffer.c[i].array);
783 		buffer.c[i].buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU;
784 		for (j = 1; j <= BUFFER_ITEM_PER_CPU; j++) {
785 			struct percpu_buffer_node *node;
786 
787 			expected_sum += j;
788 
789 			/*
790 			 * We could theoretically put the word-sized
791 			 * "data" directly in the buffer. However, we
792 			 * want to model objects that would not fit
793 			 * within a single word, so allocate an object
794 			 * for each node.
795 			 */
796 			node = malloc(sizeof(*node));
797 			assert(node);
798 			node->data = j;
799 			buffer.c[i].array[j - 1] = node;
800 			buffer.c[i].offset++;
801 		}
802 	}
803 
804 	for (i = 0; i < num_threads; i++) {
805 		ret = pthread_create(&test_threads[i], NULL,
806 				     test_percpu_buffer_thread, &buffer);
807 		if (ret) {
808 			errno = ret;
809 			perror("pthread_create");
810 			abort();
811 		}
812 	}
813 
814 	for (i = 0; i < num_threads; i++) {
815 		ret = pthread_join(test_threads[i], NULL);
816 		if (ret) {
817 			errno = ret;
818 			perror("pthread_join");
819 			abort();
820 		}
821 	}
822 
823 	for (i = 0; i < CPU_SETSIZE; i++) {
824 		struct percpu_buffer_node *node;
825 
826 		if (!CPU_ISSET(i, &allowed_cpus))
827 			continue;
828 
829 		while ((node = __percpu_buffer_pop(&buffer, i))) {
830 			sum += node->data;
831 			free(node);
832 		}
833 		free(buffer.c[i].array);
834 	}
835 
836 	/*
837 	 * All entries should now be accounted for (unless some external
838 	 * actor is interfering with our allowed affinity while this
839 	 * test is running).
840 	 */
841 	assert(sum == expected_sum);
842 }
843 
844 bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer,
845 				 struct percpu_memcpy_buffer_node item,
846 				 int *_cpu)
847 {
848 	bool result = false;
849 	int cpu;
850 
851 	for (;;) {
852 		intptr_t *targetptr_final, newval_final, offset;
853 		char *destptr, *srcptr;
854 		size_t copylen;
855 		int ret;
856 
857 		cpu = rseq_cpu_start();
858 		/* Load offset with single-copy atomicity. */
859 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
860 		if (offset == buffer->c[cpu].buflen)
861 			break;
862 		destptr = (char *)&buffer->c[cpu].array[offset];
863 		srcptr = (char *)&item;
864 		/* copylen must be <= 4kB. */
865 		copylen = sizeof(item);
866 		newval_final = offset + 1;
867 		targetptr_final = &buffer->c[cpu].offset;
868 		if (opt_mb)
869 			ret = rseq_cmpeqv_trymemcpy_storev_release(
870 				targetptr_final, offset,
871 				destptr, srcptr, copylen,
872 				newval_final, cpu);
873 		else
874 			ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final,
875 				offset, destptr, srcptr, copylen,
876 				newval_final, cpu);
877 		if (rseq_likely(!ret)) {
878 			result = true;
879 			break;
880 		}
881 		/* Retry if comparison fails or rseq aborts. */
882 	}
883 	if (_cpu)
884 		*_cpu = cpu;
885 	return result;
886 }
887 
888 bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,
889 				struct percpu_memcpy_buffer_node *item,
890 				int *_cpu)
891 {
892 	bool result = false;
893 	int cpu;
894 
895 	for (;;) {
896 		intptr_t *targetptr_final, newval_final, offset;
897 		char *destptr, *srcptr;
898 		size_t copylen;
899 		int ret;
900 
901 		cpu = rseq_cpu_start();
902 		/* Load offset with single-copy atomicity. */
903 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
904 		if (offset == 0)
905 			break;
906 		destptr = (char *)item;
907 		srcptr = (char *)&buffer->c[cpu].array[offset - 1];
908 		/* copylen must be <= 4kB. */
909 		copylen = sizeof(*item);
910 		newval_final = offset - 1;
911 		targetptr_final = &buffer->c[cpu].offset;
912 		ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final,
913 			offset, destptr, srcptr, copylen,
914 			newval_final, cpu);
915 		if (rseq_likely(!ret)) {
916 			result = true;
917 			break;
918 		}
919 		/* Retry if comparison fails or rseq aborts. */
920 	}
921 	if (_cpu)
922 		*_cpu = cpu;
923 	return result;
924 }
925 
926 /*
927  * __percpu_memcpy_buffer_pop is not safe against concurrent accesses. Should
928  * only be used on buffers that are not concurrently modified.
929  */
930 bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,
931 				struct percpu_memcpy_buffer_node *item,
932 				int cpu)
933 {
934 	intptr_t offset;
935 
936 	offset = buffer->c[cpu].offset;
937 	if (offset == 0)
938 		return false;
939 	memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item));
940 	buffer->c[cpu].offset = offset - 1;
941 	return true;
942 }
943 
944 void *test_percpu_memcpy_buffer_thread(void *arg)
945 {
946 	long long i, reps;
947 	struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg;
948 
949 	if (!opt_disable_rseq && rseq_register_current_thread())
950 		abort();
951 
952 	reps = opt_reps;
953 	for (i = 0; i < reps; i++) {
954 		struct percpu_memcpy_buffer_node item;
955 		bool result;
956 
957 		result = this_cpu_memcpy_buffer_pop(buffer, &item, NULL);
958 		if (opt_yield)
959 			sched_yield();  /* encourage shuffling */
960 		if (result) {
961 			if (!this_cpu_memcpy_buffer_push(buffer, item, NULL)) {
962 				/* Should increase buffer size. */
963 				abort();
964 			}
965 		}
966 	}
967 
968 	printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
969 		       (int) gettid(), nr_abort, signals_delivered);
970 	if (!opt_disable_rseq && rseq_unregister_current_thread())
971 		abort();
972 
973 	return NULL;
974 }
975 
976 /* Simultaneous modification to a per-cpu buffer from many threads.  */
977 void test_percpu_memcpy_buffer(void)
978 {
979 	const int num_threads = opt_threads;
980 	int i, j, ret;
981 	uint64_t sum = 0, expected_sum = 0;
982 	struct percpu_memcpy_buffer buffer;
983 	pthread_t test_threads[num_threads];
984 	cpu_set_t allowed_cpus;
985 
986 	memset(&buffer, 0, sizeof(buffer));
987 
988 	/* Generate list entries for every usable cpu. */
989 	sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
990 	for (i = 0; i < CPU_SETSIZE; i++) {
991 		if (!CPU_ISSET(i, &allowed_cpus))
992 			continue;
993 		/* Worse-case is every item in same CPU. */
994 		buffer.c[i].array =
995 			malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *
996 			       MEMCPY_BUFFER_ITEM_PER_CPU);
997 		assert(buffer.c[i].array);
998 		buffer.c[i].buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU;
999 		for (j = 1; j <= MEMCPY_BUFFER_ITEM_PER_CPU; j++) {
1000 			expected_sum += 2 * j + 1;
1001 
1002 			/*
1003 			 * We could theoretically put the word-sized
1004 			 * "data" directly in the buffer. However, we
1005 			 * want to model objects that would not fit
1006 			 * within a single word, so allocate an object
1007 			 * for each node.
1008 			 */
1009 			buffer.c[i].array[j - 1].data1 = j;
1010 			buffer.c[i].array[j - 1].data2 = j + 1;
1011 			buffer.c[i].offset++;
1012 		}
1013 	}
1014 
1015 	for (i = 0; i < num_threads; i++) {
1016 		ret = pthread_create(&test_threads[i], NULL,
1017 				     test_percpu_memcpy_buffer_thread,
1018 				     &buffer);
1019 		if (ret) {
1020 			errno = ret;
1021 			perror("pthread_create");
1022 			abort();
1023 		}
1024 	}
1025 
1026 	for (i = 0; i < num_threads; i++) {
1027 		ret = pthread_join(test_threads[i], NULL);
1028 		if (ret) {
1029 			errno = ret;
1030 			perror("pthread_join");
1031 			abort();
1032 		}
1033 	}
1034 
1035 	for (i = 0; i < CPU_SETSIZE; i++) {
1036 		struct percpu_memcpy_buffer_node item;
1037 
1038 		if (!CPU_ISSET(i, &allowed_cpus))
1039 			continue;
1040 
1041 		while (__percpu_memcpy_buffer_pop(&buffer, &item, i)) {
1042 			sum += item.data1;
1043 			sum += item.data2;
1044 		}
1045 		free(buffer.c[i].array);
1046 	}
1047 
1048 	/*
1049 	 * All entries should now be accounted for (unless some external
1050 	 * actor is interfering with our allowed affinity while this
1051 	 * test is running).
1052 	 */
1053 	assert(sum == expected_sum);
1054 }
1055 
1056 static void test_signal_interrupt_handler(int signo)
1057 {
1058 	signals_delivered++;
1059 }
1060 
1061 static int set_signal_handler(void)
1062 {
1063 	int ret = 0;
1064 	struct sigaction sa;
1065 	sigset_t sigset;
1066 
1067 	ret = sigemptyset(&sigset);
1068 	if (ret < 0) {
1069 		perror("sigemptyset");
1070 		return ret;
1071 	}
1072 
1073 	sa.sa_handler = test_signal_interrupt_handler;
1074 	sa.sa_mask = sigset;
1075 	sa.sa_flags = 0;
1076 	ret = sigaction(SIGUSR1, &sa, NULL);
1077 	if (ret < 0) {
1078 		perror("sigaction");
1079 		return ret;
1080 	}
1081 
1082 	printf_verbose("Signal handler set for SIGUSR1\n");
1083 
1084 	return ret;
1085 }
1086 
1087 static void show_usage(int argc, char **argv)
1088 {
1089 	printf("Usage : %s <OPTIONS>\n",
1090 		argv[0]);
1091 	printf("OPTIONS:\n");
1092 	printf("	[-1 loops] Number of loops for delay injection 1\n");
1093 	printf("	[-2 loops] Number of loops for delay injection 2\n");
1094 	printf("	[-3 loops] Number of loops for delay injection 3\n");
1095 	printf("	[-4 loops] Number of loops for delay injection 4\n");
1096 	printf("	[-5 loops] Number of loops for delay injection 5\n");
1097 	printf("	[-6 loops] Number of loops for delay injection 6\n");
1098 	printf("	[-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n");
1099 	printf("	[-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n");
1100 	printf("	[-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n");
1101 	printf("	[-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n");
1102 	printf("	[-y] Yield\n");
1103 	printf("	[-k] Kill thread with signal\n");
1104 	printf("	[-s S] S: =0: disabled (default), >0: sleep time (ms)\n");
1105 	printf("	[-t N] Number of threads (default 200)\n");
1106 	printf("	[-r N] Number of repetitions per thread (default 5000)\n");
1107 	printf("	[-d] Disable rseq system call (no initialization)\n");
1108 	printf("	[-D M] Disable rseq for each M threads\n");
1109 	printf("	[-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement\n");
1110 	printf("	[-M] Push into buffer and memcpy buffer with memory barriers.\n");
1111 	printf("	[-v] Verbose output.\n");
1112 	printf("	[-h] Show this help.\n");
1113 	printf("\n");
1114 }
1115 
1116 int main(int argc, char **argv)
1117 {
1118 	int i;
1119 
1120 	for (i = 1; i < argc; i++) {
1121 		if (argv[i][0] != '-')
1122 			continue;
1123 		switch (argv[i][1]) {
1124 		case '1':
1125 		case '2':
1126 		case '3':
1127 		case '4':
1128 		case '5':
1129 		case '6':
1130 		case '7':
1131 		case '8':
1132 		case '9':
1133 			if (argc < i + 2) {
1134 				show_usage(argc, argv);
1135 				goto error;
1136 			}
1137 			loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]);
1138 			i++;
1139 			break;
1140 		case 'm':
1141 			if (argc < i + 2) {
1142 				show_usage(argc, argv);
1143 				goto error;
1144 			}
1145 			opt_modulo = atol(argv[i + 1]);
1146 			if (opt_modulo < 0) {
1147 				show_usage(argc, argv);
1148 				goto error;
1149 			}
1150 			i++;
1151 			break;
1152 		case 's':
1153 			if (argc < i + 2) {
1154 				show_usage(argc, argv);
1155 				goto error;
1156 			}
1157 			opt_sleep = atol(argv[i + 1]);
1158 			if (opt_sleep < 0) {
1159 				show_usage(argc, argv);
1160 				goto error;
1161 			}
1162 			i++;
1163 			break;
1164 		case 'y':
1165 			opt_yield = 1;
1166 			break;
1167 		case 'k':
1168 			opt_signal = 1;
1169 			break;
1170 		case 'd':
1171 			opt_disable_rseq = 1;
1172 			break;
1173 		case 'D':
1174 			if (argc < i + 2) {
1175 				show_usage(argc, argv);
1176 				goto error;
1177 			}
1178 			opt_disable_mod = atol(argv[i + 1]);
1179 			if (opt_disable_mod < 0) {
1180 				show_usage(argc, argv);
1181 				goto error;
1182 			}
1183 			i++;
1184 			break;
1185 		case 't':
1186 			if (argc < i + 2) {
1187 				show_usage(argc, argv);
1188 				goto error;
1189 			}
1190 			opt_threads = atol(argv[i + 1]);
1191 			if (opt_threads < 0) {
1192 				show_usage(argc, argv);
1193 				goto error;
1194 			}
1195 			i++;
1196 			break;
1197 		case 'r':
1198 			if (argc < i + 2) {
1199 				show_usage(argc, argv);
1200 				goto error;
1201 			}
1202 			opt_reps = atoll(argv[i + 1]);
1203 			if (opt_reps < 0) {
1204 				show_usage(argc, argv);
1205 				goto error;
1206 			}
1207 			i++;
1208 			break;
1209 		case 'h':
1210 			show_usage(argc, argv);
1211 			goto end;
1212 		case 'T':
1213 			if (argc < i + 2) {
1214 				show_usage(argc, argv);
1215 				goto error;
1216 			}
1217 			opt_test = *argv[i + 1];
1218 			switch (opt_test) {
1219 			case 's':
1220 			case 'l':
1221 			case 'i':
1222 			case 'b':
1223 			case 'm':
1224 				break;
1225 			default:
1226 				show_usage(argc, argv);
1227 				goto error;
1228 			}
1229 			i++;
1230 			break;
1231 		case 'v':
1232 			verbose = 1;
1233 			break;
1234 		case 'M':
1235 			opt_mb = 1;
1236 			break;
1237 		default:
1238 			show_usage(argc, argv);
1239 			goto error;
1240 		}
1241 	}
1242 
1243 	loop_cnt_1 = loop_cnt[1];
1244 	loop_cnt_2 = loop_cnt[2];
1245 	loop_cnt_3 = loop_cnt[3];
1246 	loop_cnt_4 = loop_cnt[4];
1247 	loop_cnt_5 = loop_cnt[5];
1248 	loop_cnt_6 = loop_cnt[6];
1249 
1250 	if (set_signal_handler())
1251 		goto error;
1252 
1253 	if (!opt_disable_rseq && rseq_register_current_thread())
1254 		goto error;
1255 	switch (opt_test) {
1256 	case 's':
1257 		printf_verbose("spinlock\n");
1258 		test_percpu_spinlock();
1259 		break;
1260 	case 'l':
1261 		printf_verbose("linked list\n");
1262 		test_percpu_list();
1263 		break;
1264 	case 'b':
1265 		printf_verbose("buffer\n");
1266 		test_percpu_buffer();
1267 		break;
1268 	case 'm':
1269 		printf_verbose("memcpy buffer\n");
1270 		test_percpu_memcpy_buffer();
1271 		break;
1272 	case 'i':
1273 		printf_verbose("counter increment\n");
1274 		test_percpu_inc();
1275 		break;
1276 	}
1277 	if (!opt_disable_rseq && rseq_unregister_current_thread())
1278 		abort();
1279 end:
1280 	return 0;
1281 
1282 error:
1283 	return -1;
1284 }
1285