1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3 
4 #include <argp.h>
5 
6 #include <sys/prctl.h>
7 #include "local_storage_rcu_tasks_trace_bench.skel.h"
8 #include "bench.h"
9 
10 #include <signal.h>
11 
12 static struct {
13 	__u32 nr_procs;
14 	__u32 kthread_pid;
15 	bool quiet;
16 } args = {
17 	.nr_procs = 1000,
18 	.kthread_pid = 0,
19 	.quiet = false,
20 };
21 
22 enum {
23 	ARG_NR_PROCS = 7000,
24 	ARG_KTHREAD_PID = 7001,
25 	ARG_QUIET = 7002,
26 };
27 
28 static const struct argp_option opts[] = {
29 	{ "nr_procs", ARG_NR_PROCS, "NR_PROCS", 0,
30 		"Set number of user processes to spin up"},
31 	{ "kthread_pid", ARG_KTHREAD_PID, "PID", 0,
32 		"Pid of rcu_tasks_trace kthread for ticks tracking"},
33 	{ "quiet", ARG_QUIET, "{0,1}", 0,
34 		"If true, don't report progress"},
35 	{},
36 };
37 
38 static error_t parse_arg(int key, char *arg, struct argp_state *state)
39 {
40 	long ret;
41 
42 	switch (key) {
43 	case ARG_NR_PROCS:
44 		ret = strtol(arg, NULL, 10);
45 		if (ret < 1 || ret > UINT_MAX) {
46 			fprintf(stderr, "invalid nr_procs\n");
47 			argp_usage(state);
48 		}
49 		args.nr_procs = ret;
50 		break;
51 	case ARG_KTHREAD_PID:
52 		ret = strtol(arg, NULL, 10);
53 		if (ret < 1) {
54 			fprintf(stderr, "invalid kthread_pid\n");
55 			argp_usage(state);
56 		}
57 		args.kthread_pid = ret;
58 		break;
59 	case ARG_QUIET:
60 		ret = strtol(arg, NULL, 10);
61 		if (ret < 0 || ret > 1) {
62 			fprintf(stderr, "invalid quiet %ld\n", ret);
63 			argp_usage(state);
64 		}
65 		args.quiet = ret;
66 		break;
67 break;
68 	default:
69 		return ARGP_ERR_UNKNOWN;
70 	}
71 
72 	return 0;
73 }
74 
75 const struct argp bench_local_storage_rcu_tasks_trace_argp = {
76 	.options = opts,
77 	.parser = parse_arg,
78 };
79 
80 #define MAX_SLEEP_PROCS 150000
81 
82 static void validate(void)
83 {
84 	if (env.producer_cnt != 1) {
85 		fprintf(stderr, "benchmark doesn't support multi-producer!\n");
86 		exit(1);
87 	}
88 	if (env.consumer_cnt != 1) {
89 		fprintf(stderr, "benchmark doesn't support multi-consumer!\n");
90 		exit(1);
91 	}
92 
93 	if (args.nr_procs > MAX_SLEEP_PROCS) {
94 		fprintf(stderr, "benchmark supports up to %u sleeper procs!\n",
95 			MAX_SLEEP_PROCS);
96 		exit(1);
97 	}
98 }
99 
100 static long kthread_pid_ticks(void)
101 {
102 	char procfs_path[100];
103 	long stime;
104 	FILE *f;
105 
106 	if (!args.kthread_pid)
107 		return -1;
108 
109 	sprintf(procfs_path, "/proc/%u/stat", args.kthread_pid);
110 	f = fopen(procfs_path, "r");
111 	if (!f) {
112 		fprintf(stderr, "couldn't open %s, exiting\n", procfs_path);
113 		goto err_out;
114 	}
115 	if (fscanf(f, "%*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %ld", &stime) != 1) {
116 		fprintf(stderr, "fscanf of %s failed, exiting\n", procfs_path);
117 		goto err_out;
118 	}
119 	fclose(f);
120 	return stime;
121 
122 err_out:
123 	if (f)
124 		fclose(f);
125 	exit(1);
126 	return 0;
127 }
128 
129 static struct {
130 	struct local_storage_rcu_tasks_trace_bench *skel;
131 	long prev_kthread_stime;
132 } ctx;
133 
134 static void sleep_and_loop(void)
135 {
136 	while (true) {
137 		sleep(rand() % 4);
138 		syscall(__NR_getpgid);
139 	}
140 }
141 
142 static void local_storage_tasks_trace_setup(void)
143 {
144 	int i, err, forkret, runner_pid;
145 
146 	runner_pid = getpid();
147 
148 	for (i = 0; i < args.nr_procs; i++) {
149 		forkret = fork();
150 		if (forkret < 0) {
151 			fprintf(stderr, "Error forking sleeper proc %u of %u, exiting\n", i,
152 				args.nr_procs);
153 			goto err_out;
154 		}
155 
156 		if (!forkret) {
157 			err = prctl(PR_SET_PDEATHSIG, SIGKILL);
158 			if (err < 0) {
159 				fprintf(stderr, "prctl failed with err %d, exiting\n", errno);
160 				goto err_out;
161 			}
162 
163 			if (getppid() != runner_pid) {
164 				fprintf(stderr, "Runner died while spinning up procs, exiting\n");
165 				goto err_out;
166 			}
167 			sleep_and_loop();
168 		}
169 	}
170 	printf("Spun up %u procs (our pid %d)\n", args.nr_procs, runner_pid);
171 
172 	setup_libbpf();
173 
174 	ctx.skel = local_storage_rcu_tasks_trace_bench__open_and_load();
175 	if (!ctx.skel) {
176 		fprintf(stderr, "Error doing open_and_load, exiting\n");
177 		goto err_out;
178 	}
179 
180 	ctx.prev_kthread_stime = kthread_pid_ticks();
181 
182 	if (!bpf_program__attach(ctx.skel->progs.get_local)) {
183 		fprintf(stderr, "Error attaching bpf program\n");
184 		goto err_out;
185 	}
186 
187 	if (!bpf_program__attach(ctx.skel->progs.pregp_step)) {
188 		fprintf(stderr, "Error attaching bpf program\n");
189 		goto err_out;
190 	}
191 
192 	if (!bpf_program__attach(ctx.skel->progs.postgp)) {
193 		fprintf(stderr, "Error attaching bpf program\n");
194 		goto err_out;
195 	}
196 
197 	return;
198 err_out:
199 	exit(1);
200 }
201 
202 static void measure(struct bench_res *res)
203 {
204 	long ticks;
205 
206 	res->gp_ct = atomic_swap(&ctx.skel->bss->gp_hits, 0);
207 	res->gp_ns = atomic_swap(&ctx.skel->bss->gp_times, 0);
208 	ticks = kthread_pid_ticks();
209 	res->stime = ticks - ctx.prev_kthread_stime;
210 	ctx.prev_kthread_stime = ticks;
211 }
212 
213 static void *consumer(void *input)
214 {
215 	return NULL;
216 }
217 
218 static void *producer(void *input)
219 {
220 	while (true)
221 		syscall(__NR_getpgid);
222 	return NULL;
223 }
224 
225 static void report_progress(int iter, struct bench_res *res, long delta_ns)
226 {
227 	if (ctx.skel->bss->unexpected) {
228 		fprintf(stderr, "Error: Unexpected order of bpf prog calls (postgp after pregp).");
229 		fprintf(stderr, "Data can't be trusted, exiting\n");
230 		exit(1);
231 	}
232 
233 	if (args.quiet)
234 		return;
235 
236 	printf("Iter %d\t avg tasks_trace grace period latency\t%lf ns\n",
237 	       iter, res->gp_ns / (double)res->gp_ct);
238 	printf("Iter %d\t avg ticks per tasks_trace grace period\t%lf\n",
239 	       iter, res->stime / (double)res->gp_ct);
240 }
241 
242 static void report_final(struct bench_res res[], int res_cnt)
243 {
244 	struct basic_stats gp_stat;
245 
246 	grace_period_latency_basic_stats(res, res_cnt, &gp_stat);
247 	printf("SUMMARY tasks_trace grace period latency");
248 	printf("\tavg %.3lf us\tstddev %.3lf us\n", gp_stat.mean, gp_stat.stddev);
249 	grace_period_ticks_basic_stats(res, res_cnt, &gp_stat);
250 	printf("SUMMARY ticks per tasks_trace grace period");
251 	printf("\tavg %.3lf\tstddev %.3lf\n", gp_stat.mean, gp_stat.stddev);
252 }
253 
254 /* local-storage-tasks-trace: Benchmark performance of BPF local_storage's use
255  * of RCU Tasks-Trace.
256  *
257  * Stress RCU Tasks Trace by forking many tasks, all of which do no work aside
258  * from sleep() loop, and creating/destroying BPF task-local storage on wakeup.
259  * The number of forked tasks is configurable.
260  *
261  * exercising code paths which call call_rcu_tasks_trace while there are many
262  * thousands of tasks on the system should result in RCU Tasks-Trace having to
263  * do a noticeable amount of work.
264  *
265  * This should be observable by measuring rcu_tasks_trace_kthread CPU usage
266  * after the grace period has ended, or by measuring grace period latency.
267  *
268  * This benchmark uses both approaches, attaching to rcu_tasks_trace_pregp_step
269  * and rcu_tasks_trace_postgp functions to measure grace period latency and
270  * using /proc/PID/stat to measure rcu_tasks_trace_kthread kernel ticks
271  */
272 const struct bench bench_local_storage_tasks_trace = {
273 	.name = "local-storage-tasks-trace",
274 	.validate = validate,
275 	.setup = local_storage_tasks_trace_setup,
276 	.producer_thread = producer,
277 	.consumer_thread = consumer,
278 	.measure = measure,
279 	.report_progress = report_progress,
280 	.report_final = report_final,
281 };
282