1 /* SPDX-License-Identifier: MIT */
2 
3 /*
4  * Copyright © 2019 Intel Corporation
5  */
6 
7 #include <linux/delay.h>
8 #include <linux/dma-fence.h>
9 #include <linux/kernel.h>
10 #include <linux/kthread.h>
11 #include <linux/sched/signal.h>
12 #include <linux/slab.h>
13 #include <linux/spinlock.h>
14 
15 #include "selftest.h"
16 
17 static struct kmem_cache *slab_fences;
18 
19 static struct mock_fence {
20 	struct dma_fence base;
21 	struct spinlock lock;
22 } *to_mock_fence(struct dma_fence *f) {
23 	return container_of(f, struct mock_fence, base);
24 }
25 
26 static const char *mock_name(struct dma_fence *f)
27 {
28 	return "mock";
29 }
30 
31 static void mock_fence_release(struct dma_fence *f)
32 {
33 	kmem_cache_free(slab_fences, to_mock_fence(f));
34 }
35 
36 struct wait_cb {
37 	struct dma_fence_cb cb;
38 	struct task_struct *task;
39 };
40 
41 static void mock_wakeup(struct dma_fence *f, struct dma_fence_cb *cb)
42 {
43 	wake_up_process(container_of(cb, struct wait_cb, cb)->task);
44 }
45 
46 static long mock_wait(struct dma_fence *f, bool intr, long timeout)
47 {
48 	const int state = intr ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
49 	struct wait_cb cb = { .task = current };
50 
51 	if (dma_fence_add_callback(f, &cb.cb, mock_wakeup))
52 		return timeout;
53 
54 	while (timeout) {
55 		set_current_state(state);
56 
57 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags))
58 			break;
59 
60 		if (signal_pending_state(state, current))
61 			break;
62 
63 		timeout = schedule_timeout(timeout);
64 	}
65 	__set_current_state(TASK_RUNNING);
66 
67 	if (!dma_fence_remove_callback(f, &cb.cb))
68 		return timeout;
69 
70 	if (signal_pending_state(state, current))
71 		return -ERESTARTSYS;
72 
73 	return -ETIME;
74 }
75 
76 static const struct dma_fence_ops mock_ops = {
77 	.get_driver_name = mock_name,
78 	.get_timeline_name = mock_name,
79 	.wait = mock_wait,
80 	.release = mock_fence_release,
81 };
82 
83 static struct dma_fence *mock_fence(void)
84 {
85 	struct mock_fence *f;
86 
87 	f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
88 	if (!f)
89 		return NULL;
90 
91 	spin_lock_init(&f->lock);
92 	dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
93 
94 	return &f->base;
95 }
96 
97 static int sanitycheck(void *arg)
98 {
99 	struct dma_fence *f;
100 
101 	f = mock_fence();
102 	if (!f)
103 		return -ENOMEM;
104 
105 	dma_fence_enable_sw_signaling(f);
106 
107 	dma_fence_signal(f);
108 	dma_fence_put(f);
109 
110 	return 0;
111 }
112 
113 static int test_signaling(void *arg)
114 {
115 	struct dma_fence *f;
116 	int err = -EINVAL;
117 
118 	f = mock_fence();
119 	if (!f)
120 		return -ENOMEM;
121 
122 	dma_fence_enable_sw_signaling(f);
123 
124 	if (dma_fence_is_signaled(f)) {
125 		pr_err("Fence unexpectedly signaled on creation\n");
126 		goto err_free;
127 	}
128 
129 	if (dma_fence_signal(f)) {
130 		pr_err("Fence reported being already signaled\n");
131 		goto err_free;
132 	}
133 
134 	if (!dma_fence_is_signaled(f)) {
135 		pr_err("Fence not reporting signaled\n");
136 		goto err_free;
137 	}
138 
139 	if (!dma_fence_signal(f)) {
140 		pr_err("Fence reported not being already signaled\n");
141 		goto err_free;
142 	}
143 
144 	err = 0;
145 err_free:
146 	dma_fence_put(f);
147 	return err;
148 }
149 
150 struct simple_cb {
151 	struct dma_fence_cb cb;
152 	bool seen;
153 };
154 
155 static void simple_callback(struct dma_fence *f, struct dma_fence_cb *cb)
156 {
157 	smp_store_mb(container_of(cb, struct simple_cb, cb)->seen, true);
158 }
159 
160 static int test_add_callback(void *arg)
161 {
162 	struct simple_cb cb = {};
163 	struct dma_fence *f;
164 	int err = -EINVAL;
165 
166 	f = mock_fence();
167 	if (!f)
168 		return -ENOMEM;
169 
170 	if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
171 		pr_err("Failed to add callback, fence already signaled!\n");
172 		goto err_free;
173 	}
174 
175 	dma_fence_signal(f);
176 	if (!cb.seen) {
177 		pr_err("Callback failed!\n");
178 		goto err_free;
179 	}
180 
181 	err = 0;
182 err_free:
183 	dma_fence_put(f);
184 	return err;
185 }
186 
187 static int test_late_add_callback(void *arg)
188 {
189 	struct simple_cb cb = {};
190 	struct dma_fence *f;
191 	int err = -EINVAL;
192 
193 	f = mock_fence();
194 	if (!f)
195 		return -ENOMEM;
196 
197 	dma_fence_enable_sw_signaling(f);
198 
199 	dma_fence_signal(f);
200 
201 	if (!dma_fence_add_callback(f, &cb.cb, simple_callback)) {
202 		pr_err("Added callback, but fence was already signaled!\n");
203 		goto err_free;
204 	}
205 
206 	dma_fence_signal(f);
207 	if (cb.seen) {
208 		pr_err("Callback called after failed attachment !\n");
209 		goto err_free;
210 	}
211 
212 	err = 0;
213 err_free:
214 	dma_fence_put(f);
215 	return err;
216 }
217 
218 static int test_rm_callback(void *arg)
219 {
220 	struct simple_cb cb = {};
221 	struct dma_fence *f;
222 	int err = -EINVAL;
223 
224 	f = mock_fence();
225 	if (!f)
226 		return -ENOMEM;
227 
228 	if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
229 		pr_err("Failed to add callback, fence already signaled!\n");
230 		goto err_free;
231 	}
232 
233 	if (!dma_fence_remove_callback(f, &cb.cb)) {
234 		pr_err("Failed to remove callback!\n");
235 		goto err_free;
236 	}
237 
238 	dma_fence_signal(f);
239 	if (cb.seen) {
240 		pr_err("Callback still signaled after removal!\n");
241 		goto err_free;
242 	}
243 
244 	err = 0;
245 err_free:
246 	dma_fence_put(f);
247 	return err;
248 }
249 
250 static int test_late_rm_callback(void *arg)
251 {
252 	struct simple_cb cb = {};
253 	struct dma_fence *f;
254 	int err = -EINVAL;
255 
256 	f = mock_fence();
257 	if (!f)
258 		return -ENOMEM;
259 
260 	if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
261 		pr_err("Failed to add callback, fence already signaled!\n");
262 		goto err_free;
263 	}
264 
265 	dma_fence_signal(f);
266 	if (!cb.seen) {
267 		pr_err("Callback failed!\n");
268 		goto err_free;
269 	}
270 
271 	if (dma_fence_remove_callback(f, &cb.cb)) {
272 		pr_err("Callback removal succeed after being executed!\n");
273 		goto err_free;
274 	}
275 
276 	err = 0;
277 err_free:
278 	dma_fence_put(f);
279 	return err;
280 }
281 
282 static int test_status(void *arg)
283 {
284 	struct dma_fence *f;
285 	int err = -EINVAL;
286 
287 	f = mock_fence();
288 	if (!f)
289 		return -ENOMEM;
290 
291 	dma_fence_enable_sw_signaling(f);
292 
293 	if (dma_fence_get_status(f)) {
294 		pr_err("Fence unexpectedly has signaled status on creation\n");
295 		goto err_free;
296 	}
297 
298 	dma_fence_signal(f);
299 	if (!dma_fence_get_status(f)) {
300 		pr_err("Fence not reporting signaled status\n");
301 		goto err_free;
302 	}
303 
304 	err = 0;
305 err_free:
306 	dma_fence_put(f);
307 	return err;
308 }
309 
310 static int test_error(void *arg)
311 {
312 	struct dma_fence *f;
313 	int err = -EINVAL;
314 
315 	f = mock_fence();
316 	if (!f)
317 		return -ENOMEM;
318 
319 	dma_fence_enable_sw_signaling(f);
320 
321 	dma_fence_set_error(f, -EIO);
322 
323 	if (dma_fence_get_status(f)) {
324 		pr_err("Fence unexpectedly has error status before signal\n");
325 		goto err_free;
326 	}
327 
328 	dma_fence_signal(f);
329 	if (dma_fence_get_status(f) != -EIO) {
330 		pr_err("Fence not reporting error status, got %d\n",
331 		       dma_fence_get_status(f));
332 		goto err_free;
333 	}
334 
335 	err = 0;
336 err_free:
337 	dma_fence_put(f);
338 	return err;
339 }
340 
341 static int test_wait(void *arg)
342 {
343 	struct dma_fence *f;
344 	int err = -EINVAL;
345 
346 	f = mock_fence();
347 	if (!f)
348 		return -ENOMEM;
349 
350 	dma_fence_enable_sw_signaling(f);
351 
352 	if (dma_fence_wait_timeout(f, false, 0) != -ETIME) {
353 		pr_err("Wait reported complete before being signaled\n");
354 		goto err_free;
355 	}
356 
357 	dma_fence_signal(f);
358 
359 	if (dma_fence_wait_timeout(f, false, 0) != 0) {
360 		pr_err("Wait reported incomplete after being signaled\n");
361 		goto err_free;
362 	}
363 
364 	err = 0;
365 err_free:
366 	dma_fence_signal(f);
367 	dma_fence_put(f);
368 	return err;
369 }
370 
371 struct wait_timer {
372 	struct timer_list timer;
373 	struct dma_fence *f;
374 };
375 
376 static void wait_timer(struct timer_list *timer)
377 {
378 	struct wait_timer *wt = from_timer(wt, timer, timer);
379 
380 	dma_fence_signal(wt->f);
381 }
382 
383 static int test_wait_timeout(void *arg)
384 {
385 	struct wait_timer wt;
386 	int err = -EINVAL;
387 
388 	timer_setup_on_stack(&wt.timer, wait_timer, 0);
389 
390 	wt.f = mock_fence();
391 	if (!wt.f)
392 		return -ENOMEM;
393 
394 	dma_fence_enable_sw_signaling(wt.f);
395 
396 	if (dma_fence_wait_timeout(wt.f, false, 1) != -ETIME) {
397 		pr_err("Wait reported complete before being signaled\n");
398 		goto err_free;
399 	}
400 
401 	mod_timer(&wt.timer, jiffies + 1);
402 
403 	if (dma_fence_wait_timeout(wt.f, false, 2) == -ETIME) {
404 		if (timer_pending(&wt.timer)) {
405 			pr_notice("Timer did not fire within the jiffie!\n");
406 			err = 0; /* not our fault! */
407 		} else {
408 			pr_err("Wait reported incomplete after timeout\n");
409 		}
410 		goto err_free;
411 	}
412 
413 	err = 0;
414 err_free:
415 	del_timer_sync(&wt.timer);
416 	destroy_timer_on_stack(&wt.timer);
417 	dma_fence_signal(wt.f);
418 	dma_fence_put(wt.f);
419 	return err;
420 }
421 
422 static int test_stub(void *arg)
423 {
424 	struct dma_fence *f[64];
425 	int err = -EINVAL;
426 	int i;
427 
428 	for (i = 0; i < ARRAY_SIZE(f); i++) {
429 		f[i] = dma_fence_get_stub();
430 		if (!dma_fence_is_signaled(f[i])) {
431 			pr_err("Obtained unsignaled stub fence!\n");
432 			goto err;
433 		}
434 	}
435 
436 	err = 0;
437 err:
438 	while (i--)
439 		dma_fence_put(f[i]);
440 	return err;
441 }
442 
443 /* Now off to the races! */
444 
445 struct race_thread {
446 	struct dma_fence __rcu **fences;
447 	struct task_struct *task;
448 	bool before;
449 	int id;
450 };
451 
452 static void __wait_for_callbacks(struct dma_fence *f)
453 {
454 	spin_lock_irq(f->lock);
455 	spin_unlock_irq(f->lock);
456 }
457 
458 static int thread_signal_callback(void *arg)
459 {
460 	const struct race_thread *t = arg;
461 	unsigned long pass = 0;
462 	unsigned long miss = 0;
463 	int err = 0;
464 
465 	while (!err && !kthread_should_stop()) {
466 		struct dma_fence *f1, *f2;
467 		struct simple_cb cb;
468 
469 		f1 = mock_fence();
470 		if (!f1) {
471 			err = -ENOMEM;
472 			break;
473 		}
474 
475 		dma_fence_enable_sw_signaling(f1);
476 
477 		rcu_assign_pointer(t->fences[t->id], f1);
478 		smp_wmb();
479 
480 		rcu_read_lock();
481 		do {
482 			f2 = dma_fence_get_rcu_safe(&t->fences[!t->id]);
483 		} while (!f2 && !kthread_should_stop());
484 		rcu_read_unlock();
485 
486 		if (t->before)
487 			dma_fence_signal(f1);
488 
489 		smp_store_mb(cb.seen, false);
490 		if (!f2 ||
491 		    dma_fence_add_callback(f2, &cb.cb, simple_callback)) {
492 			miss++;
493 			cb.seen = true;
494 		}
495 
496 		if (!t->before)
497 			dma_fence_signal(f1);
498 
499 		if (!cb.seen) {
500 			dma_fence_wait(f2, false);
501 			__wait_for_callbacks(f2);
502 		}
503 
504 		if (!READ_ONCE(cb.seen)) {
505 			pr_err("Callback not seen on thread %d, pass %lu (%lu misses), signaling %s add_callback; fence signaled? %s\n",
506 			       t->id, pass, miss,
507 			       t->before ? "before" : "after",
508 			       dma_fence_is_signaled(f2) ? "yes" : "no");
509 			err = -EINVAL;
510 		}
511 
512 		dma_fence_put(f2);
513 
514 		rcu_assign_pointer(t->fences[t->id], NULL);
515 		smp_wmb();
516 
517 		dma_fence_put(f1);
518 
519 		pass++;
520 	}
521 
522 	pr_info("%s[%d] completed %lu passes, %lu misses\n",
523 		__func__, t->id, pass, miss);
524 	return err;
525 }
526 
527 static int race_signal_callback(void *arg)
528 {
529 	struct dma_fence __rcu *f[2] = {};
530 	int ret = 0;
531 	int pass;
532 
533 	for (pass = 0; !ret && pass <= 1; pass++) {
534 		struct race_thread t[2];
535 		int i;
536 
537 		for (i = 0; i < ARRAY_SIZE(t); i++) {
538 			t[i].fences = f;
539 			t[i].id = i;
540 			t[i].before = pass;
541 			t[i].task = kthread_run(thread_signal_callback, &t[i],
542 						"dma-fence:%d", i);
543 			get_task_struct(t[i].task);
544 		}
545 
546 		msleep(50);
547 
548 		for (i = 0; i < ARRAY_SIZE(t); i++) {
549 			int err;
550 
551 			err = kthread_stop_put(t[i].task);
552 			if (err && !ret)
553 				ret = err;
554 		}
555 	}
556 
557 	return ret;
558 }
559 
560 int dma_fence(void)
561 {
562 	static const struct subtest tests[] = {
563 		SUBTEST(sanitycheck),
564 		SUBTEST(test_signaling),
565 		SUBTEST(test_add_callback),
566 		SUBTEST(test_late_add_callback),
567 		SUBTEST(test_rm_callback),
568 		SUBTEST(test_late_rm_callback),
569 		SUBTEST(test_status),
570 		SUBTEST(test_error),
571 		SUBTEST(test_wait),
572 		SUBTEST(test_wait_timeout),
573 		SUBTEST(test_stub),
574 		SUBTEST(race_signal_callback),
575 	};
576 	int ret;
577 
578 	pr_info("sizeof(dma_fence)=%zu\n", sizeof(struct dma_fence));
579 
580 	slab_fences = KMEM_CACHE(mock_fence,
581 				 SLAB_TYPESAFE_BY_RCU |
582 				 SLAB_HWCACHE_ALIGN);
583 	if (!slab_fences)
584 		return -ENOMEM;
585 
586 	ret = subtests(tests, NULL);
587 
588 	kmem_cache_destroy(slab_fences);
589 
590 	return ret;
591 }
592