1 // SPDX-License-Identifier: MIT 2 3 /* 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include <linux/delay.h> 8 #include <linux/dma-fence.h> 9 #include <linux/dma-fence-chain.h> 10 #include <linux/kernel.h> 11 #include <linux/kthread.h> 12 #include <linux/mm.h> 13 #include <linux/sched/signal.h> 14 #include <linux/slab.h> 15 #include <linux/spinlock.h> 16 #include <linux/random.h> 17 18 #include "selftest.h" 19 20 #define CHAIN_SZ (4 << 10) 21 22 static struct kmem_cache *slab_fences; 23 24 static inline struct mock_fence { 25 struct dma_fence base; 26 spinlock_t lock; 27 } *to_mock_fence(struct dma_fence *f) { 28 return container_of(f, struct mock_fence, base); 29 } 30 31 static const char *mock_name(struct dma_fence *f) 32 { 33 return "mock"; 34 } 35 36 static void mock_fence_release(struct dma_fence *f) 37 { 38 kmem_cache_free(slab_fences, to_mock_fence(f)); 39 } 40 41 static const struct dma_fence_ops mock_ops = { 42 .get_driver_name = mock_name, 43 .get_timeline_name = mock_name, 44 .release = mock_fence_release, 45 }; 46 47 static struct dma_fence *mock_fence(void) 48 { 49 struct mock_fence *f; 50 51 f = kmem_cache_alloc(slab_fences, GFP_KERNEL); 52 if (!f) 53 return NULL; 54 55 spin_lock_init(&f->lock); 56 dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0); 57 58 return &f->base; 59 } 60 61 static inline struct mock_chain { 62 struct dma_fence_chain base; 63 } *to_mock_chain(struct dma_fence *f) { 64 return container_of(f, struct mock_chain, base.base); 65 } 66 67 static struct dma_fence *mock_chain(struct dma_fence *prev, 68 struct dma_fence *fence, 69 u64 seqno) 70 { 71 struct mock_chain *f; 72 73 f = kmalloc(sizeof(*f), GFP_KERNEL); 74 if (!f) 75 return NULL; 76 77 dma_fence_chain_init(&f->base, 78 dma_fence_get(prev), 79 dma_fence_get(fence), 80 seqno); 81 82 return &f->base.base; 83 } 84 85 static int sanitycheck(void *arg) 86 { 87 struct dma_fence *f, *chain; 88 int err = 0; 89 90 f = mock_fence(); 91 if (!f) 92 return -ENOMEM; 93 94 chain = mock_chain(NULL, f, 1); 95 if (!chain) 96 err = -ENOMEM; 97 98 dma_fence_signal(f); 99 dma_fence_put(f); 100 101 dma_fence_put(chain); 102 103 return err; 104 } 105 106 struct fence_chains { 107 unsigned int chain_length; 108 struct dma_fence **fences; 109 struct dma_fence **chains; 110 111 struct dma_fence *tail; 112 }; 113 114 static uint64_t seqno_inc(unsigned int i) 115 { 116 return i + 1; 117 } 118 119 static int fence_chains_init(struct fence_chains *fc, unsigned int count, 120 uint64_t (*seqno_fn)(unsigned int)) 121 { 122 unsigned int i; 123 int err = 0; 124 125 fc->chains = kvmalloc_array(count, sizeof(*fc->chains), 126 GFP_KERNEL | __GFP_ZERO); 127 if (!fc->chains) 128 return -ENOMEM; 129 130 fc->fences = kvmalloc_array(count, sizeof(*fc->fences), 131 GFP_KERNEL | __GFP_ZERO); 132 if (!fc->fences) { 133 err = -ENOMEM; 134 goto err_chains; 135 } 136 137 fc->tail = NULL; 138 for (i = 0; i < count; i++) { 139 fc->fences[i] = mock_fence(); 140 if (!fc->fences[i]) { 141 err = -ENOMEM; 142 goto unwind; 143 } 144 145 fc->chains[i] = mock_chain(fc->tail, 146 fc->fences[i], 147 seqno_fn(i)); 148 if (!fc->chains[i]) { 149 err = -ENOMEM; 150 goto unwind; 151 } 152 153 fc->tail = fc->chains[i]; 154 } 155 156 fc->chain_length = i; 157 return 0; 158 159 unwind: 160 for (i = 0; i < count; i++) { 161 dma_fence_put(fc->fences[i]); 162 dma_fence_put(fc->chains[i]); 163 } 164 kvfree(fc->fences); 165 err_chains: 166 kvfree(fc->chains); 167 return err; 168 } 169 170 static void fence_chains_fini(struct fence_chains *fc) 171 { 172 unsigned int i; 173 174 for (i = 0; i < fc->chain_length; i++) { 175 dma_fence_signal(fc->fences[i]); 176 dma_fence_put(fc->fences[i]); 177 } 178 kvfree(fc->fences); 179 180 for (i = 0; i < fc->chain_length; i++) 181 dma_fence_put(fc->chains[i]); 182 kvfree(fc->chains); 183 } 184 185 static int find_seqno(void *arg) 186 { 187 struct fence_chains fc; 188 struct dma_fence *fence; 189 int err; 190 int i; 191 192 err = fence_chains_init(&fc, 64, seqno_inc); 193 if (err) 194 return err; 195 196 fence = dma_fence_get(fc.tail); 197 err = dma_fence_chain_find_seqno(&fence, 0); 198 dma_fence_put(fence); 199 if (err) { 200 pr_err("Reported %d for find_seqno(0)!\n", err); 201 goto err; 202 } 203 204 for (i = 0; i < fc.chain_length; i++) { 205 fence = dma_fence_get(fc.tail); 206 err = dma_fence_chain_find_seqno(&fence, i + 1); 207 dma_fence_put(fence); 208 if (err) { 209 pr_err("Reported %d for find_seqno(%d:%d)!\n", 210 err, fc.chain_length + 1, i + 1); 211 goto err; 212 } 213 if (fence != fc.chains[i]) { 214 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n", 215 fc.chain_length + 1, i + 1); 216 err = -EINVAL; 217 goto err; 218 } 219 220 dma_fence_get(fence); 221 err = dma_fence_chain_find_seqno(&fence, i + 1); 222 dma_fence_put(fence); 223 if (err) { 224 pr_err("Error reported for finding self\n"); 225 goto err; 226 } 227 if (fence != fc.chains[i]) { 228 pr_err("Incorrect fence reported by find self\n"); 229 err = -EINVAL; 230 goto err; 231 } 232 233 dma_fence_get(fence); 234 err = dma_fence_chain_find_seqno(&fence, i + 2); 235 dma_fence_put(fence); 236 if (!err) { 237 pr_err("Error not reported for future fence: find_seqno(%d:%d)!\n", 238 i + 1, i + 2); 239 err = -EINVAL; 240 goto err; 241 } 242 243 dma_fence_get(fence); 244 err = dma_fence_chain_find_seqno(&fence, i); 245 dma_fence_put(fence); 246 if (err) { 247 pr_err("Error reported for previous fence!\n"); 248 goto err; 249 } 250 if (i > 0 && fence != fc.chains[i - 1]) { 251 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n", 252 i + 1, i); 253 err = -EINVAL; 254 goto err; 255 } 256 } 257 258 err: 259 fence_chains_fini(&fc); 260 return err; 261 } 262 263 static int find_signaled(void *arg) 264 { 265 struct fence_chains fc; 266 struct dma_fence *fence; 267 int err; 268 269 err = fence_chains_init(&fc, 2, seqno_inc); 270 if (err) 271 return err; 272 273 dma_fence_signal(fc.fences[0]); 274 275 fence = dma_fence_get(fc.tail); 276 err = dma_fence_chain_find_seqno(&fence, 1); 277 dma_fence_put(fence); 278 if (err) { 279 pr_err("Reported %d for find_seqno()!\n", err); 280 goto err; 281 } 282 283 if (fence && fence != fc.chains[0]) { 284 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:1\n", 285 fence->seqno); 286 287 dma_fence_get(fence); 288 err = dma_fence_chain_find_seqno(&fence, 1); 289 dma_fence_put(fence); 290 if (err) 291 pr_err("Reported %d for finding self!\n", err); 292 293 err = -EINVAL; 294 } 295 296 err: 297 fence_chains_fini(&fc); 298 return err; 299 } 300 301 static int find_out_of_order(void *arg) 302 { 303 struct fence_chains fc; 304 struct dma_fence *fence; 305 int err; 306 307 err = fence_chains_init(&fc, 3, seqno_inc); 308 if (err) 309 return err; 310 311 dma_fence_signal(fc.fences[1]); 312 313 fence = dma_fence_get(fc.tail); 314 err = dma_fence_chain_find_seqno(&fence, 2); 315 dma_fence_put(fence); 316 if (err) { 317 pr_err("Reported %d for find_seqno()!\n", err); 318 goto err; 319 } 320 321 if (fence && fence != fc.chains[1]) { 322 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:2\n", 323 fence->seqno); 324 325 dma_fence_get(fence); 326 err = dma_fence_chain_find_seqno(&fence, 2); 327 dma_fence_put(fence); 328 if (err) 329 pr_err("Reported %d for finding self!\n", err); 330 331 err = -EINVAL; 332 } 333 334 err: 335 fence_chains_fini(&fc); 336 return err; 337 } 338 339 static uint64_t seqno_inc2(unsigned int i) 340 { 341 return 2 * i + 2; 342 } 343 344 static int find_gap(void *arg) 345 { 346 struct fence_chains fc; 347 struct dma_fence *fence; 348 int err; 349 int i; 350 351 err = fence_chains_init(&fc, 64, seqno_inc2); 352 if (err) 353 return err; 354 355 for (i = 0; i < fc.chain_length; i++) { 356 fence = dma_fence_get(fc.tail); 357 err = dma_fence_chain_find_seqno(&fence, 2 * i + 1); 358 dma_fence_put(fence); 359 if (err) { 360 pr_err("Reported %d for find_seqno(%d:%d)!\n", 361 err, fc.chain_length + 1, 2 * i + 1); 362 goto err; 363 } 364 if (fence != fc.chains[i]) { 365 pr_err("Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)\n", 366 fence->seqno, 367 fc.chain_length + 1, 368 2 * i + 1); 369 err = -EINVAL; 370 goto err; 371 } 372 373 dma_fence_get(fence); 374 err = dma_fence_chain_find_seqno(&fence, 2 * i + 2); 375 dma_fence_put(fence); 376 if (err) { 377 pr_err("Error reported for finding self\n"); 378 goto err; 379 } 380 if (fence != fc.chains[i]) { 381 pr_err("Incorrect fence reported by find self\n"); 382 err = -EINVAL; 383 goto err; 384 } 385 } 386 387 err: 388 fence_chains_fini(&fc); 389 return err; 390 } 391 392 struct find_race { 393 struct fence_chains fc; 394 atomic_t children; 395 }; 396 397 static int __find_race(void *arg) 398 { 399 struct find_race *data = arg; 400 int err = 0; 401 402 while (!kthread_should_stop()) { 403 struct dma_fence *fence = dma_fence_get(data->fc.tail); 404 int seqno; 405 406 seqno = prandom_u32_max(data->fc.chain_length) + 1; 407 408 err = dma_fence_chain_find_seqno(&fence, seqno); 409 if (err) { 410 pr_err("Failed to find fence seqno:%d\n", 411 seqno); 412 dma_fence_put(fence); 413 break; 414 } 415 if (!fence) 416 goto signal; 417 418 err = dma_fence_chain_find_seqno(&fence, seqno); 419 if (err) { 420 pr_err("Reported an invalid fence for find-self:%d\n", 421 seqno); 422 dma_fence_put(fence); 423 break; 424 } 425 426 if (fence->seqno < seqno) { 427 pr_err("Reported an earlier fence.seqno:%lld for seqno:%d\n", 428 fence->seqno, seqno); 429 err = -EINVAL; 430 dma_fence_put(fence); 431 break; 432 } 433 434 dma_fence_put(fence); 435 436 signal: 437 seqno = prandom_u32_max(data->fc.chain_length - 1); 438 dma_fence_signal(data->fc.fences[seqno]); 439 cond_resched(); 440 } 441 442 if (atomic_dec_and_test(&data->children)) 443 wake_up_var(&data->children); 444 return err; 445 } 446 447 static int find_race(void *arg) 448 { 449 struct find_race data; 450 int ncpus = num_online_cpus(); 451 struct task_struct **threads; 452 unsigned long count; 453 int err; 454 int i; 455 456 err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc); 457 if (err) 458 return err; 459 460 threads = kmalloc_array(ncpus, sizeof(*threads), GFP_KERNEL); 461 if (!threads) { 462 err = -ENOMEM; 463 goto err; 464 } 465 466 atomic_set(&data.children, 0); 467 for (i = 0; i < ncpus; i++) { 468 threads[i] = kthread_run(__find_race, &data, "dmabuf/%d", i); 469 if (IS_ERR(threads[i])) { 470 ncpus = i; 471 break; 472 } 473 atomic_inc(&data.children); 474 get_task_struct(threads[i]); 475 } 476 477 wait_var_event_timeout(&data.children, 478 !atomic_read(&data.children), 479 5 * HZ); 480 481 for (i = 0; i < ncpus; i++) { 482 int ret; 483 484 ret = kthread_stop(threads[i]); 485 if (ret && !err) 486 err = ret; 487 put_task_struct(threads[i]); 488 } 489 kfree(threads); 490 491 count = 0; 492 for (i = 0; i < data.fc.chain_length; i++) 493 if (dma_fence_is_signaled(data.fc.fences[i])) 494 count++; 495 pr_info("Completed %lu cycles\n", count); 496 497 err: 498 fence_chains_fini(&data.fc); 499 return err; 500 } 501 502 static int signal_forward(void *arg) 503 { 504 struct fence_chains fc; 505 int err; 506 int i; 507 508 err = fence_chains_init(&fc, 64, seqno_inc); 509 if (err) 510 return err; 511 512 for (i = 0; i < fc.chain_length; i++) { 513 dma_fence_signal(fc.fences[i]); 514 515 if (!dma_fence_is_signaled(fc.chains[i])) { 516 pr_err("chain[%d] not signaled!\n", i); 517 err = -EINVAL; 518 goto err; 519 } 520 521 if (i + 1 < fc.chain_length && 522 dma_fence_is_signaled(fc.chains[i + 1])) { 523 pr_err("chain[%d] is signaled!\n", i); 524 err = -EINVAL; 525 goto err; 526 } 527 } 528 529 err: 530 fence_chains_fini(&fc); 531 return err; 532 } 533 534 static int signal_backward(void *arg) 535 { 536 struct fence_chains fc; 537 int err; 538 int i; 539 540 err = fence_chains_init(&fc, 64, seqno_inc); 541 if (err) 542 return err; 543 544 for (i = fc.chain_length; i--; ) { 545 dma_fence_signal(fc.fences[i]); 546 547 if (i > 0 && dma_fence_is_signaled(fc.chains[i])) { 548 pr_err("chain[%d] is signaled!\n", i); 549 err = -EINVAL; 550 goto err; 551 } 552 } 553 554 for (i = 0; i < fc.chain_length; i++) { 555 if (!dma_fence_is_signaled(fc.chains[i])) { 556 pr_err("chain[%d] was not signaled!\n", i); 557 err = -EINVAL; 558 goto err; 559 } 560 } 561 562 err: 563 fence_chains_fini(&fc); 564 return err; 565 } 566 567 static int __wait_fence_chains(void *arg) 568 { 569 struct fence_chains *fc = arg; 570 571 if (dma_fence_wait(fc->tail, false)) 572 return -EIO; 573 574 return 0; 575 } 576 577 static int wait_forward(void *arg) 578 { 579 struct fence_chains fc; 580 struct task_struct *tsk; 581 int err; 582 int i; 583 584 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 585 if (err) 586 return err; 587 588 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 589 if (IS_ERR(tsk)) { 590 err = PTR_ERR(tsk); 591 goto err; 592 } 593 get_task_struct(tsk); 594 yield_to(tsk, true); 595 596 for (i = 0; i < fc.chain_length; i++) 597 dma_fence_signal(fc.fences[i]); 598 599 err = kthread_stop(tsk); 600 put_task_struct(tsk); 601 602 err: 603 fence_chains_fini(&fc); 604 return err; 605 } 606 607 static int wait_backward(void *arg) 608 { 609 struct fence_chains fc; 610 struct task_struct *tsk; 611 int err; 612 int i; 613 614 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 615 if (err) 616 return err; 617 618 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 619 if (IS_ERR(tsk)) { 620 err = PTR_ERR(tsk); 621 goto err; 622 } 623 get_task_struct(tsk); 624 yield_to(tsk, true); 625 626 for (i = fc.chain_length; i--; ) 627 dma_fence_signal(fc.fences[i]); 628 629 err = kthread_stop(tsk); 630 put_task_struct(tsk); 631 632 err: 633 fence_chains_fini(&fc); 634 return err; 635 } 636 637 static void randomise_fences(struct fence_chains *fc) 638 { 639 unsigned int count = fc->chain_length; 640 641 /* Fisher-Yates shuffle courtesy of Knuth */ 642 while (--count) { 643 unsigned int swp; 644 645 swp = prandom_u32_max(count + 1); 646 if (swp == count) 647 continue; 648 649 swap(fc->fences[count], fc->fences[swp]); 650 } 651 } 652 653 static int wait_random(void *arg) 654 { 655 struct fence_chains fc; 656 struct task_struct *tsk; 657 int err; 658 int i; 659 660 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 661 if (err) 662 return err; 663 664 randomise_fences(&fc); 665 666 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 667 if (IS_ERR(tsk)) { 668 err = PTR_ERR(tsk); 669 goto err; 670 } 671 get_task_struct(tsk); 672 yield_to(tsk, true); 673 674 for (i = 0; i < fc.chain_length; i++) 675 dma_fence_signal(fc.fences[i]); 676 677 err = kthread_stop(tsk); 678 put_task_struct(tsk); 679 680 err: 681 fence_chains_fini(&fc); 682 return err; 683 } 684 685 int dma_fence_chain(void) 686 { 687 static const struct subtest tests[] = { 688 SUBTEST(sanitycheck), 689 SUBTEST(find_seqno), 690 SUBTEST(find_signaled), 691 SUBTEST(find_out_of_order), 692 SUBTEST(find_gap), 693 SUBTEST(find_race), 694 SUBTEST(signal_forward), 695 SUBTEST(signal_backward), 696 SUBTEST(wait_forward), 697 SUBTEST(wait_backward), 698 SUBTEST(wait_random), 699 }; 700 int ret; 701 702 pr_info("sizeof(dma_fence_chain)=%zu\n", 703 sizeof(struct dma_fence_chain)); 704 705 slab_fences = KMEM_CACHE(mock_fence, 706 SLAB_TYPESAFE_BY_RCU | 707 SLAB_HWCACHE_ALIGN); 708 if (!slab_fences) 709 return -ENOMEM; 710 711 ret = subtests(tests, NULL); 712 713 kmem_cache_destroy(slab_fences); 714 return ret; 715 } 716