1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2018 Intel Corporation 5 */ 6 7 #include <linux/prime_numbers.h> 8 9 #include "gem/i915_gem_pm.h" 10 #include "gt/intel_reset.h" 11 12 #include "i915_selftest.h" 13 #include "selftests/i915_random.h" 14 #include "selftests/igt_flush_test.h" 15 #include "selftests/igt_live_test.h" 16 #include "selftests/igt_spinner.h" 17 #include "selftests/lib_sw_fence.h" 18 19 #include "gem/selftests/igt_gem_utils.h" 20 #include "gem/selftests/mock_context.h" 21 22 static int live_sanitycheck(void *arg) 23 { 24 struct drm_i915_private *i915 = arg; 25 struct intel_engine_cs *engine; 26 struct i915_gem_context *ctx; 27 enum intel_engine_id id; 28 struct igt_spinner spin; 29 intel_wakeref_t wakeref; 30 int err = -ENOMEM; 31 32 if (!HAS_LOGICAL_RING_CONTEXTS(i915)) 33 return 0; 34 35 mutex_lock(&i915->drm.struct_mutex); 36 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 37 38 if (igt_spinner_init(&spin, i915)) 39 goto err_unlock; 40 41 ctx = kernel_context(i915); 42 if (!ctx) 43 goto err_spin; 44 45 for_each_engine(engine, i915, id) { 46 struct i915_request *rq; 47 48 rq = igt_spinner_create_request(&spin, ctx, engine, MI_NOOP); 49 if (IS_ERR(rq)) { 50 err = PTR_ERR(rq); 51 goto err_ctx; 52 } 53 54 i915_request_add(rq); 55 if (!igt_wait_for_spinner(&spin, rq)) { 56 GEM_TRACE("spinner failed to start\n"); 57 GEM_TRACE_DUMP(); 58 intel_gt_set_wedged(&i915->gt); 59 err = -EIO; 60 goto err_ctx; 61 } 62 63 igt_spinner_end(&spin); 64 if (igt_flush_test(i915, I915_WAIT_LOCKED)) { 65 err = -EIO; 66 goto err_ctx; 67 } 68 } 69 70 err = 0; 71 err_ctx: 72 kernel_context_close(ctx); 73 err_spin: 74 igt_spinner_fini(&spin); 75 err_unlock: 76 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 77 mutex_unlock(&i915->drm.struct_mutex); 78 return err; 79 } 80 81 static int 82 emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx) 83 { 84 u32 *cs; 85 86 cs = intel_ring_begin(rq, 10); 87 if (IS_ERR(cs)) 88 return PTR_ERR(cs); 89 90 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 91 92 *cs++ = MI_SEMAPHORE_WAIT | 93 MI_SEMAPHORE_GLOBAL_GTT | 94 MI_SEMAPHORE_POLL | 95 MI_SEMAPHORE_SAD_NEQ_SDD; 96 *cs++ = 0; 97 *cs++ = i915_ggtt_offset(vma) + 4 * idx; 98 *cs++ = 0; 99 100 if (idx > 0) { 101 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 102 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1); 103 *cs++ = 0; 104 *cs++ = 1; 105 } else { 106 *cs++ = MI_NOOP; 107 *cs++ = MI_NOOP; 108 *cs++ = MI_NOOP; 109 *cs++ = MI_NOOP; 110 } 111 112 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 113 114 intel_ring_advance(rq, cs); 115 return 0; 116 } 117 118 static struct i915_request * 119 semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx) 120 { 121 struct i915_gem_context *ctx; 122 struct i915_request *rq; 123 int err; 124 125 ctx = kernel_context(engine->i915); 126 if (!ctx) 127 return ERR_PTR(-ENOMEM); 128 129 rq = igt_request_alloc(ctx, engine); 130 if (IS_ERR(rq)) 131 goto out_ctx; 132 133 err = emit_semaphore_chain(rq, vma, idx); 134 i915_request_add(rq); 135 if (err) 136 rq = ERR_PTR(err); 137 138 out_ctx: 139 kernel_context_close(ctx); 140 return rq; 141 } 142 143 static int 144 release_queue(struct intel_engine_cs *engine, 145 struct i915_vma *vma, 146 int idx) 147 { 148 struct i915_sched_attr attr = { 149 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX), 150 }; 151 struct i915_request *rq; 152 u32 *cs; 153 154 rq = i915_request_create(engine->kernel_context); 155 if (IS_ERR(rq)) 156 return PTR_ERR(rq); 157 158 cs = intel_ring_begin(rq, 4); 159 if (IS_ERR(cs)) { 160 i915_request_add(rq); 161 return PTR_ERR(cs); 162 } 163 164 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 165 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1); 166 *cs++ = 0; 167 *cs++ = 1; 168 169 intel_ring_advance(rq, cs); 170 i915_request_add(rq); 171 172 engine->schedule(rq, &attr); 173 174 return 0; 175 } 176 177 static int 178 slice_semaphore_queue(struct intel_engine_cs *outer, 179 struct i915_vma *vma, 180 int count) 181 { 182 struct intel_engine_cs *engine; 183 struct i915_request *head; 184 enum intel_engine_id id; 185 int err, i, n = 0; 186 187 head = semaphore_queue(outer, vma, n++); 188 if (IS_ERR(head)) 189 return PTR_ERR(head); 190 191 i915_request_get(head); 192 for_each_engine(engine, outer->i915, id) { 193 for (i = 0; i < count; i++) { 194 struct i915_request *rq; 195 196 rq = semaphore_queue(engine, vma, n++); 197 if (IS_ERR(rq)) { 198 err = PTR_ERR(rq); 199 goto out; 200 } 201 } 202 } 203 204 err = release_queue(outer, vma, n); 205 if (err) 206 goto out; 207 208 if (i915_request_wait(head, 209 I915_WAIT_LOCKED, 210 2 * RUNTIME_INFO(outer->i915)->num_engines * (count + 2) * (count + 3)) < 0) { 211 pr_err("Failed to slice along semaphore chain of length (%d, %d)!\n", 212 count, n); 213 GEM_TRACE_DUMP(); 214 intel_gt_set_wedged(outer->gt); 215 err = -EIO; 216 } 217 218 out: 219 i915_request_put(head); 220 return err; 221 } 222 223 static int live_timeslice_preempt(void *arg) 224 { 225 struct drm_i915_private *i915 = arg; 226 struct drm_i915_gem_object *obj; 227 intel_wakeref_t wakeref; 228 struct i915_vma *vma; 229 void *vaddr; 230 int err = 0; 231 int count; 232 233 /* 234 * If a request takes too long, we would like to give other users 235 * a fair go on the GPU. In particular, users may create batches 236 * that wait upon external input, where that input may even be 237 * supplied by another GPU job. To avoid blocking forever, we 238 * need to preempt the current task and replace it with another 239 * ready task. 240 */ 241 242 mutex_lock(&i915->drm.struct_mutex); 243 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 244 245 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 246 if (IS_ERR(obj)) { 247 err = PTR_ERR(obj); 248 goto err_unlock; 249 } 250 251 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); 252 if (IS_ERR(vma)) { 253 err = PTR_ERR(vma); 254 goto err_obj; 255 } 256 257 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC); 258 if (IS_ERR(vaddr)) { 259 err = PTR_ERR(vaddr); 260 goto err_obj; 261 } 262 263 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 264 if (err) 265 goto err_map; 266 267 for_each_prime_number_from(count, 1, 16) { 268 struct intel_engine_cs *engine; 269 enum intel_engine_id id; 270 271 for_each_engine(engine, i915, id) { 272 if (!intel_engine_has_preemption(engine)) 273 continue; 274 275 memset(vaddr, 0, PAGE_SIZE); 276 277 err = slice_semaphore_queue(engine, vma, count); 278 if (err) 279 goto err_pin; 280 281 if (igt_flush_test(i915, I915_WAIT_LOCKED)) { 282 err = -EIO; 283 goto err_pin; 284 } 285 } 286 } 287 288 err_pin: 289 i915_vma_unpin(vma); 290 err_map: 291 i915_gem_object_unpin_map(obj); 292 err_obj: 293 i915_gem_object_put(obj); 294 err_unlock: 295 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 296 mutex_unlock(&i915->drm.struct_mutex); 297 298 return err; 299 } 300 301 static int live_busywait_preempt(void *arg) 302 { 303 struct drm_i915_private *i915 = arg; 304 struct i915_gem_context *ctx_hi, *ctx_lo; 305 struct intel_engine_cs *engine; 306 struct drm_i915_gem_object *obj; 307 struct i915_vma *vma; 308 enum intel_engine_id id; 309 intel_wakeref_t wakeref; 310 int err = -ENOMEM; 311 u32 *map; 312 313 /* 314 * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can 315 * preempt the busywaits used to synchronise between rings. 316 */ 317 318 mutex_lock(&i915->drm.struct_mutex); 319 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 320 321 ctx_hi = kernel_context(i915); 322 if (!ctx_hi) 323 goto err_unlock; 324 ctx_hi->sched.priority = 325 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY); 326 327 ctx_lo = kernel_context(i915); 328 if (!ctx_lo) 329 goto err_ctx_hi; 330 ctx_lo->sched.priority = 331 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY); 332 333 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 334 if (IS_ERR(obj)) { 335 err = PTR_ERR(obj); 336 goto err_ctx_lo; 337 } 338 339 map = i915_gem_object_pin_map(obj, I915_MAP_WC); 340 if (IS_ERR(map)) { 341 err = PTR_ERR(map); 342 goto err_obj; 343 } 344 345 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); 346 if (IS_ERR(vma)) { 347 err = PTR_ERR(vma); 348 goto err_map; 349 } 350 351 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 352 if (err) 353 goto err_map; 354 355 for_each_engine(engine, i915, id) { 356 struct i915_request *lo, *hi; 357 struct igt_live_test t; 358 u32 *cs; 359 360 if (!intel_engine_has_preemption(engine)) 361 continue; 362 363 if (!intel_engine_can_store_dword(engine)) 364 continue; 365 366 if (igt_live_test_begin(&t, i915, __func__, engine->name)) { 367 err = -EIO; 368 goto err_vma; 369 } 370 371 /* 372 * We create two requests. The low priority request 373 * busywaits on a semaphore (inside the ringbuffer where 374 * is should be preemptible) and the high priority requests 375 * uses a MI_STORE_DWORD_IMM to update the semaphore value 376 * allowing the first request to complete. If preemption 377 * fails, we hang instead. 378 */ 379 380 lo = igt_request_alloc(ctx_lo, engine); 381 if (IS_ERR(lo)) { 382 err = PTR_ERR(lo); 383 goto err_vma; 384 } 385 386 cs = intel_ring_begin(lo, 8); 387 if (IS_ERR(cs)) { 388 err = PTR_ERR(cs); 389 i915_request_add(lo); 390 goto err_vma; 391 } 392 393 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 394 *cs++ = i915_ggtt_offset(vma); 395 *cs++ = 0; 396 *cs++ = 1; 397 398 /* XXX Do we need a flush + invalidate here? */ 399 400 *cs++ = MI_SEMAPHORE_WAIT | 401 MI_SEMAPHORE_GLOBAL_GTT | 402 MI_SEMAPHORE_POLL | 403 MI_SEMAPHORE_SAD_EQ_SDD; 404 *cs++ = 0; 405 *cs++ = i915_ggtt_offset(vma); 406 *cs++ = 0; 407 408 intel_ring_advance(lo, cs); 409 i915_request_add(lo); 410 411 if (wait_for(READ_ONCE(*map), 10)) { 412 err = -ETIMEDOUT; 413 goto err_vma; 414 } 415 416 /* Low priority request should be busywaiting now */ 417 if (i915_request_wait(lo, 0, 1) != -ETIME) { 418 pr_err("%s: Busywaiting request did not!\n", 419 engine->name); 420 err = -EIO; 421 goto err_vma; 422 } 423 424 hi = igt_request_alloc(ctx_hi, engine); 425 if (IS_ERR(hi)) { 426 err = PTR_ERR(hi); 427 goto err_vma; 428 } 429 430 cs = intel_ring_begin(hi, 4); 431 if (IS_ERR(cs)) { 432 err = PTR_ERR(cs); 433 i915_request_add(hi); 434 goto err_vma; 435 } 436 437 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 438 *cs++ = i915_ggtt_offset(vma); 439 *cs++ = 0; 440 *cs++ = 0; 441 442 intel_ring_advance(hi, cs); 443 i915_request_add(hi); 444 445 if (i915_request_wait(lo, 0, HZ / 5) < 0) { 446 struct drm_printer p = drm_info_printer(i915->drm.dev); 447 448 pr_err("%s: Failed to preempt semaphore busywait!\n", 449 engine->name); 450 451 intel_engine_dump(engine, &p, "%s\n", engine->name); 452 GEM_TRACE_DUMP(); 453 454 intel_gt_set_wedged(&i915->gt); 455 err = -EIO; 456 goto err_vma; 457 } 458 GEM_BUG_ON(READ_ONCE(*map)); 459 460 if (igt_live_test_end(&t)) { 461 err = -EIO; 462 goto err_vma; 463 } 464 } 465 466 err = 0; 467 err_vma: 468 i915_vma_unpin(vma); 469 err_map: 470 i915_gem_object_unpin_map(obj); 471 err_obj: 472 i915_gem_object_put(obj); 473 err_ctx_lo: 474 kernel_context_close(ctx_lo); 475 err_ctx_hi: 476 kernel_context_close(ctx_hi); 477 err_unlock: 478 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 479 mutex_unlock(&i915->drm.struct_mutex); 480 return err; 481 } 482 483 static int live_preempt(void *arg) 484 { 485 struct drm_i915_private *i915 = arg; 486 struct i915_gem_context *ctx_hi, *ctx_lo; 487 struct igt_spinner spin_hi, spin_lo; 488 struct intel_engine_cs *engine; 489 enum intel_engine_id id; 490 intel_wakeref_t wakeref; 491 int err = -ENOMEM; 492 493 if (!HAS_LOGICAL_RING_PREEMPTION(i915)) 494 return 0; 495 496 if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION)) 497 pr_err("Logical preemption supported, but not exposed\n"); 498 499 mutex_lock(&i915->drm.struct_mutex); 500 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 501 502 if (igt_spinner_init(&spin_hi, i915)) 503 goto err_unlock; 504 505 if (igt_spinner_init(&spin_lo, i915)) 506 goto err_spin_hi; 507 508 ctx_hi = kernel_context(i915); 509 if (!ctx_hi) 510 goto err_spin_lo; 511 ctx_hi->sched.priority = 512 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY); 513 514 ctx_lo = kernel_context(i915); 515 if (!ctx_lo) 516 goto err_ctx_hi; 517 ctx_lo->sched.priority = 518 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY); 519 520 for_each_engine(engine, i915, id) { 521 struct igt_live_test t; 522 struct i915_request *rq; 523 524 if (!intel_engine_has_preemption(engine)) 525 continue; 526 527 if (igt_live_test_begin(&t, i915, __func__, engine->name)) { 528 err = -EIO; 529 goto err_ctx_lo; 530 } 531 532 rq = igt_spinner_create_request(&spin_lo, ctx_lo, engine, 533 MI_ARB_CHECK); 534 if (IS_ERR(rq)) { 535 err = PTR_ERR(rq); 536 goto err_ctx_lo; 537 } 538 539 i915_request_add(rq); 540 if (!igt_wait_for_spinner(&spin_lo, rq)) { 541 GEM_TRACE("lo spinner failed to start\n"); 542 GEM_TRACE_DUMP(); 543 intel_gt_set_wedged(&i915->gt); 544 err = -EIO; 545 goto err_ctx_lo; 546 } 547 548 rq = igt_spinner_create_request(&spin_hi, ctx_hi, engine, 549 MI_ARB_CHECK); 550 if (IS_ERR(rq)) { 551 igt_spinner_end(&spin_lo); 552 err = PTR_ERR(rq); 553 goto err_ctx_lo; 554 } 555 556 i915_request_add(rq); 557 if (!igt_wait_for_spinner(&spin_hi, rq)) { 558 GEM_TRACE("hi spinner failed to start\n"); 559 GEM_TRACE_DUMP(); 560 intel_gt_set_wedged(&i915->gt); 561 err = -EIO; 562 goto err_ctx_lo; 563 } 564 565 igt_spinner_end(&spin_hi); 566 igt_spinner_end(&spin_lo); 567 568 if (igt_live_test_end(&t)) { 569 err = -EIO; 570 goto err_ctx_lo; 571 } 572 } 573 574 err = 0; 575 err_ctx_lo: 576 kernel_context_close(ctx_lo); 577 err_ctx_hi: 578 kernel_context_close(ctx_hi); 579 err_spin_lo: 580 igt_spinner_fini(&spin_lo); 581 err_spin_hi: 582 igt_spinner_fini(&spin_hi); 583 err_unlock: 584 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 585 mutex_unlock(&i915->drm.struct_mutex); 586 return err; 587 } 588 589 static int live_late_preempt(void *arg) 590 { 591 struct drm_i915_private *i915 = arg; 592 struct i915_gem_context *ctx_hi, *ctx_lo; 593 struct igt_spinner spin_hi, spin_lo; 594 struct intel_engine_cs *engine; 595 struct i915_sched_attr attr = {}; 596 enum intel_engine_id id; 597 intel_wakeref_t wakeref; 598 int err = -ENOMEM; 599 600 if (!HAS_LOGICAL_RING_PREEMPTION(i915)) 601 return 0; 602 603 mutex_lock(&i915->drm.struct_mutex); 604 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 605 606 if (igt_spinner_init(&spin_hi, i915)) 607 goto err_unlock; 608 609 if (igt_spinner_init(&spin_lo, i915)) 610 goto err_spin_hi; 611 612 ctx_hi = kernel_context(i915); 613 if (!ctx_hi) 614 goto err_spin_lo; 615 616 ctx_lo = kernel_context(i915); 617 if (!ctx_lo) 618 goto err_ctx_hi; 619 620 /* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */ 621 ctx_lo->sched.priority = I915_USER_PRIORITY(1); 622 623 for_each_engine(engine, i915, id) { 624 struct igt_live_test t; 625 struct i915_request *rq; 626 627 if (!intel_engine_has_preemption(engine)) 628 continue; 629 630 if (igt_live_test_begin(&t, i915, __func__, engine->name)) { 631 err = -EIO; 632 goto err_ctx_lo; 633 } 634 635 rq = igt_spinner_create_request(&spin_lo, ctx_lo, engine, 636 MI_ARB_CHECK); 637 if (IS_ERR(rq)) { 638 err = PTR_ERR(rq); 639 goto err_ctx_lo; 640 } 641 642 i915_request_add(rq); 643 if (!igt_wait_for_spinner(&spin_lo, rq)) { 644 pr_err("First context failed to start\n"); 645 goto err_wedged; 646 } 647 648 rq = igt_spinner_create_request(&spin_hi, ctx_hi, engine, 649 MI_NOOP); 650 if (IS_ERR(rq)) { 651 igt_spinner_end(&spin_lo); 652 err = PTR_ERR(rq); 653 goto err_ctx_lo; 654 } 655 656 i915_request_add(rq); 657 if (igt_wait_for_spinner(&spin_hi, rq)) { 658 pr_err("Second context overtook first?\n"); 659 goto err_wedged; 660 } 661 662 attr.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX); 663 engine->schedule(rq, &attr); 664 665 if (!igt_wait_for_spinner(&spin_hi, rq)) { 666 pr_err("High priority context failed to preempt the low priority context\n"); 667 GEM_TRACE_DUMP(); 668 goto err_wedged; 669 } 670 671 igt_spinner_end(&spin_hi); 672 igt_spinner_end(&spin_lo); 673 674 if (igt_live_test_end(&t)) { 675 err = -EIO; 676 goto err_ctx_lo; 677 } 678 } 679 680 err = 0; 681 err_ctx_lo: 682 kernel_context_close(ctx_lo); 683 err_ctx_hi: 684 kernel_context_close(ctx_hi); 685 err_spin_lo: 686 igt_spinner_fini(&spin_lo); 687 err_spin_hi: 688 igt_spinner_fini(&spin_hi); 689 err_unlock: 690 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 691 mutex_unlock(&i915->drm.struct_mutex); 692 return err; 693 694 err_wedged: 695 igt_spinner_end(&spin_hi); 696 igt_spinner_end(&spin_lo); 697 intel_gt_set_wedged(&i915->gt); 698 err = -EIO; 699 goto err_ctx_lo; 700 } 701 702 struct preempt_client { 703 struct igt_spinner spin; 704 struct i915_gem_context *ctx; 705 }; 706 707 static int preempt_client_init(struct drm_i915_private *i915, 708 struct preempt_client *c) 709 { 710 c->ctx = kernel_context(i915); 711 if (!c->ctx) 712 return -ENOMEM; 713 714 if (igt_spinner_init(&c->spin, i915)) 715 goto err_ctx; 716 717 return 0; 718 719 err_ctx: 720 kernel_context_close(c->ctx); 721 return -ENOMEM; 722 } 723 724 static void preempt_client_fini(struct preempt_client *c) 725 { 726 igt_spinner_fini(&c->spin); 727 kernel_context_close(c->ctx); 728 } 729 730 static int live_nopreempt(void *arg) 731 { 732 struct drm_i915_private *i915 = arg; 733 struct intel_engine_cs *engine; 734 struct preempt_client a, b; 735 enum intel_engine_id id; 736 intel_wakeref_t wakeref; 737 int err = -ENOMEM; 738 739 /* 740 * Verify that we can disable preemption for an individual request 741 * that may be being observed and not want to be interrupted. 742 */ 743 744 if (!HAS_LOGICAL_RING_PREEMPTION(i915)) 745 return 0; 746 747 mutex_lock(&i915->drm.struct_mutex); 748 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 749 750 if (preempt_client_init(i915, &a)) 751 goto err_unlock; 752 if (preempt_client_init(i915, &b)) 753 goto err_client_a; 754 b.ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX); 755 756 for_each_engine(engine, i915, id) { 757 struct i915_request *rq_a, *rq_b; 758 759 if (!intel_engine_has_preemption(engine)) 760 continue; 761 762 engine->execlists.preempt_hang.count = 0; 763 764 rq_a = igt_spinner_create_request(&a.spin, 765 a.ctx, engine, 766 MI_ARB_CHECK); 767 if (IS_ERR(rq_a)) { 768 err = PTR_ERR(rq_a); 769 goto err_client_b; 770 } 771 772 /* Low priority client, but unpreemptable! */ 773 rq_a->flags |= I915_REQUEST_NOPREEMPT; 774 775 i915_request_add(rq_a); 776 if (!igt_wait_for_spinner(&a.spin, rq_a)) { 777 pr_err("First client failed to start\n"); 778 goto err_wedged; 779 } 780 781 rq_b = igt_spinner_create_request(&b.spin, 782 b.ctx, engine, 783 MI_ARB_CHECK); 784 if (IS_ERR(rq_b)) { 785 err = PTR_ERR(rq_b); 786 goto err_client_b; 787 } 788 789 i915_request_add(rq_b); 790 791 /* B is much more important than A! (But A is unpreemptable.) */ 792 GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a)); 793 794 /* Wait long enough for preemption and timeslicing */ 795 if (igt_wait_for_spinner(&b.spin, rq_b)) { 796 pr_err("Second client started too early!\n"); 797 goto err_wedged; 798 } 799 800 igt_spinner_end(&a.spin); 801 802 if (!igt_wait_for_spinner(&b.spin, rq_b)) { 803 pr_err("Second client failed to start\n"); 804 goto err_wedged; 805 } 806 807 igt_spinner_end(&b.spin); 808 809 if (engine->execlists.preempt_hang.count) { 810 pr_err("Preemption recorded x%d; should have been suppressed!\n", 811 engine->execlists.preempt_hang.count); 812 err = -EINVAL; 813 goto err_wedged; 814 } 815 816 if (igt_flush_test(i915, I915_WAIT_LOCKED)) 817 goto err_wedged; 818 } 819 820 err = 0; 821 err_client_b: 822 preempt_client_fini(&b); 823 err_client_a: 824 preempt_client_fini(&a); 825 err_unlock: 826 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 827 mutex_unlock(&i915->drm.struct_mutex); 828 return err; 829 830 err_wedged: 831 igt_spinner_end(&b.spin); 832 igt_spinner_end(&a.spin); 833 intel_gt_set_wedged(&i915->gt); 834 err = -EIO; 835 goto err_client_b; 836 } 837 838 static int live_suppress_self_preempt(void *arg) 839 { 840 struct drm_i915_private *i915 = arg; 841 struct intel_engine_cs *engine; 842 struct i915_sched_attr attr = { 843 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX) 844 }; 845 struct preempt_client a, b; 846 enum intel_engine_id id; 847 intel_wakeref_t wakeref; 848 int err = -ENOMEM; 849 850 /* 851 * Verify that if a preemption request does not cause a change in 852 * the current execution order, the preempt-to-idle injection is 853 * skipped and that we do not accidentally apply it after the CS 854 * completion event. 855 */ 856 857 if (!HAS_LOGICAL_RING_PREEMPTION(i915)) 858 return 0; 859 860 if (USES_GUC_SUBMISSION(i915)) 861 return 0; /* presume black blox */ 862 863 if (intel_vgpu_active(i915)) 864 return 0; /* GVT forces single port & request submission */ 865 866 mutex_lock(&i915->drm.struct_mutex); 867 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 868 869 if (preempt_client_init(i915, &a)) 870 goto err_unlock; 871 if (preempt_client_init(i915, &b)) 872 goto err_client_a; 873 874 for_each_engine(engine, i915, id) { 875 struct i915_request *rq_a, *rq_b; 876 int depth; 877 878 if (!intel_engine_has_preemption(engine)) 879 continue; 880 881 engine->execlists.preempt_hang.count = 0; 882 883 rq_a = igt_spinner_create_request(&a.spin, 884 a.ctx, engine, 885 MI_NOOP); 886 if (IS_ERR(rq_a)) { 887 err = PTR_ERR(rq_a); 888 goto err_client_b; 889 } 890 891 i915_request_add(rq_a); 892 if (!igt_wait_for_spinner(&a.spin, rq_a)) { 893 pr_err("First client failed to start\n"); 894 goto err_wedged; 895 } 896 897 for (depth = 0; depth < 8; depth++) { 898 rq_b = igt_spinner_create_request(&b.spin, 899 b.ctx, engine, 900 MI_NOOP); 901 if (IS_ERR(rq_b)) { 902 err = PTR_ERR(rq_b); 903 goto err_client_b; 904 } 905 i915_request_add(rq_b); 906 907 GEM_BUG_ON(i915_request_completed(rq_a)); 908 engine->schedule(rq_a, &attr); 909 igt_spinner_end(&a.spin); 910 911 if (!igt_wait_for_spinner(&b.spin, rq_b)) { 912 pr_err("Second client failed to start\n"); 913 goto err_wedged; 914 } 915 916 swap(a, b); 917 rq_a = rq_b; 918 } 919 igt_spinner_end(&a.spin); 920 921 if (engine->execlists.preempt_hang.count) { 922 pr_err("Preemption recorded x%d, depth %d; should have been suppressed!\n", 923 engine->execlists.preempt_hang.count, 924 depth); 925 err = -EINVAL; 926 goto err_client_b; 927 } 928 929 if (igt_flush_test(i915, I915_WAIT_LOCKED)) 930 goto err_wedged; 931 } 932 933 err = 0; 934 err_client_b: 935 preempt_client_fini(&b); 936 err_client_a: 937 preempt_client_fini(&a); 938 err_unlock: 939 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 940 mutex_unlock(&i915->drm.struct_mutex); 941 return err; 942 943 err_wedged: 944 igt_spinner_end(&b.spin); 945 igt_spinner_end(&a.spin); 946 intel_gt_set_wedged(&i915->gt); 947 err = -EIO; 948 goto err_client_b; 949 } 950 951 static int __i915_sw_fence_call 952 dummy_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) 953 { 954 return NOTIFY_DONE; 955 } 956 957 static struct i915_request *dummy_request(struct intel_engine_cs *engine) 958 { 959 struct i915_request *rq; 960 961 rq = kzalloc(sizeof(*rq), GFP_KERNEL); 962 if (!rq) 963 return NULL; 964 965 INIT_LIST_HEAD(&rq->active_list); 966 rq->engine = engine; 967 968 i915_sched_node_init(&rq->sched); 969 970 /* mark this request as permanently incomplete */ 971 rq->fence.seqno = 1; 972 BUILD_BUG_ON(sizeof(rq->fence.seqno) != 8); /* upper 32b == 0 */ 973 rq->hwsp_seqno = (u32 *)&rq->fence.seqno + 1; 974 GEM_BUG_ON(i915_request_completed(rq)); 975 976 i915_sw_fence_init(&rq->submit, dummy_notify); 977 set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); 978 979 spin_lock_init(&rq->lock); 980 rq->fence.lock = &rq->lock; 981 INIT_LIST_HEAD(&rq->fence.cb_list); 982 983 return rq; 984 } 985 986 static void dummy_request_free(struct i915_request *dummy) 987 { 988 /* We have to fake the CS interrupt to kick the next request */ 989 i915_sw_fence_commit(&dummy->submit); 990 991 i915_request_mark_complete(dummy); 992 dma_fence_signal(&dummy->fence); 993 994 i915_sched_node_fini(&dummy->sched); 995 i915_sw_fence_fini(&dummy->submit); 996 997 dma_fence_free(&dummy->fence); 998 } 999 1000 static int live_suppress_wait_preempt(void *arg) 1001 { 1002 struct drm_i915_private *i915 = arg; 1003 struct preempt_client client[4]; 1004 struct intel_engine_cs *engine; 1005 enum intel_engine_id id; 1006 intel_wakeref_t wakeref; 1007 int err = -ENOMEM; 1008 int i; 1009 1010 /* 1011 * Waiters are given a little priority nudge, but not enough 1012 * to actually cause any preemption. Double check that we do 1013 * not needlessly generate preempt-to-idle cycles. 1014 */ 1015 1016 if (!HAS_LOGICAL_RING_PREEMPTION(i915)) 1017 return 0; 1018 1019 mutex_lock(&i915->drm.struct_mutex); 1020 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1021 1022 if (preempt_client_init(i915, &client[0])) /* ELSP[0] */ 1023 goto err_unlock; 1024 if (preempt_client_init(i915, &client[1])) /* ELSP[1] */ 1025 goto err_client_0; 1026 if (preempt_client_init(i915, &client[2])) /* head of queue */ 1027 goto err_client_1; 1028 if (preempt_client_init(i915, &client[3])) /* bystander */ 1029 goto err_client_2; 1030 1031 for_each_engine(engine, i915, id) { 1032 int depth; 1033 1034 if (!intel_engine_has_preemption(engine)) 1035 continue; 1036 1037 if (!engine->emit_init_breadcrumb) 1038 continue; 1039 1040 for (depth = 0; depth < ARRAY_SIZE(client); depth++) { 1041 struct i915_request *rq[ARRAY_SIZE(client)]; 1042 struct i915_request *dummy; 1043 1044 engine->execlists.preempt_hang.count = 0; 1045 1046 dummy = dummy_request(engine); 1047 if (!dummy) 1048 goto err_client_3; 1049 1050 for (i = 0; i < ARRAY_SIZE(client); i++) { 1051 rq[i] = igt_spinner_create_request(&client[i].spin, 1052 client[i].ctx, engine, 1053 MI_NOOP); 1054 if (IS_ERR(rq[i])) { 1055 err = PTR_ERR(rq[i]); 1056 goto err_wedged; 1057 } 1058 1059 /* Disable NEWCLIENT promotion */ 1060 __i915_active_request_set(&rq[i]->timeline->last_request, 1061 dummy); 1062 i915_request_add(rq[i]); 1063 } 1064 1065 dummy_request_free(dummy); 1066 1067 GEM_BUG_ON(i915_request_completed(rq[0])); 1068 if (!igt_wait_for_spinner(&client[0].spin, rq[0])) { 1069 pr_err("%s: First client failed to start\n", 1070 engine->name); 1071 goto err_wedged; 1072 } 1073 GEM_BUG_ON(!i915_request_started(rq[0])); 1074 1075 if (i915_request_wait(rq[depth], 1076 I915_WAIT_PRIORITY, 1077 1) != -ETIME) { 1078 pr_err("%s: Waiter depth:%d completed!\n", 1079 engine->name, depth); 1080 goto err_wedged; 1081 } 1082 1083 for (i = 0; i < ARRAY_SIZE(client); i++) 1084 igt_spinner_end(&client[i].spin); 1085 1086 if (igt_flush_test(i915, I915_WAIT_LOCKED)) 1087 goto err_wedged; 1088 1089 if (engine->execlists.preempt_hang.count) { 1090 pr_err("%s: Preemption recorded x%d, depth %d; should have been suppressed!\n", 1091 engine->name, 1092 engine->execlists.preempt_hang.count, 1093 depth); 1094 err = -EINVAL; 1095 goto err_client_3; 1096 } 1097 } 1098 } 1099 1100 err = 0; 1101 err_client_3: 1102 preempt_client_fini(&client[3]); 1103 err_client_2: 1104 preempt_client_fini(&client[2]); 1105 err_client_1: 1106 preempt_client_fini(&client[1]); 1107 err_client_0: 1108 preempt_client_fini(&client[0]); 1109 err_unlock: 1110 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 1111 mutex_unlock(&i915->drm.struct_mutex); 1112 return err; 1113 1114 err_wedged: 1115 for (i = 0; i < ARRAY_SIZE(client); i++) 1116 igt_spinner_end(&client[i].spin); 1117 intel_gt_set_wedged(&i915->gt); 1118 err = -EIO; 1119 goto err_client_3; 1120 } 1121 1122 static int live_chain_preempt(void *arg) 1123 { 1124 struct drm_i915_private *i915 = arg; 1125 struct intel_engine_cs *engine; 1126 struct preempt_client hi, lo; 1127 enum intel_engine_id id; 1128 intel_wakeref_t wakeref; 1129 int err = -ENOMEM; 1130 1131 /* 1132 * Build a chain AB...BA between two contexts (A, B) and request 1133 * preemption of the last request. It should then complete before 1134 * the previously submitted spinner in B. 1135 */ 1136 1137 if (!HAS_LOGICAL_RING_PREEMPTION(i915)) 1138 return 0; 1139 1140 mutex_lock(&i915->drm.struct_mutex); 1141 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1142 1143 if (preempt_client_init(i915, &hi)) 1144 goto err_unlock; 1145 1146 if (preempt_client_init(i915, &lo)) 1147 goto err_client_hi; 1148 1149 for_each_engine(engine, i915, id) { 1150 struct i915_sched_attr attr = { 1151 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX), 1152 }; 1153 struct igt_live_test t; 1154 struct i915_request *rq; 1155 int ring_size, count, i; 1156 1157 if (!intel_engine_has_preemption(engine)) 1158 continue; 1159 1160 rq = igt_spinner_create_request(&lo.spin, 1161 lo.ctx, engine, 1162 MI_ARB_CHECK); 1163 if (IS_ERR(rq)) 1164 goto err_wedged; 1165 i915_request_add(rq); 1166 1167 ring_size = rq->wa_tail - rq->head; 1168 if (ring_size < 0) 1169 ring_size += rq->ring->size; 1170 ring_size = rq->ring->size / ring_size; 1171 pr_debug("%s(%s): Using maximum of %d requests\n", 1172 __func__, engine->name, ring_size); 1173 1174 igt_spinner_end(&lo.spin); 1175 if (i915_request_wait(rq, 0, HZ / 2) < 0) { 1176 pr_err("Timed out waiting to flush %s\n", engine->name); 1177 goto err_wedged; 1178 } 1179 1180 if (igt_live_test_begin(&t, i915, __func__, engine->name)) { 1181 err = -EIO; 1182 goto err_wedged; 1183 } 1184 1185 for_each_prime_number_from(count, 1, ring_size) { 1186 rq = igt_spinner_create_request(&hi.spin, 1187 hi.ctx, engine, 1188 MI_ARB_CHECK); 1189 if (IS_ERR(rq)) 1190 goto err_wedged; 1191 i915_request_add(rq); 1192 if (!igt_wait_for_spinner(&hi.spin, rq)) 1193 goto err_wedged; 1194 1195 rq = igt_spinner_create_request(&lo.spin, 1196 lo.ctx, engine, 1197 MI_ARB_CHECK); 1198 if (IS_ERR(rq)) 1199 goto err_wedged; 1200 i915_request_add(rq); 1201 1202 for (i = 0; i < count; i++) { 1203 rq = igt_request_alloc(lo.ctx, engine); 1204 if (IS_ERR(rq)) 1205 goto err_wedged; 1206 i915_request_add(rq); 1207 } 1208 1209 rq = igt_request_alloc(hi.ctx, engine); 1210 if (IS_ERR(rq)) 1211 goto err_wedged; 1212 i915_request_add(rq); 1213 engine->schedule(rq, &attr); 1214 1215 igt_spinner_end(&hi.spin); 1216 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 1217 struct drm_printer p = 1218 drm_info_printer(i915->drm.dev); 1219 1220 pr_err("Failed to preempt over chain of %d\n", 1221 count); 1222 intel_engine_dump(engine, &p, 1223 "%s\n", engine->name); 1224 goto err_wedged; 1225 } 1226 igt_spinner_end(&lo.spin); 1227 1228 rq = igt_request_alloc(lo.ctx, engine); 1229 if (IS_ERR(rq)) 1230 goto err_wedged; 1231 i915_request_add(rq); 1232 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 1233 struct drm_printer p = 1234 drm_info_printer(i915->drm.dev); 1235 1236 pr_err("Failed to flush low priority chain of %d requests\n", 1237 count); 1238 intel_engine_dump(engine, &p, 1239 "%s\n", engine->name); 1240 goto err_wedged; 1241 } 1242 } 1243 1244 if (igt_live_test_end(&t)) { 1245 err = -EIO; 1246 goto err_wedged; 1247 } 1248 } 1249 1250 err = 0; 1251 err_client_lo: 1252 preempt_client_fini(&lo); 1253 err_client_hi: 1254 preempt_client_fini(&hi); 1255 err_unlock: 1256 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 1257 mutex_unlock(&i915->drm.struct_mutex); 1258 return err; 1259 1260 err_wedged: 1261 igt_spinner_end(&hi.spin); 1262 igt_spinner_end(&lo.spin); 1263 intel_gt_set_wedged(&i915->gt); 1264 err = -EIO; 1265 goto err_client_lo; 1266 } 1267 1268 static int live_preempt_hang(void *arg) 1269 { 1270 struct drm_i915_private *i915 = arg; 1271 struct i915_gem_context *ctx_hi, *ctx_lo; 1272 struct igt_spinner spin_hi, spin_lo; 1273 struct intel_engine_cs *engine; 1274 enum intel_engine_id id; 1275 intel_wakeref_t wakeref; 1276 int err = -ENOMEM; 1277 1278 if (!HAS_LOGICAL_RING_PREEMPTION(i915)) 1279 return 0; 1280 1281 if (!intel_has_reset_engine(i915)) 1282 return 0; 1283 1284 mutex_lock(&i915->drm.struct_mutex); 1285 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1286 1287 if (igt_spinner_init(&spin_hi, i915)) 1288 goto err_unlock; 1289 1290 if (igt_spinner_init(&spin_lo, i915)) 1291 goto err_spin_hi; 1292 1293 ctx_hi = kernel_context(i915); 1294 if (!ctx_hi) 1295 goto err_spin_lo; 1296 ctx_hi->sched.priority = 1297 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY); 1298 1299 ctx_lo = kernel_context(i915); 1300 if (!ctx_lo) 1301 goto err_ctx_hi; 1302 ctx_lo->sched.priority = 1303 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY); 1304 1305 for_each_engine(engine, i915, id) { 1306 struct i915_request *rq; 1307 1308 if (!intel_engine_has_preemption(engine)) 1309 continue; 1310 1311 rq = igt_spinner_create_request(&spin_lo, ctx_lo, engine, 1312 MI_ARB_CHECK); 1313 if (IS_ERR(rq)) { 1314 err = PTR_ERR(rq); 1315 goto err_ctx_lo; 1316 } 1317 1318 i915_request_add(rq); 1319 if (!igt_wait_for_spinner(&spin_lo, rq)) { 1320 GEM_TRACE("lo spinner failed to start\n"); 1321 GEM_TRACE_DUMP(); 1322 intel_gt_set_wedged(&i915->gt); 1323 err = -EIO; 1324 goto err_ctx_lo; 1325 } 1326 1327 rq = igt_spinner_create_request(&spin_hi, ctx_hi, engine, 1328 MI_ARB_CHECK); 1329 if (IS_ERR(rq)) { 1330 igt_spinner_end(&spin_lo); 1331 err = PTR_ERR(rq); 1332 goto err_ctx_lo; 1333 } 1334 1335 init_completion(&engine->execlists.preempt_hang.completion); 1336 engine->execlists.preempt_hang.inject_hang = true; 1337 1338 i915_request_add(rq); 1339 1340 if (!wait_for_completion_timeout(&engine->execlists.preempt_hang.completion, 1341 HZ / 10)) { 1342 pr_err("Preemption did not occur within timeout!"); 1343 GEM_TRACE_DUMP(); 1344 intel_gt_set_wedged(&i915->gt); 1345 err = -EIO; 1346 goto err_ctx_lo; 1347 } 1348 1349 set_bit(I915_RESET_ENGINE + id, &i915->gt.reset.flags); 1350 intel_engine_reset(engine, NULL); 1351 clear_bit(I915_RESET_ENGINE + id, &i915->gt.reset.flags); 1352 1353 engine->execlists.preempt_hang.inject_hang = false; 1354 1355 if (!igt_wait_for_spinner(&spin_hi, rq)) { 1356 GEM_TRACE("hi spinner failed to start\n"); 1357 GEM_TRACE_DUMP(); 1358 intel_gt_set_wedged(&i915->gt); 1359 err = -EIO; 1360 goto err_ctx_lo; 1361 } 1362 1363 igt_spinner_end(&spin_hi); 1364 igt_spinner_end(&spin_lo); 1365 if (igt_flush_test(i915, I915_WAIT_LOCKED)) { 1366 err = -EIO; 1367 goto err_ctx_lo; 1368 } 1369 } 1370 1371 err = 0; 1372 err_ctx_lo: 1373 kernel_context_close(ctx_lo); 1374 err_ctx_hi: 1375 kernel_context_close(ctx_hi); 1376 err_spin_lo: 1377 igt_spinner_fini(&spin_lo); 1378 err_spin_hi: 1379 igt_spinner_fini(&spin_hi); 1380 err_unlock: 1381 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 1382 mutex_unlock(&i915->drm.struct_mutex); 1383 return err; 1384 } 1385 1386 static int random_range(struct rnd_state *rnd, int min, int max) 1387 { 1388 return i915_prandom_u32_max_state(max - min, rnd) + min; 1389 } 1390 1391 static int random_priority(struct rnd_state *rnd) 1392 { 1393 return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX); 1394 } 1395 1396 struct preempt_smoke { 1397 struct drm_i915_private *i915; 1398 struct i915_gem_context **contexts; 1399 struct intel_engine_cs *engine; 1400 struct drm_i915_gem_object *batch; 1401 unsigned int ncontext; 1402 struct rnd_state prng; 1403 unsigned long count; 1404 }; 1405 1406 static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke) 1407 { 1408 return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext, 1409 &smoke->prng)]; 1410 } 1411 1412 static int smoke_submit(struct preempt_smoke *smoke, 1413 struct i915_gem_context *ctx, int prio, 1414 struct drm_i915_gem_object *batch) 1415 { 1416 struct i915_request *rq; 1417 struct i915_vma *vma = NULL; 1418 int err = 0; 1419 1420 if (batch) { 1421 vma = i915_vma_instance(batch, ctx->vm, NULL); 1422 if (IS_ERR(vma)) 1423 return PTR_ERR(vma); 1424 1425 err = i915_vma_pin(vma, 0, 0, PIN_USER); 1426 if (err) 1427 return err; 1428 } 1429 1430 ctx->sched.priority = prio; 1431 1432 rq = igt_request_alloc(ctx, smoke->engine); 1433 if (IS_ERR(rq)) { 1434 err = PTR_ERR(rq); 1435 goto unpin; 1436 } 1437 1438 if (vma) { 1439 i915_vma_lock(vma); 1440 err = rq->engine->emit_bb_start(rq, 1441 vma->node.start, 1442 PAGE_SIZE, 0); 1443 if (!err) 1444 err = i915_vma_move_to_active(vma, rq, 0); 1445 i915_vma_unlock(vma); 1446 } 1447 1448 i915_request_add(rq); 1449 1450 unpin: 1451 if (vma) 1452 i915_vma_unpin(vma); 1453 1454 return err; 1455 } 1456 1457 static int smoke_crescendo_thread(void *arg) 1458 { 1459 struct preempt_smoke *smoke = arg; 1460 IGT_TIMEOUT(end_time); 1461 unsigned long count; 1462 1463 count = 0; 1464 do { 1465 struct i915_gem_context *ctx = smoke_context(smoke); 1466 int err; 1467 1468 mutex_lock(&smoke->i915->drm.struct_mutex); 1469 err = smoke_submit(smoke, 1470 ctx, count % I915_PRIORITY_MAX, 1471 smoke->batch); 1472 mutex_unlock(&smoke->i915->drm.struct_mutex); 1473 if (err) 1474 return err; 1475 1476 count++; 1477 } while (!__igt_timeout(end_time, NULL)); 1478 1479 smoke->count = count; 1480 return 0; 1481 } 1482 1483 static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags) 1484 #define BATCH BIT(0) 1485 { 1486 struct task_struct *tsk[I915_NUM_ENGINES] = {}; 1487 struct preempt_smoke arg[I915_NUM_ENGINES]; 1488 struct intel_engine_cs *engine; 1489 enum intel_engine_id id; 1490 unsigned long count; 1491 int err = 0; 1492 1493 mutex_unlock(&smoke->i915->drm.struct_mutex); 1494 1495 for_each_engine(engine, smoke->i915, id) { 1496 arg[id] = *smoke; 1497 arg[id].engine = engine; 1498 if (!(flags & BATCH)) 1499 arg[id].batch = NULL; 1500 arg[id].count = 0; 1501 1502 tsk[id] = kthread_run(smoke_crescendo_thread, &arg, 1503 "igt/smoke:%d", id); 1504 if (IS_ERR(tsk[id])) { 1505 err = PTR_ERR(tsk[id]); 1506 break; 1507 } 1508 get_task_struct(tsk[id]); 1509 } 1510 1511 count = 0; 1512 for_each_engine(engine, smoke->i915, id) { 1513 int status; 1514 1515 if (IS_ERR_OR_NULL(tsk[id])) 1516 continue; 1517 1518 status = kthread_stop(tsk[id]); 1519 if (status && !err) 1520 err = status; 1521 1522 count += arg[id].count; 1523 1524 put_task_struct(tsk[id]); 1525 } 1526 1527 mutex_lock(&smoke->i915->drm.struct_mutex); 1528 1529 pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n", 1530 count, flags, 1531 RUNTIME_INFO(smoke->i915)->num_engines, smoke->ncontext); 1532 return 0; 1533 } 1534 1535 static int smoke_random(struct preempt_smoke *smoke, unsigned int flags) 1536 { 1537 enum intel_engine_id id; 1538 IGT_TIMEOUT(end_time); 1539 unsigned long count; 1540 1541 count = 0; 1542 do { 1543 for_each_engine(smoke->engine, smoke->i915, id) { 1544 struct i915_gem_context *ctx = smoke_context(smoke); 1545 int err; 1546 1547 err = smoke_submit(smoke, 1548 ctx, random_priority(&smoke->prng), 1549 flags & BATCH ? smoke->batch : NULL); 1550 if (err) 1551 return err; 1552 1553 count++; 1554 } 1555 } while (!__igt_timeout(end_time, NULL)); 1556 1557 pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n", 1558 count, flags, 1559 RUNTIME_INFO(smoke->i915)->num_engines, smoke->ncontext); 1560 return 0; 1561 } 1562 1563 static int live_preempt_smoke(void *arg) 1564 { 1565 struct preempt_smoke smoke = { 1566 .i915 = arg, 1567 .prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed), 1568 .ncontext = 1024, 1569 }; 1570 const unsigned int phase[] = { 0, BATCH }; 1571 intel_wakeref_t wakeref; 1572 struct igt_live_test t; 1573 int err = -ENOMEM; 1574 u32 *cs; 1575 int n; 1576 1577 if (!HAS_LOGICAL_RING_PREEMPTION(smoke.i915)) 1578 return 0; 1579 1580 smoke.contexts = kmalloc_array(smoke.ncontext, 1581 sizeof(*smoke.contexts), 1582 GFP_KERNEL); 1583 if (!smoke.contexts) 1584 return -ENOMEM; 1585 1586 mutex_lock(&smoke.i915->drm.struct_mutex); 1587 wakeref = intel_runtime_pm_get(&smoke.i915->runtime_pm); 1588 1589 smoke.batch = i915_gem_object_create_internal(smoke.i915, PAGE_SIZE); 1590 if (IS_ERR(smoke.batch)) { 1591 err = PTR_ERR(smoke.batch); 1592 goto err_unlock; 1593 } 1594 1595 cs = i915_gem_object_pin_map(smoke.batch, I915_MAP_WB); 1596 if (IS_ERR(cs)) { 1597 err = PTR_ERR(cs); 1598 goto err_batch; 1599 } 1600 for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++) 1601 cs[n] = MI_ARB_CHECK; 1602 cs[n] = MI_BATCH_BUFFER_END; 1603 i915_gem_object_flush_map(smoke.batch); 1604 i915_gem_object_unpin_map(smoke.batch); 1605 1606 if (igt_live_test_begin(&t, smoke.i915, __func__, "all")) { 1607 err = -EIO; 1608 goto err_batch; 1609 } 1610 1611 for (n = 0; n < smoke.ncontext; n++) { 1612 smoke.contexts[n] = kernel_context(smoke.i915); 1613 if (!smoke.contexts[n]) 1614 goto err_ctx; 1615 } 1616 1617 for (n = 0; n < ARRAY_SIZE(phase); n++) { 1618 err = smoke_crescendo(&smoke, phase[n]); 1619 if (err) 1620 goto err_ctx; 1621 1622 err = smoke_random(&smoke, phase[n]); 1623 if (err) 1624 goto err_ctx; 1625 } 1626 1627 err_ctx: 1628 if (igt_live_test_end(&t)) 1629 err = -EIO; 1630 1631 for (n = 0; n < smoke.ncontext; n++) { 1632 if (!smoke.contexts[n]) 1633 break; 1634 kernel_context_close(smoke.contexts[n]); 1635 } 1636 1637 err_batch: 1638 i915_gem_object_put(smoke.batch); 1639 err_unlock: 1640 intel_runtime_pm_put(&smoke.i915->runtime_pm, wakeref); 1641 mutex_unlock(&smoke.i915->drm.struct_mutex); 1642 kfree(smoke.contexts); 1643 1644 return err; 1645 } 1646 1647 static int nop_virtual_engine(struct drm_i915_private *i915, 1648 struct intel_engine_cs **siblings, 1649 unsigned int nsibling, 1650 unsigned int nctx, 1651 unsigned int flags) 1652 #define CHAIN BIT(0) 1653 { 1654 IGT_TIMEOUT(end_time); 1655 struct i915_request *request[16]; 1656 struct i915_gem_context *ctx[16]; 1657 struct intel_context *ve[16]; 1658 unsigned long n, prime, nc; 1659 struct igt_live_test t; 1660 ktime_t times[2] = {}; 1661 int err; 1662 1663 GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ctx)); 1664 1665 for (n = 0; n < nctx; n++) { 1666 ctx[n] = kernel_context(i915); 1667 if (!ctx[n]) { 1668 err = -ENOMEM; 1669 nctx = n; 1670 goto out; 1671 } 1672 1673 ve[n] = intel_execlists_create_virtual(ctx[n], 1674 siblings, nsibling); 1675 if (IS_ERR(ve[n])) { 1676 kernel_context_close(ctx[n]); 1677 err = PTR_ERR(ve[n]); 1678 nctx = n; 1679 goto out; 1680 } 1681 1682 err = intel_context_pin(ve[n]); 1683 if (err) { 1684 intel_context_put(ve[n]); 1685 kernel_context_close(ctx[n]); 1686 nctx = n; 1687 goto out; 1688 } 1689 } 1690 1691 err = igt_live_test_begin(&t, i915, __func__, ve[0]->engine->name); 1692 if (err) 1693 goto out; 1694 1695 for_each_prime_number_from(prime, 1, 8192) { 1696 times[1] = ktime_get_raw(); 1697 1698 if (flags & CHAIN) { 1699 for (nc = 0; nc < nctx; nc++) { 1700 for (n = 0; n < prime; n++) { 1701 request[nc] = 1702 i915_request_create(ve[nc]); 1703 if (IS_ERR(request[nc])) { 1704 err = PTR_ERR(request[nc]); 1705 goto out; 1706 } 1707 1708 i915_request_add(request[nc]); 1709 } 1710 } 1711 } else { 1712 for (n = 0; n < prime; n++) { 1713 for (nc = 0; nc < nctx; nc++) { 1714 request[nc] = 1715 i915_request_create(ve[nc]); 1716 if (IS_ERR(request[nc])) { 1717 err = PTR_ERR(request[nc]); 1718 goto out; 1719 } 1720 1721 i915_request_add(request[nc]); 1722 } 1723 } 1724 } 1725 1726 for (nc = 0; nc < nctx; nc++) { 1727 if (i915_request_wait(request[nc], 0, HZ / 10) < 0) { 1728 pr_err("%s(%s): wait for %llx:%lld timed out\n", 1729 __func__, ve[0]->engine->name, 1730 request[nc]->fence.context, 1731 request[nc]->fence.seqno); 1732 1733 GEM_TRACE("%s(%s) failed at request %llx:%lld\n", 1734 __func__, ve[0]->engine->name, 1735 request[nc]->fence.context, 1736 request[nc]->fence.seqno); 1737 GEM_TRACE_DUMP(); 1738 intel_gt_set_wedged(&i915->gt); 1739 break; 1740 } 1741 } 1742 1743 times[1] = ktime_sub(ktime_get_raw(), times[1]); 1744 if (prime == 1) 1745 times[0] = times[1]; 1746 1747 if (__igt_timeout(end_time, NULL)) 1748 break; 1749 } 1750 1751 err = igt_live_test_end(&t); 1752 if (err) 1753 goto out; 1754 1755 pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n", 1756 nctx, ve[0]->engine->name, ktime_to_ns(times[0]), 1757 prime, div64_u64(ktime_to_ns(times[1]), prime)); 1758 1759 out: 1760 if (igt_flush_test(i915, I915_WAIT_LOCKED)) 1761 err = -EIO; 1762 1763 for (nc = 0; nc < nctx; nc++) { 1764 intel_context_unpin(ve[nc]); 1765 intel_context_put(ve[nc]); 1766 kernel_context_close(ctx[nc]); 1767 } 1768 return err; 1769 } 1770 1771 static int live_virtual_engine(void *arg) 1772 { 1773 struct drm_i915_private *i915 = arg; 1774 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 1775 struct intel_engine_cs *engine; 1776 enum intel_engine_id id; 1777 unsigned int class, inst; 1778 int err = -ENODEV; 1779 1780 if (USES_GUC_SUBMISSION(i915)) 1781 return 0; 1782 1783 mutex_lock(&i915->drm.struct_mutex); 1784 1785 for_each_engine(engine, i915, id) { 1786 err = nop_virtual_engine(i915, &engine, 1, 1, 0); 1787 if (err) { 1788 pr_err("Failed to wrap engine %s: err=%d\n", 1789 engine->name, err); 1790 goto out_unlock; 1791 } 1792 } 1793 1794 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 1795 int nsibling, n; 1796 1797 nsibling = 0; 1798 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) { 1799 if (!i915->engine_class[class][inst]) 1800 continue; 1801 1802 siblings[nsibling++] = i915->engine_class[class][inst]; 1803 } 1804 if (nsibling < 2) 1805 continue; 1806 1807 for (n = 1; n <= nsibling + 1; n++) { 1808 err = nop_virtual_engine(i915, siblings, nsibling, 1809 n, 0); 1810 if (err) 1811 goto out_unlock; 1812 } 1813 1814 err = nop_virtual_engine(i915, siblings, nsibling, n, CHAIN); 1815 if (err) 1816 goto out_unlock; 1817 } 1818 1819 out_unlock: 1820 mutex_unlock(&i915->drm.struct_mutex); 1821 return err; 1822 } 1823 1824 static int mask_virtual_engine(struct drm_i915_private *i915, 1825 struct intel_engine_cs **siblings, 1826 unsigned int nsibling) 1827 { 1828 struct i915_request *request[MAX_ENGINE_INSTANCE + 1]; 1829 struct i915_gem_context *ctx; 1830 struct intel_context *ve; 1831 struct igt_live_test t; 1832 unsigned int n; 1833 int err; 1834 1835 /* 1836 * Check that by setting the execution mask on a request, we can 1837 * restrict it to our desired engine within the virtual engine. 1838 */ 1839 1840 ctx = kernel_context(i915); 1841 if (!ctx) 1842 return -ENOMEM; 1843 1844 ve = intel_execlists_create_virtual(ctx, siblings, nsibling); 1845 if (IS_ERR(ve)) { 1846 err = PTR_ERR(ve); 1847 goto out_close; 1848 } 1849 1850 err = intel_context_pin(ve); 1851 if (err) 1852 goto out_put; 1853 1854 err = igt_live_test_begin(&t, i915, __func__, ve->engine->name); 1855 if (err) 1856 goto out_unpin; 1857 1858 for (n = 0; n < nsibling; n++) { 1859 request[n] = i915_request_create(ve); 1860 if (IS_ERR(request[n])) { 1861 err = PTR_ERR(request[n]); 1862 nsibling = n; 1863 goto out; 1864 } 1865 1866 /* Reverse order as it's more likely to be unnatural */ 1867 request[n]->execution_mask = siblings[nsibling - n - 1]->mask; 1868 1869 i915_request_get(request[n]); 1870 i915_request_add(request[n]); 1871 } 1872 1873 for (n = 0; n < nsibling; n++) { 1874 if (i915_request_wait(request[n], 0, HZ / 10) < 0) { 1875 pr_err("%s(%s): wait for %llx:%lld timed out\n", 1876 __func__, ve->engine->name, 1877 request[n]->fence.context, 1878 request[n]->fence.seqno); 1879 1880 GEM_TRACE("%s(%s) failed at request %llx:%lld\n", 1881 __func__, ve->engine->name, 1882 request[n]->fence.context, 1883 request[n]->fence.seqno); 1884 GEM_TRACE_DUMP(); 1885 intel_gt_set_wedged(&i915->gt); 1886 err = -EIO; 1887 goto out; 1888 } 1889 1890 if (request[n]->engine != siblings[nsibling - n - 1]) { 1891 pr_err("Executed on wrong sibling '%s', expected '%s'\n", 1892 request[n]->engine->name, 1893 siblings[nsibling - n - 1]->name); 1894 err = -EINVAL; 1895 goto out; 1896 } 1897 } 1898 1899 err = igt_live_test_end(&t); 1900 if (err) 1901 goto out; 1902 1903 out: 1904 if (igt_flush_test(i915, I915_WAIT_LOCKED)) 1905 err = -EIO; 1906 1907 for (n = 0; n < nsibling; n++) 1908 i915_request_put(request[n]); 1909 1910 out_unpin: 1911 intel_context_unpin(ve); 1912 out_put: 1913 intel_context_put(ve); 1914 out_close: 1915 kernel_context_close(ctx); 1916 return err; 1917 } 1918 1919 static int live_virtual_mask(void *arg) 1920 { 1921 struct drm_i915_private *i915 = arg; 1922 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 1923 unsigned int class, inst; 1924 int err = 0; 1925 1926 if (USES_GUC_SUBMISSION(i915)) 1927 return 0; 1928 1929 mutex_lock(&i915->drm.struct_mutex); 1930 1931 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 1932 unsigned int nsibling; 1933 1934 nsibling = 0; 1935 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) { 1936 if (!i915->engine_class[class][inst]) 1937 break; 1938 1939 siblings[nsibling++] = i915->engine_class[class][inst]; 1940 } 1941 if (nsibling < 2) 1942 continue; 1943 1944 err = mask_virtual_engine(i915, siblings, nsibling); 1945 if (err) 1946 goto out_unlock; 1947 } 1948 1949 out_unlock: 1950 mutex_unlock(&i915->drm.struct_mutex); 1951 return err; 1952 } 1953 1954 static int bond_virtual_engine(struct drm_i915_private *i915, 1955 unsigned int class, 1956 struct intel_engine_cs **siblings, 1957 unsigned int nsibling, 1958 unsigned int flags) 1959 #define BOND_SCHEDULE BIT(0) 1960 { 1961 struct intel_engine_cs *master; 1962 struct i915_gem_context *ctx; 1963 struct i915_request *rq[16]; 1964 enum intel_engine_id id; 1965 unsigned long n; 1966 int err; 1967 1968 GEM_BUG_ON(nsibling >= ARRAY_SIZE(rq) - 1); 1969 1970 ctx = kernel_context(i915); 1971 if (!ctx) 1972 return -ENOMEM; 1973 1974 err = 0; 1975 rq[0] = ERR_PTR(-ENOMEM); 1976 for_each_engine(master, i915, id) { 1977 struct i915_sw_fence fence = {}; 1978 1979 if (master->class == class) 1980 continue; 1981 1982 memset_p((void *)rq, ERR_PTR(-EINVAL), ARRAY_SIZE(rq)); 1983 1984 rq[0] = igt_request_alloc(ctx, master); 1985 if (IS_ERR(rq[0])) { 1986 err = PTR_ERR(rq[0]); 1987 goto out; 1988 } 1989 i915_request_get(rq[0]); 1990 1991 if (flags & BOND_SCHEDULE) { 1992 onstack_fence_init(&fence); 1993 err = i915_sw_fence_await_sw_fence_gfp(&rq[0]->submit, 1994 &fence, 1995 GFP_KERNEL); 1996 } 1997 i915_request_add(rq[0]); 1998 if (err < 0) 1999 goto out; 2000 2001 for (n = 0; n < nsibling; n++) { 2002 struct intel_context *ve; 2003 2004 ve = intel_execlists_create_virtual(ctx, 2005 siblings, 2006 nsibling); 2007 if (IS_ERR(ve)) { 2008 err = PTR_ERR(ve); 2009 onstack_fence_fini(&fence); 2010 goto out; 2011 } 2012 2013 err = intel_virtual_engine_attach_bond(ve->engine, 2014 master, 2015 siblings[n]); 2016 if (err) { 2017 intel_context_put(ve); 2018 onstack_fence_fini(&fence); 2019 goto out; 2020 } 2021 2022 err = intel_context_pin(ve); 2023 intel_context_put(ve); 2024 if (err) { 2025 onstack_fence_fini(&fence); 2026 goto out; 2027 } 2028 2029 rq[n + 1] = i915_request_create(ve); 2030 intel_context_unpin(ve); 2031 if (IS_ERR(rq[n + 1])) { 2032 err = PTR_ERR(rq[n + 1]); 2033 onstack_fence_fini(&fence); 2034 goto out; 2035 } 2036 i915_request_get(rq[n + 1]); 2037 2038 err = i915_request_await_execution(rq[n + 1], 2039 &rq[0]->fence, 2040 ve->engine->bond_execute); 2041 i915_request_add(rq[n + 1]); 2042 if (err < 0) { 2043 onstack_fence_fini(&fence); 2044 goto out; 2045 } 2046 } 2047 onstack_fence_fini(&fence); 2048 2049 if (i915_request_wait(rq[0], 0, HZ / 10) < 0) { 2050 pr_err("Master request did not execute (on %s)!\n", 2051 rq[0]->engine->name); 2052 err = -EIO; 2053 goto out; 2054 } 2055 2056 for (n = 0; n < nsibling; n++) { 2057 if (i915_request_wait(rq[n + 1], 0, 2058 MAX_SCHEDULE_TIMEOUT) < 0) { 2059 err = -EIO; 2060 goto out; 2061 } 2062 2063 if (rq[n + 1]->engine != siblings[n]) { 2064 pr_err("Bonded request did not execute on target engine: expected %s, used %s; master was %s\n", 2065 siblings[n]->name, 2066 rq[n + 1]->engine->name, 2067 rq[0]->engine->name); 2068 err = -EINVAL; 2069 goto out; 2070 } 2071 } 2072 2073 for (n = 0; !IS_ERR(rq[n]); n++) 2074 i915_request_put(rq[n]); 2075 rq[0] = ERR_PTR(-ENOMEM); 2076 } 2077 2078 out: 2079 for (n = 0; !IS_ERR(rq[n]); n++) 2080 i915_request_put(rq[n]); 2081 if (igt_flush_test(i915, I915_WAIT_LOCKED)) 2082 err = -EIO; 2083 2084 kernel_context_close(ctx); 2085 return err; 2086 } 2087 2088 static int live_virtual_bond(void *arg) 2089 { 2090 static const struct phase { 2091 const char *name; 2092 unsigned int flags; 2093 } phases[] = { 2094 { "", 0 }, 2095 { "schedule", BOND_SCHEDULE }, 2096 { }, 2097 }; 2098 struct drm_i915_private *i915 = arg; 2099 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 2100 unsigned int class, inst; 2101 int err = 0; 2102 2103 if (USES_GUC_SUBMISSION(i915)) 2104 return 0; 2105 2106 mutex_lock(&i915->drm.struct_mutex); 2107 2108 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 2109 const struct phase *p; 2110 int nsibling; 2111 2112 nsibling = 0; 2113 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) { 2114 if (!i915->engine_class[class][inst]) 2115 break; 2116 2117 GEM_BUG_ON(nsibling == ARRAY_SIZE(siblings)); 2118 siblings[nsibling++] = i915->engine_class[class][inst]; 2119 } 2120 if (nsibling < 2) 2121 continue; 2122 2123 for (p = phases; p->name; p++) { 2124 err = bond_virtual_engine(i915, 2125 class, siblings, nsibling, 2126 p->flags); 2127 if (err) { 2128 pr_err("%s(%s): failed class=%d, nsibling=%d, err=%d\n", 2129 __func__, p->name, class, nsibling, err); 2130 goto out_unlock; 2131 } 2132 } 2133 } 2134 2135 out_unlock: 2136 mutex_unlock(&i915->drm.struct_mutex); 2137 return err; 2138 } 2139 2140 int intel_execlists_live_selftests(struct drm_i915_private *i915) 2141 { 2142 static const struct i915_subtest tests[] = { 2143 SUBTEST(live_sanitycheck), 2144 SUBTEST(live_timeslice_preempt), 2145 SUBTEST(live_busywait_preempt), 2146 SUBTEST(live_preempt), 2147 SUBTEST(live_late_preempt), 2148 SUBTEST(live_nopreempt), 2149 SUBTEST(live_suppress_self_preempt), 2150 SUBTEST(live_suppress_wait_preempt), 2151 SUBTEST(live_chain_preempt), 2152 SUBTEST(live_preempt_hang), 2153 SUBTEST(live_preempt_smoke), 2154 SUBTEST(live_virtual_engine), 2155 SUBTEST(live_virtual_mask), 2156 SUBTEST(live_virtual_bond), 2157 }; 2158 2159 if (!HAS_EXECLISTS(i915)) 2160 return 0; 2161 2162 if (intel_gt_is_wedged(&i915->gt)) 2163 return 0; 2164 2165 return i915_live_subtests(tests, i915); 2166 } 2167