1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2018 Intel Corporation 4 */ 5 6 #include <linux/prime_numbers.h> 7 8 #include "gem/i915_gem_internal.h" 9 #include "gem/i915_gem_pm.h" 10 #include "gt/intel_engine_heartbeat.h" 11 #include "gt/intel_reset.h" 12 #include "gt/selftest_engine_heartbeat.h" 13 14 #include "i915_selftest.h" 15 #include "selftests/i915_random.h" 16 #include "selftests/igt_flush_test.h" 17 #include "selftests/igt_live_test.h" 18 #include "selftests/igt_spinner.h" 19 #include "selftests/lib_sw_fence.h" 20 21 #include "gem/selftests/igt_gem_utils.h" 22 #include "gem/selftests/mock_context.h" 23 24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4) 25 #define NUM_GPR 16 26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */ 27 28 static bool is_active(struct i915_request *rq) 29 { 30 if (i915_request_is_active(rq)) 31 return true; 32 33 if (i915_request_on_hold(rq)) 34 return true; 35 36 if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq)) 37 return true; 38 39 return false; 40 } 41 42 static int wait_for_submit(struct intel_engine_cs *engine, 43 struct i915_request *rq, 44 unsigned long timeout) 45 { 46 /* Ignore our own attempts to suppress excess tasklets */ 47 tasklet_hi_schedule(&engine->sched_engine->tasklet); 48 49 timeout += jiffies; 50 do { 51 bool done = time_after(jiffies, timeout); 52 53 if (i915_request_completed(rq)) /* that was quick! */ 54 return 0; 55 56 /* Wait until the HW has acknowleged the submission (or err) */ 57 intel_engine_flush_submission(engine); 58 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq)) 59 return 0; 60 61 if (done) 62 return -ETIME; 63 64 cond_resched(); 65 } while (1); 66 } 67 68 static int wait_for_reset(struct intel_engine_cs *engine, 69 struct i915_request *rq, 70 unsigned long timeout) 71 { 72 timeout += jiffies; 73 74 do { 75 cond_resched(); 76 intel_engine_flush_submission(engine); 77 78 if (READ_ONCE(engine->execlists.pending[0])) 79 continue; 80 81 if (i915_request_completed(rq)) 82 break; 83 84 if (READ_ONCE(rq->fence.error)) 85 break; 86 } while (time_before(jiffies, timeout)); 87 88 flush_scheduled_work(); 89 90 if (rq->fence.error != -EIO) { 91 pr_err("%s: hanging request %llx:%lld not reset\n", 92 engine->name, 93 rq->fence.context, 94 rq->fence.seqno); 95 return -EINVAL; 96 } 97 98 /* Give the request a jiffie to complete after flushing the worker */ 99 if (i915_request_wait(rq, 0, 100 max(0l, (long)(timeout - jiffies)) + 1) < 0) { 101 pr_err("%s: hanging request %llx:%lld did not complete\n", 102 engine->name, 103 rq->fence.context, 104 rq->fence.seqno); 105 return -ETIME; 106 } 107 108 return 0; 109 } 110 111 static int live_sanitycheck(void *arg) 112 { 113 struct intel_gt *gt = arg; 114 struct intel_engine_cs *engine; 115 enum intel_engine_id id; 116 struct igt_spinner spin; 117 int err = 0; 118 119 if (!HAS_LOGICAL_RING_CONTEXTS(gt->i915)) 120 return 0; 121 122 if (igt_spinner_init(&spin, gt)) 123 return -ENOMEM; 124 125 for_each_engine(engine, gt, id) { 126 struct intel_context *ce; 127 struct i915_request *rq; 128 129 ce = intel_context_create(engine); 130 if (IS_ERR(ce)) { 131 err = PTR_ERR(ce); 132 break; 133 } 134 135 rq = igt_spinner_create_request(&spin, ce, MI_NOOP); 136 if (IS_ERR(rq)) { 137 err = PTR_ERR(rq); 138 goto out_ctx; 139 } 140 141 i915_request_add(rq); 142 if (!igt_wait_for_spinner(&spin, rq)) { 143 GEM_TRACE("spinner failed to start\n"); 144 GEM_TRACE_DUMP(); 145 intel_gt_set_wedged(gt); 146 err = -EIO; 147 goto out_ctx; 148 } 149 150 igt_spinner_end(&spin); 151 if (igt_flush_test(gt->i915)) { 152 err = -EIO; 153 goto out_ctx; 154 } 155 156 out_ctx: 157 intel_context_put(ce); 158 if (err) 159 break; 160 } 161 162 igt_spinner_fini(&spin); 163 return err; 164 } 165 166 static int live_unlite_restore(struct intel_gt *gt, int prio) 167 { 168 struct intel_engine_cs *engine; 169 enum intel_engine_id id; 170 struct igt_spinner spin; 171 int err = -ENOMEM; 172 173 /* 174 * Check that we can correctly context switch between 2 instances 175 * on the same engine from the same parent context. 176 */ 177 178 if (igt_spinner_init(&spin, gt)) 179 return err; 180 181 err = 0; 182 for_each_engine(engine, gt, id) { 183 struct intel_context *ce[2] = {}; 184 struct i915_request *rq[2]; 185 struct igt_live_test t; 186 int n; 187 188 if (prio && !intel_engine_has_preemption(engine)) 189 continue; 190 191 if (!intel_engine_can_store_dword(engine)) 192 continue; 193 194 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 195 err = -EIO; 196 break; 197 } 198 st_engine_heartbeat_disable(engine); 199 200 for (n = 0; n < ARRAY_SIZE(ce); n++) { 201 struct intel_context *tmp; 202 203 tmp = intel_context_create(engine); 204 if (IS_ERR(tmp)) { 205 err = PTR_ERR(tmp); 206 goto err_ce; 207 } 208 209 err = intel_context_pin(tmp); 210 if (err) { 211 intel_context_put(tmp); 212 goto err_ce; 213 } 214 215 /* 216 * Setup the pair of contexts such that if we 217 * lite-restore using the RING_TAIL from ce[1] it 218 * will execute garbage from ce[0]->ring. 219 */ 220 memset(tmp->ring->vaddr, 221 POISON_INUSE, /* IPEHR: 0x5a5a5a5a [hung!] */ 222 tmp->ring->vma->size); 223 224 ce[n] = tmp; 225 } 226 GEM_BUG_ON(!ce[1]->ring->size); 227 intel_ring_reset(ce[1]->ring, ce[1]->ring->size / 2); 228 lrc_update_regs(ce[1], engine, ce[1]->ring->head); 229 230 rq[0] = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK); 231 if (IS_ERR(rq[0])) { 232 err = PTR_ERR(rq[0]); 233 goto err_ce; 234 } 235 236 i915_request_get(rq[0]); 237 i915_request_add(rq[0]); 238 GEM_BUG_ON(rq[0]->postfix > ce[1]->ring->emit); 239 240 if (!igt_wait_for_spinner(&spin, rq[0])) { 241 i915_request_put(rq[0]); 242 goto err_ce; 243 } 244 245 rq[1] = i915_request_create(ce[1]); 246 if (IS_ERR(rq[1])) { 247 err = PTR_ERR(rq[1]); 248 i915_request_put(rq[0]); 249 goto err_ce; 250 } 251 252 if (!prio) { 253 /* 254 * Ensure we do the switch to ce[1] on completion. 255 * 256 * rq[0] is already submitted, so this should reduce 257 * to a no-op (a wait on a request on the same engine 258 * uses the submit fence, not the completion fence), 259 * but it will install a dependency on rq[1] for rq[0] 260 * that will prevent the pair being reordered by 261 * timeslicing. 262 */ 263 i915_request_await_dma_fence(rq[1], &rq[0]->fence); 264 } 265 266 i915_request_get(rq[1]); 267 i915_request_add(rq[1]); 268 GEM_BUG_ON(rq[1]->postfix <= rq[0]->postfix); 269 i915_request_put(rq[0]); 270 271 if (prio) { 272 struct i915_sched_attr attr = { 273 .priority = prio, 274 }; 275 276 /* Alternatively preempt the spinner with ce[1] */ 277 engine->sched_engine->schedule(rq[1], &attr); 278 } 279 280 /* And switch back to ce[0] for good measure */ 281 rq[0] = i915_request_create(ce[0]); 282 if (IS_ERR(rq[0])) { 283 err = PTR_ERR(rq[0]); 284 i915_request_put(rq[1]); 285 goto err_ce; 286 } 287 288 i915_request_await_dma_fence(rq[0], &rq[1]->fence); 289 i915_request_get(rq[0]); 290 i915_request_add(rq[0]); 291 GEM_BUG_ON(rq[0]->postfix > rq[1]->postfix); 292 i915_request_put(rq[1]); 293 i915_request_put(rq[0]); 294 295 err_ce: 296 intel_engine_flush_submission(engine); 297 igt_spinner_end(&spin); 298 for (n = 0; n < ARRAY_SIZE(ce); n++) { 299 if (IS_ERR_OR_NULL(ce[n])) 300 break; 301 302 intel_context_unpin(ce[n]); 303 intel_context_put(ce[n]); 304 } 305 306 st_engine_heartbeat_enable(engine); 307 if (igt_live_test_end(&t)) 308 err = -EIO; 309 if (err) 310 break; 311 } 312 313 igt_spinner_fini(&spin); 314 return err; 315 } 316 317 static int live_unlite_switch(void *arg) 318 { 319 return live_unlite_restore(arg, 0); 320 } 321 322 static int live_unlite_preempt(void *arg) 323 { 324 return live_unlite_restore(arg, I915_PRIORITY_MAX); 325 } 326 327 static int live_unlite_ring(void *arg) 328 { 329 struct intel_gt *gt = arg; 330 struct intel_engine_cs *engine; 331 struct igt_spinner spin; 332 enum intel_engine_id id; 333 int err = 0; 334 335 /* 336 * Setup a preemption event that will cause almost the entire ring 337 * to be unwound, potentially fooling our intel_ring_direction() 338 * into emitting a forward lite-restore instead of the rollback. 339 */ 340 341 if (igt_spinner_init(&spin, gt)) 342 return -ENOMEM; 343 344 for_each_engine(engine, gt, id) { 345 struct intel_context *ce[2] = {}; 346 struct i915_request *rq; 347 struct igt_live_test t; 348 int n; 349 350 if (!intel_engine_has_preemption(engine)) 351 continue; 352 353 if (!intel_engine_can_store_dword(engine)) 354 continue; 355 356 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 357 err = -EIO; 358 break; 359 } 360 st_engine_heartbeat_disable(engine); 361 362 for (n = 0; n < ARRAY_SIZE(ce); n++) { 363 struct intel_context *tmp; 364 365 tmp = intel_context_create(engine); 366 if (IS_ERR(tmp)) { 367 err = PTR_ERR(tmp); 368 goto err_ce; 369 } 370 371 err = intel_context_pin(tmp); 372 if (err) { 373 intel_context_put(tmp); 374 goto err_ce; 375 } 376 377 memset32(tmp->ring->vaddr, 378 0xdeadbeef, /* trigger a hang if executed */ 379 tmp->ring->vma->size / sizeof(u32)); 380 381 ce[n] = tmp; 382 } 383 384 /* Create max prio spinner, followed by N low prio nops */ 385 rq = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK); 386 if (IS_ERR(rq)) { 387 err = PTR_ERR(rq); 388 goto err_ce; 389 } 390 391 i915_request_get(rq); 392 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 393 i915_request_add(rq); 394 395 if (!igt_wait_for_spinner(&spin, rq)) { 396 intel_gt_set_wedged(gt); 397 i915_request_put(rq); 398 err = -ETIME; 399 goto err_ce; 400 } 401 402 /* Fill the ring, until we will cause a wrap */ 403 n = 0; 404 while (intel_ring_direction(ce[0]->ring, 405 rq->wa_tail, 406 ce[0]->ring->tail) <= 0) { 407 struct i915_request *tmp; 408 409 tmp = intel_context_create_request(ce[0]); 410 if (IS_ERR(tmp)) { 411 err = PTR_ERR(tmp); 412 i915_request_put(rq); 413 goto err_ce; 414 } 415 416 i915_request_add(tmp); 417 intel_engine_flush_submission(engine); 418 n++; 419 } 420 intel_engine_flush_submission(engine); 421 pr_debug("%s: Filled ring with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n", 422 engine->name, n, 423 ce[0]->ring->size, 424 ce[0]->ring->tail, 425 ce[0]->ring->emit, 426 rq->tail); 427 GEM_BUG_ON(intel_ring_direction(ce[0]->ring, 428 rq->tail, 429 ce[0]->ring->tail) <= 0); 430 i915_request_put(rq); 431 432 /* Create a second ring to preempt the first ring after rq[0] */ 433 rq = intel_context_create_request(ce[1]); 434 if (IS_ERR(rq)) { 435 err = PTR_ERR(rq); 436 goto err_ce; 437 } 438 439 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 440 i915_request_get(rq); 441 i915_request_add(rq); 442 443 err = wait_for_submit(engine, rq, HZ / 2); 444 i915_request_put(rq); 445 if (err) { 446 pr_err("%s: preemption request was not submitted\n", 447 engine->name); 448 err = -ETIME; 449 } 450 451 pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n", 452 engine->name, 453 ce[0]->ring->tail, ce[0]->ring->emit, 454 ce[1]->ring->tail, ce[1]->ring->emit); 455 456 err_ce: 457 intel_engine_flush_submission(engine); 458 igt_spinner_end(&spin); 459 for (n = 0; n < ARRAY_SIZE(ce); n++) { 460 if (IS_ERR_OR_NULL(ce[n])) 461 break; 462 463 intel_context_unpin(ce[n]); 464 intel_context_put(ce[n]); 465 } 466 st_engine_heartbeat_enable(engine); 467 if (igt_live_test_end(&t)) 468 err = -EIO; 469 if (err) 470 break; 471 } 472 473 igt_spinner_fini(&spin); 474 return err; 475 } 476 477 static int live_pin_rewind(void *arg) 478 { 479 struct intel_gt *gt = arg; 480 struct intel_engine_cs *engine; 481 enum intel_engine_id id; 482 int err = 0; 483 484 /* 485 * We have to be careful not to trust intel_ring too much, for example 486 * ring->head is updated upon retire which is out of sync with pinning 487 * the context. Thus we cannot use ring->head to set CTX_RING_HEAD, 488 * or else we risk writing an older, stale value. 489 * 490 * To simulate this, let's apply a bit of deliberate sabotague. 491 */ 492 493 for_each_engine(engine, gt, id) { 494 struct intel_context *ce; 495 struct i915_request *rq; 496 struct intel_ring *ring; 497 struct igt_live_test t; 498 499 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 500 err = -EIO; 501 break; 502 } 503 504 ce = intel_context_create(engine); 505 if (IS_ERR(ce)) { 506 err = PTR_ERR(ce); 507 break; 508 } 509 510 err = intel_context_pin(ce); 511 if (err) { 512 intel_context_put(ce); 513 break; 514 } 515 516 /* Keep the context awake while we play games */ 517 err = i915_active_acquire(&ce->active); 518 if (err) { 519 intel_context_unpin(ce); 520 intel_context_put(ce); 521 break; 522 } 523 ring = ce->ring; 524 525 /* Poison the ring, and offset the next request from HEAD */ 526 memset32(ring->vaddr, STACK_MAGIC, ring->size / sizeof(u32)); 527 ring->emit = ring->size / 2; 528 ring->tail = ring->emit; 529 GEM_BUG_ON(ring->head); 530 531 intel_context_unpin(ce); 532 533 /* Submit a simple nop request */ 534 GEM_BUG_ON(intel_context_is_pinned(ce)); 535 rq = intel_context_create_request(ce); 536 i915_active_release(&ce->active); /* e.g. async retire */ 537 intel_context_put(ce); 538 if (IS_ERR(rq)) { 539 err = PTR_ERR(rq); 540 break; 541 } 542 GEM_BUG_ON(!rq->head); 543 i915_request_add(rq); 544 545 /* Expect not to hang! */ 546 if (igt_live_test_end(&t)) { 547 err = -EIO; 548 break; 549 } 550 } 551 552 return err; 553 } 554 555 static int engine_lock_reset_tasklet(struct intel_engine_cs *engine) 556 { 557 tasklet_disable(&engine->sched_engine->tasklet); 558 local_bh_disable(); 559 560 if (test_and_set_bit(I915_RESET_ENGINE + engine->id, 561 &engine->gt->reset.flags)) { 562 local_bh_enable(); 563 tasklet_enable(&engine->sched_engine->tasklet); 564 565 intel_gt_set_wedged(engine->gt); 566 return -EBUSY; 567 } 568 569 return 0; 570 } 571 572 static void engine_unlock_reset_tasklet(struct intel_engine_cs *engine) 573 { 574 clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id, 575 &engine->gt->reset.flags); 576 577 local_bh_enable(); 578 tasklet_enable(&engine->sched_engine->tasklet); 579 } 580 581 static int live_hold_reset(void *arg) 582 { 583 struct intel_gt *gt = arg; 584 struct intel_engine_cs *engine; 585 enum intel_engine_id id; 586 struct igt_spinner spin; 587 int err = 0; 588 589 /* 590 * In order to support offline error capture for fast preempt reset, 591 * we need to decouple the guilty request and ensure that it and its 592 * descendents are not executed while the capture is in progress. 593 */ 594 595 if (!intel_has_reset_engine(gt)) 596 return 0; 597 598 if (igt_spinner_init(&spin, gt)) 599 return -ENOMEM; 600 601 for_each_engine(engine, gt, id) { 602 struct intel_context *ce; 603 struct i915_request *rq; 604 605 ce = intel_context_create(engine); 606 if (IS_ERR(ce)) { 607 err = PTR_ERR(ce); 608 break; 609 } 610 611 st_engine_heartbeat_disable(engine); 612 613 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK); 614 if (IS_ERR(rq)) { 615 err = PTR_ERR(rq); 616 goto out; 617 } 618 i915_request_add(rq); 619 620 if (!igt_wait_for_spinner(&spin, rq)) { 621 intel_gt_set_wedged(gt); 622 err = -ETIME; 623 goto out; 624 } 625 626 /* We have our request executing, now remove it and reset */ 627 628 err = engine_lock_reset_tasklet(engine); 629 if (err) 630 goto out; 631 632 engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet); 633 GEM_BUG_ON(execlists_active(&engine->execlists) != rq); 634 635 i915_request_get(rq); 636 execlists_hold(engine, rq); 637 GEM_BUG_ON(!i915_request_on_hold(rq)); 638 639 __intel_engine_reset_bh(engine, NULL); 640 GEM_BUG_ON(rq->fence.error != -EIO); 641 642 engine_unlock_reset_tasklet(engine); 643 644 /* Check that we do not resubmit the held request */ 645 if (!i915_request_wait(rq, 0, HZ / 5)) { 646 pr_err("%s: on hold request completed!\n", 647 engine->name); 648 i915_request_put(rq); 649 err = -EIO; 650 goto out; 651 } 652 GEM_BUG_ON(!i915_request_on_hold(rq)); 653 654 /* But is resubmitted on release */ 655 execlists_unhold(engine, rq); 656 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 657 pr_err("%s: held request did not complete!\n", 658 engine->name); 659 intel_gt_set_wedged(gt); 660 err = -ETIME; 661 } 662 i915_request_put(rq); 663 664 out: 665 st_engine_heartbeat_enable(engine); 666 intel_context_put(ce); 667 if (err) 668 break; 669 } 670 671 igt_spinner_fini(&spin); 672 return err; 673 } 674 675 static const char *error_repr(int err) 676 { 677 return err ? "bad" : "good"; 678 } 679 680 static int live_error_interrupt(void *arg) 681 { 682 static const struct error_phase { 683 enum { GOOD = 0, BAD = -EIO } error[2]; 684 } phases[] = { 685 { { BAD, GOOD } }, 686 { { BAD, BAD } }, 687 { { BAD, GOOD } }, 688 { { GOOD, GOOD } }, /* sentinel */ 689 }; 690 struct intel_gt *gt = arg; 691 struct intel_engine_cs *engine; 692 enum intel_engine_id id; 693 694 /* 695 * We hook up the CS_MASTER_ERROR_INTERRUPT to have forewarning 696 * of invalid commands in user batches that will cause a GPU hang. 697 * This is a faster mechanism than using hangcheck/heartbeats, but 698 * only detects problems the HW knows about -- it will not warn when 699 * we kill the HW! 700 * 701 * To verify our detection and reset, we throw some invalid commands 702 * at the HW and wait for the interrupt. 703 */ 704 705 if (!intel_has_reset_engine(gt)) 706 return 0; 707 708 for_each_engine(engine, gt, id) { 709 const struct error_phase *p; 710 int err = 0; 711 712 st_engine_heartbeat_disable(engine); 713 714 for (p = phases; p->error[0] != GOOD; p++) { 715 struct i915_request *client[ARRAY_SIZE(phases->error)]; 716 u32 *cs; 717 int i; 718 719 memset(client, 0, sizeof(*client)); 720 for (i = 0; i < ARRAY_SIZE(client); i++) { 721 struct intel_context *ce; 722 struct i915_request *rq; 723 724 ce = intel_context_create(engine); 725 if (IS_ERR(ce)) { 726 err = PTR_ERR(ce); 727 goto out; 728 } 729 730 rq = intel_context_create_request(ce); 731 intel_context_put(ce); 732 if (IS_ERR(rq)) { 733 err = PTR_ERR(rq); 734 goto out; 735 } 736 737 if (rq->engine->emit_init_breadcrumb) { 738 err = rq->engine->emit_init_breadcrumb(rq); 739 if (err) { 740 i915_request_add(rq); 741 goto out; 742 } 743 } 744 745 cs = intel_ring_begin(rq, 2); 746 if (IS_ERR(cs)) { 747 i915_request_add(rq); 748 err = PTR_ERR(cs); 749 goto out; 750 } 751 752 if (p->error[i]) { 753 *cs++ = 0xdeadbeef; 754 *cs++ = 0xdeadbeef; 755 } else { 756 *cs++ = MI_NOOP; 757 *cs++ = MI_NOOP; 758 } 759 760 client[i] = i915_request_get(rq); 761 i915_request_add(rq); 762 } 763 764 err = wait_for_submit(engine, client[0], HZ / 2); 765 if (err) { 766 pr_err("%s: first request did not start within time!\n", 767 engine->name); 768 err = -ETIME; 769 goto out; 770 } 771 772 for (i = 0; i < ARRAY_SIZE(client); i++) { 773 if (i915_request_wait(client[i], 0, HZ / 5) < 0) 774 pr_debug("%s: %s request incomplete!\n", 775 engine->name, 776 error_repr(p->error[i])); 777 778 if (!i915_request_started(client[i])) { 779 pr_err("%s: %s request not started!\n", 780 engine->name, 781 error_repr(p->error[i])); 782 err = -ETIME; 783 goto out; 784 } 785 786 /* Kick the tasklet to process the error */ 787 intel_engine_flush_submission(engine); 788 if (client[i]->fence.error != p->error[i]) { 789 pr_err("%s: %s request (%s) with wrong error code: %d\n", 790 engine->name, 791 error_repr(p->error[i]), 792 i915_request_completed(client[i]) ? "completed" : "running", 793 client[i]->fence.error); 794 err = -EINVAL; 795 goto out; 796 } 797 } 798 799 out: 800 for (i = 0; i < ARRAY_SIZE(client); i++) 801 if (client[i]) 802 i915_request_put(client[i]); 803 if (err) { 804 pr_err("%s: failed at phase[%zd] { %d, %d }\n", 805 engine->name, p - phases, 806 p->error[0], p->error[1]); 807 break; 808 } 809 } 810 811 st_engine_heartbeat_enable(engine); 812 if (err) { 813 intel_gt_set_wedged(gt); 814 return err; 815 } 816 } 817 818 return 0; 819 } 820 821 static int 822 emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx) 823 { 824 u32 *cs; 825 826 cs = intel_ring_begin(rq, 10); 827 if (IS_ERR(cs)) 828 return PTR_ERR(cs); 829 830 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 831 832 *cs++ = MI_SEMAPHORE_WAIT | 833 MI_SEMAPHORE_GLOBAL_GTT | 834 MI_SEMAPHORE_POLL | 835 MI_SEMAPHORE_SAD_NEQ_SDD; 836 *cs++ = 0; 837 *cs++ = i915_ggtt_offset(vma) + 4 * idx; 838 *cs++ = 0; 839 840 if (idx > 0) { 841 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 842 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1); 843 *cs++ = 0; 844 *cs++ = 1; 845 } else { 846 *cs++ = MI_NOOP; 847 *cs++ = MI_NOOP; 848 *cs++ = MI_NOOP; 849 *cs++ = MI_NOOP; 850 } 851 852 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 853 854 intel_ring_advance(rq, cs); 855 return 0; 856 } 857 858 static struct i915_request * 859 semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx) 860 { 861 struct intel_context *ce; 862 struct i915_request *rq; 863 int err; 864 865 ce = intel_context_create(engine); 866 if (IS_ERR(ce)) 867 return ERR_CAST(ce); 868 869 rq = intel_context_create_request(ce); 870 if (IS_ERR(rq)) 871 goto out_ce; 872 873 err = 0; 874 if (rq->engine->emit_init_breadcrumb) 875 err = rq->engine->emit_init_breadcrumb(rq); 876 if (err == 0) 877 err = emit_semaphore_chain(rq, vma, idx); 878 if (err == 0) 879 i915_request_get(rq); 880 i915_request_add(rq); 881 if (err) 882 rq = ERR_PTR(err); 883 884 out_ce: 885 intel_context_put(ce); 886 return rq; 887 } 888 889 static int 890 release_queue(struct intel_engine_cs *engine, 891 struct i915_vma *vma, 892 int idx, int prio) 893 { 894 struct i915_sched_attr attr = { 895 .priority = prio, 896 }; 897 struct i915_request *rq; 898 u32 *cs; 899 900 rq = intel_engine_create_kernel_request(engine); 901 if (IS_ERR(rq)) 902 return PTR_ERR(rq); 903 904 cs = intel_ring_begin(rq, 4); 905 if (IS_ERR(cs)) { 906 i915_request_add(rq); 907 return PTR_ERR(cs); 908 } 909 910 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 911 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1); 912 *cs++ = 0; 913 *cs++ = 1; 914 915 intel_ring_advance(rq, cs); 916 917 i915_request_get(rq); 918 i915_request_add(rq); 919 920 local_bh_disable(); 921 engine->sched_engine->schedule(rq, &attr); 922 local_bh_enable(); /* kick tasklet */ 923 924 i915_request_put(rq); 925 926 return 0; 927 } 928 929 static int 930 slice_semaphore_queue(struct intel_engine_cs *outer, 931 struct i915_vma *vma, 932 int count) 933 { 934 struct intel_engine_cs *engine; 935 struct i915_request *head; 936 enum intel_engine_id id; 937 int err, i, n = 0; 938 939 head = semaphore_queue(outer, vma, n++); 940 if (IS_ERR(head)) 941 return PTR_ERR(head); 942 943 for_each_engine(engine, outer->gt, id) { 944 if (!intel_engine_has_preemption(engine)) 945 continue; 946 947 for (i = 0; i < count; i++) { 948 struct i915_request *rq; 949 950 rq = semaphore_queue(engine, vma, n++); 951 if (IS_ERR(rq)) { 952 err = PTR_ERR(rq); 953 goto out; 954 } 955 956 i915_request_put(rq); 957 } 958 } 959 960 err = release_queue(outer, vma, n, I915_PRIORITY_BARRIER); 961 if (err) 962 goto out; 963 964 if (i915_request_wait(head, 0, 965 2 * outer->gt->info.num_engines * (count + 2) * (count + 3)) < 0) { 966 pr_err("%s: Failed to slice along semaphore chain of length (%d, %d)!\n", 967 outer->name, count, n); 968 GEM_TRACE_DUMP(); 969 intel_gt_set_wedged(outer->gt); 970 err = -EIO; 971 } 972 973 out: 974 i915_request_put(head); 975 return err; 976 } 977 978 static int live_timeslice_preempt(void *arg) 979 { 980 struct intel_gt *gt = arg; 981 struct drm_i915_gem_object *obj; 982 struct intel_engine_cs *engine; 983 enum intel_engine_id id; 984 struct i915_vma *vma; 985 void *vaddr; 986 int err = 0; 987 988 /* 989 * If a request takes too long, we would like to give other users 990 * a fair go on the GPU. In particular, users may create batches 991 * that wait upon external input, where that input may even be 992 * supplied by another GPU job. To avoid blocking forever, we 993 * need to preempt the current task and replace it with another 994 * ready task. 995 */ 996 if (!CONFIG_DRM_I915_TIMESLICE_DURATION) 997 return 0; 998 999 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); 1000 if (IS_ERR(obj)) 1001 return PTR_ERR(obj); 1002 1003 vma = i915_vma_instance(obj, >->ggtt->vm, NULL); 1004 if (IS_ERR(vma)) { 1005 err = PTR_ERR(vma); 1006 goto err_obj; 1007 } 1008 1009 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 1010 if (IS_ERR(vaddr)) { 1011 err = PTR_ERR(vaddr); 1012 goto err_obj; 1013 } 1014 1015 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 1016 if (err) 1017 goto err_map; 1018 1019 err = i915_vma_sync(vma); 1020 if (err) 1021 goto err_pin; 1022 1023 for_each_engine(engine, gt, id) { 1024 if (!intel_engine_has_preemption(engine)) 1025 continue; 1026 1027 memset(vaddr, 0, PAGE_SIZE); 1028 1029 st_engine_heartbeat_disable(engine); 1030 err = slice_semaphore_queue(engine, vma, 5); 1031 st_engine_heartbeat_enable(engine); 1032 if (err) 1033 goto err_pin; 1034 1035 if (igt_flush_test(gt->i915)) { 1036 err = -EIO; 1037 goto err_pin; 1038 } 1039 } 1040 1041 err_pin: 1042 i915_vma_unpin(vma); 1043 err_map: 1044 i915_gem_object_unpin_map(obj); 1045 err_obj: 1046 i915_gem_object_put(obj); 1047 return err; 1048 } 1049 1050 static struct i915_request * 1051 create_rewinder(struct intel_context *ce, 1052 struct i915_request *wait, 1053 void *slot, int idx) 1054 { 1055 const u32 offset = 1056 i915_ggtt_offset(ce->engine->status_page.vma) + 1057 offset_in_page(slot); 1058 struct i915_request *rq; 1059 u32 *cs; 1060 int err; 1061 1062 rq = intel_context_create_request(ce); 1063 if (IS_ERR(rq)) 1064 return rq; 1065 1066 if (wait) { 1067 err = i915_request_await_dma_fence(rq, &wait->fence); 1068 if (err) 1069 goto err; 1070 } 1071 1072 cs = intel_ring_begin(rq, 14); 1073 if (IS_ERR(cs)) { 1074 err = PTR_ERR(cs); 1075 goto err; 1076 } 1077 1078 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1079 *cs++ = MI_NOOP; 1080 1081 *cs++ = MI_SEMAPHORE_WAIT | 1082 MI_SEMAPHORE_GLOBAL_GTT | 1083 MI_SEMAPHORE_POLL | 1084 MI_SEMAPHORE_SAD_GTE_SDD; 1085 *cs++ = idx; 1086 *cs++ = offset; 1087 *cs++ = 0; 1088 1089 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 1090 *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(rq->engine->mmio_base)); 1091 *cs++ = offset + idx * sizeof(u32); 1092 *cs++ = 0; 1093 1094 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 1095 *cs++ = offset; 1096 *cs++ = 0; 1097 *cs++ = idx + 1; 1098 1099 intel_ring_advance(rq, cs); 1100 1101 err = 0; 1102 err: 1103 i915_request_get(rq); 1104 i915_request_add(rq); 1105 if (err) { 1106 i915_request_put(rq); 1107 return ERR_PTR(err); 1108 } 1109 1110 return rq; 1111 } 1112 1113 static int live_timeslice_rewind(void *arg) 1114 { 1115 struct intel_gt *gt = arg; 1116 struct intel_engine_cs *engine; 1117 enum intel_engine_id id; 1118 1119 /* 1120 * The usual presumption on timeslice expiration is that we replace 1121 * the active context with another. However, given a chain of 1122 * dependencies we may end up with replacing the context with itself, 1123 * but only a few of those requests, forcing us to rewind the 1124 * RING_TAIL of the original request. 1125 */ 1126 if (!CONFIG_DRM_I915_TIMESLICE_DURATION) 1127 return 0; 1128 1129 for_each_engine(engine, gt, id) { 1130 enum { A1, A2, B1 }; 1131 enum { X = 1, Z, Y }; 1132 struct i915_request *rq[3] = {}; 1133 struct intel_context *ce; 1134 unsigned long timeslice; 1135 int i, err = 0; 1136 u32 *slot; 1137 1138 if (!intel_engine_has_timeslices(engine)) 1139 continue; 1140 1141 /* 1142 * A:rq1 -- semaphore wait, timestamp X 1143 * A:rq2 -- write timestamp Y 1144 * 1145 * B:rq1 [await A:rq1] -- write timestamp Z 1146 * 1147 * Force timeslice, release semaphore. 1148 * 1149 * Expect execution/evaluation order XZY 1150 */ 1151 1152 st_engine_heartbeat_disable(engine); 1153 timeslice = xchg(&engine->props.timeslice_duration_ms, 1); 1154 1155 slot = memset32(engine->status_page.addr + 1000, 0, 4); 1156 1157 ce = intel_context_create(engine); 1158 if (IS_ERR(ce)) { 1159 err = PTR_ERR(ce); 1160 goto err; 1161 } 1162 1163 rq[A1] = create_rewinder(ce, NULL, slot, X); 1164 if (IS_ERR(rq[A1])) { 1165 intel_context_put(ce); 1166 goto err; 1167 } 1168 1169 rq[A2] = create_rewinder(ce, NULL, slot, Y); 1170 intel_context_put(ce); 1171 if (IS_ERR(rq[A2])) 1172 goto err; 1173 1174 err = wait_for_submit(engine, rq[A2], HZ / 2); 1175 if (err) { 1176 pr_err("%s: failed to submit first context\n", 1177 engine->name); 1178 goto err; 1179 } 1180 1181 ce = intel_context_create(engine); 1182 if (IS_ERR(ce)) { 1183 err = PTR_ERR(ce); 1184 goto err; 1185 } 1186 1187 rq[B1] = create_rewinder(ce, rq[A1], slot, Z); 1188 intel_context_put(ce); 1189 if (IS_ERR(rq[2])) 1190 goto err; 1191 1192 err = wait_for_submit(engine, rq[B1], HZ / 2); 1193 if (err) { 1194 pr_err("%s: failed to submit second context\n", 1195 engine->name); 1196 goto err; 1197 } 1198 1199 /* ELSP[] = { { A:rq1, A:rq2 }, { B:rq1 } } */ 1200 ENGINE_TRACE(engine, "forcing tasklet for rewind\n"); 1201 while (i915_request_is_active(rq[A2])) { /* semaphore yield! */ 1202 /* Wait for the timeslice to kick in */ 1203 del_timer(&engine->execlists.timer); 1204 tasklet_hi_schedule(&engine->sched_engine->tasklet); 1205 intel_engine_flush_submission(engine); 1206 } 1207 /* -> ELSP[] = { { A:rq1 }, { B:rq1 } } */ 1208 GEM_BUG_ON(!i915_request_is_active(rq[A1])); 1209 GEM_BUG_ON(!i915_request_is_active(rq[B1])); 1210 GEM_BUG_ON(i915_request_is_active(rq[A2])); 1211 1212 /* Release the hounds! */ 1213 slot[0] = 1; 1214 wmb(); /* "pairs" with GPU; paranoid kick of internal CPU$ */ 1215 1216 for (i = 1; i <= 3; i++) { 1217 unsigned long timeout = jiffies + HZ / 2; 1218 1219 while (!READ_ONCE(slot[i]) && 1220 time_before(jiffies, timeout)) 1221 ; 1222 1223 if (!time_before(jiffies, timeout)) { 1224 pr_err("%s: rq[%d] timed out\n", 1225 engine->name, i - 1); 1226 err = -ETIME; 1227 goto err; 1228 } 1229 1230 pr_debug("%s: slot[%d]:%x\n", engine->name, i, slot[i]); 1231 } 1232 1233 /* XZY: XZ < XY */ 1234 if (slot[Z] - slot[X] >= slot[Y] - slot[X]) { 1235 pr_err("%s: timeslicing did not run context B [%u] before A [%u]!\n", 1236 engine->name, 1237 slot[Z] - slot[X], 1238 slot[Y] - slot[X]); 1239 err = -EINVAL; 1240 } 1241 1242 err: 1243 memset32(&slot[0], -1, 4); 1244 wmb(); 1245 1246 engine->props.timeslice_duration_ms = timeslice; 1247 st_engine_heartbeat_enable(engine); 1248 for (i = 0; i < 3; i++) 1249 i915_request_put(rq[i]); 1250 if (igt_flush_test(gt->i915)) 1251 err = -EIO; 1252 if (err) 1253 return err; 1254 } 1255 1256 return 0; 1257 } 1258 1259 static struct i915_request *nop_request(struct intel_engine_cs *engine) 1260 { 1261 struct i915_request *rq; 1262 1263 rq = intel_engine_create_kernel_request(engine); 1264 if (IS_ERR(rq)) 1265 return rq; 1266 1267 i915_request_get(rq); 1268 i915_request_add(rq); 1269 1270 return rq; 1271 } 1272 1273 static long slice_timeout(struct intel_engine_cs *engine) 1274 { 1275 long timeout; 1276 1277 /* Enough time for a timeslice to kick in, and kick out */ 1278 timeout = 2 * msecs_to_jiffies_timeout(timeslice(engine)); 1279 1280 /* Enough time for the nop request to complete */ 1281 timeout += HZ / 5; 1282 1283 return timeout + 1; 1284 } 1285 1286 static int live_timeslice_queue(void *arg) 1287 { 1288 struct intel_gt *gt = arg; 1289 struct drm_i915_gem_object *obj; 1290 struct intel_engine_cs *engine; 1291 enum intel_engine_id id; 1292 struct i915_vma *vma; 1293 void *vaddr; 1294 int err = 0; 1295 1296 /* 1297 * Make sure that even if ELSP[0] and ELSP[1] are filled with 1298 * timeslicing between them disabled, we *do* enable timeslicing 1299 * if the queue demands it. (Normally, we do not submit if 1300 * ELSP[1] is already occupied, so must rely on timeslicing to 1301 * eject ELSP[0] in favour of the queue.) 1302 */ 1303 if (!CONFIG_DRM_I915_TIMESLICE_DURATION) 1304 return 0; 1305 1306 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); 1307 if (IS_ERR(obj)) 1308 return PTR_ERR(obj); 1309 1310 vma = i915_vma_instance(obj, >->ggtt->vm, NULL); 1311 if (IS_ERR(vma)) { 1312 err = PTR_ERR(vma); 1313 goto err_obj; 1314 } 1315 1316 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 1317 if (IS_ERR(vaddr)) { 1318 err = PTR_ERR(vaddr); 1319 goto err_obj; 1320 } 1321 1322 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 1323 if (err) 1324 goto err_map; 1325 1326 err = i915_vma_sync(vma); 1327 if (err) 1328 goto err_pin; 1329 1330 for_each_engine(engine, gt, id) { 1331 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX }; 1332 struct i915_request *rq, *nop; 1333 1334 if (!intel_engine_has_preemption(engine)) 1335 continue; 1336 1337 st_engine_heartbeat_disable(engine); 1338 memset(vaddr, 0, PAGE_SIZE); 1339 1340 /* ELSP[0]: semaphore wait */ 1341 rq = semaphore_queue(engine, vma, 0); 1342 if (IS_ERR(rq)) { 1343 err = PTR_ERR(rq); 1344 goto err_heartbeat; 1345 } 1346 engine->sched_engine->schedule(rq, &attr); 1347 err = wait_for_submit(engine, rq, HZ / 2); 1348 if (err) { 1349 pr_err("%s: Timed out trying to submit semaphores\n", 1350 engine->name); 1351 goto err_rq; 1352 } 1353 1354 /* ELSP[1]: nop request */ 1355 nop = nop_request(engine); 1356 if (IS_ERR(nop)) { 1357 err = PTR_ERR(nop); 1358 goto err_rq; 1359 } 1360 err = wait_for_submit(engine, nop, HZ / 2); 1361 i915_request_put(nop); 1362 if (err) { 1363 pr_err("%s: Timed out trying to submit nop\n", 1364 engine->name); 1365 goto err_rq; 1366 } 1367 1368 GEM_BUG_ON(i915_request_completed(rq)); 1369 GEM_BUG_ON(execlists_active(&engine->execlists) != rq); 1370 1371 /* Queue: semaphore signal, matching priority as semaphore */ 1372 err = release_queue(engine, vma, 1, effective_prio(rq)); 1373 if (err) 1374 goto err_rq; 1375 1376 /* Wait until we ack the release_queue and start timeslicing */ 1377 do { 1378 cond_resched(); 1379 intel_engine_flush_submission(engine); 1380 } while (READ_ONCE(engine->execlists.pending[0])); 1381 1382 /* Timeslice every jiffy, so within 2 we should signal */ 1383 if (i915_request_wait(rq, 0, slice_timeout(engine)) < 0) { 1384 struct drm_printer p = 1385 drm_info_printer(gt->i915->drm.dev); 1386 1387 pr_err("%s: Failed to timeslice into queue\n", 1388 engine->name); 1389 intel_engine_dump(engine, &p, 1390 "%s\n", engine->name); 1391 1392 memset(vaddr, 0xff, PAGE_SIZE); 1393 err = -EIO; 1394 } 1395 err_rq: 1396 i915_request_put(rq); 1397 err_heartbeat: 1398 st_engine_heartbeat_enable(engine); 1399 if (err) 1400 break; 1401 } 1402 1403 err_pin: 1404 i915_vma_unpin(vma); 1405 err_map: 1406 i915_gem_object_unpin_map(obj); 1407 err_obj: 1408 i915_gem_object_put(obj); 1409 return err; 1410 } 1411 1412 static int live_timeslice_nopreempt(void *arg) 1413 { 1414 struct intel_gt *gt = arg; 1415 struct intel_engine_cs *engine; 1416 enum intel_engine_id id; 1417 struct igt_spinner spin; 1418 int err = 0; 1419 1420 /* 1421 * We should not timeslice into a request that is marked with 1422 * I915_REQUEST_NOPREEMPT. 1423 */ 1424 if (!CONFIG_DRM_I915_TIMESLICE_DURATION) 1425 return 0; 1426 1427 if (igt_spinner_init(&spin, gt)) 1428 return -ENOMEM; 1429 1430 for_each_engine(engine, gt, id) { 1431 struct intel_context *ce; 1432 struct i915_request *rq; 1433 unsigned long timeslice; 1434 1435 if (!intel_engine_has_preemption(engine)) 1436 continue; 1437 1438 ce = intel_context_create(engine); 1439 if (IS_ERR(ce)) { 1440 err = PTR_ERR(ce); 1441 break; 1442 } 1443 1444 st_engine_heartbeat_disable(engine); 1445 timeslice = xchg(&engine->props.timeslice_duration_ms, 1); 1446 1447 /* Create an unpreemptible spinner */ 1448 1449 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK); 1450 intel_context_put(ce); 1451 if (IS_ERR(rq)) { 1452 err = PTR_ERR(rq); 1453 goto out_heartbeat; 1454 } 1455 1456 i915_request_get(rq); 1457 i915_request_add(rq); 1458 1459 if (!igt_wait_for_spinner(&spin, rq)) { 1460 i915_request_put(rq); 1461 err = -ETIME; 1462 goto out_spin; 1463 } 1464 1465 set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags); 1466 i915_request_put(rq); 1467 1468 /* Followed by a maximum priority barrier (heartbeat) */ 1469 1470 ce = intel_context_create(engine); 1471 if (IS_ERR(ce)) { 1472 err = PTR_ERR(ce); 1473 goto out_spin; 1474 } 1475 1476 rq = intel_context_create_request(ce); 1477 intel_context_put(ce); 1478 if (IS_ERR(rq)) { 1479 err = PTR_ERR(rq); 1480 goto out_spin; 1481 } 1482 1483 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 1484 i915_request_get(rq); 1485 i915_request_add(rq); 1486 1487 /* 1488 * Wait until the barrier is in ELSP, and we know timeslicing 1489 * will have been activated. 1490 */ 1491 if (wait_for_submit(engine, rq, HZ / 2)) { 1492 i915_request_put(rq); 1493 err = -ETIME; 1494 goto out_spin; 1495 } 1496 1497 /* 1498 * Since the ELSP[0] request is unpreemptible, it should not 1499 * allow the maximum priority barrier through. Wait long 1500 * enough to see if it is timesliced in by mistake. 1501 */ 1502 if (i915_request_wait(rq, 0, slice_timeout(engine)) >= 0) { 1503 pr_err("%s: I915_PRIORITY_BARRIER request completed, bypassing no-preempt request\n", 1504 engine->name); 1505 err = -EINVAL; 1506 } 1507 i915_request_put(rq); 1508 1509 out_spin: 1510 igt_spinner_end(&spin); 1511 out_heartbeat: 1512 xchg(&engine->props.timeslice_duration_ms, timeslice); 1513 st_engine_heartbeat_enable(engine); 1514 if (err) 1515 break; 1516 1517 if (igt_flush_test(gt->i915)) { 1518 err = -EIO; 1519 break; 1520 } 1521 } 1522 1523 igt_spinner_fini(&spin); 1524 return err; 1525 } 1526 1527 static int live_busywait_preempt(void *arg) 1528 { 1529 struct intel_gt *gt = arg; 1530 struct i915_gem_context *ctx_hi, *ctx_lo; 1531 struct intel_engine_cs *engine; 1532 struct drm_i915_gem_object *obj; 1533 struct i915_vma *vma; 1534 enum intel_engine_id id; 1535 int err = -ENOMEM; 1536 u32 *map; 1537 1538 /* 1539 * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can 1540 * preempt the busywaits used to synchronise between rings. 1541 */ 1542 1543 ctx_hi = kernel_context(gt->i915, NULL); 1544 if (!ctx_hi) 1545 return -ENOMEM; 1546 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY; 1547 1548 ctx_lo = kernel_context(gt->i915, NULL); 1549 if (!ctx_lo) 1550 goto err_ctx_hi; 1551 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY; 1552 1553 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); 1554 if (IS_ERR(obj)) { 1555 err = PTR_ERR(obj); 1556 goto err_ctx_lo; 1557 } 1558 1559 map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 1560 if (IS_ERR(map)) { 1561 err = PTR_ERR(map); 1562 goto err_obj; 1563 } 1564 1565 vma = i915_vma_instance(obj, >->ggtt->vm, NULL); 1566 if (IS_ERR(vma)) { 1567 err = PTR_ERR(vma); 1568 goto err_map; 1569 } 1570 1571 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 1572 if (err) 1573 goto err_map; 1574 1575 err = i915_vma_sync(vma); 1576 if (err) 1577 goto err_vma; 1578 1579 for_each_engine(engine, gt, id) { 1580 struct i915_request *lo, *hi; 1581 struct igt_live_test t; 1582 u32 *cs; 1583 1584 if (!intel_engine_has_preemption(engine)) 1585 continue; 1586 1587 if (!intel_engine_can_store_dword(engine)) 1588 continue; 1589 1590 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 1591 err = -EIO; 1592 goto err_vma; 1593 } 1594 1595 /* 1596 * We create two requests. The low priority request 1597 * busywaits on a semaphore (inside the ringbuffer where 1598 * is should be preemptible) and the high priority requests 1599 * uses a MI_STORE_DWORD_IMM to update the semaphore value 1600 * allowing the first request to complete. If preemption 1601 * fails, we hang instead. 1602 */ 1603 1604 lo = igt_request_alloc(ctx_lo, engine); 1605 if (IS_ERR(lo)) { 1606 err = PTR_ERR(lo); 1607 goto err_vma; 1608 } 1609 1610 cs = intel_ring_begin(lo, 8); 1611 if (IS_ERR(cs)) { 1612 err = PTR_ERR(cs); 1613 i915_request_add(lo); 1614 goto err_vma; 1615 } 1616 1617 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 1618 *cs++ = i915_ggtt_offset(vma); 1619 *cs++ = 0; 1620 *cs++ = 1; 1621 1622 /* XXX Do we need a flush + invalidate here? */ 1623 1624 *cs++ = MI_SEMAPHORE_WAIT | 1625 MI_SEMAPHORE_GLOBAL_GTT | 1626 MI_SEMAPHORE_POLL | 1627 MI_SEMAPHORE_SAD_EQ_SDD; 1628 *cs++ = 0; 1629 *cs++ = i915_ggtt_offset(vma); 1630 *cs++ = 0; 1631 1632 intel_ring_advance(lo, cs); 1633 1634 i915_request_get(lo); 1635 i915_request_add(lo); 1636 1637 if (wait_for(READ_ONCE(*map), 10)) { 1638 i915_request_put(lo); 1639 err = -ETIMEDOUT; 1640 goto err_vma; 1641 } 1642 1643 /* Low priority request should be busywaiting now */ 1644 if (i915_request_wait(lo, 0, 1) != -ETIME) { 1645 i915_request_put(lo); 1646 pr_err("%s: Busywaiting request did not!\n", 1647 engine->name); 1648 err = -EIO; 1649 goto err_vma; 1650 } 1651 1652 hi = igt_request_alloc(ctx_hi, engine); 1653 if (IS_ERR(hi)) { 1654 err = PTR_ERR(hi); 1655 i915_request_put(lo); 1656 goto err_vma; 1657 } 1658 1659 cs = intel_ring_begin(hi, 4); 1660 if (IS_ERR(cs)) { 1661 err = PTR_ERR(cs); 1662 i915_request_add(hi); 1663 i915_request_put(lo); 1664 goto err_vma; 1665 } 1666 1667 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 1668 *cs++ = i915_ggtt_offset(vma); 1669 *cs++ = 0; 1670 *cs++ = 0; 1671 1672 intel_ring_advance(hi, cs); 1673 i915_request_add(hi); 1674 1675 if (i915_request_wait(lo, 0, HZ / 5) < 0) { 1676 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 1677 1678 pr_err("%s: Failed to preempt semaphore busywait!\n", 1679 engine->name); 1680 1681 intel_engine_dump(engine, &p, "%s\n", engine->name); 1682 GEM_TRACE_DUMP(); 1683 1684 i915_request_put(lo); 1685 intel_gt_set_wedged(gt); 1686 err = -EIO; 1687 goto err_vma; 1688 } 1689 GEM_BUG_ON(READ_ONCE(*map)); 1690 i915_request_put(lo); 1691 1692 if (igt_live_test_end(&t)) { 1693 err = -EIO; 1694 goto err_vma; 1695 } 1696 } 1697 1698 err = 0; 1699 err_vma: 1700 i915_vma_unpin(vma); 1701 err_map: 1702 i915_gem_object_unpin_map(obj); 1703 err_obj: 1704 i915_gem_object_put(obj); 1705 err_ctx_lo: 1706 kernel_context_close(ctx_lo); 1707 err_ctx_hi: 1708 kernel_context_close(ctx_hi); 1709 return err; 1710 } 1711 1712 static struct i915_request * 1713 spinner_create_request(struct igt_spinner *spin, 1714 struct i915_gem_context *ctx, 1715 struct intel_engine_cs *engine, 1716 u32 arb) 1717 { 1718 struct intel_context *ce; 1719 struct i915_request *rq; 1720 1721 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx); 1722 if (IS_ERR(ce)) 1723 return ERR_CAST(ce); 1724 1725 rq = igt_spinner_create_request(spin, ce, arb); 1726 intel_context_put(ce); 1727 return rq; 1728 } 1729 1730 static int live_preempt(void *arg) 1731 { 1732 struct intel_gt *gt = arg; 1733 struct i915_gem_context *ctx_hi, *ctx_lo; 1734 struct igt_spinner spin_hi, spin_lo; 1735 struct intel_engine_cs *engine; 1736 enum intel_engine_id id; 1737 int err = -ENOMEM; 1738 1739 ctx_hi = kernel_context(gt->i915, NULL); 1740 if (!ctx_hi) 1741 return -ENOMEM; 1742 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY; 1743 1744 ctx_lo = kernel_context(gt->i915, NULL); 1745 if (!ctx_lo) 1746 goto err_ctx_hi; 1747 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY; 1748 1749 if (igt_spinner_init(&spin_hi, gt)) 1750 goto err_ctx_lo; 1751 1752 if (igt_spinner_init(&spin_lo, gt)) 1753 goto err_spin_hi; 1754 1755 for_each_engine(engine, gt, id) { 1756 struct igt_live_test t; 1757 struct i915_request *rq; 1758 1759 if (!intel_engine_has_preemption(engine)) 1760 continue; 1761 1762 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 1763 err = -EIO; 1764 goto err_spin_lo; 1765 } 1766 1767 rq = spinner_create_request(&spin_lo, ctx_lo, engine, 1768 MI_ARB_CHECK); 1769 if (IS_ERR(rq)) { 1770 err = PTR_ERR(rq); 1771 goto err_spin_lo; 1772 } 1773 1774 i915_request_add(rq); 1775 if (!igt_wait_for_spinner(&spin_lo, rq)) { 1776 GEM_TRACE("lo spinner failed to start\n"); 1777 GEM_TRACE_DUMP(); 1778 intel_gt_set_wedged(gt); 1779 err = -EIO; 1780 goto err_spin_lo; 1781 } 1782 1783 rq = spinner_create_request(&spin_hi, ctx_hi, engine, 1784 MI_ARB_CHECK); 1785 if (IS_ERR(rq)) { 1786 igt_spinner_end(&spin_lo); 1787 err = PTR_ERR(rq); 1788 goto err_spin_lo; 1789 } 1790 1791 i915_request_add(rq); 1792 if (!igt_wait_for_spinner(&spin_hi, rq)) { 1793 GEM_TRACE("hi spinner failed to start\n"); 1794 GEM_TRACE_DUMP(); 1795 intel_gt_set_wedged(gt); 1796 err = -EIO; 1797 goto err_spin_lo; 1798 } 1799 1800 igt_spinner_end(&spin_hi); 1801 igt_spinner_end(&spin_lo); 1802 1803 if (igt_live_test_end(&t)) { 1804 err = -EIO; 1805 goto err_spin_lo; 1806 } 1807 } 1808 1809 err = 0; 1810 err_spin_lo: 1811 igt_spinner_fini(&spin_lo); 1812 err_spin_hi: 1813 igt_spinner_fini(&spin_hi); 1814 err_ctx_lo: 1815 kernel_context_close(ctx_lo); 1816 err_ctx_hi: 1817 kernel_context_close(ctx_hi); 1818 return err; 1819 } 1820 1821 static int live_late_preempt(void *arg) 1822 { 1823 struct intel_gt *gt = arg; 1824 struct i915_gem_context *ctx_hi, *ctx_lo; 1825 struct igt_spinner spin_hi, spin_lo; 1826 struct intel_engine_cs *engine; 1827 struct i915_sched_attr attr = {}; 1828 enum intel_engine_id id; 1829 int err = -ENOMEM; 1830 1831 ctx_hi = kernel_context(gt->i915, NULL); 1832 if (!ctx_hi) 1833 return -ENOMEM; 1834 1835 ctx_lo = kernel_context(gt->i915, NULL); 1836 if (!ctx_lo) 1837 goto err_ctx_hi; 1838 1839 if (igt_spinner_init(&spin_hi, gt)) 1840 goto err_ctx_lo; 1841 1842 if (igt_spinner_init(&spin_lo, gt)) 1843 goto err_spin_hi; 1844 1845 /* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */ 1846 ctx_lo->sched.priority = 1; 1847 1848 for_each_engine(engine, gt, id) { 1849 struct igt_live_test t; 1850 struct i915_request *rq; 1851 1852 if (!intel_engine_has_preemption(engine)) 1853 continue; 1854 1855 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 1856 err = -EIO; 1857 goto err_spin_lo; 1858 } 1859 1860 rq = spinner_create_request(&spin_lo, ctx_lo, engine, 1861 MI_ARB_CHECK); 1862 if (IS_ERR(rq)) { 1863 err = PTR_ERR(rq); 1864 goto err_spin_lo; 1865 } 1866 1867 i915_request_add(rq); 1868 if (!igt_wait_for_spinner(&spin_lo, rq)) { 1869 pr_err("First context failed to start\n"); 1870 goto err_wedged; 1871 } 1872 1873 rq = spinner_create_request(&spin_hi, ctx_hi, engine, 1874 MI_NOOP); 1875 if (IS_ERR(rq)) { 1876 igt_spinner_end(&spin_lo); 1877 err = PTR_ERR(rq); 1878 goto err_spin_lo; 1879 } 1880 1881 i915_request_add(rq); 1882 if (igt_wait_for_spinner(&spin_hi, rq)) { 1883 pr_err("Second context overtook first?\n"); 1884 goto err_wedged; 1885 } 1886 1887 attr.priority = I915_PRIORITY_MAX; 1888 engine->sched_engine->schedule(rq, &attr); 1889 1890 if (!igt_wait_for_spinner(&spin_hi, rq)) { 1891 pr_err("High priority context failed to preempt the low priority context\n"); 1892 GEM_TRACE_DUMP(); 1893 goto err_wedged; 1894 } 1895 1896 igt_spinner_end(&spin_hi); 1897 igt_spinner_end(&spin_lo); 1898 1899 if (igt_live_test_end(&t)) { 1900 err = -EIO; 1901 goto err_spin_lo; 1902 } 1903 } 1904 1905 err = 0; 1906 err_spin_lo: 1907 igt_spinner_fini(&spin_lo); 1908 err_spin_hi: 1909 igt_spinner_fini(&spin_hi); 1910 err_ctx_lo: 1911 kernel_context_close(ctx_lo); 1912 err_ctx_hi: 1913 kernel_context_close(ctx_hi); 1914 return err; 1915 1916 err_wedged: 1917 igt_spinner_end(&spin_hi); 1918 igt_spinner_end(&spin_lo); 1919 intel_gt_set_wedged(gt); 1920 err = -EIO; 1921 goto err_spin_lo; 1922 } 1923 1924 struct preempt_client { 1925 struct igt_spinner spin; 1926 struct i915_gem_context *ctx; 1927 }; 1928 1929 static int preempt_client_init(struct intel_gt *gt, struct preempt_client *c) 1930 { 1931 c->ctx = kernel_context(gt->i915, NULL); 1932 if (!c->ctx) 1933 return -ENOMEM; 1934 1935 if (igt_spinner_init(&c->spin, gt)) 1936 goto err_ctx; 1937 1938 return 0; 1939 1940 err_ctx: 1941 kernel_context_close(c->ctx); 1942 return -ENOMEM; 1943 } 1944 1945 static void preempt_client_fini(struct preempt_client *c) 1946 { 1947 igt_spinner_fini(&c->spin); 1948 kernel_context_close(c->ctx); 1949 } 1950 1951 static int live_nopreempt(void *arg) 1952 { 1953 struct intel_gt *gt = arg; 1954 struct intel_engine_cs *engine; 1955 struct preempt_client a, b; 1956 enum intel_engine_id id; 1957 int err = -ENOMEM; 1958 1959 /* 1960 * Verify that we can disable preemption for an individual request 1961 * that may be being observed and not want to be interrupted. 1962 */ 1963 1964 if (preempt_client_init(gt, &a)) 1965 return -ENOMEM; 1966 if (preempt_client_init(gt, &b)) 1967 goto err_client_a; 1968 b.ctx->sched.priority = I915_PRIORITY_MAX; 1969 1970 for_each_engine(engine, gt, id) { 1971 struct i915_request *rq_a, *rq_b; 1972 1973 if (!intel_engine_has_preemption(engine)) 1974 continue; 1975 1976 engine->execlists.preempt_hang.count = 0; 1977 1978 rq_a = spinner_create_request(&a.spin, 1979 a.ctx, engine, 1980 MI_ARB_CHECK); 1981 if (IS_ERR(rq_a)) { 1982 err = PTR_ERR(rq_a); 1983 goto err_client_b; 1984 } 1985 1986 /* Low priority client, but unpreemptable! */ 1987 __set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq_a->fence.flags); 1988 1989 i915_request_add(rq_a); 1990 if (!igt_wait_for_spinner(&a.spin, rq_a)) { 1991 pr_err("First client failed to start\n"); 1992 goto err_wedged; 1993 } 1994 1995 rq_b = spinner_create_request(&b.spin, 1996 b.ctx, engine, 1997 MI_ARB_CHECK); 1998 if (IS_ERR(rq_b)) { 1999 err = PTR_ERR(rq_b); 2000 goto err_client_b; 2001 } 2002 2003 i915_request_add(rq_b); 2004 2005 /* B is much more important than A! (But A is unpreemptable.) */ 2006 GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a)); 2007 2008 /* Wait long enough for preemption and timeslicing */ 2009 if (igt_wait_for_spinner(&b.spin, rq_b)) { 2010 pr_err("Second client started too early!\n"); 2011 goto err_wedged; 2012 } 2013 2014 igt_spinner_end(&a.spin); 2015 2016 if (!igt_wait_for_spinner(&b.spin, rq_b)) { 2017 pr_err("Second client failed to start\n"); 2018 goto err_wedged; 2019 } 2020 2021 igt_spinner_end(&b.spin); 2022 2023 if (engine->execlists.preempt_hang.count) { 2024 pr_err("Preemption recorded x%d; should have been suppressed!\n", 2025 engine->execlists.preempt_hang.count); 2026 err = -EINVAL; 2027 goto err_wedged; 2028 } 2029 2030 if (igt_flush_test(gt->i915)) 2031 goto err_wedged; 2032 } 2033 2034 err = 0; 2035 err_client_b: 2036 preempt_client_fini(&b); 2037 err_client_a: 2038 preempt_client_fini(&a); 2039 return err; 2040 2041 err_wedged: 2042 igt_spinner_end(&b.spin); 2043 igt_spinner_end(&a.spin); 2044 intel_gt_set_wedged(gt); 2045 err = -EIO; 2046 goto err_client_b; 2047 } 2048 2049 struct live_preempt_cancel { 2050 struct intel_engine_cs *engine; 2051 struct preempt_client a, b; 2052 }; 2053 2054 static int __cancel_active0(struct live_preempt_cancel *arg) 2055 { 2056 struct i915_request *rq; 2057 struct igt_live_test t; 2058 int err; 2059 2060 /* Preempt cancel of ELSP0 */ 2061 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name); 2062 if (igt_live_test_begin(&t, arg->engine->i915, 2063 __func__, arg->engine->name)) 2064 return -EIO; 2065 2066 rq = spinner_create_request(&arg->a.spin, 2067 arg->a.ctx, arg->engine, 2068 MI_ARB_CHECK); 2069 if (IS_ERR(rq)) 2070 return PTR_ERR(rq); 2071 2072 clear_bit(CONTEXT_BANNED, &rq->context->flags); 2073 i915_request_get(rq); 2074 i915_request_add(rq); 2075 if (!igt_wait_for_spinner(&arg->a.spin, rq)) { 2076 err = -EIO; 2077 goto out; 2078 } 2079 2080 intel_context_set_banned(rq->context); 2081 err = intel_engine_pulse(arg->engine); 2082 if (err) 2083 goto out; 2084 2085 err = wait_for_reset(arg->engine, rq, HZ / 2); 2086 if (err) { 2087 pr_err("Cancelled inflight0 request did not reset\n"); 2088 goto out; 2089 } 2090 2091 out: 2092 i915_request_put(rq); 2093 if (igt_live_test_end(&t)) 2094 err = -EIO; 2095 return err; 2096 } 2097 2098 static int __cancel_active1(struct live_preempt_cancel *arg) 2099 { 2100 struct i915_request *rq[2] = {}; 2101 struct igt_live_test t; 2102 int err; 2103 2104 /* Preempt cancel of ELSP1 */ 2105 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name); 2106 if (igt_live_test_begin(&t, arg->engine->i915, 2107 __func__, arg->engine->name)) 2108 return -EIO; 2109 2110 rq[0] = spinner_create_request(&arg->a.spin, 2111 arg->a.ctx, arg->engine, 2112 MI_NOOP); /* no preemption */ 2113 if (IS_ERR(rq[0])) 2114 return PTR_ERR(rq[0]); 2115 2116 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags); 2117 i915_request_get(rq[0]); 2118 i915_request_add(rq[0]); 2119 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) { 2120 err = -EIO; 2121 goto out; 2122 } 2123 2124 rq[1] = spinner_create_request(&arg->b.spin, 2125 arg->b.ctx, arg->engine, 2126 MI_ARB_CHECK); 2127 if (IS_ERR(rq[1])) { 2128 err = PTR_ERR(rq[1]); 2129 goto out; 2130 } 2131 2132 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags); 2133 i915_request_get(rq[1]); 2134 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence); 2135 i915_request_add(rq[1]); 2136 if (err) 2137 goto out; 2138 2139 intel_context_set_banned(rq[1]->context); 2140 err = intel_engine_pulse(arg->engine); 2141 if (err) 2142 goto out; 2143 2144 igt_spinner_end(&arg->a.spin); 2145 err = wait_for_reset(arg->engine, rq[1], HZ / 2); 2146 if (err) 2147 goto out; 2148 2149 if (rq[0]->fence.error != 0) { 2150 pr_err("Normal inflight0 request did not complete\n"); 2151 err = -EINVAL; 2152 goto out; 2153 } 2154 2155 if (rq[1]->fence.error != -EIO) { 2156 pr_err("Cancelled inflight1 request did not report -EIO\n"); 2157 err = -EINVAL; 2158 goto out; 2159 } 2160 2161 out: 2162 i915_request_put(rq[1]); 2163 i915_request_put(rq[0]); 2164 if (igt_live_test_end(&t)) 2165 err = -EIO; 2166 return err; 2167 } 2168 2169 static int __cancel_queued(struct live_preempt_cancel *arg) 2170 { 2171 struct i915_request *rq[3] = {}; 2172 struct igt_live_test t; 2173 int err; 2174 2175 /* Full ELSP and one in the wings */ 2176 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name); 2177 if (igt_live_test_begin(&t, arg->engine->i915, 2178 __func__, arg->engine->name)) 2179 return -EIO; 2180 2181 rq[0] = spinner_create_request(&arg->a.spin, 2182 arg->a.ctx, arg->engine, 2183 MI_ARB_CHECK); 2184 if (IS_ERR(rq[0])) 2185 return PTR_ERR(rq[0]); 2186 2187 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags); 2188 i915_request_get(rq[0]); 2189 i915_request_add(rq[0]); 2190 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) { 2191 err = -EIO; 2192 goto out; 2193 } 2194 2195 rq[1] = igt_request_alloc(arg->b.ctx, arg->engine); 2196 if (IS_ERR(rq[1])) { 2197 err = PTR_ERR(rq[1]); 2198 goto out; 2199 } 2200 2201 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags); 2202 i915_request_get(rq[1]); 2203 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence); 2204 i915_request_add(rq[1]); 2205 if (err) 2206 goto out; 2207 2208 rq[2] = spinner_create_request(&arg->b.spin, 2209 arg->a.ctx, arg->engine, 2210 MI_ARB_CHECK); 2211 if (IS_ERR(rq[2])) { 2212 err = PTR_ERR(rq[2]); 2213 goto out; 2214 } 2215 2216 i915_request_get(rq[2]); 2217 err = i915_request_await_dma_fence(rq[2], &rq[1]->fence); 2218 i915_request_add(rq[2]); 2219 if (err) 2220 goto out; 2221 2222 intel_context_set_banned(rq[2]->context); 2223 err = intel_engine_pulse(arg->engine); 2224 if (err) 2225 goto out; 2226 2227 err = wait_for_reset(arg->engine, rq[2], HZ / 2); 2228 if (err) 2229 goto out; 2230 2231 if (rq[0]->fence.error != -EIO) { 2232 pr_err("Cancelled inflight0 request did not report -EIO\n"); 2233 err = -EINVAL; 2234 goto out; 2235 } 2236 2237 if (rq[1]->fence.error != 0) { 2238 pr_err("Normal inflight1 request did not complete\n"); 2239 err = -EINVAL; 2240 goto out; 2241 } 2242 2243 if (rq[2]->fence.error != -EIO) { 2244 pr_err("Cancelled queued request did not report -EIO\n"); 2245 err = -EINVAL; 2246 goto out; 2247 } 2248 2249 out: 2250 i915_request_put(rq[2]); 2251 i915_request_put(rq[1]); 2252 i915_request_put(rq[0]); 2253 if (igt_live_test_end(&t)) 2254 err = -EIO; 2255 return err; 2256 } 2257 2258 static int __cancel_hostile(struct live_preempt_cancel *arg) 2259 { 2260 struct i915_request *rq; 2261 int err; 2262 2263 /* Preempt cancel non-preemptible spinner in ELSP0 */ 2264 if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT) 2265 return 0; 2266 2267 if (!intel_has_reset_engine(arg->engine->gt)) 2268 return 0; 2269 2270 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name); 2271 rq = spinner_create_request(&arg->a.spin, 2272 arg->a.ctx, arg->engine, 2273 MI_NOOP); /* preemption disabled */ 2274 if (IS_ERR(rq)) 2275 return PTR_ERR(rq); 2276 2277 clear_bit(CONTEXT_BANNED, &rq->context->flags); 2278 i915_request_get(rq); 2279 i915_request_add(rq); 2280 if (!igt_wait_for_spinner(&arg->a.spin, rq)) { 2281 err = -EIO; 2282 goto out; 2283 } 2284 2285 intel_context_set_banned(rq->context); 2286 err = intel_engine_pulse(arg->engine); /* force reset */ 2287 if (err) 2288 goto out; 2289 2290 err = wait_for_reset(arg->engine, rq, HZ / 2); 2291 if (err) { 2292 pr_err("Cancelled inflight0 request did not reset\n"); 2293 goto out; 2294 } 2295 2296 out: 2297 i915_request_put(rq); 2298 if (igt_flush_test(arg->engine->i915)) 2299 err = -EIO; 2300 return err; 2301 } 2302 2303 static void force_reset_timeout(struct intel_engine_cs *engine) 2304 { 2305 engine->reset_timeout.probability = 999; 2306 atomic_set(&engine->reset_timeout.times, -1); 2307 } 2308 2309 static void cancel_reset_timeout(struct intel_engine_cs *engine) 2310 { 2311 memset(&engine->reset_timeout, 0, sizeof(engine->reset_timeout)); 2312 } 2313 2314 static int __cancel_fail(struct live_preempt_cancel *arg) 2315 { 2316 struct intel_engine_cs *engine = arg->engine; 2317 struct i915_request *rq; 2318 int err; 2319 2320 if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT) 2321 return 0; 2322 2323 if (!intel_has_reset_engine(engine->gt)) 2324 return 0; 2325 2326 GEM_TRACE("%s(%s)\n", __func__, engine->name); 2327 rq = spinner_create_request(&arg->a.spin, 2328 arg->a.ctx, engine, 2329 MI_NOOP); /* preemption disabled */ 2330 if (IS_ERR(rq)) 2331 return PTR_ERR(rq); 2332 2333 clear_bit(CONTEXT_BANNED, &rq->context->flags); 2334 i915_request_get(rq); 2335 i915_request_add(rq); 2336 if (!igt_wait_for_spinner(&arg->a.spin, rq)) { 2337 err = -EIO; 2338 goto out; 2339 } 2340 2341 intel_context_set_banned(rq->context); 2342 2343 err = intel_engine_pulse(engine); 2344 if (err) 2345 goto out; 2346 2347 force_reset_timeout(engine); 2348 2349 /* force preempt reset [failure] */ 2350 while (!engine->execlists.pending[0]) 2351 intel_engine_flush_submission(engine); 2352 del_timer_sync(&engine->execlists.preempt); 2353 intel_engine_flush_submission(engine); 2354 2355 cancel_reset_timeout(engine); 2356 2357 /* after failure, require heartbeats to reset device */ 2358 intel_engine_set_heartbeat(engine, 1); 2359 err = wait_for_reset(engine, rq, HZ / 2); 2360 intel_engine_set_heartbeat(engine, 2361 engine->defaults.heartbeat_interval_ms); 2362 if (err) { 2363 pr_err("Cancelled inflight0 request did not reset\n"); 2364 goto out; 2365 } 2366 2367 out: 2368 i915_request_put(rq); 2369 if (igt_flush_test(engine->i915)) 2370 err = -EIO; 2371 return err; 2372 } 2373 2374 static int live_preempt_cancel(void *arg) 2375 { 2376 struct intel_gt *gt = arg; 2377 struct live_preempt_cancel data; 2378 enum intel_engine_id id; 2379 int err = -ENOMEM; 2380 2381 /* 2382 * To cancel an inflight context, we need to first remove it from the 2383 * GPU. That sounds like preemption! Plus a little bit of bookkeeping. 2384 */ 2385 2386 if (preempt_client_init(gt, &data.a)) 2387 return -ENOMEM; 2388 if (preempt_client_init(gt, &data.b)) 2389 goto err_client_a; 2390 2391 for_each_engine(data.engine, gt, id) { 2392 if (!intel_engine_has_preemption(data.engine)) 2393 continue; 2394 2395 err = __cancel_active0(&data); 2396 if (err) 2397 goto err_wedged; 2398 2399 err = __cancel_active1(&data); 2400 if (err) 2401 goto err_wedged; 2402 2403 err = __cancel_queued(&data); 2404 if (err) 2405 goto err_wedged; 2406 2407 err = __cancel_hostile(&data); 2408 if (err) 2409 goto err_wedged; 2410 2411 err = __cancel_fail(&data); 2412 if (err) 2413 goto err_wedged; 2414 } 2415 2416 err = 0; 2417 err_client_b: 2418 preempt_client_fini(&data.b); 2419 err_client_a: 2420 preempt_client_fini(&data.a); 2421 return err; 2422 2423 err_wedged: 2424 GEM_TRACE_DUMP(); 2425 igt_spinner_end(&data.b.spin); 2426 igt_spinner_end(&data.a.spin); 2427 intel_gt_set_wedged(gt); 2428 goto err_client_b; 2429 } 2430 2431 static int live_suppress_self_preempt(void *arg) 2432 { 2433 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX }; 2434 struct intel_gt *gt = arg; 2435 struct intel_engine_cs *engine; 2436 struct preempt_client a, b; 2437 enum intel_engine_id id; 2438 int err = -ENOMEM; 2439 2440 /* 2441 * Verify that if a preemption request does not cause a change in 2442 * the current execution order, the preempt-to-idle injection is 2443 * skipped and that we do not accidentally apply it after the CS 2444 * completion event. 2445 */ 2446 2447 if (intel_uc_uses_guc_submission(>->uc)) 2448 return 0; /* presume black blox */ 2449 2450 if (intel_vgpu_active(gt->i915)) 2451 return 0; /* GVT forces single port & request submission */ 2452 2453 if (preempt_client_init(gt, &a)) 2454 return -ENOMEM; 2455 if (preempt_client_init(gt, &b)) 2456 goto err_client_a; 2457 2458 for_each_engine(engine, gt, id) { 2459 struct i915_request *rq_a, *rq_b; 2460 int depth; 2461 2462 if (!intel_engine_has_preemption(engine)) 2463 continue; 2464 2465 if (igt_flush_test(gt->i915)) 2466 goto err_wedged; 2467 2468 st_engine_heartbeat_disable(engine); 2469 engine->execlists.preempt_hang.count = 0; 2470 2471 rq_a = spinner_create_request(&a.spin, 2472 a.ctx, engine, 2473 MI_NOOP); 2474 if (IS_ERR(rq_a)) { 2475 err = PTR_ERR(rq_a); 2476 st_engine_heartbeat_enable(engine); 2477 goto err_client_b; 2478 } 2479 2480 i915_request_add(rq_a); 2481 if (!igt_wait_for_spinner(&a.spin, rq_a)) { 2482 pr_err("First client failed to start\n"); 2483 st_engine_heartbeat_enable(engine); 2484 goto err_wedged; 2485 } 2486 2487 /* Keep postponing the timer to avoid premature slicing */ 2488 mod_timer(&engine->execlists.timer, jiffies + HZ); 2489 for (depth = 0; depth < 8; depth++) { 2490 rq_b = spinner_create_request(&b.spin, 2491 b.ctx, engine, 2492 MI_NOOP); 2493 if (IS_ERR(rq_b)) { 2494 err = PTR_ERR(rq_b); 2495 st_engine_heartbeat_enable(engine); 2496 goto err_client_b; 2497 } 2498 i915_request_add(rq_b); 2499 2500 GEM_BUG_ON(i915_request_completed(rq_a)); 2501 engine->sched_engine->schedule(rq_a, &attr); 2502 igt_spinner_end(&a.spin); 2503 2504 if (!igt_wait_for_spinner(&b.spin, rq_b)) { 2505 pr_err("Second client failed to start\n"); 2506 st_engine_heartbeat_enable(engine); 2507 goto err_wedged; 2508 } 2509 2510 swap(a, b); 2511 rq_a = rq_b; 2512 } 2513 igt_spinner_end(&a.spin); 2514 2515 if (engine->execlists.preempt_hang.count) { 2516 pr_err("Preemption on %s recorded x%d, depth %d; should have been suppressed!\n", 2517 engine->name, 2518 engine->execlists.preempt_hang.count, 2519 depth); 2520 st_engine_heartbeat_enable(engine); 2521 err = -EINVAL; 2522 goto err_client_b; 2523 } 2524 2525 st_engine_heartbeat_enable(engine); 2526 if (igt_flush_test(gt->i915)) 2527 goto err_wedged; 2528 } 2529 2530 err = 0; 2531 err_client_b: 2532 preempt_client_fini(&b); 2533 err_client_a: 2534 preempt_client_fini(&a); 2535 return err; 2536 2537 err_wedged: 2538 igt_spinner_end(&b.spin); 2539 igt_spinner_end(&a.spin); 2540 intel_gt_set_wedged(gt); 2541 err = -EIO; 2542 goto err_client_b; 2543 } 2544 2545 static int live_chain_preempt(void *arg) 2546 { 2547 struct intel_gt *gt = arg; 2548 struct intel_engine_cs *engine; 2549 struct preempt_client hi, lo; 2550 enum intel_engine_id id; 2551 int err = -ENOMEM; 2552 2553 /* 2554 * Build a chain AB...BA between two contexts (A, B) and request 2555 * preemption of the last request. It should then complete before 2556 * the previously submitted spinner in B. 2557 */ 2558 2559 if (preempt_client_init(gt, &hi)) 2560 return -ENOMEM; 2561 2562 if (preempt_client_init(gt, &lo)) 2563 goto err_client_hi; 2564 2565 for_each_engine(engine, gt, id) { 2566 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX }; 2567 struct igt_live_test t; 2568 struct i915_request *rq; 2569 int ring_size, count, i; 2570 2571 if (!intel_engine_has_preemption(engine)) 2572 continue; 2573 2574 rq = spinner_create_request(&lo.spin, 2575 lo.ctx, engine, 2576 MI_ARB_CHECK); 2577 if (IS_ERR(rq)) 2578 goto err_wedged; 2579 2580 i915_request_get(rq); 2581 i915_request_add(rq); 2582 2583 ring_size = rq->wa_tail - rq->head; 2584 if (ring_size < 0) 2585 ring_size += rq->ring->size; 2586 ring_size = rq->ring->size / ring_size; 2587 pr_debug("%s(%s): Using maximum of %d requests\n", 2588 __func__, engine->name, ring_size); 2589 2590 igt_spinner_end(&lo.spin); 2591 if (i915_request_wait(rq, 0, HZ / 2) < 0) { 2592 pr_err("Timed out waiting to flush %s\n", engine->name); 2593 i915_request_put(rq); 2594 goto err_wedged; 2595 } 2596 i915_request_put(rq); 2597 2598 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 2599 err = -EIO; 2600 goto err_wedged; 2601 } 2602 2603 for_each_prime_number_from(count, 1, ring_size) { 2604 rq = spinner_create_request(&hi.spin, 2605 hi.ctx, engine, 2606 MI_ARB_CHECK); 2607 if (IS_ERR(rq)) 2608 goto err_wedged; 2609 i915_request_add(rq); 2610 if (!igt_wait_for_spinner(&hi.spin, rq)) 2611 goto err_wedged; 2612 2613 rq = spinner_create_request(&lo.spin, 2614 lo.ctx, engine, 2615 MI_ARB_CHECK); 2616 if (IS_ERR(rq)) 2617 goto err_wedged; 2618 i915_request_add(rq); 2619 2620 for (i = 0; i < count; i++) { 2621 rq = igt_request_alloc(lo.ctx, engine); 2622 if (IS_ERR(rq)) 2623 goto err_wedged; 2624 i915_request_add(rq); 2625 } 2626 2627 rq = igt_request_alloc(hi.ctx, engine); 2628 if (IS_ERR(rq)) 2629 goto err_wedged; 2630 2631 i915_request_get(rq); 2632 i915_request_add(rq); 2633 engine->sched_engine->schedule(rq, &attr); 2634 2635 igt_spinner_end(&hi.spin); 2636 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 2637 struct drm_printer p = 2638 drm_info_printer(gt->i915->drm.dev); 2639 2640 pr_err("Failed to preempt over chain of %d\n", 2641 count); 2642 intel_engine_dump(engine, &p, 2643 "%s\n", engine->name); 2644 i915_request_put(rq); 2645 goto err_wedged; 2646 } 2647 igt_spinner_end(&lo.spin); 2648 i915_request_put(rq); 2649 2650 rq = igt_request_alloc(lo.ctx, engine); 2651 if (IS_ERR(rq)) 2652 goto err_wedged; 2653 2654 i915_request_get(rq); 2655 i915_request_add(rq); 2656 2657 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 2658 struct drm_printer p = 2659 drm_info_printer(gt->i915->drm.dev); 2660 2661 pr_err("Failed to flush low priority chain of %d requests\n", 2662 count); 2663 intel_engine_dump(engine, &p, 2664 "%s\n", engine->name); 2665 2666 i915_request_put(rq); 2667 goto err_wedged; 2668 } 2669 i915_request_put(rq); 2670 } 2671 2672 if (igt_live_test_end(&t)) { 2673 err = -EIO; 2674 goto err_wedged; 2675 } 2676 } 2677 2678 err = 0; 2679 err_client_lo: 2680 preempt_client_fini(&lo); 2681 err_client_hi: 2682 preempt_client_fini(&hi); 2683 return err; 2684 2685 err_wedged: 2686 igt_spinner_end(&hi.spin); 2687 igt_spinner_end(&lo.spin); 2688 intel_gt_set_wedged(gt); 2689 err = -EIO; 2690 goto err_client_lo; 2691 } 2692 2693 static int create_gang(struct intel_engine_cs *engine, 2694 struct i915_request **prev) 2695 { 2696 struct drm_i915_gem_object *obj; 2697 struct intel_context *ce; 2698 struct i915_request *rq; 2699 struct i915_vma *vma; 2700 u32 *cs; 2701 int err; 2702 2703 ce = intel_context_create(engine); 2704 if (IS_ERR(ce)) 2705 return PTR_ERR(ce); 2706 2707 obj = i915_gem_object_create_internal(engine->i915, 4096); 2708 if (IS_ERR(obj)) { 2709 err = PTR_ERR(obj); 2710 goto err_ce; 2711 } 2712 2713 vma = i915_vma_instance(obj, ce->vm, NULL); 2714 if (IS_ERR(vma)) { 2715 err = PTR_ERR(vma); 2716 goto err_obj; 2717 } 2718 2719 err = i915_vma_pin(vma, 0, 0, PIN_USER); 2720 if (err) 2721 goto err_obj; 2722 2723 cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 2724 if (IS_ERR(cs)) { 2725 err = PTR_ERR(cs); 2726 goto err_obj; 2727 } 2728 2729 /* Semaphore target: spin until zero */ 2730 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 2731 2732 *cs++ = MI_SEMAPHORE_WAIT | 2733 MI_SEMAPHORE_POLL | 2734 MI_SEMAPHORE_SAD_EQ_SDD; 2735 *cs++ = 0; 2736 *cs++ = lower_32_bits(vma->node.start); 2737 *cs++ = upper_32_bits(vma->node.start); 2738 2739 if (*prev) { 2740 u64 offset = (*prev)->batch->node.start; 2741 2742 /* Terminate the spinner in the next lower priority batch. */ 2743 *cs++ = MI_STORE_DWORD_IMM_GEN4; 2744 *cs++ = lower_32_bits(offset); 2745 *cs++ = upper_32_bits(offset); 2746 *cs++ = 0; 2747 } 2748 2749 *cs++ = MI_BATCH_BUFFER_END; 2750 i915_gem_object_flush_map(obj); 2751 i915_gem_object_unpin_map(obj); 2752 2753 rq = intel_context_create_request(ce); 2754 if (IS_ERR(rq)) { 2755 err = PTR_ERR(rq); 2756 goto err_obj; 2757 } 2758 2759 rq->batch = i915_vma_get(vma); 2760 i915_request_get(rq); 2761 2762 i915_vma_lock(vma); 2763 err = i915_request_await_object(rq, vma->obj, false); 2764 if (!err) 2765 err = i915_vma_move_to_active(vma, rq, 0); 2766 if (!err) 2767 err = rq->engine->emit_bb_start(rq, 2768 vma->node.start, 2769 PAGE_SIZE, 0); 2770 i915_vma_unlock(vma); 2771 i915_request_add(rq); 2772 if (err) 2773 goto err_rq; 2774 2775 i915_gem_object_put(obj); 2776 intel_context_put(ce); 2777 2778 rq->mock.link.next = &(*prev)->mock.link; 2779 *prev = rq; 2780 return 0; 2781 2782 err_rq: 2783 i915_vma_put(rq->batch); 2784 i915_request_put(rq); 2785 err_obj: 2786 i915_gem_object_put(obj); 2787 err_ce: 2788 intel_context_put(ce); 2789 return err; 2790 } 2791 2792 static int __live_preempt_ring(struct intel_engine_cs *engine, 2793 struct igt_spinner *spin, 2794 int queue_sz, int ring_sz) 2795 { 2796 struct intel_context *ce[2] = {}; 2797 struct i915_request *rq; 2798 struct igt_live_test t; 2799 int err = 0; 2800 int n; 2801 2802 if (igt_live_test_begin(&t, engine->i915, __func__, engine->name)) 2803 return -EIO; 2804 2805 for (n = 0; n < ARRAY_SIZE(ce); n++) { 2806 struct intel_context *tmp; 2807 2808 tmp = intel_context_create(engine); 2809 if (IS_ERR(tmp)) { 2810 err = PTR_ERR(tmp); 2811 goto err_ce; 2812 } 2813 2814 tmp->ring_size = ring_sz; 2815 2816 err = intel_context_pin(tmp); 2817 if (err) { 2818 intel_context_put(tmp); 2819 goto err_ce; 2820 } 2821 2822 memset32(tmp->ring->vaddr, 2823 0xdeadbeef, /* trigger a hang if executed */ 2824 tmp->ring->vma->size / sizeof(u32)); 2825 2826 ce[n] = tmp; 2827 } 2828 2829 rq = igt_spinner_create_request(spin, ce[0], MI_ARB_CHECK); 2830 if (IS_ERR(rq)) { 2831 err = PTR_ERR(rq); 2832 goto err_ce; 2833 } 2834 2835 i915_request_get(rq); 2836 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 2837 i915_request_add(rq); 2838 2839 if (!igt_wait_for_spinner(spin, rq)) { 2840 intel_gt_set_wedged(engine->gt); 2841 i915_request_put(rq); 2842 err = -ETIME; 2843 goto err_ce; 2844 } 2845 2846 /* Fill the ring, until we will cause a wrap */ 2847 n = 0; 2848 while (ce[0]->ring->tail - rq->wa_tail <= queue_sz) { 2849 struct i915_request *tmp; 2850 2851 tmp = intel_context_create_request(ce[0]); 2852 if (IS_ERR(tmp)) { 2853 err = PTR_ERR(tmp); 2854 i915_request_put(rq); 2855 goto err_ce; 2856 } 2857 2858 i915_request_add(tmp); 2859 intel_engine_flush_submission(engine); 2860 n++; 2861 } 2862 intel_engine_flush_submission(engine); 2863 pr_debug("%s: Filled %d with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n", 2864 engine->name, queue_sz, n, 2865 ce[0]->ring->size, 2866 ce[0]->ring->tail, 2867 ce[0]->ring->emit, 2868 rq->tail); 2869 i915_request_put(rq); 2870 2871 /* Create a second request to preempt the first ring */ 2872 rq = intel_context_create_request(ce[1]); 2873 if (IS_ERR(rq)) { 2874 err = PTR_ERR(rq); 2875 goto err_ce; 2876 } 2877 2878 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 2879 i915_request_get(rq); 2880 i915_request_add(rq); 2881 2882 err = wait_for_submit(engine, rq, HZ / 2); 2883 i915_request_put(rq); 2884 if (err) { 2885 pr_err("%s: preemption request was not submitted\n", 2886 engine->name); 2887 err = -ETIME; 2888 } 2889 2890 pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n", 2891 engine->name, 2892 ce[0]->ring->tail, ce[0]->ring->emit, 2893 ce[1]->ring->tail, ce[1]->ring->emit); 2894 2895 err_ce: 2896 intel_engine_flush_submission(engine); 2897 igt_spinner_end(spin); 2898 for (n = 0; n < ARRAY_SIZE(ce); n++) { 2899 if (IS_ERR_OR_NULL(ce[n])) 2900 break; 2901 2902 intel_context_unpin(ce[n]); 2903 intel_context_put(ce[n]); 2904 } 2905 if (igt_live_test_end(&t)) 2906 err = -EIO; 2907 return err; 2908 } 2909 2910 static int live_preempt_ring(void *arg) 2911 { 2912 struct intel_gt *gt = arg; 2913 struct intel_engine_cs *engine; 2914 struct igt_spinner spin; 2915 enum intel_engine_id id; 2916 int err = 0; 2917 2918 /* 2919 * Check that we rollback large chunks of a ring in order to do a 2920 * preemption event. Similar to live_unlite_ring, but looking at 2921 * ring size rather than the impact of intel_ring_direction(). 2922 */ 2923 2924 if (igt_spinner_init(&spin, gt)) 2925 return -ENOMEM; 2926 2927 for_each_engine(engine, gt, id) { 2928 int n; 2929 2930 if (!intel_engine_has_preemption(engine)) 2931 continue; 2932 2933 if (!intel_engine_can_store_dword(engine)) 2934 continue; 2935 2936 st_engine_heartbeat_disable(engine); 2937 2938 for (n = 0; n <= 3; n++) { 2939 err = __live_preempt_ring(engine, &spin, 2940 n * SZ_4K / 4, SZ_4K); 2941 if (err) 2942 break; 2943 } 2944 2945 st_engine_heartbeat_enable(engine); 2946 if (err) 2947 break; 2948 } 2949 2950 igt_spinner_fini(&spin); 2951 return err; 2952 } 2953 2954 static int live_preempt_gang(void *arg) 2955 { 2956 struct intel_gt *gt = arg; 2957 struct intel_engine_cs *engine; 2958 enum intel_engine_id id; 2959 2960 /* 2961 * Build as long a chain of preempters as we can, with each 2962 * request higher priority than the last. Once we are ready, we release 2963 * the last batch which then precolates down the chain, each releasing 2964 * the next oldest in turn. The intent is to simply push as hard as we 2965 * can with the number of preemptions, trying to exceed narrow HW 2966 * limits. At a minimum, we insist that we can sort all the user 2967 * high priority levels into execution order. 2968 */ 2969 2970 for_each_engine(engine, gt, id) { 2971 struct i915_request *rq = NULL; 2972 struct igt_live_test t; 2973 IGT_TIMEOUT(end_time); 2974 int prio = 0; 2975 int err = 0; 2976 u32 *cs; 2977 2978 if (!intel_engine_has_preemption(engine)) 2979 continue; 2980 2981 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) 2982 return -EIO; 2983 2984 do { 2985 struct i915_sched_attr attr = { .priority = prio++ }; 2986 2987 err = create_gang(engine, &rq); 2988 if (err) 2989 break; 2990 2991 /* Submit each spinner at increasing priority */ 2992 engine->sched_engine->schedule(rq, &attr); 2993 } while (prio <= I915_PRIORITY_MAX && 2994 !__igt_timeout(end_time, NULL)); 2995 pr_debug("%s: Preempt chain of %d requests\n", 2996 engine->name, prio); 2997 2998 /* 2999 * Such that the last spinner is the highest priority and 3000 * should execute first. When that spinner completes, 3001 * it will terminate the next lowest spinner until there 3002 * are no more spinners and the gang is complete. 3003 */ 3004 cs = i915_gem_object_pin_map_unlocked(rq->batch->obj, I915_MAP_WC); 3005 if (!IS_ERR(cs)) { 3006 *cs = 0; 3007 i915_gem_object_unpin_map(rq->batch->obj); 3008 } else { 3009 err = PTR_ERR(cs); 3010 intel_gt_set_wedged(gt); 3011 } 3012 3013 while (rq) { /* wait for each rq from highest to lowest prio */ 3014 struct i915_request *n = list_next_entry(rq, mock.link); 3015 3016 if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) { 3017 struct drm_printer p = 3018 drm_info_printer(engine->i915->drm.dev); 3019 3020 pr_err("Failed to flush chain of %d requests, at %d\n", 3021 prio, rq_prio(rq)); 3022 intel_engine_dump(engine, &p, 3023 "%s\n", engine->name); 3024 3025 err = -ETIME; 3026 } 3027 3028 i915_vma_put(rq->batch); 3029 i915_request_put(rq); 3030 rq = n; 3031 } 3032 3033 if (igt_live_test_end(&t)) 3034 err = -EIO; 3035 if (err) 3036 return err; 3037 } 3038 3039 return 0; 3040 } 3041 3042 static struct i915_vma * 3043 create_gpr_user(struct intel_engine_cs *engine, 3044 struct i915_vma *result, 3045 unsigned int offset) 3046 { 3047 struct drm_i915_gem_object *obj; 3048 struct i915_vma *vma; 3049 u32 *cs; 3050 int err; 3051 int i; 3052 3053 obj = i915_gem_object_create_internal(engine->i915, 4096); 3054 if (IS_ERR(obj)) 3055 return ERR_CAST(obj); 3056 3057 vma = i915_vma_instance(obj, result->vm, NULL); 3058 if (IS_ERR(vma)) { 3059 i915_gem_object_put(obj); 3060 return vma; 3061 } 3062 3063 err = i915_vma_pin(vma, 0, 0, PIN_USER); 3064 if (err) { 3065 i915_vma_put(vma); 3066 return ERR_PTR(err); 3067 } 3068 3069 cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 3070 if (IS_ERR(cs)) { 3071 i915_vma_put(vma); 3072 return ERR_CAST(cs); 3073 } 3074 3075 /* All GPR are clear for new contexts. We use GPR(0) as a constant */ 3076 *cs++ = MI_LOAD_REGISTER_IMM(1); 3077 *cs++ = CS_GPR(engine, 0); 3078 *cs++ = 1; 3079 3080 for (i = 1; i < NUM_GPR; i++) { 3081 u64 addr; 3082 3083 /* 3084 * Perform: GPR[i]++ 3085 * 3086 * As we read and write into the context saved GPR[i], if 3087 * we restart this batch buffer from an earlier point, we 3088 * will repeat the increment and store a value > 1. 3089 */ 3090 *cs++ = MI_MATH(4); 3091 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(i)); 3092 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(0)); 3093 *cs++ = MI_MATH_ADD; 3094 *cs++ = MI_MATH_STORE(MI_MATH_REG(i), MI_MATH_REG_ACCU); 3095 3096 addr = result->node.start + offset + i * sizeof(*cs); 3097 *cs++ = MI_STORE_REGISTER_MEM_GEN8; 3098 *cs++ = CS_GPR(engine, 2 * i); 3099 *cs++ = lower_32_bits(addr); 3100 *cs++ = upper_32_bits(addr); 3101 3102 *cs++ = MI_SEMAPHORE_WAIT | 3103 MI_SEMAPHORE_POLL | 3104 MI_SEMAPHORE_SAD_GTE_SDD; 3105 *cs++ = i; 3106 *cs++ = lower_32_bits(result->node.start); 3107 *cs++ = upper_32_bits(result->node.start); 3108 } 3109 3110 *cs++ = MI_BATCH_BUFFER_END; 3111 i915_gem_object_flush_map(obj); 3112 i915_gem_object_unpin_map(obj); 3113 3114 return vma; 3115 } 3116 3117 static struct i915_vma *create_global(struct intel_gt *gt, size_t sz) 3118 { 3119 struct drm_i915_gem_object *obj; 3120 struct i915_vma *vma; 3121 int err; 3122 3123 obj = i915_gem_object_create_internal(gt->i915, sz); 3124 if (IS_ERR(obj)) 3125 return ERR_CAST(obj); 3126 3127 vma = i915_vma_instance(obj, >->ggtt->vm, NULL); 3128 if (IS_ERR(vma)) { 3129 i915_gem_object_put(obj); 3130 return vma; 3131 } 3132 3133 err = i915_ggtt_pin(vma, NULL, 0, 0); 3134 if (err) { 3135 i915_vma_put(vma); 3136 return ERR_PTR(err); 3137 } 3138 3139 return vma; 3140 } 3141 3142 static struct i915_request * 3143 create_gpr_client(struct intel_engine_cs *engine, 3144 struct i915_vma *global, 3145 unsigned int offset) 3146 { 3147 struct i915_vma *batch, *vma; 3148 struct intel_context *ce; 3149 struct i915_request *rq; 3150 int err; 3151 3152 ce = intel_context_create(engine); 3153 if (IS_ERR(ce)) 3154 return ERR_CAST(ce); 3155 3156 vma = i915_vma_instance(global->obj, ce->vm, NULL); 3157 if (IS_ERR(vma)) { 3158 err = PTR_ERR(vma); 3159 goto out_ce; 3160 } 3161 3162 err = i915_vma_pin(vma, 0, 0, PIN_USER); 3163 if (err) 3164 goto out_ce; 3165 3166 batch = create_gpr_user(engine, vma, offset); 3167 if (IS_ERR(batch)) { 3168 err = PTR_ERR(batch); 3169 goto out_vma; 3170 } 3171 3172 rq = intel_context_create_request(ce); 3173 if (IS_ERR(rq)) { 3174 err = PTR_ERR(rq); 3175 goto out_batch; 3176 } 3177 3178 i915_vma_lock(vma); 3179 err = i915_request_await_object(rq, vma->obj, false); 3180 if (!err) 3181 err = i915_vma_move_to_active(vma, rq, 0); 3182 i915_vma_unlock(vma); 3183 3184 i915_vma_lock(batch); 3185 if (!err) 3186 err = i915_request_await_object(rq, batch->obj, false); 3187 if (!err) 3188 err = i915_vma_move_to_active(batch, rq, 0); 3189 if (!err) 3190 err = rq->engine->emit_bb_start(rq, 3191 batch->node.start, 3192 PAGE_SIZE, 0); 3193 i915_vma_unlock(batch); 3194 i915_vma_unpin(batch); 3195 3196 if (!err) 3197 i915_request_get(rq); 3198 i915_request_add(rq); 3199 3200 out_batch: 3201 i915_vma_put(batch); 3202 out_vma: 3203 i915_vma_unpin(vma); 3204 out_ce: 3205 intel_context_put(ce); 3206 return err ? ERR_PTR(err) : rq; 3207 } 3208 3209 static int preempt_user(struct intel_engine_cs *engine, 3210 struct i915_vma *global, 3211 int id) 3212 { 3213 struct i915_sched_attr attr = { 3214 .priority = I915_PRIORITY_MAX 3215 }; 3216 struct i915_request *rq; 3217 int err = 0; 3218 u32 *cs; 3219 3220 rq = intel_engine_create_kernel_request(engine); 3221 if (IS_ERR(rq)) 3222 return PTR_ERR(rq); 3223 3224 cs = intel_ring_begin(rq, 4); 3225 if (IS_ERR(cs)) { 3226 i915_request_add(rq); 3227 return PTR_ERR(cs); 3228 } 3229 3230 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 3231 *cs++ = i915_ggtt_offset(global); 3232 *cs++ = 0; 3233 *cs++ = id; 3234 3235 intel_ring_advance(rq, cs); 3236 3237 i915_request_get(rq); 3238 i915_request_add(rq); 3239 3240 engine->sched_engine->schedule(rq, &attr); 3241 3242 if (i915_request_wait(rq, 0, HZ / 2) < 0) 3243 err = -ETIME; 3244 i915_request_put(rq); 3245 3246 return err; 3247 } 3248 3249 static int live_preempt_user(void *arg) 3250 { 3251 struct intel_gt *gt = arg; 3252 struct intel_engine_cs *engine; 3253 struct i915_vma *global; 3254 enum intel_engine_id id; 3255 u32 *result; 3256 int err = 0; 3257 3258 /* 3259 * In our other tests, we look at preemption in carefully 3260 * controlled conditions in the ringbuffer. Since most of the 3261 * time is spent in user batches, most of our preemptions naturally 3262 * occur there. We want to verify that when we preempt inside a batch 3263 * we continue on from the current instruction and do not roll back 3264 * to the start, or another earlier arbitration point. 3265 * 3266 * To verify this, we create a batch which is a mixture of 3267 * MI_MATH (gpr++) MI_SRM (gpr) and preemption points. Then with 3268 * a few preempting contexts thrown into the mix, we look for any 3269 * repeated instructions (which show up as incorrect values). 3270 */ 3271 3272 global = create_global(gt, 4096); 3273 if (IS_ERR(global)) 3274 return PTR_ERR(global); 3275 3276 result = i915_gem_object_pin_map_unlocked(global->obj, I915_MAP_WC); 3277 if (IS_ERR(result)) { 3278 i915_vma_unpin_and_release(&global, 0); 3279 return PTR_ERR(result); 3280 } 3281 3282 for_each_engine(engine, gt, id) { 3283 struct i915_request *client[3] = {}; 3284 struct igt_live_test t; 3285 int i; 3286 3287 if (!intel_engine_has_preemption(engine)) 3288 continue; 3289 3290 if (GRAPHICS_VER(gt->i915) == 8 && engine->class != RENDER_CLASS) 3291 continue; /* we need per-context GPR */ 3292 3293 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 3294 err = -EIO; 3295 break; 3296 } 3297 3298 memset(result, 0, 4096); 3299 3300 for (i = 0; i < ARRAY_SIZE(client); i++) { 3301 struct i915_request *rq; 3302 3303 rq = create_gpr_client(engine, global, 3304 NUM_GPR * i * sizeof(u32)); 3305 if (IS_ERR(rq)) { 3306 err = PTR_ERR(rq); 3307 goto end_test; 3308 } 3309 3310 client[i] = rq; 3311 } 3312 3313 /* Continuously preempt the set of 3 running contexts */ 3314 for (i = 1; i <= NUM_GPR; i++) { 3315 err = preempt_user(engine, global, i); 3316 if (err) 3317 goto end_test; 3318 } 3319 3320 if (READ_ONCE(result[0]) != NUM_GPR) { 3321 pr_err("%s: Failed to release semaphore\n", 3322 engine->name); 3323 err = -EIO; 3324 goto end_test; 3325 } 3326 3327 for (i = 0; i < ARRAY_SIZE(client); i++) { 3328 int gpr; 3329 3330 if (i915_request_wait(client[i], 0, HZ / 2) < 0) { 3331 err = -ETIME; 3332 goto end_test; 3333 } 3334 3335 for (gpr = 1; gpr < NUM_GPR; gpr++) { 3336 if (result[NUM_GPR * i + gpr] != 1) { 3337 pr_err("%s: Invalid result, client %d, gpr %d, result: %d\n", 3338 engine->name, 3339 i, gpr, result[NUM_GPR * i + gpr]); 3340 err = -EINVAL; 3341 goto end_test; 3342 } 3343 } 3344 } 3345 3346 end_test: 3347 for (i = 0; i < ARRAY_SIZE(client); i++) { 3348 if (!client[i]) 3349 break; 3350 3351 i915_request_put(client[i]); 3352 } 3353 3354 /* Flush the semaphores on error */ 3355 smp_store_mb(result[0], -1); 3356 if (igt_live_test_end(&t)) 3357 err = -EIO; 3358 if (err) 3359 break; 3360 } 3361 3362 i915_vma_unpin_and_release(&global, I915_VMA_RELEASE_MAP); 3363 return err; 3364 } 3365 3366 static int live_preempt_timeout(void *arg) 3367 { 3368 struct intel_gt *gt = arg; 3369 struct i915_gem_context *ctx_hi, *ctx_lo; 3370 struct igt_spinner spin_lo; 3371 struct intel_engine_cs *engine; 3372 enum intel_engine_id id; 3373 int err = -ENOMEM; 3374 3375 /* 3376 * Check that we force preemption to occur by cancelling the previous 3377 * context if it refuses to yield the GPU. 3378 */ 3379 if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT) 3380 return 0; 3381 3382 if (!intel_has_reset_engine(gt)) 3383 return 0; 3384 3385 ctx_hi = kernel_context(gt->i915, NULL); 3386 if (!ctx_hi) 3387 return -ENOMEM; 3388 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY; 3389 3390 ctx_lo = kernel_context(gt->i915, NULL); 3391 if (!ctx_lo) 3392 goto err_ctx_hi; 3393 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY; 3394 3395 if (igt_spinner_init(&spin_lo, gt)) 3396 goto err_ctx_lo; 3397 3398 for_each_engine(engine, gt, id) { 3399 unsigned long saved_timeout; 3400 struct i915_request *rq; 3401 3402 if (!intel_engine_has_preemption(engine)) 3403 continue; 3404 3405 rq = spinner_create_request(&spin_lo, ctx_lo, engine, 3406 MI_NOOP); /* preemption disabled */ 3407 if (IS_ERR(rq)) { 3408 err = PTR_ERR(rq); 3409 goto err_spin_lo; 3410 } 3411 3412 i915_request_add(rq); 3413 if (!igt_wait_for_spinner(&spin_lo, rq)) { 3414 intel_gt_set_wedged(gt); 3415 err = -EIO; 3416 goto err_spin_lo; 3417 } 3418 3419 rq = igt_request_alloc(ctx_hi, engine); 3420 if (IS_ERR(rq)) { 3421 igt_spinner_end(&spin_lo); 3422 err = PTR_ERR(rq); 3423 goto err_spin_lo; 3424 } 3425 3426 /* Flush the previous CS ack before changing timeouts */ 3427 while (READ_ONCE(engine->execlists.pending[0])) 3428 cpu_relax(); 3429 3430 saved_timeout = engine->props.preempt_timeout_ms; 3431 engine->props.preempt_timeout_ms = 1; /* in ms, -> 1 jiffie */ 3432 3433 i915_request_get(rq); 3434 i915_request_add(rq); 3435 3436 intel_engine_flush_submission(engine); 3437 engine->props.preempt_timeout_ms = saved_timeout; 3438 3439 if (i915_request_wait(rq, 0, HZ / 10) < 0) { 3440 intel_gt_set_wedged(gt); 3441 i915_request_put(rq); 3442 err = -ETIME; 3443 goto err_spin_lo; 3444 } 3445 3446 igt_spinner_end(&spin_lo); 3447 i915_request_put(rq); 3448 } 3449 3450 err = 0; 3451 err_spin_lo: 3452 igt_spinner_fini(&spin_lo); 3453 err_ctx_lo: 3454 kernel_context_close(ctx_lo); 3455 err_ctx_hi: 3456 kernel_context_close(ctx_hi); 3457 return err; 3458 } 3459 3460 static int random_range(struct rnd_state *rnd, int min, int max) 3461 { 3462 return i915_prandom_u32_max_state(max - min, rnd) + min; 3463 } 3464 3465 static int random_priority(struct rnd_state *rnd) 3466 { 3467 return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX); 3468 } 3469 3470 struct preempt_smoke { 3471 struct intel_gt *gt; 3472 struct i915_gem_context **contexts; 3473 struct intel_engine_cs *engine; 3474 struct drm_i915_gem_object *batch; 3475 unsigned int ncontext; 3476 struct rnd_state prng; 3477 unsigned long count; 3478 }; 3479 3480 static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke) 3481 { 3482 return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext, 3483 &smoke->prng)]; 3484 } 3485 3486 static int smoke_submit(struct preempt_smoke *smoke, 3487 struct i915_gem_context *ctx, int prio, 3488 struct drm_i915_gem_object *batch) 3489 { 3490 struct i915_request *rq; 3491 struct i915_vma *vma = NULL; 3492 int err = 0; 3493 3494 if (batch) { 3495 struct i915_address_space *vm; 3496 3497 vm = i915_gem_context_get_eb_vm(ctx); 3498 vma = i915_vma_instance(batch, vm, NULL); 3499 i915_vm_put(vm); 3500 if (IS_ERR(vma)) 3501 return PTR_ERR(vma); 3502 3503 err = i915_vma_pin(vma, 0, 0, PIN_USER); 3504 if (err) 3505 return err; 3506 } 3507 3508 ctx->sched.priority = prio; 3509 3510 rq = igt_request_alloc(ctx, smoke->engine); 3511 if (IS_ERR(rq)) { 3512 err = PTR_ERR(rq); 3513 goto unpin; 3514 } 3515 3516 if (vma) { 3517 i915_vma_lock(vma); 3518 err = i915_request_await_object(rq, vma->obj, false); 3519 if (!err) 3520 err = i915_vma_move_to_active(vma, rq, 0); 3521 if (!err) 3522 err = rq->engine->emit_bb_start(rq, 3523 vma->node.start, 3524 PAGE_SIZE, 0); 3525 i915_vma_unlock(vma); 3526 } 3527 3528 i915_request_add(rq); 3529 3530 unpin: 3531 if (vma) 3532 i915_vma_unpin(vma); 3533 3534 return err; 3535 } 3536 3537 static int smoke_crescendo_thread(void *arg) 3538 { 3539 struct preempt_smoke *smoke = arg; 3540 IGT_TIMEOUT(end_time); 3541 unsigned long count; 3542 3543 count = 0; 3544 do { 3545 struct i915_gem_context *ctx = smoke_context(smoke); 3546 int err; 3547 3548 err = smoke_submit(smoke, 3549 ctx, count % I915_PRIORITY_MAX, 3550 smoke->batch); 3551 if (err) 3552 return err; 3553 3554 count++; 3555 } while (count < smoke->ncontext && !__igt_timeout(end_time, NULL)); 3556 3557 smoke->count = count; 3558 return 0; 3559 } 3560 3561 static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags) 3562 #define BATCH BIT(0) 3563 { 3564 struct task_struct *tsk[I915_NUM_ENGINES] = {}; 3565 struct preempt_smoke *arg; 3566 struct intel_engine_cs *engine; 3567 enum intel_engine_id id; 3568 unsigned long count; 3569 int err = 0; 3570 3571 arg = kmalloc_array(I915_NUM_ENGINES, sizeof(*arg), GFP_KERNEL); 3572 if (!arg) 3573 return -ENOMEM; 3574 3575 for_each_engine(engine, smoke->gt, id) { 3576 arg[id] = *smoke; 3577 arg[id].engine = engine; 3578 if (!(flags & BATCH)) 3579 arg[id].batch = NULL; 3580 arg[id].count = 0; 3581 3582 tsk[id] = kthread_run(smoke_crescendo_thread, arg, 3583 "igt/smoke:%d", id); 3584 if (IS_ERR(tsk[id])) { 3585 err = PTR_ERR(tsk[id]); 3586 break; 3587 } 3588 get_task_struct(tsk[id]); 3589 } 3590 3591 yield(); /* start all threads before we kthread_stop() */ 3592 3593 count = 0; 3594 for_each_engine(engine, smoke->gt, id) { 3595 int status; 3596 3597 if (IS_ERR_OR_NULL(tsk[id])) 3598 continue; 3599 3600 status = kthread_stop(tsk[id]); 3601 if (status && !err) 3602 err = status; 3603 3604 count += arg[id].count; 3605 3606 put_task_struct(tsk[id]); 3607 } 3608 3609 pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n", 3610 count, flags, smoke->gt->info.num_engines, smoke->ncontext); 3611 3612 kfree(arg); 3613 return 0; 3614 } 3615 3616 static int smoke_random(struct preempt_smoke *smoke, unsigned int flags) 3617 { 3618 enum intel_engine_id id; 3619 IGT_TIMEOUT(end_time); 3620 unsigned long count; 3621 3622 count = 0; 3623 do { 3624 for_each_engine(smoke->engine, smoke->gt, id) { 3625 struct i915_gem_context *ctx = smoke_context(smoke); 3626 int err; 3627 3628 err = smoke_submit(smoke, 3629 ctx, random_priority(&smoke->prng), 3630 flags & BATCH ? smoke->batch : NULL); 3631 if (err) 3632 return err; 3633 3634 count++; 3635 } 3636 } while (count < smoke->ncontext && !__igt_timeout(end_time, NULL)); 3637 3638 pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n", 3639 count, flags, smoke->gt->info.num_engines, smoke->ncontext); 3640 return 0; 3641 } 3642 3643 static int live_preempt_smoke(void *arg) 3644 { 3645 struct preempt_smoke smoke = { 3646 .gt = arg, 3647 .prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed), 3648 .ncontext = 256, 3649 }; 3650 const unsigned int phase[] = { 0, BATCH }; 3651 struct igt_live_test t; 3652 int err = -ENOMEM; 3653 u32 *cs; 3654 int n; 3655 3656 smoke.contexts = kmalloc_array(smoke.ncontext, 3657 sizeof(*smoke.contexts), 3658 GFP_KERNEL); 3659 if (!smoke.contexts) 3660 return -ENOMEM; 3661 3662 smoke.batch = 3663 i915_gem_object_create_internal(smoke.gt->i915, PAGE_SIZE); 3664 if (IS_ERR(smoke.batch)) { 3665 err = PTR_ERR(smoke.batch); 3666 goto err_free; 3667 } 3668 3669 cs = i915_gem_object_pin_map_unlocked(smoke.batch, I915_MAP_WB); 3670 if (IS_ERR(cs)) { 3671 err = PTR_ERR(cs); 3672 goto err_batch; 3673 } 3674 for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++) 3675 cs[n] = MI_ARB_CHECK; 3676 cs[n] = MI_BATCH_BUFFER_END; 3677 i915_gem_object_flush_map(smoke.batch); 3678 i915_gem_object_unpin_map(smoke.batch); 3679 3680 if (igt_live_test_begin(&t, smoke.gt->i915, __func__, "all")) { 3681 err = -EIO; 3682 goto err_batch; 3683 } 3684 3685 for (n = 0; n < smoke.ncontext; n++) { 3686 smoke.contexts[n] = kernel_context(smoke.gt->i915, NULL); 3687 if (!smoke.contexts[n]) 3688 goto err_ctx; 3689 } 3690 3691 for (n = 0; n < ARRAY_SIZE(phase); n++) { 3692 err = smoke_crescendo(&smoke, phase[n]); 3693 if (err) 3694 goto err_ctx; 3695 3696 err = smoke_random(&smoke, phase[n]); 3697 if (err) 3698 goto err_ctx; 3699 } 3700 3701 err_ctx: 3702 if (igt_live_test_end(&t)) 3703 err = -EIO; 3704 3705 for (n = 0; n < smoke.ncontext; n++) { 3706 if (!smoke.contexts[n]) 3707 break; 3708 kernel_context_close(smoke.contexts[n]); 3709 } 3710 3711 err_batch: 3712 i915_gem_object_put(smoke.batch); 3713 err_free: 3714 kfree(smoke.contexts); 3715 3716 return err; 3717 } 3718 3719 static int nop_virtual_engine(struct intel_gt *gt, 3720 struct intel_engine_cs **siblings, 3721 unsigned int nsibling, 3722 unsigned int nctx, 3723 unsigned int flags) 3724 #define CHAIN BIT(0) 3725 { 3726 IGT_TIMEOUT(end_time); 3727 struct i915_request *request[16] = {}; 3728 struct intel_context *ve[16]; 3729 unsigned long n, prime, nc; 3730 struct igt_live_test t; 3731 ktime_t times[2] = {}; 3732 int err; 3733 3734 GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ve)); 3735 3736 for (n = 0; n < nctx; n++) { 3737 ve[n] = intel_engine_create_virtual(siblings, nsibling, 0); 3738 if (IS_ERR(ve[n])) { 3739 err = PTR_ERR(ve[n]); 3740 nctx = n; 3741 goto out; 3742 } 3743 3744 err = intel_context_pin(ve[n]); 3745 if (err) { 3746 intel_context_put(ve[n]); 3747 nctx = n; 3748 goto out; 3749 } 3750 } 3751 3752 err = igt_live_test_begin(&t, gt->i915, __func__, ve[0]->engine->name); 3753 if (err) 3754 goto out; 3755 3756 for_each_prime_number_from(prime, 1, 8192) { 3757 times[1] = ktime_get_raw(); 3758 3759 if (flags & CHAIN) { 3760 for (nc = 0; nc < nctx; nc++) { 3761 for (n = 0; n < prime; n++) { 3762 struct i915_request *rq; 3763 3764 rq = i915_request_create(ve[nc]); 3765 if (IS_ERR(rq)) { 3766 err = PTR_ERR(rq); 3767 goto out; 3768 } 3769 3770 if (request[nc]) 3771 i915_request_put(request[nc]); 3772 request[nc] = i915_request_get(rq); 3773 i915_request_add(rq); 3774 } 3775 } 3776 } else { 3777 for (n = 0; n < prime; n++) { 3778 for (nc = 0; nc < nctx; nc++) { 3779 struct i915_request *rq; 3780 3781 rq = i915_request_create(ve[nc]); 3782 if (IS_ERR(rq)) { 3783 err = PTR_ERR(rq); 3784 goto out; 3785 } 3786 3787 if (request[nc]) 3788 i915_request_put(request[nc]); 3789 request[nc] = i915_request_get(rq); 3790 i915_request_add(rq); 3791 } 3792 } 3793 } 3794 3795 for (nc = 0; nc < nctx; nc++) { 3796 if (i915_request_wait(request[nc], 0, HZ / 10) < 0) { 3797 pr_err("%s(%s): wait for %llx:%lld timed out\n", 3798 __func__, ve[0]->engine->name, 3799 request[nc]->fence.context, 3800 request[nc]->fence.seqno); 3801 3802 GEM_TRACE("%s(%s) failed at request %llx:%lld\n", 3803 __func__, ve[0]->engine->name, 3804 request[nc]->fence.context, 3805 request[nc]->fence.seqno); 3806 GEM_TRACE_DUMP(); 3807 intel_gt_set_wedged(gt); 3808 break; 3809 } 3810 } 3811 3812 times[1] = ktime_sub(ktime_get_raw(), times[1]); 3813 if (prime == 1) 3814 times[0] = times[1]; 3815 3816 for (nc = 0; nc < nctx; nc++) { 3817 i915_request_put(request[nc]); 3818 request[nc] = NULL; 3819 } 3820 3821 if (__igt_timeout(end_time, NULL)) 3822 break; 3823 } 3824 3825 err = igt_live_test_end(&t); 3826 if (err) 3827 goto out; 3828 3829 pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n", 3830 nctx, ve[0]->engine->name, ktime_to_ns(times[0]), 3831 prime, div64_u64(ktime_to_ns(times[1]), prime)); 3832 3833 out: 3834 if (igt_flush_test(gt->i915)) 3835 err = -EIO; 3836 3837 for (nc = 0; nc < nctx; nc++) { 3838 i915_request_put(request[nc]); 3839 intel_context_unpin(ve[nc]); 3840 intel_context_put(ve[nc]); 3841 } 3842 return err; 3843 } 3844 3845 static unsigned int 3846 __select_siblings(struct intel_gt *gt, 3847 unsigned int class, 3848 struct intel_engine_cs **siblings, 3849 bool (*filter)(const struct intel_engine_cs *)) 3850 { 3851 unsigned int n = 0; 3852 unsigned int inst; 3853 3854 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) { 3855 if (!gt->engine_class[class][inst]) 3856 continue; 3857 3858 if (filter && !filter(gt->engine_class[class][inst])) 3859 continue; 3860 3861 siblings[n++] = gt->engine_class[class][inst]; 3862 } 3863 3864 return n; 3865 } 3866 3867 static unsigned int 3868 select_siblings(struct intel_gt *gt, 3869 unsigned int class, 3870 struct intel_engine_cs **siblings) 3871 { 3872 return __select_siblings(gt, class, siblings, NULL); 3873 } 3874 3875 static int live_virtual_engine(void *arg) 3876 { 3877 struct intel_gt *gt = arg; 3878 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 3879 struct intel_engine_cs *engine; 3880 enum intel_engine_id id; 3881 unsigned int class; 3882 int err; 3883 3884 if (intel_uc_uses_guc_submission(>->uc)) 3885 return 0; 3886 3887 for_each_engine(engine, gt, id) { 3888 err = nop_virtual_engine(gt, &engine, 1, 1, 0); 3889 if (err) { 3890 pr_err("Failed to wrap engine %s: err=%d\n", 3891 engine->name, err); 3892 return err; 3893 } 3894 } 3895 3896 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 3897 int nsibling, n; 3898 3899 nsibling = select_siblings(gt, class, siblings); 3900 if (nsibling < 2) 3901 continue; 3902 3903 for (n = 1; n <= nsibling + 1; n++) { 3904 err = nop_virtual_engine(gt, siblings, nsibling, 3905 n, 0); 3906 if (err) 3907 return err; 3908 } 3909 3910 err = nop_virtual_engine(gt, siblings, nsibling, n, CHAIN); 3911 if (err) 3912 return err; 3913 } 3914 3915 return 0; 3916 } 3917 3918 static int mask_virtual_engine(struct intel_gt *gt, 3919 struct intel_engine_cs **siblings, 3920 unsigned int nsibling) 3921 { 3922 struct i915_request *request[MAX_ENGINE_INSTANCE + 1]; 3923 struct intel_context *ve; 3924 struct igt_live_test t; 3925 unsigned int n; 3926 int err; 3927 3928 /* 3929 * Check that by setting the execution mask on a request, we can 3930 * restrict it to our desired engine within the virtual engine. 3931 */ 3932 3933 ve = intel_engine_create_virtual(siblings, nsibling, 0); 3934 if (IS_ERR(ve)) { 3935 err = PTR_ERR(ve); 3936 goto out_close; 3937 } 3938 3939 err = intel_context_pin(ve); 3940 if (err) 3941 goto out_put; 3942 3943 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name); 3944 if (err) 3945 goto out_unpin; 3946 3947 for (n = 0; n < nsibling; n++) { 3948 request[n] = i915_request_create(ve); 3949 if (IS_ERR(request[n])) { 3950 err = PTR_ERR(request[n]); 3951 nsibling = n; 3952 goto out; 3953 } 3954 3955 /* Reverse order as it's more likely to be unnatural */ 3956 request[n]->execution_mask = siblings[nsibling - n - 1]->mask; 3957 3958 i915_request_get(request[n]); 3959 i915_request_add(request[n]); 3960 } 3961 3962 for (n = 0; n < nsibling; n++) { 3963 if (i915_request_wait(request[n], 0, HZ / 10) < 0) { 3964 pr_err("%s(%s): wait for %llx:%lld timed out\n", 3965 __func__, ve->engine->name, 3966 request[n]->fence.context, 3967 request[n]->fence.seqno); 3968 3969 GEM_TRACE("%s(%s) failed at request %llx:%lld\n", 3970 __func__, ve->engine->name, 3971 request[n]->fence.context, 3972 request[n]->fence.seqno); 3973 GEM_TRACE_DUMP(); 3974 intel_gt_set_wedged(gt); 3975 err = -EIO; 3976 goto out; 3977 } 3978 3979 if (request[n]->engine != siblings[nsibling - n - 1]) { 3980 pr_err("Executed on wrong sibling '%s', expected '%s'\n", 3981 request[n]->engine->name, 3982 siblings[nsibling - n - 1]->name); 3983 err = -EINVAL; 3984 goto out; 3985 } 3986 } 3987 3988 err = igt_live_test_end(&t); 3989 out: 3990 if (igt_flush_test(gt->i915)) 3991 err = -EIO; 3992 3993 for (n = 0; n < nsibling; n++) 3994 i915_request_put(request[n]); 3995 3996 out_unpin: 3997 intel_context_unpin(ve); 3998 out_put: 3999 intel_context_put(ve); 4000 out_close: 4001 return err; 4002 } 4003 4004 static int live_virtual_mask(void *arg) 4005 { 4006 struct intel_gt *gt = arg; 4007 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 4008 unsigned int class; 4009 int err; 4010 4011 if (intel_uc_uses_guc_submission(>->uc)) 4012 return 0; 4013 4014 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 4015 unsigned int nsibling; 4016 4017 nsibling = select_siblings(gt, class, siblings); 4018 if (nsibling < 2) 4019 continue; 4020 4021 err = mask_virtual_engine(gt, siblings, nsibling); 4022 if (err) 4023 return err; 4024 } 4025 4026 return 0; 4027 } 4028 4029 static int slicein_virtual_engine(struct intel_gt *gt, 4030 struct intel_engine_cs **siblings, 4031 unsigned int nsibling) 4032 { 4033 const long timeout = slice_timeout(siblings[0]); 4034 struct intel_context *ce; 4035 struct i915_request *rq; 4036 struct igt_spinner spin; 4037 unsigned int n; 4038 int err = 0; 4039 4040 /* 4041 * Virtual requests must take part in timeslicing on the target engines. 4042 */ 4043 4044 if (igt_spinner_init(&spin, gt)) 4045 return -ENOMEM; 4046 4047 for (n = 0; n < nsibling; n++) { 4048 ce = intel_context_create(siblings[n]); 4049 if (IS_ERR(ce)) { 4050 err = PTR_ERR(ce); 4051 goto out; 4052 } 4053 4054 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK); 4055 intel_context_put(ce); 4056 if (IS_ERR(rq)) { 4057 err = PTR_ERR(rq); 4058 goto out; 4059 } 4060 4061 i915_request_add(rq); 4062 } 4063 4064 ce = intel_engine_create_virtual(siblings, nsibling, 0); 4065 if (IS_ERR(ce)) { 4066 err = PTR_ERR(ce); 4067 goto out; 4068 } 4069 4070 rq = intel_context_create_request(ce); 4071 intel_context_put(ce); 4072 if (IS_ERR(rq)) { 4073 err = PTR_ERR(rq); 4074 goto out; 4075 } 4076 4077 i915_request_get(rq); 4078 i915_request_add(rq); 4079 if (i915_request_wait(rq, 0, timeout) < 0) { 4080 GEM_TRACE_ERR("%s(%s) failed to slice in virtual request\n", 4081 __func__, rq->engine->name); 4082 GEM_TRACE_DUMP(); 4083 intel_gt_set_wedged(gt); 4084 err = -EIO; 4085 } 4086 i915_request_put(rq); 4087 4088 out: 4089 igt_spinner_end(&spin); 4090 if (igt_flush_test(gt->i915)) 4091 err = -EIO; 4092 igt_spinner_fini(&spin); 4093 return err; 4094 } 4095 4096 static int sliceout_virtual_engine(struct intel_gt *gt, 4097 struct intel_engine_cs **siblings, 4098 unsigned int nsibling) 4099 { 4100 const long timeout = slice_timeout(siblings[0]); 4101 struct intel_context *ce; 4102 struct i915_request *rq; 4103 struct igt_spinner spin; 4104 unsigned int n; 4105 int err = 0; 4106 4107 /* 4108 * Virtual requests must allow others a fair timeslice. 4109 */ 4110 4111 if (igt_spinner_init(&spin, gt)) 4112 return -ENOMEM; 4113 4114 /* XXX We do not handle oversubscription and fairness with normal rq */ 4115 for (n = 0; n < nsibling; n++) { 4116 ce = intel_engine_create_virtual(siblings, nsibling, 0); 4117 if (IS_ERR(ce)) { 4118 err = PTR_ERR(ce); 4119 goto out; 4120 } 4121 4122 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK); 4123 intel_context_put(ce); 4124 if (IS_ERR(rq)) { 4125 err = PTR_ERR(rq); 4126 goto out; 4127 } 4128 4129 i915_request_add(rq); 4130 } 4131 4132 for (n = 0; !err && n < nsibling; n++) { 4133 ce = intel_context_create(siblings[n]); 4134 if (IS_ERR(ce)) { 4135 err = PTR_ERR(ce); 4136 goto out; 4137 } 4138 4139 rq = intel_context_create_request(ce); 4140 intel_context_put(ce); 4141 if (IS_ERR(rq)) { 4142 err = PTR_ERR(rq); 4143 goto out; 4144 } 4145 4146 i915_request_get(rq); 4147 i915_request_add(rq); 4148 if (i915_request_wait(rq, 0, timeout) < 0) { 4149 GEM_TRACE_ERR("%s(%s) failed to slice out virtual request\n", 4150 __func__, siblings[n]->name); 4151 GEM_TRACE_DUMP(); 4152 intel_gt_set_wedged(gt); 4153 err = -EIO; 4154 } 4155 i915_request_put(rq); 4156 } 4157 4158 out: 4159 igt_spinner_end(&spin); 4160 if (igt_flush_test(gt->i915)) 4161 err = -EIO; 4162 igt_spinner_fini(&spin); 4163 return err; 4164 } 4165 4166 static int live_virtual_slice(void *arg) 4167 { 4168 struct intel_gt *gt = arg; 4169 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 4170 unsigned int class; 4171 int err; 4172 4173 if (intel_uc_uses_guc_submission(>->uc)) 4174 return 0; 4175 4176 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 4177 unsigned int nsibling; 4178 4179 nsibling = __select_siblings(gt, class, siblings, 4180 intel_engine_has_timeslices); 4181 if (nsibling < 2) 4182 continue; 4183 4184 err = slicein_virtual_engine(gt, siblings, nsibling); 4185 if (err) 4186 return err; 4187 4188 err = sliceout_virtual_engine(gt, siblings, nsibling); 4189 if (err) 4190 return err; 4191 } 4192 4193 return 0; 4194 } 4195 4196 static int preserved_virtual_engine(struct intel_gt *gt, 4197 struct intel_engine_cs **siblings, 4198 unsigned int nsibling) 4199 { 4200 struct i915_request *last = NULL; 4201 struct intel_context *ve; 4202 struct i915_vma *scratch; 4203 struct igt_live_test t; 4204 unsigned int n; 4205 int err = 0; 4206 u32 *cs; 4207 4208 scratch = 4209 __vm_create_scratch_for_read_pinned(&siblings[0]->gt->ggtt->vm, 4210 PAGE_SIZE); 4211 if (IS_ERR(scratch)) 4212 return PTR_ERR(scratch); 4213 4214 err = i915_vma_sync(scratch); 4215 if (err) 4216 goto out_scratch; 4217 4218 ve = intel_engine_create_virtual(siblings, nsibling, 0); 4219 if (IS_ERR(ve)) { 4220 err = PTR_ERR(ve); 4221 goto out_scratch; 4222 } 4223 4224 err = intel_context_pin(ve); 4225 if (err) 4226 goto out_put; 4227 4228 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name); 4229 if (err) 4230 goto out_unpin; 4231 4232 for (n = 0; n < NUM_GPR_DW; n++) { 4233 struct intel_engine_cs *engine = siblings[n % nsibling]; 4234 struct i915_request *rq; 4235 4236 rq = i915_request_create(ve); 4237 if (IS_ERR(rq)) { 4238 err = PTR_ERR(rq); 4239 goto out_end; 4240 } 4241 4242 i915_request_put(last); 4243 last = i915_request_get(rq); 4244 4245 cs = intel_ring_begin(rq, 8); 4246 if (IS_ERR(cs)) { 4247 i915_request_add(rq); 4248 err = PTR_ERR(cs); 4249 goto out_end; 4250 } 4251 4252 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 4253 *cs++ = CS_GPR(engine, n); 4254 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32); 4255 *cs++ = 0; 4256 4257 *cs++ = MI_LOAD_REGISTER_IMM(1); 4258 *cs++ = CS_GPR(engine, (n + 1) % NUM_GPR_DW); 4259 *cs++ = n + 1; 4260 4261 *cs++ = MI_NOOP; 4262 intel_ring_advance(rq, cs); 4263 4264 /* Restrict this request to run on a particular engine */ 4265 rq->execution_mask = engine->mask; 4266 i915_request_add(rq); 4267 } 4268 4269 if (i915_request_wait(last, 0, HZ / 5) < 0) { 4270 err = -ETIME; 4271 goto out_end; 4272 } 4273 4274 cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB); 4275 if (IS_ERR(cs)) { 4276 err = PTR_ERR(cs); 4277 goto out_end; 4278 } 4279 4280 for (n = 0; n < NUM_GPR_DW; n++) { 4281 if (cs[n] != n) { 4282 pr_err("Incorrect value[%d] found for GPR[%d]\n", 4283 cs[n], n); 4284 err = -EINVAL; 4285 break; 4286 } 4287 } 4288 4289 i915_gem_object_unpin_map(scratch->obj); 4290 4291 out_end: 4292 if (igt_live_test_end(&t)) 4293 err = -EIO; 4294 i915_request_put(last); 4295 out_unpin: 4296 intel_context_unpin(ve); 4297 out_put: 4298 intel_context_put(ve); 4299 out_scratch: 4300 i915_vma_unpin_and_release(&scratch, 0); 4301 return err; 4302 } 4303 4304 static int live_virtual_preserved(void *arg) 4305 { 4306 struct intel_gt *gt = arg; 4307 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 4308 unsigned int class; 4309 4310 /* 4311 * Check that the context image retains non-privileged (user) registers 4312 * from one engine to the next. For this we check that the CS_GPR 4313 * are preserved. 4314 */ 4315 4316 if (intel_uc_uses_guc_submission(>->uc)) 4317 return 0; 4318 4319 /* As we use CS_GPR we cannot run before they existed on all engines. */ 4320 if (GRAPHICS_VER(gt->i915) < 9) 4321 return 0; 4322 4323 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 4324 int nsibling, err; 4325 4326 nsibling = select_siblings(gt, class, siblings); 4327 if (nsibling < 2) 4328 continue; 4329 4330 err = preserved_virtual_engine(gt, siblings, nsibling); 4331 if (err) 4332 return err; 4333 } 4334 4335 return 0; 4336 } 4337 4338 static int reset_virtual_engine(struct intel_gt *gt, 4339 struct intel_engine_cs **siblings, 4340 unsigned int nsibling) 4341 { 4342 struct intel_engine_cs *engine; 4343 struct intel_context *ve; 4344 struct igt_spinner spin; 4345 struct i915_request *rq; 4346 unsigned int n; 4347 int err = 0; 4348 4349 /* 4350 * In order to support offline error capture for fast preempt reset, 4351 * we need to decouple the guilty request and ensure that it and its 4352 * descendents are not executed while the capture is in progress. 4353 */ 4354 4355 if (igt_spinner_init(&spin, gt)) 4356 return -ENOMEM; 4357 4358 ve = intel_engine_create_virtual(siblings, nsibling, 0); 4359 if (IS_ERR(ve)) { 4360 err = PTR_ERR(ve); 4361 goto out_spin; 4362 } 4363 4364 for (n = 0; n < nsibling; n++) 4365 st_engine_heartbeat_disable(siblings[n]); 4366 4367 rq = igt_spinner_create_request(&spin, ve, MI_ARB_CHECK); 4368 if (IS_ERR(rq)) { 4369 err = PTR_ERR(rq); 4370 goto out_heartbeat; 4371 } 4372 i915_request_add(rq); 4373 4374 if (!igt_wait_for_spinner(&spin, rq)) { 4375 intel_gt_set_wedged(gt); 4376 err = -ETIME; 4377 goto out_heartbeat; 4378 } 4379 4380 engine = rq->engine; 4381 GEM_BUG_ON(engine == ve->engine); 4382 4383 /* Take ownership of the reset and tasklet */ 4384 err = engine_lock_reset_tasklet(engine); 4385 if (err) 4386 goto out_heartbeat; 4387 4388 engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet); 4389 GEM_BUG_ON(execlists_active(&engine->execlists) != rq); 4390 4391 /* Fake a preemption event; failed of course */ 4392 spin_lock_irq(&engine->sched_engine->lock); 4393 __unwind_incomplete_requests(engine); 4394 spin_unlock_irq(&engine->sched_engine->lock); 4395 GEM_BUG_ON(rq->engine != engine); 4396 4397 /* Reset the engine while keeping our active request on hold */ 4398 execlists_hold(engine, rq); 4399 GEM_BUG_ON(!i915_request_on_hold(rq)); 4400 4401 __intel_engine_reset_bh(engine, NULL); 4402 GEM_BUG_ON(rq->fence.error != -EIO); 4403 4404 /* Release our grasp on the engine, letting CS flow again */ 4405 engine_unlock_reset_tasklet(engine); 4406 4407 /* Check that we do not resubmit the held request */ 4408 i915_request_get(rq); 4409 if (!i915_request_wait(rq, 0, HZ / 5)) { 4410 pr_err("%s: on hold request completed!\n", 4411 engine->name); 4412 intel_gt_set_wedged(gt); 4413 err = -EIO; 4414 goto out_rq; 4415 } 4416 GEM_BUG_ON(!i915_request_on_hold(rq)); 4417 4418 /* But is resubmitted on release */ 4419 execlists_unhold(engine, rq); 4420 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 4421 pr_err("%s: held request did not complete!\n", 4422 engine->name); 4423 intel_gt_set_wedged(gt); 4424 err = -ETIME; 4425 } 4426 4427 out_rq: 4428 i915_request_put(rq); 4429 out_heartbeat: 4430 for (n = 0; n < nsibling; n++) 4431 st_engine_heartbeat_enable(siblings[n]); 4432 4433 intel_context_put(ve); 4434 out_spin: 4435 igt_spinner_fini(&spin); 4436 return err; 4437 } 4438 4439 static int live_virtual_reset(void *arg) 4440 { 4441 struct intel_gt *gt = arg; 4442 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 4443 unsigned int class; 4444 4445 /* 4446 * Check that we handle a reset event within a virtual engine. 4447 * Only the physical engine is reset, but we have to check the flow 4448 * of the virtual requests around the reset, and make sure it is not 4449 * forgotten. 4450 */ 4451 4452 if (intel_uc_uses_guc_submission(>->uc)) 4453 return 0; 4454 4455 if (!intel_has_reset_engine(gt)) 4456 return 0; 4457 4458 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 4459 int nsibling, err; 4460 4461 nsibling = select_siblings(gt, class, siblings); 4462 if (nsibling < 2) 4463 continue; 4464 4465 err = reset_virtual_engine(gt, siblings, nsibling); 4466 if (err) 4467 return err; 4468 } 4469 4470 return 0; 4471 } 4472 4473 int intel_execlists_live_selftests(struct drm_i915_private *i915) 4474 { 4475 static const struct i915_subtest tests[] = { 4476 SUBTEST(live_sanitycheck), 4477 SUBTEST(live_unlite_switch), 4478 SUBTEST(live_unlite_preempt), 4479 SUBTEST(live_unlite_ring), 4480 SUBTEST(live_pin_rewind), 4481 SUBTEST(live_hold_reset), 4482 SUBTEST(live_error_interrupt), 4483 SUBTEST(live_timeslice_preempt), 4484 SUBTEST(live_timeslice_rewind), 4485 SUBTEST(live_timeslice_queue), 4486 SUBTEST(live_timeslice_nopreempt), 4487 SUBTEST(live_busywait_preempt), 4488 SUBTEST(live_preempt), 4489 SUBTEST(live_late_preempt), 4490 SUBTEST(live_nopreempt), 4491 SUBTEST(live_preempt_cancel), 4492 SUBTEST(live_suppress_self_preempt), 4493 SUBTEST(live_chain_preempt), 4494 SUBTEST(live_preempt_ring), 4495 SUBTEST(live_preempt_gang), 4496 SUBTEST(live_preempt_timeout), 4497 SUBTEST(live_preempt_user), 4498 SUBTEST(live_preempt_smoke), 4499 SUBTEST(live_virtual_engine), 4500 SUBTEST(live_virtual_mask), 4501 SUBTEST(live_virtual_preserved), 4502 SUBTEST(live_virtual_slice), 4503 SUBTEST(live_virtual_reset), 4504 }; 4505 4506 if (to_gt(i915)->submission_method != INTEL_SUBMISSION_ELSP) 4507 return 0; 4508 4509 if (intel_gt_is_wedged(to_gt(i915))) 4510 return 0; 4511 4512 return intel_gt_live_subtests(tests, to_gt(i915)); 4513 } 4514