1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2018 Intel Corporation 4 */ 5 6 #include <linux/prime_numbers.h> 7 8 #include "gem/i915_gem_internal.h" 9 #include "gem/i915_gem_pm.h" 10 #include "gt/intel_engine_heartbeat.h" 11 #include "gt/intel_reset.h" 12 #include "gt/selftest_engine_heartbeat.h" 13 14 #include "i915_selftest.h" 15 #include "selftests/i915_random.h" 16 #include "selftests/igt_flush_test.h" 17 #include "selftests/igt_live_test.h" 18 #include "selftests/igt_spinner.h" 19 #include "selftests/lib_sw_fence.h" 20 21 #include "gem/selftests/igt_gem_utils.h" 22 #include "gem/selftests/mock_context.h" 23 24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4) 25 #define NUM_GPR 16 26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */ 27 28 static bool is_active(struct i915_request *rq) 29 { 30 if (i915_request_is_active(rq)) 31 return true; 32 33 if (i915_request_on_hold(rq)) 34 return true; 35 36 if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq)) 37 return true; 38 39 return false; 40 } 41 42 static int wait_for_submit(struct intel_engine_cs *engine, 43 struct i915_request *rq, 44 unsigned long timeout) 45 { 46 /* Ignore our own attempts to suppress excess tasklets */ 47 tasklet_hi_schedule(&engine->sched_engine->tasklet); 48 49 timeout += jiffies; 50 do { 51 bool done = time_after(jiffies, timeout); 52 53 if (i915_request_completed(rq)) /* that was quick! */ 54 return 0; 55 56 /* Wait until the HW has acknowleged the submission (or err) */ 57 intel_engine_flush_submission(engine); 58 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq)) 59 return 0; 60 61 if (done) 62 return -ETIME; 63 64 cond_resched(); 65 } while (1); 66 } 67 68 static int wait_for_reset(struct intel_engine_cs *engine, 69 struct i915_request *rq, 70 unsigned long timeout) 71 { 72 timeout += jiffies; 73 74 do { 75 cond_resched(); 76 intel_engine_flush_submission(engine); 77 78 if (READ_ONCE(engine->execlists.pending[0])) 79 continue; 80 81 if (i915_request_completed(rq)) 82 break; 83 84 if (READ_ONCE(rq->fence.error)) 85 break; 86 } while (time_before(jiffies, timeout)); 87 88 if (rq->fence.error != -EIO) { 89 pr_err("%s: hanging request %llx:%lld not reset\n", 90 engine->name, 91 rq->fence.context, 92 rq->fence.seqno); 93 return -EINVAL; 94 } 95 96 /* Give the request a jiffie to complete after flushing the worker */ 97 if (i915_request_wait(rq, 0, 98 max(0l, (long)(timeout - jiffies)) + 1) < 0) { 99 pr_err("%s: hanging request %llx:%lld did not complete\n", 100 engine->name, 101 rq->fence.context, 102 rq->fence.seqno); 103 return -ETIME; 104 } 105 106 return 0; 107 } 108 109 static int live_sanitycheck(void *arg) 110 { 111 struct intel_gt *gt = arg; 112 struct intel_engine_cs *engine; 113 enum intel_engine_id id; 114 struct igt_spinner spin; 115 int err = 0; 116 117 if (!HAS_LOGICAL_RING_CONTEXTS(gt->i915)) 118 return 0; 119 120 if (igt_spinner_init(&spin, gt)) 121 return -ENOMEM; 122 123 for_each_engine(engine, gt, id) { 124 struct intel_context *ce; 125 struct i915_request *rq; 126 127 ce = intel_context_create(engine); 128 if (IS_ERR(ce)) { 129 err = PTR_ERR(ce); 130 break; 131 } 132 133 rq = igt_spinner_create_request(&spin, ce, MI_NOOP); 134 if (IS_ERR(rq)) { 135 err = PTR_ERR(rq); 136 goto out_ctx; 137 } 138 139 i915_request_add(rq); 140 if (!igt_wait_for_spinner(&spin, rq)) { 141 GEM_TRACE("spinner failed to start\n"); 142 GEM_TRACE_DUMP(); 143 intel_gt_set_wedged(gt); 144 err = -EIO; 145 goto out_ctx; 146 } 147 148 igt_spinner_end(&spin); 149 if (igt_flush_test(gt->i915)) { 150 err = -EIO; 151 goto out_ctx; 152 } 153 154 out_ctx: 155 intel_context_put(ce); 156 if (err) 157 break; 158 } 159 160 igt_spinner_fini(&spin); 161 return err; 162 } 163 164 static int live_unlite_restore(struct intel_gt *gt, int prio) 165 { 166 struct intel_engine_cs *engine; 167 enum intel_engine_id id; 168 struct igt_spinner spin; 169 int err = -ENOMEM; 170 171 /* 172 * Check that we can correctly context switch between 2 instances 173 * on the same engine from the same parent context. 174 */ 175 176 if (igt_spinner_init(&spin, gt)) 177 return err; 178 179 err = 0; 180 for_each_engine(engine, gt, id) { 181 struct intel_context *ce[2] = {}; 182 struct i915_request *rq[2]; 183 struct igt_live_test t; 184 int n; 185 186 if (prio && !intel_engine_has_preemption(engine)) 187 continue; 188 189 if (!intel_engine_can_store_dword(engine)) 190 continue; 191 192 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 193 err = -EIO; 194 break; 195 } 196 st_engine_heartbeat_disable(engine); 197 198 for (n = 0; n < ARRAY_SIZE(ce); n++) { 199 struct intel_context *tmp; 200 201 tmp = intel_context_create(engine); 202 if (IS_ERR(tmp)) { 203 err = PTR_ERR(tmp); 204 goto err_ce; 205 } 206 207 err = intel_context_pin(tmp); 208 if (err) { 209 intel_context_put(tmp); 210 goto err_ce; 211 } 212 213 /* 214 * Setup the pair of contexts such that if we 215 * lite-restore using the RING_TAIL from ce[1] it 216 * will execute garbage from ce[0]->ring. 217 */ 218 memset(tmp->ring->vaddr, 219 POISON_INUSE, /* IPEHR: 0x5a5a5a5a [hung!] */ 220 tmp->ring->vma->size); 221 222 ce[n] = tmp; 223 } 224 GEM_BUG_ON(!ce[1]->ring->size); 225 intel_ring_reset(ce[1]->ring, ce[1]->ring->size / 2); 226 lrc_update_regs(ce[1], engine, ce[1]->ring->head); 227 228 rq[0] = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK); 229 if (IS_ERR(rq[0])) { 230 err = PTR_ERR(rq[0]); 231 goto err_ce; 232 } 233 234 i915_request_get(rq[0]); 235 i915_request_add(rq[0]); 236 GEM_BUG_ON(rq[0]->postfix > ce[1]->ring->emit); 237 238 if (!igt_wait_for_spinner(&spin, rq[0])) { 239 i915_request_put(rq[0]); 240 goto err_ce; 241 } 242 243 rq[1] = i915_request_create(ce[1]); 244 if (IS_ERR(rq[1])) { 245 err = PTR_ERR(rq[1]); 246 i915_request_put(rq[0]); 247 goto err_ce; 248 } 249 250 if (!prio) { 251 /* 252 * Ensure we do the switch to ce[1] on completion. 253 * 254 * rq[0] is already submitted, so this should reduce 255 * to a no-op (a wait on a request on the same engine 256 * uses the submit fence, not the completion fence), 257 * but it will install a dependency on rq[1] for rq[0] 258 * that will prevent the pair being reordered by 259 * timeslicing. 260 */ 261 i915_request_await_dma_fence(rq[1], &rq[0]->fence); 262 } 263 264 i915_request_get(rq[1]); 265 i915_request_add(rq[1]); 266 GEM_BUG_ON(rq[1]->postfix <= rq[0]->postfix); 267 i915_request_put(rq[0]); 268 269 if (prio) { 270 struct i915_sched_attr attr = { 271 .priority = prio, 272 }; 273 274 /* Alternatively preempt the spinner with ce[1] */ 275 engine->sched_engine->schedule(rq[1], &attr); 276 } 277 278 /* And switch back to ce[0] for good measure */ 279 rq[0] = i915_request_create(ce[0]); 280 if (IS_ERR(rq[0])) { 281 err = PTR_ERR(rq[0]); 282 i915_request_put(rq[1]); 283 goto err_ce; 284 } 285 286 i915_request_await_dma_fence(rq[0], &rq[1]->fence); 287 i915_request_get(rq[0]); 288 i915_request_add(rq[0]); 289 GEM_BUG_ON(rq[0]->postfix > rq[1]->postfix); 290 i915_request_put(rq[1]); 291 i915_request_put(rq[0]); 292 293 err_ce: 294 intel_engine_flush_submission(engine); 295 igt_spinner_end(&spin); 296 for (n = 0; n < ARRAY_SIZE(ce); n++) { 297 if (IS_ERR_OR_NULL(ce[n])) 298 break; 299 300 intel_context_unpin(ce[n]); 301 intel_context_put(ce[n]); 302 } 303 304 st_engine_heartbeat_enable(engine); 305 if (igt_live_test_end(&t)) 306 err = -EIO; 307 if (err) 308 break; 309 } 310 311 igt_spinner_fini(&spin); 312 return err; 313 } 314 315 static int live_unlite_switch(void *arg) 316 { 317 return live_unlite_restore(arg, 0); 318 } 319 320 static int live_unlite_preempt(void *arg) 321 { 322 return live_unlite_restore(arg, I915_PRIORITY_MAX); 323 } 324 325 static int live_unlite_ring(void *arg) 326 { 327 struct intel_gt *gt = arg; 328 struct intel_engine_cs *engine; 329 struct igt_spinner spin; 330 enum intel_engine_id id; 331 int err = 0; 332 333 /* 334 * Setup a preemption event that will cause almost the entire ring 335 * to be unwound, potentially fooling our intel_ring_direction() 336 * into emitting a forward lite-restore instead of the rollback. 337 */ 338 339 if (igt_spinner_init(&spin, gt)) 340 return -ENOMEM; 341 342 for_each_engine(engine, gt, id) { 343 struct intel_context *ce[2] = {}; 344 struct i915_request *rq; 345 struct igt_live_test t; 346 int n; 347 348 if (!intel_engine_has_preemption(engine)) 349 continue; 350 351 if (!intel_engine_can_store_dword(engine)) 352 continue; 353 354 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 355 err = -EIO; 356 break; 357 } 358 st_engine_heartbeat_disable(engine); 359 360 for (n = 0; n < ARRAY_SIZE(ce); n++) { 361 struct intel_context *tmp; 362 363 tmp = intel_context_create(engine); 364 if (IS_ERR(tmp)) { 365 err = PTR_ERR(tmp); 366 goto err_ce; 367 } 368 369 err = intel_context_pin(tmp); 370 if (err) { 371 intel_context_put(tmp); 372 goto err_ce; 373 } 374 375 memset32(tmp->ring->vaddr, 376 0xdeadbeef, /* trigger a hang if executed */ 377 tmp->ring->vma->size / sizeof(u32)); 378 379 ce[n] = tmp; 380 } 381 382 /* Create max prio spinner, followed by N low prio nops */ 383 rq = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK); 384 if (IS_ERR(rq)) { 385 err = PTR_ERR(rq); 386 goto err_ce; 387 } 388 389 i915_request_get(rq); 390 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 391 i915_request_add(rq); 392 393 if (!igt_wait_for_spinner(&spin, rq)) { 394 intel_gt_set_wedged(gt); 395 i915_request_put(rq); 396 err = -ETIME; 397 goto err_ce; 398 } 399 400 /* Fill the ring, until we will cause a wrap */ 401 n = 0; 402 while (intel_ring_direction(ce[0]->ring, 403 rq->wa_tail, 404 ce[0]->ring->tail) <= 0) { 405 struct i915_request *tmp; 406 407 tmp = intel_context_create_request(ce[0]); 408 if (IS_ERR(tmp)) { 409 err = PTR_ERR(tmp); 410 i915_request_put(rq); 411 goto err_ce; 412 } 413 414 i915_request_add(tmp); 415 intel_engine_flush_submission(engine); 416 n++; 417 } 418 intel_engine_flush_submission(engine); 419 pr_debug("%s: Filled ring with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n", 420 engine->name, n, 421 ce[0]->ring->size, 422 ce[0]->ring->tail, 423 ce[0]->ring->emit, 424 rq->tail); 425 GEM_BUG_ON(intel_ring_direction(ce[0]->ring, 426 rq->tail, 427 ce[0]->ring->tail) <= 0); 428 i915_request_put(rq); 429 430 /* Create a second ring to preempt the first ring after rq[0] */ 431 rq = intel_context_create_request(ce[1]); 432 if (IS_ERR(rq)) { 433 err = PTR_ERR(rq); 434 goto err_ce; 435 } 436 437 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 438 i915_request_get(rq); 439 i915_request_add(rq); 440 441 err = wait_for_submit(engine, rq, HZ / 2); 442 i915_request_put(rq); 443 if (err) { 444 pr_err("%s: preemption request was not submitted\n", 445 engine->name); 446 err = -ETIME; 447 } 448 449 pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n", 450 engine->name, 451 ce[0]->ring->tail, ce[0]->ring->emit, 452 ce[1]->ring->tail, ce[1]->ring->emit); 453 454 err_ce: 455 intel_engine_flush_submission(engine); 456 igt_spinner_end(&spin); 457 for (n = 0; n < ARRAY_SIZE(ce); n++) { 458 if (IS_ERR_OR_NULL(ce[n])) 459 break; 460 461 intel_context_unpin(ce[n]); 462 intel_context_put(ce[n]); 463 } 464 st_engine_heartbeat_enable(engine); 465 if (igt_live_test_end(&t)) 466 err = -EIO; 467 if (err) 468 break; 469 } 470 471 igt_spinner_fini(&spin); 472 return err; 473 } 474 475 static int live_pin_rewind(void *arg) 476 { 477 struct intel_gt *gt = arg; 478 struct intel_engine_cs *engine; 479 enum intel_engine_id id; 480 int err = 0; 481 482 /* 483 * We have to be careful not to trust intel_ring too much, for example 484 * ring->head is updated upon retire which is out of sync with pinning 485 * the context. Thus we cannot use ring->head to set CTX_RING_HEAD, 486 * or else we risk writing an older, stale value. 487 * 488 * To simulate this, let's apply a bit of deliberate sabotague. 489 */ 490 491 for_each_engine(engine, gt, id) { 492 struct intel_context *ce; 493 struct i915_request *rq; 494 struct intel_ring *ring; 495 struct igt_live_test t; 496 497 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 498 err = -EIO; 499 break; 500 } 501 502 ce = intel_context_create(engine); 503 if (IS_ERR(ce)) { 504 err = PTR_ERR(ce); 505 break; 506 } 507 508 err = intel_context_pin(ce); 509 if (err) { 510 intel_context_put(ce); 511 break; 512 } 513 514 /* Keep the context awake while we play games */ 515 err = i915_active_acquire(&ce->active); 516 if (err) { 517 intel_context_unpin(ce); 518 intel_context_put(ce); 519 break; 520 } 521 ring = ce->ring; 522 523 /* Poison the ring, and offset the next request from HEAD */ 524 memset32(ring->vaddr, STACK_MAGIC, ring->size / sizeof(u32)); 525 ring->emit = ring->size / 2; 526 ring->tail = ring->emit; 527 GEM_BUG_ON(ring->head); 528 529 intel_context_unpin(ce); 530 531 /* Submit a simple nop request */ 532 GEM_BUG_ON(intel_context_is_pinned(ce)); 533 rq = intel_context_create_request(ce); 534 i915_active_release(&ce->active); /* e.g. async retire */ 535 intel_context_put(ce); 536 if (IS_ERR(rq)) { 537 err = PTR_ERR(rq); 538 break; 539 } 540 GEM_BUG_ON(!rq->head); 541 i915_request_add(rq); 542 543 /* Expect not to hang! */ 544 if (igt_live_test_end(&t)) { 545 err = -EIO; 546 break; 547 } 548 } 549 550 return err; 551 } 552 553 static int engine_lock_reset_tasklet(struct intel_engine_cs *engine) 554 { 555 tasklet_disable(&engine->sched_engine->tasklet); 556 local_bh_disable(); 557 558 if (test_and_set_bit(I915_RESET_ENGINE + engine->id, 559 &engine->gt->reset.flags)) { 560 local_bh_enable(); 561 tasklet_enable(&engine->sched_engine->tasklet); 562 563 intel_gt_set_wedged(engine->gt); 564 return -EBUSY; 565 } 566 567 return 0; 568 } 569 570 static void engine_unlock_reset_tasklet(struct intel_engine_cs *engine) 571 { 572 clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id, 573 &engine->gt->reset.flags); 574 575 local_bh_enable(); 576 tasklet_enable(&engine->sched_engine->tasklet); 577 } 578 579 static int live_hold_reset(void *arg) 580 { 581 struct intel_gt *gt = arg; 582 struct intel_engine_cs *engine; 583 enum intel_engine_id id; 584 struct igt_spinner spin; 585 int err = 0; 586 587 /* 588 * In order to support offline error capture for fast preempt reset, 589 * we need to decouple the guilty request and ensure that it and its 590 * descendents are not executed while the capture is in progress. 591 */ 592 593 if (!intel_has_reset_engine(gt)) 594 return 0; 595 596 if (igt_spinner_init(&spin, gt)) 597 return -ENOMEM; 598 599 for_each_engine(engine, gt, id) { 600 struct intel_context *ce; 601 struct i915_request *rq; 602 603 ce = intel_context_create(engine); 604 if (IS_ERR(ce)) { 605 err = PTR_ERR(ce); 606 break; 607 } 608 609 st_engine_heartbeat_disable(engine); 610 611 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK); 612 if (IS_ERR(rq)) { 613 err = PTR_ERR(rq); 614 goto out; 615 } 616 i915_request_add(rq); 617 618 if (!igt_wait_for_spinner(&spin, rq)) { 619 intel_gt_set_wedged(gt); 620 err = -ETIME; 621 goto out; 622 } 623 624 /* We have our request executing, now remove it and reset */ 625 626 err = engine_lock_reset_tasklet(engine); 627 if (err) 628 goto out; 629 630 engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet); 631 GEM_BUG_ON(execlists_active(&engine->execlists) != rq); 632 633 i915_request_get(rq); 634 execlists_hold(engine, rq); 635 GEM_BUG_ON(!i915_request_on_hold(rq)); 636 637 __intel_engine_reset_bh(engine, NULL); 638 GEM_BUG_ON(rq->fence.error != -EIO); 639 640 engine_unlock_reset_tasklet(engine); 641 642 /* Check that we do not resubmit the held request */ 643 if (!i915_request_wait(rq, 0, HZ / 5)) { 644 pr_err("%s: on hold request completed!\n", 645 engine->name); 646 i915_request_put(rq); 647 err = -EIO; 648 goto out; 649 } 650 GEM_BUG_ON(!i915_request_on_hold(rq)); 651 652 /* But is resubmitted on release */ 653 execlists_unhold(engine, rq); 654 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 655 pr_err("%s: held request did not complete!\n", 656 engine->name); 657 intel_gt_set_wedged(gt); 658 err = -ETIME; 659 } 660 i915_request_put(rq); 661 662 out: 663 st_engine_heartbeat_enable(engine); 664 intel_context_put(ce); 665 if (err) 666 break; 667 } 668 669 igt_spinner_fini(&spin); 670 return err; 671 } 672 673 static const char *error_repr(int err) 674 { 675 return err ? "bad" : "good"; 676 } 677 678 static int live_error_interrupt(void *arg) 679 { 680 static const struct error_phase { 681 enum { GOOD = 0, BAD = -EIO } error[2]; 682 } phases[] = { 683 { { BAD, GOOD } }, 684 { { BAD, BAD } }, 685 { { BAD, GOOD } }, 686 { { GOOD, GOOD } }, /* sentinel */ 687 }; 688 struct intel_gt *gt = arg; 689 struct intel_engine_cs *engine; 690 enum intel_engine_id id; 691 692 /* 693 * We hook up the CS_MASTER_ERROR_INTERRUPT to have forewarning 694 * of invalid commands in user batches that will cause a GPU hang. 695 * This is a faster mechanism than using hangcheck/heartbeats, but 696 * only detects problems the HW knows about -- it will not warn when 697 * we kill the HW! 698 * 699 * To verify our detection and reset, we throw some invalid commands 700 * at the HW and wait for the interrupt. 701 */ 702 703 if (!intel_has_reset_engine(gt)) 704 return 0; 705 706 for_each_engine(engine, gt, id) { 707 const struct error_phase *p; 708 int err = 0; 709 710 st_engine_heartbeat_disable(engine); 711 712 for (p = phases; p->error[0] != GOOD; p++) { 713 struct i915_request *client[ARRAY_SIZE(phases->error)]; 714 u32 *cs; 715 int i; 716 717 memset(client, 0, sizeof(*client)); 718 for (i = 0; i < ARRAY_SIZE(client); i++) { 719 struct intel_context *ce; 720 struct i915_request *rq; 721 722 ce = intel_context_create(engine); 723 if (IS_ERR(ce)) { 724 err = PTR_ERR(ce); 725 goto out; 726 } 727 728 rq = intel_context_create_request(ce); 729 intel_context_put(ce); 730 if (IS_ERR(rq)) { 731 err = PTR_ERR(rq); 732 goto out; 733 } 734 735 if (rq->engine->emit_init_breadcrumb) { 736 err = rq->engine->emit_init_breadcrumb(rq); 737 if (err) { 738 i915_request_add(rq); 739 goto out; 740 } 741 } 742 743 cs = intel_ring_begin(rq, 2); 744 if (IS_ERR(cs)) { 745 i915_request_add(rq); 746 err = PTR_ERR(cs); 747 goto out; 748 } 749 750 if (p->error[i]) { 751 *cs++ = 0xdeadbeef; 752 *cs++ = 0xdeadbeef; 753 } else { 754 *cs++ = MI_NOOP; 755 *cs++ = MI_NOOP; 756 } 757 758 client[i] = i915_request_get(rq); 759 i915_request_add(rq); 760 } 761 762 err = wait_for_submit(engine, client[0], HZ / 2); 763 if (err) { 764 pr_err("%s: first request did not start within time!\n", 765 engine->name); 766 err = -ETIME; 767 goto out; 768 } 769 770 for (i = 0; i < ARRAY_SIZE(client); i++) { 771 if (i915_request_wait(client[i], 0, HZ / 5) < 0) 772 pr_debug("%s: %s request incomplete!\n", 773 engine->name, 774 error_repr(p->error[i])); 775 776 if (!i915_request_started(client[i])) { 777 pr_err("%s: %s request not started!\n", 778 engine->name, 779 error_repr(p->error[i])); 780 err = -ETIME; 781 goto out; 782 } 783 784 /* Kick the tasklet to process the error */ 785 intel_engine_flush_submission(engine); 786 if (client[i]->fence.error != p->error[i]) { 787 pr_err("%s: %s request (%s) with wrong error code: %d\n", 788 engine->name, 789 error_repr(p->error[i]), 790 i915_request_completed(client[i]) ? "completed" : "running", 791 client[i]->fence.error); 792 err = -EINVAL; 793 goto out; 794 } 795 } 796 797 out: 798 for (i = 0; i < ARRAY_SIZE(client); i++) 799 if (client[i]) 800 i915_request_put(client[i]); 801 if (err) { 802 pr_err("%s: failed at phase[%zd] { %d, %d }\n", 803 engine->name, p - phases, 804 p->error[0], p->error[1]); 805 break; 806 } 807 } 808 809 st_engine_heartbeat_enable(engine); 810 if (err) { 811 intel_gt_set_wedged(gt); 812 return err; 813 } 814 } 815 816 return 0; 817 } 818 819 static int 820 emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx) 821 { 822 u32 *cs; 823 824 cs = intel_ring_begin(rq, 10); 825 if (IS_ERR(cs)) 826 return PTR_ERR(cs); 827 828 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 829 830 *cs++ = MI_SEMAPHORE_WAIT | 831 MI_SEMAPHORE_GLOBAL_GTT | 832 MI_SEMAPHORE_POLL | 833 MI_SEMAPHORE_SAD_NEQ_SDD; 834 *cs++ = 0; 835 *cs++ = i915_ggtt_offset(vma) + 4 * idx; 836 *cs++ = 0; 837 838 if (idx > 0) { 839 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 840 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1); 841 *cs++ = 0; 842 *cs++ = 1; 843 } else { 844 *cs++ = MI_NOOP; 845 *cs++ = MI_NOOP; 846 *cs++ = MI_NOOP; 847 *cs++ = MI_NOOP; 848 } 849 850 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 851 852 intel_ring_advance(rq, cs); 853 return 0; 854 } 855 856 static struct i915_request * 857 semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx) 858 { 859 struct intel_context *ce; 860 struct i915_request *rq; 861 int err; 862 863 ce = intel_context_create(engine); 864 if (IS_ERR(ce)) 865 return ERR_CAST(ce); 866 867 rq = intel_context_create_request(ce); 868 if (IS_ERR(rq)) 869 goto out_ce; 870 871 err = 0; 872 if (rq->engine->emit_init_breadcrumb) 873 err = rq->engine->emit_init_breadcrumb(rq); 874 if (err == 0) 875 err = emit_semaphore_chain(rq, vma, idx); 876 if (err == 0) 877 i915_request_get(rq); 878 i915_request_add(rq); 879 if (err) 880 rq = ERR_PTR(err); 881 882 out_ce: 883 intel_context_put(ce); 884 return rq; 885 } 886 887 static int 888 release_queue(struct intel_engine_cs *engine, 889 struct i915_vma *vma, 890 int idx, int prio) 891 { 892 struct i915_sched_attr attr = { 893 .priority = prio, 894 }; 895 struct i915_request *rq; 896 u32 *cs; 897 898 rq = intel_engine_create_kernel_request(engine); 899 if (IS_ERR(rq)) 900 return PTR_ERR(rq); 901 902 cs = intel_ring_begin(rq, 4); 903 if (IS_ERR(cs)) { 904 i915_request_add(rq); 905 return PTR_ERR(cs); 906 } 907 908 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 909 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1); 910 *cs++ = 0; 911 *cs++ = 1; 912 913 intel_ring_advance(rq, cs); 914 915 i915_request_get(rq); 916 i915_request_add(rq); 917 918 local_bh_disable(); 919 engine->sched_engine->schedule(rq, &attr); 920 local_bh_enable(); /* kick tasklet */ 921 922 i915_request_put(rq); 923 924 return 0; 925 } 926 927 static int 928 slice_semaphore_queue(struct intel_engine_cs *outer, 929 struct i915_vma *vma, 930 int count) 931 { 932 struct intel_engine_cs *engine; 933 struct i915_request *head; 934 enum intel_engine_id id; 935 int err, i, n = 0; 936 937 head = semaphore_queue(outer, vma, n++); 938 if (IS_ERR(head)) 939 return PTR_ERR(head); 940 941 for_each_engine(engine, outer->gt, id) { 942 if (!intel_engine_has_preemption(engine)) 943 continue; 944 945 for (i = 0; i < count; i++) { 946 struct i915_request *rq; 947 948 rq = semaphore_queue(engine, vma, n++); 949 if (IS_ERR(rq)) { 950 err = PTR_ERR(rq); 951 goto out; 952 } 953 954 i915_request_put(rq); 955 } 956 } 957 958 err = release_queue(outer, vma, n, I915_PRIORITY_BARRIER); 959 if (err) 960 goto out; 961 962 if (i915_request_wait(head, 0, 963 2 * outer->gt->info.num_engines * (count + 2) * (count + 3)) < 0) { 964 pr_err("%s: Failed to slice along semaphore chain of length (%d, %d)!\n", 965 outer->name, count, n); 966 GEM_TRACE_DUMP(); 967 intel_gt_set_wedged(outer->gt); 968 err = -EIO; 969 } 970 971 out: 972 i915_request_put(head); 973 return err; 974 } 975 976 static int live_timeslice_preempt(void *arg) 977 { 978 struct intel_gt *gt = arg; 979 struct drm_i915_gem_object *obj; 980 struct intel_engine_cs *engine; 981 enum intel_engine_id id; 982 struct i915_vma *vma; 983 void *vaddr; 984 int err = 0; 985 986 /* 987 * If a request takes too long, we would like to give other users 988 * a fair go on the GPU. In particular, users may create batches 989 * that wait upon external input, where that input may even be 990 * supplied by another GPU job. To avoid blocking forever, we 991 * need to preempt the current task and replace it with another 992 * ready task. 993 */ 994 if (!CONFIG_DRM_I915_TIMESLICE_DURATION) 995 return 0; 996 997 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); 998 if (IS_ERR(obj)) 999 return PTR_ERR(obj); 1000 1001 vma = i915_vma_instance(obj, >->ggtt->vm, NULL); 1002 if (IS_ERR(vma)) { 1003 err = PTR_ERR(vma); 1004 goto err_obj; 1005 } 1006 1007 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 1008 if (IS_ERR(vaddr)) { 1009 err = PTR_ERR(vaddr); 1010 goto err_obj; 1011 } 1012 1013 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 1014 if (err) 1015 goto err_map; 1016 1017 err = i915_vma_sync(vma); 1018 if (err) 1019 goto err_pin; 1020 1021 for_each_engine(engine, gt, id) { 1022 if (!intel_engine_has_preemption(engine)) 1023 continue; 1024 1025 memset(vaddr, 0, PAGE_SIZE); 1026 1027 st_engine_heartbeat_disable(engine); 1028 err = slice_semaphore_queue(engine, vma, 5); 1029 st_engine_heartbeat_enable(engine); 1030 if (err) 1031 goto err_pin; 1032 1033 if (igt_flush_test(gt->i915)) { 1034 err = -EIO; 1035 goto err_pin; 1036 } 1037 } 1038 1039 err_pin: 1040 i915_vma_unpin(vma); 1041 err_map: 1042 i915_gem_object_unpin_map(obj); 1043 err_obj: 1044 i915_gem_object_put(obj); 1045 return err; 1046 } 1047 1048 static struct i915_request * 1049 create_rewinder(struct intel_context *ce, 1050 struct i915_request *wait, 1051 void *slot, int idx) 1052 { 1053 const u32 offset = 1054 i915_ggtt_offset(ce->engine->status_page.vma) + 1055 offset_in_page(slot); 1056 struct i915_request *rq; 1057 u32 *cs; 1058 int err; 1059 1060 rq = intel_context_create_request(ce); 1061 if (IS_ERR(rq)) 1062 return rq; 1063 1064 if (wait) { 1065 err = i915_request_await_dma_fence(rq, &wait->fence); 1066 if (err) 1067 goto err; 1068 } 1069 1070 cs = intel_ring_begin(rq, 14); 1071 if (IS_ERR(cs)) { 1072 err = PTR_ERR(cs); 1073 goto err; 1074 } 1075 1076 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1077 *cs++ = MI_NOOP; 1078 1079 *cs++ = MI_SEMAPHORE_WAIT | 1080 MI_SEMAPHORE_GLOBAL_GTT | 1081 MI_SEMAPHORE_POLL | 1082 MI_SEMAPHORE_SAD_GTE_SDD; 1083 *cs++ = idx; 1084 *cs++ = offset; 1085 *cs++ = 0; 1086 1087 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 1088 *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(rq->engine->mmio_base)); 1089 *cs++ = offset + idx * sizeof(u32); 1090 *cs++ = 0; 1091 1092 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 1093 *cs++ = offset; 1094 *cs++ = 0; 1095 *cs++ = idx + 1; 1096 1097 intel_ring_advance(rq, cs); 1098 1099 err = 0; 1100 err: 1101 i915_request_get(rq); 1102 i915_request_add(rq); 1103 if (err) { 1104 i915_request_put(rq); 1105 return ERR_PTR(err); 1106 } 1107 1108 return rq; 1109 } 1110 1111 static int live_timeslice_rewind(void *arg) 1112 { 1113 struct intel_gt *gt = arg; 1114 struct intel_engine_cs *engine; 1115 enum intel_engine_id id; 1116 1117 /* 1118 * The usual presumption on timeslice expiration is that we replace 1119 * the active context with another. However, given a chain of 1120 * dependencies we may end up with replacing the context with itself, 1121 * but only a few of those requests, forcing us to rewind the 1122 * RING_TAIL of the original request. 1123 */ 1124 if (!CONFIG_DRM_I915_TIMESLICE_DURATION) 1125 return 0; 1126 1127 for_each_engine(engine, gt, id) { 1128 enum { A1, A2, B1 }; 1129 enum { X = 1, Z, Y }; 1130 struct i915_request *rq[3] = {}; 1131 struct intel_context *ce; 1132 unsigned long timeslice; 1133 int i, err = 0; 1134 u32 *slot; 1135 1136 if (!intel_engine_has_timeslices(engine)) 1137 continue; 1138 1139 /* 1140 * A:rq1 -- semaphore wait, timestamp X 1141 * A:rq2 -- write timestamp Y 1142 * 1143 * B:rq1 [await A:rq1] -- write timestamp Z 1144 * 1145 * Force timeslice, release semaphore. 1146 * 1147 * Expect execution/evaluation order XZY 1148 */ 1149 1150 st_engine_heartbeat_disable(engine); 1151 timeslice = xchg(&engine->props.timeslice_duration_ms, 1); 1152 1153 slot = memset32(engine->status_page.addr + 1000, 0, 4); 1154 1155 ce = intel_context_create(engine); 1156 if (IS_ERR(ce)) { 1157 err = PTR_ERR(ce); 1158 goto err; 1159 } 1160 1161 rq[A1] = create_rewinder(ce, NULL, slot, X); 1162 if (IS_ERR(rq[A1])) { 1163 intel_context_put(ce); 1164 goto err; 1165 } 1166 1167 rq[A2] = create_rewinder(ce, NULL, slot, Y); 1168 intel_context_put(ce); 1169 if (IS_ERR(rq[A2])) 1170 goto err; 1171 1172 err = wait_for_submit(engine, rq[A2], HZ / 2); 1173 if (err) { 1174 pr_err("%s: failed to submit first context\n", 1175 engine->name); 1176 goto err; 1177 } 1178 1179 ce = intel_context_create(engine); 1180 if (IS_ERR(ce)) { 1181 err = PTR_ERR(ce); 1182 goto err; 1183 } 1184 1185 rq[B1] = create_rewinder(ce, rq[A1], slot, Z); 1186 intel_context_put(ce); 1187 if (IS_ERR(rq[2])) 1188 goto err; 1189 1190 err = wait_for_submit(engine, rq[B1], HZ / 2); 1191 if (err) { 1192 pr_err("%s: failed to submit second context\n", 1193 engine->name); 1194 goto err; 1195 } 1196 1197 /* ELSP[] = { { A:rq1, A:rq2 }, { B:rq1 } } */ 1198 ENGINE_TRACE(engine, "forcing tasklet for rewind\n"); 1199 while (i915_request_is_active(rq[A2])) { /* semaphore yield! */ 1200 /* Wait for the timeslice to kick in */ 1201 del_timer(&engine->execlists.timer); 1202 tasklet_hi_schedule(&engine->sched_engine->tasklet); 1203 intel_engine_flush_submission(engine); 1204 } 1205 /* -> ELSP[] = { { A:rq1 }, { B:rq1 } } */ 1206 GEM_BUG_ON(!i915_request_is_active(rq[A1])); 1207 GEM_BUG_ON(!i915_request_is_active(rq[B1])); 1208 GEM_BUG_ON(i915_request_is_active(rq[A2])); 1209 1210 /* Release the hounds! */ 1211 slot[0] = 1; 1212 wmb(); /* "pairs" with GPU; paranoid kick of internal CPU$ */ 1213 1214 for (i = 1; i <= 3; i++) { 1215 unsigned long timeout = jiffies + HZ / 2; 1216 1217 while (!READ_ONCE(slot[i]) && 1218 time_before(jiffies, timeout)) 1219 ; 1220 1221 if (!time_before(jiffies, timeout)) { 1222 pr_err("%s: rq[%d] timed out\n", 1223 engine->name, i - 1); 1224 err = -ETIME; 1225 goto err; 1226 } 1227 1228 pr_debug("%s: slot[%d]:%x\n", engine->name, i, slot[i]); 1229 } 1230 1231 /* XZY: XZ < XY */ 1232 if (slot[Z] - slot[X] >= slot[Y] - slot[X]) { 1233 pr_err("%s: timeslicing did not run context B [%u] before A [%u]!\n", 1234 engine->name, 1235 slot[Z] - slot[X], 1236 slot[Y] - slot[X]); 1237 err = -EINVAL; 1238 } 1239 1240 err: 1241 memset32(&slot[0], -1, 4); 1242 wmb(); 1243 1244 engine->props.timeslice_duration_ms = timeslice; 1245 st_engine_heartbeat_enable(engine); 1246 for (i = 0; i < 3; i++) 1247 i915_request_put(rq[i]); 1248 if (igt_flush_test(gt->i915)) 1249 err = -EIO; 1250 if (err) 1251 return err; 1252 } 1253 1254 return 0; 1255 } 1256 1257 static struct i915_request *nop_request(struct intel_engine_cs *engine) 1258 { 1259 struct i915_request *rq; 1260 1261 rq = intel_engine_create_kernel_request(engine); 1262 if (IS_ERR(rq)) 1263 return rq; 1264 1265 i915_request_get(rq); 1266 i915_request_add(rq); 1267 1268 return rq; 1269 } 1270 1271 static long slice_timeout(struct intel_engine_cs *engine) 1272 { 1273 long timeout; 1274 1275 /* Enough time for a timeslice to kick in, and kick out */ 1276 timeout = 2 * msecs_to_jiffies_timeout(timeslice(engine)); 1277 1278 /* Enough time for the nop request to complete */ 1279 timeout += HZ / 5; 1280 1281 return timeout + 1; 1282 } 1283 1284 static int live_timeslice_queue(void *arg) 1285 { 1286 struct intel_gt *gt = arg; 1287 struct drm_i915_gem_object *obj; 1288 struct intel_engine_cs *engine; 1289 enum intel_engine_id id; 1290 struct i915_vma *vma; 1291 void *vaddr; 1292 int err = 0; 1293 1294 /* 1295 * Make sure that even if ELSP[0] and ELSP[1] are filled with 1296 * timeslicing between them disabled, we *do* enable timeslicing 1297 * if the queue demands it. (Normally, we do not submit if 1298 * ELSP[1] is already occupied, so must rely on timeslicing to 1299 * eject ELSP[0] in favour of the queue.) 1300 */ 1301 if (!CONFIG_DRM_I915_TIMESLICE_DURATION) 1302 return 0; 1303 1304 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); 1305 if (IS_ERR(obj)) 1306 return PTR_ERR(obj); 1307 1308 vma = i915_vma_instance(obj, >->ggtt->vm, NULL); 1309 if (IS_ERR(vma)) { 1310 err = PTR_ERR(vma); 1311 goto err_obj; 1312 } 1313 1314 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 1315 if (IS_ERR(vaddr)) { 1316 err = PTR_ERR(vaddr); 1317 goto err_obj; 1318 } 1319 1320 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 1321 if (err) 1322 goto err_map; 1323 1324 err = i915_vma_sync(vma); 1325 if (err) 1326 goto err_pin; 1327 1328 for_each_engine(engine, gt, id) { 1329 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX }; 1330 struct i915_request *rq, *nop; 1331 1332 if (!intel_engine_has_preemption(engine)) 1333 continue; 1334 1335 st_engine_heartbeat_disable(engine); 1336 memset(vaddr, 0, PAGE_SIZE); 1337 1338 /* ELSP[0]: semaphore wait */ 1339 rq = semaphore_queue(engine, vma, 0); 1340 if (IS_ERR(rq)) { 1341 err = PTR_ERR(rq); 1342 goto err_heartbeat; 1343 } 1344 engine->sched_engine->schedule(rq, &attr); 1345 err = wait_for_submit(engine, rq, HZ / 2); 1346 if (err) { 1347 pr_err("%s: Timed out trying to submit semaphores\n", 1348 engine->name); 1349 goto err_rq; 1350 } 1351 1352 /* ELSP[1]: nop request */ 1353 nop = nop_request(engine); 1354 if (IS_ERR(nop)) { 1355 err = PTR_ERR(nop); 1356 goto err_rq; 1357 } 1358 err = wait_for_submit(engine, nop, HZ / 2); 1359 i915_request_put(nop); 1360 if (err) { 1361 pr_err("%s: Timed out trying to submit nop\n", 1362 engine->name); 1363 goto err_rq; 1364 } 1365 1366 GEM_BUG_ON(i915_request_completed(rq)); 1367 GEM_BUG_ON(execlists_active(&engine->execlists) != rq); 1368 1369 /* Queue: semaphore signal, matching priority as semaphore */ 1370 err = release_queue(engine, vma, 1, effective_prio(rq)); 1371 if (err) 1372 goto err_rq; 1373 1374 /* Wait until we ack the release_queue and start timeslicing */ 1375 do { 1376 cond_resched(); 1377 intel_engine_flush_submission(engine); 1378 } while (READ_ONCE(engine->execlists.pending[0])); 1379 1380 /* Timeslice every jiffy, so within 2 we should signal */ 1381 if (i915_request_wait(rq, 0, slice_timeout(engine)) < 0) { 1382 struct drm_printer p = 1383 drm_info_printer(gt->i915->drm.dev); 1384 1385 pr_err("%s: Failed to timeslice into queue\n", 1386 engine->name); 1387 intel_engine_dump(engine, &p, 1388 "%s\n", engine->name); 1389 1390 memset(vaddr, 0xff, PAGE_SIZE); 1391 err = -EIO; 1392 } 1393 err_rq: 1394 i915_request_put(rq); 1395 err_heartbeat: 1396 st_engine_heartbeat_enable(engine); 1397 if (err) 1398 break; 1399 } 1400 1401 err_pin: 1402 i915_vma_unpin(vma); 1403 err_map: 1404 i915_gem_object_unpin_map(obj); 1405 err_obj: 1406 i915_gem_object_put(obj); 1407 return err; 1408 } 1409 1410 static int live_timeslice_nopreempt(void *arg) 1411 { 1412 struct intel_gt *gt = arg; 1413 struct intel_engine_cs *engine; 1414 enum intel_engine_id id; 1415 struct igt_spinner spin; 1416 int err = 0; 1417 1418 /* 1419 * We should not timeslice into a request that is marked with 1420 * I915_REQUEST_NOPREEMPT. 1421 */ 1422 if (!CONFIG_DRM_I915_TIMESLICE_DURATION) 1423 return 0; 1424 1425 if (igt_spinner_init(&spin, gt)) 1426 return -ENOMEM; 1427 1428 for_each_engine(engine, gt, id) { 1429 struct intel_context *ce; 1430 struct i915_request *rq; 1431 unsigned long timeslice; 1432 1433 if (!intel_engine_has_preemption(engine)) 1434 continue; 1435 1436 ce = intel_context_create(engine); 1437 if (IS_ERR(ce)) { 1438 err = PTR_ERR(ce); 1439 break; 1440 } 1441 1442 st_engine_heartbeat_disable(engine); 1443 timeslice = xchg(&engine->props.timeslice_duration_ms, 1); 1444 1445 /* Create an unpreemptible spinner */ 1446 1447 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK); 1448 intel_context_put(ce); 1449 if (IS_ERR(rq)) { 1450 err = PTR_ERR(rq); 1451 goto out_heartbeat; 1452 } 1453 1454 i915_request_get(rq); 1455 i915_request_add(rq); 1456 1457 if (!igt_wait_for_spinner(&spin, rq)) { 1458 i915_request_put(rq); 1459 err = -ETIME; 1460 goto out_spin; 1461 } 1462 1463 set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags); 1464 i915_request_put(rq); 1465 1466 /* Followed by a maximum priority barrier (heartbeat) */ 1467 1468 ce = intel_context_create(engine); 1469 if (IS_ERR(ce)) { 1470 err = PTR_ERR(ce); 1471 goto out_spin; 1472 } 1473 1474 rq = intel_context_create_request(ce); 1475 intel_context_put(ce); 1476 if (IS_ERR(rq)) { 1477 err = PTR_ERR(rq); 1478 goto out_spin; 1479 } 1480 1481 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 1482 i915_request_get(rq); 1483 i915_request_add(rq); 1484 1485 /* 1486 * Wait until the barrier is in ELSP, and we know timeslicing 1487 * will have been activated. 1488 */ 1489 if (wait_for_submit(engine, rq, HZ / 2)) { 1490 i915_request_put(rq); 1491 err = -ETIME; 1492 goto out_spin; 1493 } 1494 1495 /* 1496 * Since the ELSP[0] request is unpreemptible, it should not 1497 * allow the maximum priority barrier through. Wait long 1498 * enough to see if it is timesliced in by mistake. 1499 */ 1500 if (i915_request_wait(rq, 0, slice_timeout(engine)) >= 0) { 1501 pr_err("%s: I915_PRIORITY_BARRIER request completed, bypassing no-preempt request\n", 1502 engine->name); 1503 err = -EINVAL; 1504 } 1505 i915_request_put(rq); 1506 1507 out_spin: 1508 igt_spinner_end(&spin); 1509 out_heartbeat: 1510 xchg(&engine->props.timeslice_duration_ms, timeslice); 1511 st_engine_heartbeat_enable(engine); 1512 if (err) 1513 break; 1514 1515 if (igt_flush_test(gt->i915)) { 1516 err = -EIO; 1517 break; 1518 } 1519 } 1520 1521 igt_spinner_fini(&spin); 1522 return err; 1523 } 1524 1525 static int live_busywait_preempt(void *arg) 1526 { 1527 struct intel_gt *gt = arg; 1528 struct i915_gem_context *ctx_hi, *ctx_lo; 1529 struct intel_engine_cs *engine; 1530 struct drm_i915_gem_object *obj; 1531 struct i915_vma *vma; 1532 enum intel_engine_id id; 1533 u32 *map; 1534 int err; 1535 1536 /* 1537 * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can 1538 * preempt the busywaits used to synchronise between rings. 1539 */ 1540 1541 ctx_hi = kernel_context(gt->i915, NULL); 1542 if (IS_ERR(ctx_hi)) 1543 return PTR_ERR(ctx_hi); 1544 1545 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY; 1546 1547 ctx_lo = kernel_context(gt->i915, NULL); 1548 if (IS_ERR(ctx_lo)) { 1549 err = PTR_ERR(ctx_lo); 1550 goto err_ctx_hi; 1551 } 1552 1553 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY; 1554 1555 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); 1556 if (IS_ERR(obj)) { 1557 err = PTR_ERR(obj); 1558 goto err_ctx_lo; 1559 } 1560 1561 map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 1562 if (IS_ERR(map)) { 1563 err = PTR_ERR(map); 1564 goto err_obj; 1565 } 1566 1567 vma = i915_vma_instance(obj, >->ggtt->vm, NULL); 1568 if (IS_ERR(vma)) { 1569 err = PTR_ERR(vma); 1570 goto err_map; 1571 } 1572 1573 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 1574 if (err) 1575 goto err_map; 1576 1577 err = i915_vma_sync(vma); 1578 if (err) 1579 goto err_vma; 1580 1581 for_each_engine(engine, gt, id) { 1582 struct i915_request *lo, *hi; 1583 struct igt_live_test t; 1584 u32 *cs; 1585 1586 if (!intel_engine_has_preemption(engine)) 1587 continue; 1588 1589 if (!intel_engine_can_store_dword(engine)) 1590 continue; 1591 1592 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 1593 err = -EIO; 1594 goto err_vma; 1595 } 1596 1597 /* 1598 * We create two requests. The low priority request 1599 * busywaits on a semaphore (inside the ringbuffer where 1600 * is should be preemptible) and the high priority requests 1601 * uses a MI_STORE_DWORD_IMM to update the semaphore value 1602 * allowing the first request to complete. If preemption 1603 * fails, we hang instead. 1604 */ 1605 1606 lo = igt_request_alloc(ctx_lo, engine); 1607 if (IS_ERR(lo)) { 1608 err = PTR_ERR(lo); 1609 goto err_vma; 1610 } 1611 1612 cs = intel_ring_begin(lo, 8); 1613 if (IS_ERR(cs)) { 1614 err = PTR_ERR(cs); 1615 i915_request_add(lo); 1616 goto err_vma; 1617 } 1618 1619 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 1620 *cs++ = i915_ggtt_offset(vma); 1621 *cs++ = 0; 1622 *cs++ = 1; 1623 1624 /* XXX Do we need a flush + invalidate here? */ 1625 1626 *cs++ = MI_SEMAPHORE_WAIT | 1627 MI_SEMAPHORE_GLOBAL_GTT | 1628 MI_SEMAPHORE_POLL | 1629 MI_SEMAPHORE_SAD_EQ_SDD; 1630 *cs++ = 0; 1631 *cs++ = i915_ggtt_offset(vma); 1632 *cs++ = 0; 1633 1634 intel_ring_advance(lo, cs); 1635 1636 i915_request_get(lo); 1637 i915_request_add(lo); 1638 1639 if (wait_for(READ_ONCE(*map), 10)) { 1640 i915_request_put(lo); 1641 err = -ETIMEDOUT; 1642 goto err_vma; 1643 } 1644 1645 /* Low priority request should be busywaiting now */ 1646 if (i915_request_wait(lo, 0, 1) != -ETIME) { 1647 i915_request_put(lo); 1648 pr_err("%s: Busywaiting request did not!\n", 1649 engine->name); 1650 err = -EIO; 1651 goto err_vma; 1652 } 1653 1654 hi = igt_request_alloc(ctx_hi, engine); 1655 if (IS_ERR(hi)) { 1656 err = PTR_ERR(hi); 1657 i915_request_put(lo); 1658 goto err_vma; 1659 } 1660 1661 cs = intel_ring_begin(hi, 4); 1662 if (IS_ERR(cs)) { 1663 err = PTR_ERR(cs); 1664 i915_request_add(hi); 1665 i915_request_put(lo); 1666 goto err_vma; 1667 } 1668 1669 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 1670 *cs++ = i915_ggtt_offset(vma); 1671 *cs++ = 0; 1672 *cs++ = 0; 1673 1674 intel_ring_advance(hi, cs); 1675 i915_request_add(hi); 1676 1677 if (i915_request_wait(lo, 0, HZ / 5) < 0) { 1678 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 1679 1680 pr_err("%s: Failed to preempt semaphore busywait!\n", 1681 engine->name); 1682 1683 intel_engine_dump(engine, &p, "%s\n", engine->name); 1684 GEM_TRACE_DUMP(); 1685 1686 i915_request_put(lo); 1687 intel_gt_set_wedged(gt); 1688 err = -EIO; 1689 goto err_vma; 1690 } 1691 GEM_BUG_ON(READ_ONCE(*map)); 1692 i915_request_put(lo); 1693 1694 if (igt_live_test_end(&t)) { 1695 err = -EIO; 1696 goto err_vma; 1697 } 1698 } 1699 1700 err = 0; 1701 err_vma: 1702 i915_vma_unpin(vma); 1703 err_map: 1704 i915_gem_object_unpin_map(obj); 1705 err_obj: 1706 i915_gem_object_put(obj); 1707 err_ctx_lo: 1708 kernel_context_close(ctx_lo); 1709 err_ctx_hi: 1710 kernel_context_close(ctx_hi); 1711 return err; 1712 } 1713 1714 static struct i915_request * 1715 spinner_create_request(struct igt_spinner *spin, 1716 struct i915_gem_context *ctx, 1717 struct intel_engine_cs *engine, 1718 u32 arb) 1719 { 1720 struct intel_context *ce; 1721 struct i915_request *rq; 1722 1723 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx); 1724 if (IS_ERR(ce)) 1725 return ERR_CAST(ce); 1726 1727 rq = igt_spinner_create_request(spin, ce, arb); 1728 intel_context_put(ce); 1729 return rq; 1730 } 1731 1732 static int live_preempt(void *arg) 1733 { 1734 struct intel_gt *gt = arg; 1735 struct i915_gem_context *ctx_hi, *ctx_lo; 1736 struct igt_spinner spin_hi, spin_lo; 1737 struct intel_engine_cs *engine; 1738 enum intel_engine_id id; 1739 int err = -ENOMEM; 1740 1741 ctx_hi = kernel_context(gt->i915, NULL); 1742 if (!ctx_hi) 1743 return -ENOMEM; 1744 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY; 1745 1746 ctx_lo = kernel_context(gt->i915, NULL); 1747 if (!ctx_lo) 1748 goto err_ctx_hi; 1749 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY; 1750 1751 if (igt_spinner_init(&spin_hi, gt)) 1752 goto err_ctx_lo; 1753 1754 if (igt_spinner_init(&spin_lo, gt)) 1755 goto err_spin_hi; 1756 1757 for_each_engine(engine, gt, id) { 1758 struct igt_live_test t; 1759 struct i915_request *rq; 1760 1761 if (!intel_engine_has_preemption(engine)) 1762 continue; 1763 1764 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 1765 err = -EIO; 1766 goto err_spin_lo; 1767 } 1768 1769 rq = spinner_create_request(&spin_lo, ctx_lo, engine, 1770 MI_ARB_CHECK); 1771 if (IS_ERR(rq)) { 1772 err = PTR_ERR(rq); 1773 goto err_spin_lo; 1774 } 1775 1776 i915_request_add(rq); 1777 if (!igt_wait_for_spinner(&spin_lo, rq)) { 1778 GEM_TRACE("lo spinner failed to start\n"); 1779 GEM_TRACE_DUMP(); 1780 intel_gt_set_wedged(gt); 1781 err = -EIO; 1782 goto err_spin_lo; 1783 } 1784 1785 rq = spinner_create_request(&spin_hi, ctx_hi, engine, 1786 MI_ARB_CHECK); 1787 if (IS_ERR(rq)) { 1788 igt_spinner_end(&spin_lo); 1789 err = PTR_ERR(rq); 1790 goto err_spin_lo; 1791 } 1792 1793 i915_request_add(rq); 1794 if (!igt_wait_for_spinner(&spin_hi, rq)) { 1795 GEM_TRACE("hi spinner failed to start\n"); 1796 GEM_TRACE_DUMP(); 1797 intel_gt_set_wedged(gt); 1798 err = -EIO; 1799 goto err_spin_lo; 1800 } 1801 1802 igt_spinner_end(&spin_hi); 1803 igt_spinner_end(&spin_lo); 1804 1805 if (igt_live_test_end(&t)) { 1806 err = -EIO; 1807 goto err_spin_lo; 1808 } 1809 } 1810 1811 err = 0; 1812 err_spin_lo: 1813 igt_spinner_fini(&spin_lo); 1814 err_spin_hi: 1815 igt_spinner_fini(&spin_hi); 1816 err_ctx_lo: 1817 kernel_context_close(ctx_lo); 1818 err_ctx_hi: 1819 kernel_context_close(ctx_hi); 1820 return err; 1821 } 1822 1823 static int live_late_preempt(void *arg) 1824 { 1825 struct intel_gt *gt = arg; 1826 struct i915_gem_context *ctx_hi, *ctx_lo; 1827 struct igt_spinner spin_hi, spin_lo; 1828 struct intel_engine_cs *engine; 1829 struct i915_sched_attr attr = {}; 1830 enum intel_engine_id id; 1831 int err = -ENOMEM; 1832 1833 ctx_hi = kernel_context(gt->i915, NULL); 1834 if (!ctx_hi) 1835 return -ENOMEM; 1836 1837 ctx_lo = kernel_context(gt->i915, NULL); 1838 if (!ctx_lo) 1839 goto err_ctx_hi; 1840 1841 if (igt_spinner_init(&spin_hi, gt)) 1842 goto err_ctx_lo; 1843 1844 if (igt_spinner_init(&spin_lo, gt)) 1845 goto err_spin_hi; 1846 1847 /* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */ 1848 ctx_lo->sched.priority = 1; 1849 1850 for_each_engine(engine, gt, id) { 1851 struct igt_live_test t; 1852 struct i915_request *rq; 1853 1854 if (!intel_engine_has_preemption(engine)) 1855 continue; 1856 1857 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 1858 err = -EIO; 1859 goto err_spin_lo; 1860 } 1861 1862 rq = spinner_create_request(&spin_lo, ctx_lo, engine, 1863 MI_ARB_CHECK); 1864 if (IS_ERR(rq)) { 1865 err = PTR_ERR(rq); 1866 goto err_spin_lo; 1867 } 1868 1869 i915_request_add(rq); 1870 if (!igt_wait_for_spinner(&spin_lo, rq)) { 1871 pr_err("First context failed to start\n"); 1872 goto err_wedged; 1873 } 1874 1875 rq = spinner_create_request(&spin_hi, ctx_hi, engine, 1876 MI_NOOP); 1877 if (IS_ERR(rq)) { 1878 igt_spinner_end(&spin_lo); 1879 err = PTR_ERR(rq); 1880 goto err_spin_lo; 1881 } 1882 1883 i915_request_add(rq); 1884 if (igt_wait_for_spinner(&spin_hi, rq)) { 1885 pr_err("Second context overtook first?\n"); 1886 goto err_wedged; 1887 } 1888 1889 attr.priority = I915_PRIORITY_MAX; 1890 engine->sched_engine->schedule(rq, &attr); 1891 1892 if (!igt_wait_for_spinner(&spin_hi, rq)) { 1893 pr_err("High priority context failed to preempt the low priority context\n"); 1894 GEM_TRACE_DUMP(); 1895 goto err_wedged; 1896 } 1897 1898 igt_spinner_end(&spin_hi); 1899 igt_spinner_end(&spin_lo); 1900 1901 if (igt_live_test_end(&t)) { 1902 err = -EIO; 1903 goto err_spin_lo; 1904 } 1905 } 1906 1907 err = 0; 1908 err_spin_lo: 1909 igt_spinner_fini(&spin_lo); 1910 err_spin_hi: 1911 igt_spinner_fini(&spin_hi); 1912 err_ctx_lo: 1913 kernel_context_close(ctx_lo); 1914 err_ctx_hi: 1915 kernel_context_close(ctx_hi); 1916 return err; 1917 1918 err_wedged: 1919 igt_spinner_end(&spin_hi); 1920 igt_spinner_end(&spin_lo); 1921 intel_gt_set_wedged(gt); 1922 err = -EIO; 1923 goto err_spin_lo; 1924 } 1925 1926 struct preempt_client { 1927 struct igt_spinner spin; 1928 struct i915_gem_context *ctx; 1929 }; 1930 1931 static int preempt_client_init(struct intel_gt *gt, struct preempt_client *c) 1932 { 1933 c->ctx = kernel_context(gt->i915, NULL); 1934 if (!c->ctx) 1935 return -ENOMEM; 1936 1937 if (igt_spinner_init(&c->spin, gt)) 1938 goto err_ctx; 1939 1940 return 0; 1941 1942 err_ctx: 1943 kernel_context_close(c->ctx); 1944 return -ENOMEM; 1945 } 1946 1947 static void preempt_client_fini(struct preempt_client *c) 1948 { 1949 igt_spinner_fini(&c->spin); 1950 kernel_context_close(c->ctx); 1951 } 1952 1953 static int live_nopreempt(void *arg) 1954 { 1955 struct intel_gt *gt = arg; 1956 struct intel_engine_cs *engine; 1957 struct preempt_client a, b; 1958 enum intel_engine_id id; 1959 int err = -ENOMEM; 1960 1961 /* 1962 * Verify that we can disable preemption for an individual request 1963 * that may be being observed and not want to be interrupted. 1964 */ 1965 1966 if (preempt_client_init(gt, &a)) 1967 return -ENOMEM; 1968 if (preempt_client_init(gt, &b)) 1969 goto err_client_a; 1970 b.ctx->sched.priority = I915_PRIORITY_MAX; 1971 1972 for_each_engine(engine, gt, id) { 1973 struct i915_request *rq_a, *rq_b; 1974 1975 if (!intel_engine_has_preemption(engine)) 1976 continue; 1977 1978 engine->execlists.preempt_hang.count = 0; 1979 1980 rq_a = spinner_create_request(&a.spin, 1981 a.ctx, engine, 1982 MI_ARB_CHECK); 1983 if (IS_ERR(rq_a)) { 1984 err = PTR_ERR(rq_a); 1985 goto err_client_b; 1986 } 1987 1988 /* Low priority client, but unpreemptable! */ 1989 __set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq_a->fence.flags); 1990 1991 i915_request_add(rq_a); 1992 if (!igt_wait_for_spinner(&a.spin, rq_a)) { 1993 pr_err("First client failed to start\n"); 1994 goto err_wedged; 1995 } 1996 1997 rq_b = spinner_create_request(&b.spin, 1998 b.ctx, engine, 1999 MI_ARB_CHECK); 2000 if (IS_ERR(rq_b)) { 2001 err = PTR_ERR(rq_b); 2002 goto err_client_b; 2003 } 2004 2005 i915_request_add(rq_b); 2006 2007 /* B is much more important than A! (But A is unpreemptable.) */ 2008 GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a)); 2009 2010 /* Wait long enough for preemption and timeslicing */ 2011 if (igt_wait_for_spinner(&b.spin, rq_b)) { 2012 pr_err("Second client started too early!\n"); 2013 goto err_wedged; 2014 } 2015 2016 igt_spinner_end(&a.spin); 2017 2018 if (!igt_wait_for_spinner(&b.spin, rq_b)) { 2019 pr_err("Second client failed to start\n"); 2020 goto err_wedged; 2021 } 2022 2023 igt_spinner_end(&b.spin); 2024 2025 if (engine->execlists.preempt_hang.count) { 2026 pr_err("Preemption recorded x%d; should have been suppressed!\n", 2027 engine->execlists.preempt_hang.count); 2028 err = -EINVAL; 2029 goto err_wedged; 2030 } 2031 2032 if (igt_flush_test(gt->i915)) 2033 goto err_wedged; 2034 } 2035 2036 err = 0; 2037 err_client_b: 2038 preempt_client_fini(&b); 2039 err_client_a: 2040 preempt_client_fini(&a); 2041 return err; 2042 2043 err_wedged: 2044 igt_spinner_end(&b.spin); 2045 igt_spinner_end(&a.spin); 2046 intel_gt_set_wedged(gt); 2047 err = -EIO; 2048 goto err_client_b; 2049 } 2050 2051 struct live_preempt_cancel { 2052 struct intel_engine_cs *engine; 2053 struct preempt_client a, b; 2054 }; 2055 2056 static int __cancel_active0(struct live_preempt_cancel *arg) 2057 { 2058 struct i915_request *rq; 2059 struct igt_live_test t; 2060 int err; 2061 2062 /* Preempt cancel of ELSP0 */ 2063 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name); 2064 if (igt_live_test_begin(&t, arg->engine->i915, 2065 __func__, arg->engine->name)) 2066 return -EIO; 2067 2068 rq = spinner_create_request(&arg->a.spin, 2069 arg->a.ctx, arg->engine, 2070 MI_ARB_CHECK); 2071 if (IS_ERR(rq)) 2072 return PTR_ERR(rq); 2073 2074 clear_bit(CONTEXT_BANNED, &rq->context->flags); 2075 i915_request_get(rq); 2076 i915_request_add(rq); 2077 if (!igt_wait_for_spinner(&arg->a.spin, rq)) { 2078 err = -EIO; 2079 goto out; 2080 } 2081 2082 intel_context_ban(rq->context, rq); 2083 err = intel_engine_pulse(arg->engine); 2084 if (err) 2085 goto out; 2086 2087 err = wait_for_reset(arg->engine, rq, HZ / 2); 2088 if (err) { 2089 pr_err("Cancelled inflight0 request did not reset\n"); 2090 goto out; 2091 } 2092 2093 out: 2094 i915_request_put(rq); 2095 if (igt_live_test_end(&t)) 2096 err = -EIO; 2097 return err; 2098 } 2099 2100 static int __cancel_active1(struct live_preempt_cancel *arg) 2101 { 2102 struct i915_request *rq[2] = {}; 2103 struct igt_live_test t; 2104 int err; 2105 2106 /* Preempt cancel of ELSP1 */ 2107 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name); 2108 if (igt_live_test_begin(&t, arg->engine->i915, 2109 __func__, arg->engine->name)) 2110 return -EIO; 2111 2112 rq[0] = spinner_create_request(&arg->a.spin, 2113 arg->a.ctx, arg->engine, 2114 MI_NOOP); /* no preemption */ 2115 if (IS_ERR(rq[0])) 2116 return PTR_ERR(rq[0]); 2117 2118 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags); 2119 i915_request_get(rq[0]); 2120 i915_request_add(rq[0]); 2121 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) { 2122 err = -EIO; 2123 goto out; 2124 } 2125 2126 rq[1] = spinner_create_request(&arg->b.spin, 2127 arg->b.ctx, arg->engine, 2128 MI_ARB_CHECK); 2129 if (IS_ERR(rq[1])) { 2130 err = PTR_ERR(rq[1]); 2131 goto out; 2132 } 2133 2134 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags); 2135 i915_request_get(rq[1]); 2136 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence); 2137 i915_request_add(rq[1]); 2138 if (err) 2139 goto out; 2140 2141 intel_context_ban(rq[1]->context, rq[1]); 2142 err = intel_engine_pulse(arg->engine); 2143 if (err) 2144 goto out; 2145 2146 igt_spinner_end(&arg->a.spin); 2147 err = wait_for_reset(arg->engine, rq[1], HZ / 2); 2148 if (err) 2149 goto out; 2150 2151 if (rq[0]->fence.error != 0) { 2152 pr_err("Normal inflight0 request did not complete\n"); 2153 err = -EINVAL; 2154 goto out; 2155 } 2156 2157 if (rq[1]->fence.error != -EIO) { 2158 pr_err("Cancelled inflight1 request did not report -EIO\n"); 2159 err = -EINVAL; 2160 goto out; 2161 } 2162 2163 out: 2164 i915_request_put(rq[1]); 2165 i915_request_put(rq[0]); 2166 if (igt_live_test_end(&t)) 2167 err = -EIO; 2168 return err; 2169 } 2170 2171 static int __cancel_queued(struct live_preempt_cancel *arg) 2172 { 2173 struct i915_request *rq[3] = {}; 2174 struct igt_live_test t; 2175 int err; 2176 2177 /* Full ELSP and one in the wings */ 2178 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name); 2179 if (igt_live_test_begin(&t, arg->engine->i915, 2180 __func__, arg->engine->name)) 2181 return -EIO; 2182 2183 rq[0] = spinner_create_request(&arg->a.spin, 2184 arg->a.ctx, arg->engine, 2185 MI_ARB_CHECK); 2186 if (IS_ERR(rq[0])) 2187 return PTR_ERR(rq[0]); 2188 2189 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags); 2190 i915_request_get(rq[0]); 2191 i915_request_add(rq[0]); 2192 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) { 2193 err = -EIO; 2194 goto out; 2195 } 2196 2197 rq[1] = igt_request_alloc(arg->b.ctx, arg->engine); 2198 if (IS_ERR(rq[1])) { 2199 err = PTR_ERR(rq[1]); 2200 goto out; 2201 } 2202 2203 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags); 2204 i915_request_get(rq[1]); 2205 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence); 2206 i915_request_add(rq[1]); 2207 if (err) 2208 goto out; 2209 2210 rq[2] = spinner_create_request(&arg->b.spin, 2211 arg->a.ctx, arg->engine, 2212 MI_ARB_CHECK); 2213 if (IS_ERR(rq[2])) { 2214 err = PTR_ERR(rq[2]); 2215 goto out; 2216 } 2217 2218 i915_request_get(rq[2]); 2219 err = i915_request_await_dma_fence(rq[2], &rq[1]->fence); 2220 i915_request_add(rq[2]); 2221 if (err) 2222 goto out; 2223 2224 intel_context_ban(rq[2]->context, rq[2]); 2225 err = intel_engine_pulse(arg->engine); 2226 if (err) 2227 goto out; 2228 2229 err = wait_for_reset(arg->engine, rq[2], HZ / 2); 2230 if (err) 2231 goto out; 2232 2233 if (rq[0]->fence.error != -EIO) { 2234 pr_err("Cancelled inflight0 request did not report -EIO\n"); 2235 err = -EINVAL; 2236 goto out; 2237 } 2238 2239 /* 2240 * The behavior between having semaphores and not is different. With 2241 * semaphores the subsequent request is on the hardware and not cancelled 2242 * while without the request is held in the driver and cancelled. 2243 */ 2244 if (intel_engine_has_semaphores(rq[1]->engine) && 2245 rq[1]->fence.error != 0) { 2246 pr_err("Normal inflight1 request did not complete\n"); 2247 err = -EINVAL; 2248 goto out; 2249 } 2250 2251 if (rq[2]->fence.error != -EIO) { 2252 pr_err("Cancelled queued request did not report -EIO\n"); 2253 err = -EINVAL; 2254 goto out; 2255 } 2256 2257 out: 2258 i915_request_put(rq[2]); 2259 i915_request_put(rq[1]); 2260 i915_request_put(rq[0]); 2261 if (igt_live_test_end(&t)) 2262 err = -EIO; 2263 return err; 2264 } 2265 2266 static int __cancel_hostile(struct live_preempt_cancel *arg) 2267 { 2268 struct i915_request *rq; 2269 int err; 2270 2271 /* Preempt cancel non-preemptible spinner in ELSP0 */ 2272 if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT) 2273 return 0; 2274 2275 if (!intel_has_reset_engine(arg->engine->gt)) 2276 return 0; 2277 2278 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name); 2279 rq = spinner_create_request(&arg->a.spin, 2280 arg->a.ctx, arg->engine, 2281 MI_NOOP); /* preemption disabled */ 2282 if (IS_ERR(rq)) 2283 return PTR_ERR(rq); 2284 2285 clear_bit(CONTEXT_BANNED, &rq->context->flags); 2286 i915_request_get(rq); 2287 i915_request_add(rq); 2288 if (!igt_wait_for_spinner(&arg->a.spin, rq)) { 2289 err = -EIO; 2290 goto out; 2291 } 2292 2293 intel_context_ban(rq->context, rq); 2294 err = intel_engine_pulse(arg->engine); /* force reset */ 2295 if (err) 2296 goto out; 2297 2298 err = wait_for_reset(arg->engine, rq, HZ / 2); 2299 if (err) { 2300 pr_err("Cancelled inflight0 request did not reset\n"); 2301 goto out; 2302 } 2303 2304 out: 2305 i915_request_put(rq); 2306 if (igt_flush_test(arg->engine->i915)) 2307 err = -EIO; 2308 return err; 2309 } 2310 2311 static void force_reset_timeout(struct intel_engine_cs *engine) 2312 { 2313 engine->reset_timeout.probability = 999; 2314 atomic_set(&engine->reset_timeout.times, -1); 2315 } 2316 2317 static void cancel_reset_timeout(struct intel_engine_cs *engine) 2318 { 2319 memset(&engine->reset_timeout, 0, sizeof(engine->reset_timeout)); 2320 } 2321 2322 static int __cancel_fail(struct live_preempt_cancel *arg) 2323 { 2324 struct intel_engine_cs *engine = arg->engine; 2325 struct i915_request *rq; 2326 int err; 2327 2328 if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT) 2329 return 0; 2330 2331 if (!intel_has_reset_engine(engine->gt)) 2332 return 0; 2333 2334 GEM_TRACE("%s(%s)\n", __func__, engine->name); 2335 rq = spinner_create_request(&arg->a.spin, 2336 arg->a.ctx, engine, 2337 MI_NOOP); /* preemption disabled */ 2338 if (IS_ERR(rq)) 2339 return PTR_ERR(rq); 2340 2341 clear_bit(CONTEXT_BANNED, &rq->context->flags); 2342 i915_request_get(rq); 2343 i915_request_add(rq); 2344 if (!igt_wait_for_spinner(&arg->a.spin, rq)) { 2345 err = -EIO; 2346 goto out; 2347 } 2348 2349 intel_context_set_banned(rq->context); 2350 2351 err = intel_engine_pulse(engine); 2352 if (err) 2353 goto out; 2354 2355 force_reset_timeout(engine); 2356 2357 /* force preempt reset [failure] */ 2358 while (!engine->execlists.pending[0]) 2359 intel_engine_flush_submission(engine); 2360 del_timer_sync(&engine->execlists.preempt); 2361 intel_engine_flush_submission(engine); 2362 2363 cancel_reset_timeout(engine); 2364 2365 /* after failure, require heartbeats to reset device */ 2366 intel_engine_set_heartbeat(engine, 1); 2367 err = wait_for_reset(engine, rq, HZ / 2); 2368 intel_engine_set_heartbeat(engine, 2369 engine->defaults.heartbeat_interval_ms); 2370 if (err) { 2371 pr_err("Cancelled inflight0 request did not reset\n"); 2372 goto out; 2373 } 2374 2375 out: 2376 i915_request_put(rq); 2377 if (igt_flush_test(engine->i915)) 2378 err = -EIO; 2379 return err; 2380 } 2381 2382 static int live_preempt_cancel(void *arg) 2383 { 2384 struct intel_gt *gt = arg; 2385 struct live_preempt_cancel data; 2386 enum intel_engine_id id; 2387 int err = -ENOMEM; 2388 2389 /* 2390 * To cancel an inflight context, we need to first remove it from the 2391 * GPU. That sounds like preemption! Plus a little bit of bookkeeping. 2392 */ 2393 2394 if (preempt_client_init(gt, &data.a)) 2395 return -ENOMEM; 2396 if (preempt_client_init(gt, &data.b)) 2397 goto err_client_a; 2398 2399 for_each_engine(data.engine, gt, id) { 2400 if (!intel_engine_has_preemption(data.engine)) 2401 continue; 2402 2403 err = __cancel_active0(&data); 2404 if (err) 2405 goto err_wedged; 2406 2407 err = __cancel_active1(&data); 2408 if (err) 2409 goto err_wedged; 2410 2411 err = __cancel_queued(&data); 2412 if (err) 2413 goto err_wedged; 2414 2415 err = __cancel_hostile(&data); 2416 if (err) 2417 goto err_wedged; 2418 2419 err = __cancel_fail(&data); 2420 if (err) 2421 goto err_wedged; 2422 } 2423 2424 err = 0; 2425 err_client_b: 2426 preempt_client_fini(&data.b); 2427 err_client_a: 2428 preempt_client_fini(&data.a); 2429 return err; 2430 2431 err_wedged: 2432 GEM_TRACE_DUMP(); 2433 igt_spinner_end(&data.b.spin); 2434 igt_spinner_end(&data.a.spin); 2435 intel_gt_set_wedged(gt); 2436 goto err_client_b; 2437 } 2438 2439 static int live_suppress_self_preempt(void *arg) 2440 { 2441 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX }; 2442 struct intel_gt *gt = arg; 2443 struct intel_engine_cs *engine; 2444 struct preempt_client a, b; 2445 enum intel_engine_id id; 2446 int err = -ENOMEM; 2447 2448 /* 2449 * Verify that if a preemption request does not cause a change in 2450 * the current execution order, the preempt-to-idle injection is 2451 * skipped and that we do not accidentally apply it after the CS 2452 * completion event. 2453 */ 2454 2455 if (intel_uc_uses_guc_submission(>->uc)) 2456 return 0; /* presume black blox */ 2457 2458 if (intel_vgpu_active(gt->i915)) 2459 return 0; /* GVT forces single port & request submission */ 2460 2461 if (preempt_client_init(gt, &a)) 2462 return -ENOMEM; 2463 if (preempt_client_init(gt, &b)) 2464 goto err_client_a; 2465 2466 for_each_engine(engine, gt, id) { 2467 struct i915_request *rq_a, *rq_b; 2468 int depth; 2469 2470 if (!intel_engine_has_preemption(engine)) 2471 continue; 2472 2473 if (igt_flush_test(gt->i915)) 2474 goto err_wedged; 2475 2476 st_engine_heartbeat_disable(engine); 2477 engine->execlists.preempt_hang.count = 0; 2478 2479 rq_a = spinner_create_request(&a.spin, 2480 a.ctx, engine, 2481 MI_NOOP); 2482 if (IS_ERR(rq_a)) { 2483 err = PTR_ERR(rq_a); 2484 st_engine_heartbeat_enable(engine); 2485 goto err_client_b; 2486 } 2487 2488 i915_request_add(rq_a); 2489 if (!igt_wait_for_spinner(&a.spin, rq_a)) { 2490 pr_err("First client failed to start\n"); 2491 st_engine_heartbeat_enable(engine); 2492 goto err_wedged; 2493 } 2494 2495 /* Keep postponing the timer to avoid premature slicing */ 2496 mod_timer(&engine->execlists.timer, jiffies + HZ); 2497 for (depth = 0; depth < 8; depth++) { 2498 rq_b = spinner_create_request(&b.spin, 2499 b.ctx, engine, 2500 MI_NOOP); 2501 if (IS_ERR(rq_b)) { 2502 err = PTR_ERR(rq_b); 2503 st_engine_heartbeat_enable(engine); 2504 goto err_client_b; 2505 } 2506 i915_request_add(rq_b); 2507 2508 GEM_BUG_ON(i915_request_completed(rq_a)); 2509 engine->sched_engine->schedule(rq_a, &attr); 2510 igt_spinner_end(&a.spin); 2511 2512 if (!igt_wait_for_spinner(&b.spin, rq_b)) { 2513 pr_err("Second client failed to start\n"); 2514 st_engine_heartbeat_enable(engine); 2515 goto err_wedged; 2516 } 2517 2518 swap(a, b); 2519 rq_a = rq_b; 2520 } 2521 igt_spinner_end(&a.spin); 2522 2523 if (engine->execlists.preempt_hang.count) { 2524 pr_err("Preemption on %s recorded x%d, depth %d; should have been suppressed!\n", 2525 engine->name, 2526 engine->execlists.preempt_hang.count, 2527 depth); 2528 st_engine_heartbeat_enable(engine); 2529 err = -EINVAL; 2530 goto err_client_b; 2531 } 2532 2533 st_engine_heartbeat_enable(engine); 2534 if (igt_flush_test(gt->i915)) 2535 goto err_wedged; 2536 } 2537 2538 err = 0; 2539 err_client_b: 2540 preempt_client_fini(&b); 2541 err_client_a: 2542 preempt_client_fini(&a); 2543 return err; 2544 2545 err_wedged: 2546 igt_spinner_end(&b.spin); 2547 igt_spinner_end(&a.spin); 2548 intel_gt_set_wedged(gt); 2549 err = -EIO; 2550 goto err_client_b; 2551 } 2552 2553 static int live_chain_preempt(void *arg) 2554 { 2555 struct intel_gt *gt = arg; 2556 struct intel_engine_cs *engine; 2557 struct preempt_client hi, lo; 2558 enum intel_engine_id id; 2559 int err = -ENOMEM; 2560 2561 /* 2562 * Build a chain AB...BA between two contexts (A, B) and request 2563 * preemption of the last request. It should then complete before 2564 * the previously submitted spinner in B. 2565 */ 2566 2567 if (preempt_client_init(gt, &hi)) 2568 return -ENOMEM; 2569 2570 if (preempt_client_init(gt, &lo)) 2571 goto err_client_hi; 2572 2573 for_each_engine(engine, gt, id) { 2574 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX }; 2575 struct igt_live_test t; 2576 struct i915_request *rq; 2577 int ring_size, count, i; 2578 2579 if (!intel_engine_has_preemption(engine)) 2580 continue; 2581 2582 rq = spinner_create_request(&lo.spin, 2583 lo.ctx, engine, 2584 MI_ARB_CHECK); 2585 if (IS_ERR(rq)) 2586 goto err_wedged; 2587 2588 i915_request_get(rq); 2589 i915_request_add(rq); 2590 2591 ring_size = rq->wa_tail - rq->head; 2592 if (ring_size < 0) 2593 ring_size += rq->ring->size; 2594 ring_size = rq->ring->size / ring_size; 2595 pr_debug("%s(%s): Using maximum of %d requests\n", 2596 __func__, engine->name, ring_size); 2597 2598 igt_spinner_end(&lo.spin); 2599 if (i915_request_wait(rq, 0, HZ / 2) < 0) { 2600 pr_err("Timed out waiting to flush %s\n", engine->name); 2601 i915_request_put(rq); 2602 goto err_wedged; 2603 } 2604 i915_request_put(rq); 2605 2606 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 2607 err = -EIO; 2608 goto err_wedged; 2609 } 2610 2611 for_each_prime_number_from(count, 1, ring_size) { 2612 rq = spinner_create_request(&hi.spin, 2613 hi.ctx, engine, 2614 MI_ARB_CHECK); 2615 if (IS_ERR(rq)) 2616 goto err_wedged; 2617 i915_request_add(rq); 2618 if (!igt_wait_for_spinner(&hi.spin, rq)) 2619 goto err_wedged; 2620 2621 rq = spinner_create_request(&lo.spin, 2622 lo.ctx, engine, 2623 MI_ARB_CHECK); 2624 if (IS_ERR(rq)) 2625 goto err_wedged; 2626 i915_request_add(rq); 2627 2628 for (i = 0; i < count; i++) { 2629 rq = igt_request_alloc(lo.ctx, engine); 2630 if (IS_ERR(rq)) 2631 goto err_wedged; 2632 i915_request_add(rq); 2633 } 2634 2635 rq = igt_request_alloc(hi.ctx, engine); 2636 if (IS_ERR(rq)) 2637 goto err_wedged; 2638 2639 i915_request_get(rq); 2640 i915_request_add(rq); 2641 engine->sched_engine->schedule(rq, &attr); 2642 2643 igt_spinner_end(&hi.spin); 2644 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 2645 struct drm_printer p = 2646 drm_info_printer(gt->i915->drm.dev); 2647 2648 pr_err("Failed to preempt over chain of %d\n", 2649 count); 2650 intel_engine_dump(engine, &p, 2651 "%s\n", engine->name); 2652 i915_request_put(rq); 2653 goto err_wedged; 2654 } 2655 igt_spinner_end(&lo.spin); 2656 i915_request_put(rq); 2657 2658 rq = igt_request_alloc(lo.ctx, engine); 2659 if (IS_ERR(rq)) 2660 goto err_wedged; 2661 2662 i915_request_get(rq); 2663 i915_request_add(rq); 2664 2665 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 2666 struct drm_printer p = 2667 drm_info_printer(gt->i915->drm.dev); 2668 2669 pr_err("Failed to flush low priority chain of %d requests\n", 2670 count); 2671 intel_engine_dump(engine, &p, 2672 "%s\n", engine->name); 2673 2674 i915_request_put(rq); 2675 goto err_wedged; 2676 } 2677 i915_request_put(rq); 2678 } 2679 2680 if (igt_live_test_end(&t)) { 2681 err = -EIO; 2682 goto err_wedged; 2683 } 2684 } 2685 2686 err = 0; 2687 err_client_lo: 2688 preempt_client_fini(&lo); 2689 err_client_hi: 2690 preempt_client_fini(&hi); 2691 return err; 2692 2693 err_wedged: 2694 igt_spinner_end(&hi.spin); 2695 igt_spinner_end(&lo.spin); 2696 intel_gt_set_wedged(gt); 2697 err = -EIO; 2698 goto err_client_lo; 2699 } 2700 2701 static int create_gang(struct intel_engine_cs *engine, 2702 struct i915_request **prev) 2703 { 2704 struct drm_i915_gem_object *obj; 2705 struct intel_context *ce; 2706 struct i915_request *rq; 2707 struct i915_vma *vma; 2708 u32 *cs; 2709 int err; 2710 2711 ce = intel_context_create(engine); 2712 if (IS_ERR(ce)) 2713 return PTR_ERR(ce); 2714 2715 obj = i915_gem_object_create_internal(engine->i915, 4096); 2716 if (IS_ERR(obj)) { 2717 err = PTR_ERR(obj); 2718 goto err_ce; 2719 } 2720 2721 vma = i915_vma_instance(obj, ce->vm, NULL); 2722 if (IS_ERR(vma)) { 2723 err = PTR_ERR(vma); 2724 goto err_obj; 2725 } 2726 2727 err = i915_vma_pin(vma, 0, 0, PIN_USER); 2728 if (err) 2729 goto err_obj; 2730 2731 cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 2732 if (IS_ERR(cs)) { 2733 err = PTR_ERR(cs); 2734 goto err_obj; 2735 } 2736 2737 /* Semaphore target: spin until zero */ 2738 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 2739 2740 *cs++ = MI_SEMAPHORE_WAIT | 2741 MI_SEMAPHORE_POLL | 2742 MI_SEMAPHORE_SAD_EQ_SDD; 2743 *cs++ = 0; 2744 *cs++ = lower_32_bits(i915_vma_offset(vma)); 2745 *cs++ = upper_32_bits(i915_vma_offset(vma)); 2746 2747 if (*prev) { 2748 u64 offset = i915_vma_offset((*prev)->batch); 2749 2750 /* Terminate the spinner in the next lower priority batch. */ 2751 *cs++ = MI_STORE_DWORD_IMM_GEN4; 2752 *cs++ = lower_32_bits(offset); 2753 *cs++ = upper_32_bits(offset); 2754 *cs++ = 0; 2755 } 2756 2757 *cs++ = MI_BATCH_BUFFER_END; 2758 i915_gem_object_flush_map(obj); 2759 i915_gem_object_unpin_map(obj); 2760 2761 rq = intel_context_create_request(ce); 2762 if (IS_ERR(rq)) { 2763 err = PTR_ERR(rq); 2764 goto err_obj; 2765 } 2766 2767 rq->batch = i915_vma_get(vma); 2768 i915_request_get(rq); 2769 2770 err = igt_vma_move_to_active_unlocked(vma, rq, 0); 2771 if (!err) 2772 err = rq->engine->emit_bb_start(rq, 2773 i915_vma_offset(vma), 2774 PAGE_SIZE, 0); 2775 i915_request_add(rq); 2776 if (err) 2777 goto err_rq; 2778 2779 i915_gem_object_put(obj); 2780 intel_context_put(ce); 2781 2782 rq->mock.link.next = &(*prev)->mock.link; 2783 *prev = rq; 2784 return 0; 2785 2786 err_rq: 2787 i915_vma_put(rq->batch); 2788 i915_request_put(rq); 2789 err_obj: 2790 i915_gem_object_put(obj); 2791 err_ce: 2792 intel_context_put(ce); 2793 return err; 2794 } 2795 2796 static int __live_preempt_ring(struct intel_engine_cs *engine, 2797 struct igt_spinner *spin, 2798 int queue_sz, int ring_sz) 2799 { 2800 struct intel_context *ce[2] = {}; 2801 struct i915_request *rq; 2802 struct igt_live_test t; 2803 int err = 0; 2804 int n; 2805 2806 if (igt_live_test_begin(&t, engine->i915, __func__, engine->name)) 2807 return -EIO; 2808 2809 for (n = 0; n < ARRAY_SIZE(ce); n++) { 2810 struct intel_context *tmp; 2811 2812 tmp = intel_context_create(engine); 2813 if (IS_ERR(tmp)) { 2814 err = PTR_ERR(tmp); 2815 goto err_ce; 2816 } 2817 2818 tmp->ring_size = ring_sz; 2819 2820 err = intel_context_pin(tmp); 2821 if (err) { 2822 intel_context_put(tmp); 2823 goto err_ce; 2824 } 2825 2826 memset32(tmp->ring->vaddr, 2827 0xdeadbeef, /* trigger a hang if executed */ 2828 tmp->ring->vma->size / sizeof(u32)); 2829 2830 ce[n] = tmp; 2831 } 2832 2833 rq = igt_spinner_create_request(spin, ce[0], MI_ARB_CHECK); 2834 if (IS_ERR(rq)) { 2835 err = PTR_ERR(rq); 2836 goto err_ce; 2837 } 2838 2839 i915_request_get(rq); 2840 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 2841 i915_request_add(rq); 2842 2843 if (!igt_wait_for_spinner(spin, rq)) { 2844 intel_gt_set_wedged(engine->gt); 2845 i915_request_put(rq); 2846 err = -ETIME; 2847 goto err_ce; 2848 } 2849 2850 /* Fill the ring, until we will cause a wrap */ 2851 n = 0; 2852 while (ce[0]->ring->tail - rq->wa_tail <= queue_sz) { 2853 struct i915_request *tmp; 2854 2855 tmp = intel_context_create_request(ce[0]); 2856 if (IS_ERR(tmp)) { 2857 err = PTR_ERR(tmp); 2858 i915_request_put(rq); 2859 goto err_ce; 2860 } 2861 2862 i915_request_add(tmp); 2863 intel_engine_flush_submission(engine); 2864 n++; 2865 } 2866 intel_engine_flush_submission(engine); 2867 pr_debug("%s: Filled %d with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n", 2868 engine->name, queue_sz, n, 2869 ce[0]->ring->size, 2870 ce[0]->ring->tail, 2871 ce[0]->ring->emit, 2872 rq->tail); 2873 i915_request_put(rq); 2874 2875 /* Create a second request to preempt the first ring */ 2876 rq = intel_context_create_request(ce[1]); 2877 if (IS_ERR(rq)) { 2878 err = PTR_ERR(rq); 2879 goto err_ce; 2880 } 2881 2882 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 2883 i915_request_get(rq); 2884 i915_request_add(rq); 2885 2886 err = wait_for_submit(engine, rq, HZ / 2); 2887 i915_request_put(rq); 2888 if (err) { 2889 pr_err("%s: preemption request was not submitted\n", 2890 engine->name); 2891 err = -ETIME; 2892 } 2893 2894 pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n", 2895 engine->name, 2896 ce[0]->ring->tail, ce[0]->ring->emit, 2897 ce[1]->ring->tail, ce[1]->ring->emit); 2898 2899 err_ce: 2900 intel_engine_flush_submission(engine); 2901 igt_spinner_end(spin); 2902 for (n = 0; n < ARRAY_SIZE(ce); n++) { 2903 if (IS_ERR_OR_NULL(ce[n])) 2904 break; 2905 2906 intel_context_unpin(ce[n]); 2907 intel_context_put(ce[n]); 2908 } 2909 if (igt_live_test_end(&t)) 2910 err = -EIO; 2911 return err; 2912 } 2913 2914 static int live_preempt_ring(void *arg) 2915 { 2916 struct intel_gt *gt = arg; 2917 struct intel_engine_cs *engine; 2918 struct igt_spinner spin; 2919 enum intel_engine_id id; 2920 int err = 0; 2921 2922 /* 2923 * Check that we rollback large chunks of a ring in order to do a 2924 * preemption event. Similar to live_unlite_ring, but looking at 2925 * ring size rather than the impact of intel_ring_direction(). 2926 */ 2927 2928 if (igt_spinner_init(&spin, gt)) 2929 return -ENOMEM; 2930 2931 for_each_engine(engine, gt, id) { 2932 int n; 2933 2934 if (!intel_engine_has_preemption(engine)) 2935 continue; 2936 2937 if (!intel_engine_can_store_dword(engine)) 2938 continue; 2939 2940 st_engine_heartbeat_disable(engine); 2941 2942 for (n = 0; n <= 3; n++) { 2943 err = __live_preempt_ring(engine, &spin, 2944 n * SZ_4K / 4, SZ_4K); 2945 if (err) 2946 break; 2947 } 2948 2949 st_engine_heartbeat_enable(engine); 2950 if (err) 2951 break; 2952 } 2953 2954 igt_spinner_fini(&spin); 2955 return err; 2956 } 2957 2958 static int live_preempt_gang(void *arg) 2959 { 2960 struct intel_gt *gt = arg; 2961 struct intel_engine_cs *engine; 2962 enum intel_engine_id id; 2963 2964 /* 2965 * Build as long a chain of preempters as we can, with each 2966 * request higher priority than the last. Once we are ready, we release 2967 * the last batch which then precolates down the chain, each releasing 2968 * the next oldest in turn. The intent is to simply push as hard as we 2969 * can with the number of preemptions, trying to exceed narrow HW 2970 * limits. At a minimum, we insist that we can sort all the user 2971 * high priority levels into execution order. 2972 */ 2973 2974 for_each_engine(engine, gt, id) { 2975 struct i915_request *rq = NULL; 2976 struct igt_live_test t; 2977 IGT_TIMEOUT(end_time); 2978 int prio = 0; 2979 int err = 0; 2980 u32 *cs; 2981 2982 if (!intel_engine_has_preemption(engine)) 2983 continue; 2984 2985 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) 2986 return -EIO; 2987 2988 do { 2989 struct i915_sched_attr attr = { .priority = prio++ }; 2990 2991 err = create_gang(engine, &rq); 2992 if (err) 2993 break; 2994 2995 /* Submit each spinner at increasing priority */ 2996 engine->sched_engine->schedule(rq, &attr); 2997 } while (prio <= I915_PRIORITY_MAX && 2998 !__igt_timeout(end_time, NULL)); 2999 pr_debug("%s: Preempt chain of %d requests\n", 3000 engine->name, prio); 3001 3002 /* 3003 * Such that the last spinner is the highest priority and 3004 * should execute first. When that spinner completes, 3005 * it will terminate the next lowest spinner until there 3006 * are no more spinners and the gang is complete. 3007 */ 3008 cs = i915_gem_object_pin_map_unlocked(rq->batch->obj, I915_MAP_WC); 3009 if (!IS_ERR(cs)) { 3010 *cs = 0; 3011 i915_gem_object_unpin_map(rq->batch->obj); 3012 } else { 3013 err = PTR_ERR(cs); 3014 intel_gt_set_wedged(gt); 3015 } 3016 3017 while (rq) { /* wait for each rq from highest to lowest prio */ 3018 struct i915_request *n = list_next_entry(rq, mock.link); 3019 3020 if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) { 3021 struct drm_printer p = 3022 drm_info_printer(engine->i915->drm.dev); 3023 3024 pr_err("Failed to flush chain of %d requests, at %d\n", 3025 prio, rq_prio(rq)); 3026 intel_engine_dump(engine, &p, 3027 "%s\n", engine->name); 3028 3029 err = -ETIME; 3030 } 3031 3032 i915_vma_put(rq->batch); 3033 i915_request_put(rq); 3034 rq = n; 3035 } 3036 3037 if (igt_live_test_end(&t)) 3038 err = -EIO; 3039 if (err) 3040 return err; 3041 } 3042 3043 return 0; 3044 } 3045 3046 static struct i915_vma * 3047 create_gpr_user(struct intel_engine_cs *engine, 3048 struct i915_vma *result, 3049 unsigned int offset) 3050 { 3051 struct drm_i915_gem_object *obj; 3052 struct i915_vma *vma; 3053 u32 *cs; 3054 int err; 3055 int i; 3056 3057 obj = i915_gem_object_create_internal(engine->i915, 4096); 3058 if (IS_ERR(obj)) 3059 return ERR_CAST(obj); 3060 3061 vma = i915_vma_instance(obj, result->vm, NULL); 3062 if (IS_ERR(vma)) { 3063 i915_gem_object_put(obj); 3064 return vma; 3065 } 3066 3067 err = i915_vma_pin(vma, 0, 0, PIN_USER); 3068 if (err) { 3069 i915_vma_put(vma); 3070 return ERR_PTR(err); 3071 } 3072 3073 cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 3074 if (IS_ERR(cs)) { 3075 i915_vma_put(vma); 3076 return ERR_CAST(cs); 3077 } 3078 3079 /* All GPR are clear for new contexts. We use GPR(0) as a constant */ 3080 *cs++ = MI_LOAD_REGISTER_IMM(1); 3081 *cs++ = CS_GPR(engine, 0); 3082 *cs++ = 1; 3083 3084 for (i = 1; i < NUM_GPR; i++) { 3085 u64 addr; 3086 3087 /* 3088 * Perform: GPR[i]++ 3089 * 3090 * As we read and write into the context saved GPR[i], if 3091 * we restart this batch buffer from an earlier point, we 3092 * will repeat the increment and store a value > 1. 3093 */ 3094 *cs++ = MI_MATH(4); 3095 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(i)); 3096 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(0)); 3097 *cs++ = MI_MATH_ADD; 3098 *cs++ = MI_MATH_STORE(MI_MATH_REG(i), MI_MATH_REG_ACCU); 3099 3100 addr = i915_vma_offset(result) + offset + i * sizeof(*cs); 3101 *cs++ = MI_STORE_REGISTER_MEM_GEN8; 3102 *cs++ = CS_GPR(engine, 2 * i); 3103 *cs++ = lower_32_bits(addr); 3104 *cs++ = upper_32_bits(addr); 3105 3106 *cs++ = MI_SEMAPHORE_WAIT | 3107 MI_SEMAPHORE_POLL | 3108 MI_SEMAPHORE_SAD_GTE_SDD; 3109 *cs++ = i; 3110 *cs++ = lower_32_bits(i915_vma_offset(result)); 3111 *cs++ = upper_32_bits(i915_vma_offset(result)); 3112 } 3113 3114 *cs++ = MI_BATCH_BUFFER_END; 3115 i915_gem_object_flush_map(obj); 3116 i915_gem_object_unpin_map(obj); 3117 3118 return vma; 3119 } 3120 3121 static struct i915_vma *create_global(struct intel_gt *gt, size_t sz) 3122 { 3123 struct drm_i915_gem_object *obj; 3124 struct i915_vma *vma; 3125 int err; 3126 3127 obj = i915_gem_object_create_internal(gt->i915, sz); 3128 if (IS_ERR(obj)) 3129 return ERR_CAST(obj); 3130 3131 vma = i915_vma_instance(obj, >->ggtt->vm, NULL); 3132 if (IS_ERR(vma)) { 3133 i915_gem_object_put(obj); 3134 return vma; 3135 } 3136 3137 err = i915_ggtt_pin(vma, NULL, 0, 0); 3138 if (err) { 3139 i915_vma_put(vma); 3140 return ERR_PTR(err); 3141 } 3142 3143 return vma; 3144 } 3145 3146 static struct i915_request * 3147 create_gpr_client(struct intel_engine_cs *engine, 3148 struct i915_vma *global, 3149 unsigned int offset) 3150 { 3151 struct i915_vma *batch, *vma; 3152 struct intel_context *ce; 3153 struct i915_request *rq; 3154 int err; 3155 3156 ce = intel_context_create(engine); 3157 if (IS_ERR(ce)) 3158 return ERR_CAST(ce); 3159 3160 vma = i915_vma_instance(global->obj, ce->vm, NULL); 3161 if (IS_ERR(vma)) { 3162 err = PTR_ERR(vma); 3163 goto out_ce; 3164 } 3165 3166 err = i915_vma_pin(vma, 0, 0, PIN_USER); 3167 if (err) 3168 goto out_ce; 3169 3170 batch = create_gpr_user(engine, vma, offset); 3171 if (IS_ERR(batch)) { 3172 err = PTR_ERR(batch); 3173 goto out_vma; 3174 } 3175 3176 rq = intel_context_create_request(ce); 3177 if (IS_ERR(rq)) { 3178 err = PTR_ERR(rq); 3179 goto out_batch; 3180 } 3181 3182 err = igt_vma_move_to_active_unlocked(vma, rq, 0); 3183 3184 i915_vma_lock(batch); 3185 if (!err) 3186 err = i915_vma_move_to_active(batch, rq, 0); 3187 if (!err) 3188 err = rq->engine->emit_bb_start(rq, 3189 i915_vma_offset(batch), 3190 PAGE_SIZE, 0); 3191 i915_vma_unlock(batch); 3192 i915_vma_unpin(batch); 3193 3194 if (!err) 3195 i915_request_get(rq); 3196 i915_request_add(rq); 3197 3198 out_batch: 3199 i915_vma_put(batch); 3200 out_vma: 3201 i915_vma_unpin(vma); 3202 out_ce: 3203 intel_context_put(ce); 3204 return err ? ERR_PTR(err) : rq; 3205 } 3206 3207 static int preempt_user(struct intel_engine_cs *engine, 3208 struct i915_vma *global, 3209 int id) 3210 { 3211 struct i915_sched_attr attr = { 3212 .priority = I915_PRIORITY_MAX 3213 }; 3214 struct i915_request *rq; 3215 int err = 0; 3216 u32 *cs; 3217 3218 rq = intel_engine_create_kernel_request(engine); 3219 if (IS_ERR(rq)) 3220 return PTR_ERR(rq); 3221 3222 cs = intel_ring_begin(rq, 4); 3223 if (IS_ERR(cs)) { 3224 i915_request_add(rq); 3225 return PTR_ERR(cs); 3226 } 3227 3228 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 3229 *cs++ = i915_ggtt_offset(global); 3230 *cs++ = 0; 3231 *cs++ = id; 3232 3233 intel_ring_advance(rq, cs); 3234 3235 i915_request_get(rq); 3236 i915_request_add(rq); 3237 3238 engine->sched_engine->schedule(rq, &attr); 3239 3240 if (i915_request_wait(rq, 0, HZ / 2) < 0) 3241 err = -ETIME; 3242 i915_request_put(rq); 3243 3244 return err; 3245 } 3246 3247 static int live_preempt_user(void *arg) 3248 { 3249 struct intel_gt *gt = arg; 3250 struct intel_engine_cs *engine; 3251 struct i915_vma *global; 3252 enum intel_engine_id id; 3253 u32 *result; 3254 int err = 0; 3255 3256 /* 3257 * In our other tests, we look at preemption in carefully 3258 * controlled conditions in the ringbuffer. Since most of the 3259 * time is spent in user batches, most of our preemptions naturally 3260 * occur there. We want to verify that when we preempt inside a batch 3261 * we continue on from the current instruction and do not roll back 3262 * to the start, or another earlier arbitration point. 3263 * 3264 * To verify this, we create a batch which is a mixture of 3265 * MI_MATH (gpr++) MI_SRM (gpr) and preemption points. Then with 3266 * a few preempting contexts thrown into the mix, we look for any 3267 * repeated instructions (which show up as incorrect values). 3268 */ 3269 3270 global = create_global(gt, 4096); 3271 if (IS_ERR(global)) 3272 return PTR_ERR(global); 3273 3274 result = i915_gem_object_pin_map_unlocked(global->obj, I915_MAP_WC); 3275 if (IS_ERR(result)) { 3276 i915_vma_unpin_and_release(&global, 0); 3277 return PTR_ERR(result); 3278 } 3279 3280 for_each_engine(engine, gt, id) { 3281 struct i915_request *client[3] = {}; 3282 struct igt_live_test t; 3283 int i; 3284 3285 if (!intel_engine_has_preemption(engine)) 3286 continue; 3287 3288 if (GRAPHICS_VER(gt->i915) == 8 && engine->class != RENDER_CLASS) 3289 continue; /* we need per-context GPR */ 3290 3291 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { 3292 err = -EIO; 3293 break; 3294 } 3295 3296 memset(result, 0, 4096); 3297 3298 for (i = 0; i < ARRAY_SIZE(client); i++) { 3299 struct i915_request *rq; 3300 3301 rq = create_gpr_client(engine, global, 3302 NUM_GPR * i * sizeof(u32)); 3303 if (IS_ERR(rq)) { 3304 err = PTR_ERR(rq); 3305 goto end_test; 3306 } 3307 3308 client[i] = rq; 3309 } 3310 3311 /* Continuously preempt the set of 3 running contexts */ 3312 for (i = 1; i <= NUM_GPR; i++) { 3313 err = preempt_user(engine, global, i); 3314 if (err) 3315 goto end_test; 3316 } 3317 3318 if (READ_ONCE(result[0]) != NUM_GPR) { 3319 pr_err("%s: Failed to release semaphore\n", 3320 engine->name); 3321 err = -EIO; 3322 goto end_test; 3323 } 3324 3325 for (i = 0; i < ARRAY_SIZE(client); i++) { 3326 int gpr; 3327 3328 if (i915_request_wait(client[i], 0, HZ / 2) < 0) { 3329 err = -ETIME; 3330 goto end_test; 3331 } 3332 3333 for (gpr = 1; gpr < NUM_GPR; gpr++) { 3334 if (result[NUM_GPR * i + gpr] != 1) { 3335 pr_err("%s: Invalid result, client %d, gpr %d, result: %d\n", 3336 engine->name, 3337 i, gpr, result[NUM_GPR * i + gpr]); 3338 err = -EINVAL; 3339 goto end_test; 3340 } 3341 } 3342 } 3343 3344 end_test: 3345 for (i = 0; i < ARRAY_SIZE(client); i++) { 3346 if (!client[i]) 3347 break; 3348 3349 i915_request_put(client[i]); 3350 } 3351 3352 /* Flush the semaphores on error */ 3353 smp_store_mb(result[0], -1); 3354 if (igt_live_test_end(&t)) 3355 err = -EIO; 3356 if (err) 3357 break; 3358 } 3359 3360 i915_vma_unpin_and_release(&global, I915_VMA_RELEASE_MAP); 3361 return err; 3362 } 3363 3364 static int live_preempt_timeout(void *arg) 3365 { 3366 struct intel_gt *gt = arg; 3367 struct i915_gem_context *ctx_hi, *ctx_lo; 3368 struct igt_spinner spin_lo; 3369 struct intel_engine_cs *engine; 3370 enum intel_engine_id id; 3371 int err = -ENOMEM; 3372 3373 /* 3374 * Check that we force preemption to occur by cancelling the previous 3375 * context if it refuses to yield the GPU. 3376 */ 3377 if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT) 3378 return 0; 3379 3380 if (!intel_has_reset_engine(gt)) 3381 return 0; 3382 3383 ctx_hi = kernel_context(gt->i915, NULL); 3384 if (!ctx_hi) 3385 return -ENOMEM; 3386 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY; 3387 3388 ctx_lo = kernel_context(gt->i915, NULL); 3389 if (!ctx_lo) 3390 goto err_ctx_hi; 3391 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY; 3392 3393 if (igt_spinner_init(&spin_lo, gt)) 3394 goto err_ctx_lo; 3395 3396 for_each_engine(engine, gt, id) { 3397 unsigned long saved_timeout; 3398 struct i915_request *rq; 3399 3400 if (!intel_engine_has_preemption(engine)) 3401 continue; 3402 3403 rq = spinner_create_request(&spin_lo, ctx_lo, engine, 3404 MI_NOOP); /* preemption disabled */ 3405 if (IS_ERR(rq)) { 3406 err = PTR_ERR(rq); 3407 goto err_spin_lo; 3408 } 3409 3410 i915_request_add(rq); 3411 if (!igt_wait_for_spinner(&spin_lo, rq)) { 3412 intel_gt_set_wedged(gt); 3413 err = -EIO; 3414 goto err_spin_lo; 3415 } 3416 3417 rq = igt_request_alloc(ctx_hi, engine); 3418 if (IS_ERR(rq)) { 3419 igt_spinner_end(&spin_lo); 3420 err = PTR_ERR(rq); 3421 goto err_spin_lo; 3422 } 3423 3424 /* Flush the previous CS ack before changing timeouts */ 3425 while (READ_ONCE(engine->execlists.pending[0])) 3426 cpu_relax(); 3427 3428 saved_timeout = engine->props.preempt_timeout_ms; 3429 engine->props.preempt_timeout_ms = 1; /* in ms, -> 1 jiffie */ 3430 3431 i915_request_get(rq); 3432 i915_request_add(rq); 3433 3434 intel_engine_flush_submission(engine); 3435 engine->props.preempt_timeout_ms = saved_timeout; 3436 3437 if (i915_request_wait(rq, 0, HZ / 10) < 0) { 3438 intel_gt_set_wedged(gt); 3439 i915_request_put(rq); 3440 err = -ETIME; 3441 goto err_spin_lo; 3442 } 3443 3444 igt_spinner_end(&spin_lo); 3445 i915_request_put(rq); 3446 } 3447 3448 err = 0; 3449 err_spin_lo: 3450 igt_spinner_fini(&spin_lo); 3451 err_ctx_lo: 3452 kernel_context_close(ctx_lo); 3453 err_ctx_hi: 3454 kernel_context_close(ctx_hi); 3455 return err; 3456 } 3457 3458 static int random_range(struct rnd_state *rnd, int min, int max) 3459 { 3460 return i915_prandom_u32_max_state(max - min, rnd) + min; 3461 } 3462 3463 static int random_priority(struct rnd_state *rnd) 3464 { 3465 return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX); 3466 } 3467 3468 struct preempt_smoke { 3469 struct intel_gt *gt; 3470 struct kthread_work work; 3471 struct i915_gem_context **contexts; 3472 struct intel_engine_cs *engine; 3473 struct drm_i915_gem_object *batch; 3474 unsigned int ncontext; 3475 struct rnd_state prng; 3476 unsigned long count; 3477 int result; 3478 }; 3479 3480 static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke) 3481 { 3482 return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext, 3483 &smoke->prng)]; 3484 } 3485 3486 static int smoke_submit(struct preempt_smoke *smoke, 3487 struct i915_gem_context *ctx, int prio, 3488 struct drm_i915_gem_object *batch) 3489 { 3490 struct i915_request *rq; 3491 struct i915_vma *vma = NULL; 3492 int err = 0; 3493 3494 if (batch) { 3495 struct i915_address_space *vm; 3496 3497 vm = i915_gem_context_get_eb_vm(ctx); 3498 vma = i915_vma_instance(batch, vm, NULL); 3499 i915_vm_put(vm); 3500 if (IS_ERR(vma)) 3501 return PTR_ERR(vma); 3502 3503 err = i915_vma_pin(vma, 0, 0, PIN_USER); 3504 if (err) 3505 return err; 3506 } 3507 3508 ctx->sched.priority = prio; 3509 3510 rq = igt_request_alloc(ctx, smoke->engine); 3511 if (IS_ERR(rq)) { 3512 err = PTR_ERR(rq); 3513 goto unpin; 3514 } 3515 3516 if (vma) { 3517 err = igt_vma_move_to_active_unlocked(vma, rq, 0); 3518 if (!err) 3519 err = rq->engine->emit_bb_start(rq, 3520 i915_vma_offset(vma), 3521 PAGE_SIZE, 0); 3522 } 3523 3524 i915_request_add(rq); 3525 3526 unpin: 3527 if (vma) 3528 i915_vma_unpin(vma); 3529 3530 return err; 3531 } 3532 3533 static void smoke_crescendo_work(struct kthread_work *work) 3534 { 3535 struct preempt_smoke *smoke = container_of(work, typeof(*smoke), work); 3536 IGT_TIMEOUT(end_time); 3537 unsigned long count; 3538 3539 count = 0; 3540 do { 3541 struct i915_gem_context *ctx = smoke_context(smoke); 3542 3543 smoke->result = smoke_submit(smoke, ctx, 3544 count % I915_PRIORITY_MAX, 3545 smoke->batch); 3546 3547 count++; 3548 } while (!smoke->result && count < smoke->ncontext && 3549 !__igt_timeout(end_time, NULL)); 3550 3551 smoke->count = count; 3552 } 3553 3554 static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags) 3555 #define BATCH BIT(0) 3556 { 3557 struct kthread_worker *worker[I915_NUM_ENGINES] = {}; 3558 struct preempt_smoke *arg; 3559 struct intel_engine_cs *engine; 3560 enum intel_engine_id id; 3561 unsigned long count; 3562 int err = 0; 3563 3564 arg = kmalloc_array(I915_NUM_ENGINES, sizeof(*arg), GFP_KERNEL); 3565 if (!arg) 3566 return -ENOMEM; 3567 3568 memset(arg, 0, I915_NUM_ENGINES * sizeof(*arg)); 3569 3570 for_each_engine(engine, smoke->gt, id) { 3571 arg[id] = *smoke; 3572 arg[id].engine = engine; 3573 if (!(flags & BATCH)) 3574 arg[id].batch = NULL; 3575 arg[id].count = 0; 3576 3577 worker[id] = kthread_create_worker(0, "igt/smoke:%d", id); 3578 if (IS_ERR(worker[id])) { 3579 err = PTR_ERR(worker[id]); 3580 break; 3581 } 3582 3583 kthread_init_work(&arg[id].work, smoke_crescendo_work); 3584 kthread_queue_work(worker[id], &arg[id].work); 3585 } 3586 3587 count = 0; 3588 for_each_engine(engine, smoke->gt, id) { 3589 if (IS_ERR_OR_NULL(worker[id])) 3590 continue; 3591 3592 kthread_flush_work(&arg[id].work); 3593 if (arg[id].result && !err) 3594 err = arg[id].result; 3595 3596 count += arg[id].count; 3597 3598 kthread_destroy_worker(worker[id]); 3599 } 3600 3601 pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n", 3602 count, flags, smoke->gt->info.num_engines, smoke->ncontext); 3603 3604 kfree(arg); 3605 return 0; 3606 } 3607 3608 static int smoke_random(struct preempt_smoke *smoke, unsigned int flags) 3609 { 3610 enum intel_engine_id id; 3611 IGT_TIMEOUT(end_time); 3612 unsigned long count; 3613 3614 count = 0; 3615 do { 3616 for_each_engine(smoke->engine, smoke->gt, id) { 3617 struct i915_gem_context *ctx = smoke_context(smoke); 3618 int err; 3619 3620 err = smoke_submit(smoke, 3621 ctx, random_priority(&smoke->prng), 3622 flags & BATCH ? smoke->batch : NULL); 3623 if (err) 3624 return err; 3625 3626 count++; 3627 } 3628 } while (count < smoke->ncontext && !__igt_timeout(end_time, NULL)); 3629 3630 pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n", 3631 count, flags, smoke->gt->info.num_engines, smoke->ncontext); 3632 return 0; 3633 } 3634 3635 static int live_preempt_smoke(void *arg) 3636 { 3637 struct preempt_smoke smoke = { 3638 .gt = arg, 3639 .prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed), 3640 .ncontext = 256, 3641 }; 3642 const unsigned int phase[] = { 0, BATCH }; 3643 struct igt_live_test t; 3644 int err = -ENOMEM; 3645 u32 *cs; 3646 int n; 3647 3648 smoke.contexts = kmalloc_array(smoke.ncontext, 3649 sizeof(*smoke.contexts), 3650 GFP_KERNEL); 3651 if (!smoke.contexts) 3652 return -ENOMEM; 3653 3654 smoke.batch = 3655 i915_gem_object_create_internal(smoke.gt->i915, PAGE_SIZE); 3656 if (IS_ERR(smoke.batch)) { 3657 err = PTR_ERR(smoke.batch); 3658 goto err_free; 3659 } 3660 3661 cs = i915_gem_object_pin_map_unlocked(smoke.batch, I915_MAP_WB); 3662 if (IS_ERR(cs)) { 3663 err = PTR_ERR(cs); 3664 goto err_batch; 3665 } 3666 for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++) 3667 cs[n] = MI_ARB_CHECK; 3668 cs[n] = MI_BATCH_BUFFER_END; 3669 i915_gem_object_flush_map(smoke.batch); 3670 i915_gem_object_unpin_map(smoke.batch); 3671 3672 if (igt_live_test_begin(&t, smoke.gt->i915, __func__, "all")) { 3673 err = -EIO; 3674 goto err_batch; 3675 } 3676 3677 for (n = 0; n < smoke.ncontext; n++) { 3678 smoke.contexts[n] = kernel_context(smoke.gt->i915, NULL); 3679 if (!smoke.contexts[n]) 3680 goto err_ctx; 3681 } 3682 3683 for (n = 0; n < ARRAY_SIZE(phase); n++) { 3684 err = smoke_crescendo(&smoke, phase[n]); 3685 if (err) 3686 goto err_ctx; 3687 3688 err = smoke_random(&smoke, phase[n]); 3689 if (err) 3690 goto err_ctx; 3691 } 3692 3693 err_ctx: 3694 if (igt_live_test_end(&t)) 3695 err = -EIO; 3696 3697 for (n = 0; n < smoke.ncontext; n++) { 3698 if (!smoke.contexts[n]) 3699 break; 3700 kernel_context_close(smoke.contexts[n]); 3701 } 3702 3703 err_batch: 3704 i915_gem_object_put(smoke.batch); 3705 err_free: 3706 kfree(smoke.contexts); 3707 3708 return err; 3709 } 3710 3711 static int nop_virtual_engine(struct intel_gt *gt, 3712 struct intel_engine_cs **siblings, 3713 unsigned int nsibling, 3714 unsigned int nctx, 3715 unsigned int flags) 3716 #define CHAIN BIT(0) 3717 { 3718 IGT_TIMEOUT(end_time); 3719 struct i915_request *request[16] = {}; 3720 struct intel_context *ve[16]; 3721 unsigned long n, prime, nc; 3722 struct igt_live_test t; 3723 ktime_t times[2] = {}; 3724 int err; 3725 3726 GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ve)); 3727 3728 for (n = 0; n < nctx; n++) { 3729 ve[n] = intel_engine_create_virtual(siblings, nsibling, 0); 3730 if (IS_ERR(ve[n])) { 3731 err = PTR_ERR(ve[n]); 3732 nctx = n; 3733 goto out; 3734 } 3735 3736 err = intel_context_pin(ve[n]); 3737 if (err) { 3738 intel_context_put(ve[n]); 3739 nctx = n; 3740 goto out; 3741 } 3742 } 3743 3744 err = igt_live_test_begin(&t, gt->i915, __func__, ve[0]->engine->name); 3745 if (err) 3746 goto out; 3747 3748 for_each_prime_number_from(prime, 1, 8192) { 3749 times[1] = ktime_get_raw(); 3750 3751 if (flags & CHAIN) { 3752 for (nc = 0; nc < nctx; nc++) { 3753 for (n = 0; n < prime; n++) { 3754 struct i915_request *rq; 3755 3756 rq = i915_request_create(ve[nc]); 3757 if (IS_ERR(rq)) { 3758 err = PTR_ERR(rq); 3759 goto out; 3760 } 3761 3762 if (request[nc]) 3763 i915_request_put(request[nc]); 3764 request[nc] = i915_request_get(rq); 3765 i915_request_add(rq); 3766 } 3767 } 3768 } else { 3769 for (n = 0; n < prime; n++) { 3770 for (nc = 0; nc < nctx; nc++) { 3771 struct i915_request *rq; 3772 3773 rq = i915_request_create(ve[nc]); 3774 if (IS_ERR(rq)) { 3775 err = PTR_ERR(rq); 3776 goto out; 3777 } 3778 3779 if (request[nc]) 3780 i915_request_put(request[nc]); 3781 request[nc] = i915_request_get(rq); 3782 i915_request_add(rq); 3783 } 3784 } 3785 } 3786 3787 for (nc = 0; nc < nctx; nc++) { 3788 if (i915_request_wait(request[nc], 0, HZ / 10) < 0) { 3789 pr_err("%s(%s): wait for %llx:%lld timed out\n", 3790 __func__, ve[0]->engine->name, 3791 request[nc]->fence.context, 3792 request[nc]->fence.seqno); 3793 3794 GEM_TRACE("%s(%s) failed at request %llx:%lld\n", 3795 __func__, ve[0]->engine->name, 3796 request[nc]->fence.context, 3797 request[nc]->fence.seqno); 3798 GEM_TRACE_DUMP(); 3799 intel_gt_set_wedged(gt); 3800 break; 3801 } 3802 } 3803 3804 times[1] = ktime_sub(ktime_get_raw(), times[1]); 3805 if (prime == 1) 3806 times[0] = times[1]; 3807 3808 for (nc = 0; nc < nctx; nc++) { 3809 i915_request_put(request[nc]); 3810 request[nc] = NULL; 3811 } 3812 3813 if (__igt_timeout(end_time, NULL)) 3814 break; 3815 } 3816 3817 err = igt_live_test_end(&t); 3818 if (err) 3819 goto out; 3820 3821 pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n", 3822 nctx, ve[0]->engine->name, ktime_to_ns(times[0]), 3823 prime, div64_u64(ktime_to_ns(times[1]), prime)); 3824 3825 out: 3826 if (igt_flush_test(gt->i915)) 3827 err = -EIO; 3828 3829 for (nc = 0; nc < nctx; nc++) { 3830 i915_request_put(request[nc]); 3831 intel_context_unpin(ve[nc]); 3832 intel_context_put(ve[nc]); 3833 } 3834 return err; 3835 } 3836 3837 static unsigned int 3838 __select_siblings(struct intel_gt *gt, 3839 unsigned int class, 3840 struct intel_engine_cs **siblings, 3841 bool (*filter)(const struct intel_engine_cs *)) 3842 { 3843 unsigned int n = 0; 3844 unsigned int inst; 3845 3846 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) { 3847 if (!gt->engine_class[class][inst]) 3848 continue; 3849 3850 if (filter && !filter(gt->engine_class[class][inst])) 3851 continue; 3852 3853 siblings[n++] = gt->engine_class[class][inst]; 3854 } 3855 3856 return n; 3857 } 3858 3859 static unsigned int 3860 select_siblings(struct intel_gt *gt, 3861 unsigned int class, 3862 struct intel_engine_cs **siblings) 3863 { 3864 return __select_siblings(gt, class, siblings, NULL); 3865 } 3866 3867 static int live_virtual_engine(void *arg) 3868 { 3869 struct intel_gt *gt = arg; 3870 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 3871 struct intel_engine_cs *engine; 3872 enum intel_engine_id id; 3873 unsigned int class; 3874 int err; 3875 3876 if (intel_uc_uses_guc_submission(>->uc)) 3877 return 0; 3878 3879 for_each_engine(engine, gt, id) { 3880 err = nop_virtual_engine(gt, &engine, 1, 1, 0); 3881 if (err) { 3882 pr_err("Failed to wrap engine %s: err=%d\n", 3883 engine->name, err); 3884 return err; 3885 } 3886 } 3887 3888 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 3889 int nsibling, n; 3890 3891 nsibling = select_siblings(gt, class, siblings); 3892 if (nsibling < 2) 3893 continue; 3894 3895 for (n = 1; n <= nsibling + 1; n++) { 3896 err = nop_virtual_engine(gt, siblings, nsibling, 3897 n, 0); 3898 if (err) 3899 return err; 3900 } 3901 3902 err = nop_virtual_engine(gt, siblings, nsibling, n, CHAIN); 3903 if (err) 3904 return err; 3905 } 3906 3907 return 0; 3908 } 3909 3910 static int mask_virtual_engine(struct intel_gt *gt, 3911 struct intel_engine_cs **siblings, 3912 unsigned int nsibling) 3913 { 3914 struct i915_request *request[MAX_ENGINE_INSTANCE + 1]; 3915 struct intel_context *ve; 3916 struct igt_live_test t; 3917 unsigned int n; 3918 int err; 3919 3920 /* 3921 * Check that by setting the execution mask on a request, we can 3922 * restrict it to our desired engine within the virtual engine. 3923 */ 3924 3925 ve = intel_engine_create_virtual(siblings, nsibling, 0); 3926 if (IS_ERR(ve)) { 3927 err = PTR_ERR(ve); 3928 goto out_close; 3929 } 3930 3931 err = intel_context_pin(ve); 3932 if (err) 3933 goto out_put; 3934 3935 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name); 3936 if (err) 3937 goto out_unpin; 3938 3939 for (n = 0; n < nsibling; n++) { 3940 request[n] = i915_request_create(ve); 3941 if (IS_ERR(request[n])) { 3942 err = PTR_ERR(request[n]); 3943 nsibling = n; 3944 goto out; 3945 } 3946 3947 /* Reverse order as it's more likely to be unnatural */ 3948 request[n]->execution_mask = siblings[nsibling - n - 1]->mask; 3949 3950 i915_request_get(request[n]); 3951 i915_request_add(request[n]); 3952 } 3953 3954 for (n = 0; n < nsibling; n++) { 3955 if (i915_request_wait(request[n], 0, HZ / 10) < 0) { 3956 pr_err("%s(%s): wait for %llx:%lld timed out\n", 3957 __func__, ve->engine->name, 3958 request[n]->fence.context, 3959 request[n]->fence.seqno); 3960 3961 GEM_TRACE("%s(%s) failed at request %llx:%lld\n", 3962 __func__, ve->engine->name, 3963 request[n]->fence.context, 3964 request[n]->fence.seqno); 3965 GEM_TRACE_DUMP(); 3966 intel_gt_set_wedged(gt); 3967 err = -EIO; 3968 goto out; 3969 } 3970 3971 if (request[n]->engine != siblings[nsibling - n - 1]) { 3972 pr_err("Executed on wrong sibling '%s', expected '%s'\n", 3973 request[n]->engine->name, 3974 siblings[nsibling - n - 1]->name); 3975 err = -EINVAL; 3976 goto out; 3977 } 3978 } 3979 3980 err = igt_live_test_end(&t); 3981 out: 3982 if (igt_flush_test(gt->i915)) 3983 err = -EIO; 3984 3985 for (n = 0; n < nsibling; n++) 3986 i915_request_put(request[n]); 3987 3988 out_unpin: 3989 intel_context_unpin(ve); 3990 out_put: 3991 intel_context_put(ve); 3992 out_close: 3993 return err; 3994 } 3995 3996 static int live_virtual_mask(void *arg) 3997 { 3998 struct intel_gt *gt = arg; 3999 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 4000 unsigned int class; 4001 int err; 4002 4003 if (intel_uc_uses_guc_submission(>->uc)) 4004 return 0; 4005 4006 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 4007 unsigned int nsibling; 4008 4009 nsibling = select_siblings(gt, class, siblings); 4010 if (nsibling < 2) 4011 continue; 4012 4013 err = mask_virtual_engine(gt, siblings, nsibling); 4014 if (err) 4015 return err; 4016 } 4017 4018 return 0; 4019 } 4020 4021 static int slicein_virtual_engine(struct intel_gt *gt, 4022 struct intel_engine_cs **siblings, 4023 unsigned int nsibling) 4024 { 4025 const long timeout = slice_timeout(siblings[0]); 4026 struct intel_context *ce; 4027 struct i915_request *rq; 4028 struct igt_spinner spin; 4029 unsigned int n; 4030 int err = 0; 4031 4032 /* 4033 * Virtual requests must take part in timeslicing on the target engines. 4034 */ 4035 4036 if (igt_spinner_init(&spin, gt)) 4037 return -ENOMEM; 4038 4039 for (n = 0; n < nsibling; n++) { 4040 ce = intel_context_create(siblings[n]); 4041 if (IS_ERR(ce)) { 4042 err = PTR_ERR(ce); 4043 goto out; 4044 } 4045 4046 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK); 4047 intel_context_put(ce); 4048 if (IS_ERR(rq)) { 4049 err = PTR_ERR(rq); 4050 goto out; 4051 } 4052 4053 i915_request_add(rq); 4054 } 4055 4056 ce = intel_engine_create_virtual(siblings, nsibling, 0); 4057 if (IS_ERR(ce)) { 4058 err = PTR_ERR(ce); 4059 goto out; 4060 } 4061 4062 rq = intel_context_create_request(ce); 4063 intel_context_put(ce); 4064 if (IS_ERR(rq)) { 4065 err = PTR_ERR(rq); 4066 goto out; 4067 } 4068 4069 i915_request_get(rq); 4070 i915_request_add(rq); 4071 if (i915_request_wait(rq, 0, timeout) < 0) { 4072 GEM_TRACE_ERR("%s(%s) failed to slice in virtual request\n", 4073 __func__, rq->engine->name); 4074 GEM_TRACE_DUMP(); 4075 intel_gt_set_wedged(gt); 4076 err = -EIO; 4077 } 4078 i915_request_put(rq); 4079 4080 out: 4081 igt_spinner_end(&spin); 4082 if (igt_flush_test(gt->i915)) 4083 err = -EIO; 4084 igt_spinner_fini(&spin); 4085 return err; 4086 } 4087 4088 static int sliceout_virtual_engine(struct intel_gt *gt, 4089 struct intel_engine_cs **siblings, 4090 unsigned int nsibling) 4091 { 4092 const long timeout = slice_timeout(siblings[0]); 4093 struct intel_context *ce; 4094 struct i915_request *rq; 4095 struct igt_spinner spin; 4096 unsigned int n; 4097 int err = 0; 4098 4099 /* 4100 * Virtual requests must allow others a fair timeslice. 4101 */ 4102 4103 if (igt_spinner_init(&spin, gt)) 4104 return -ENOMEM; 4105 4106 /* XXX We do not handle oversubscription and fairness with normal rq */ 4107 for (n = 0; n < nsibling; n++) { 4108 ce = intel_engine_create_virtual(siblings, nsibling, 0); 4109 if (IS_ERR(ce)) { 4110 err = PTR_ERR(ce); 4111 goto out; 4112 } 4113 4114 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK); 4115 intel_context_put(ce); 4116 if (IS_ERR(rq)) { 4117 err = PTR_ERR(rq); 4118 goto out; 4119 } 4120 4121 i915_request_add(rq); 4122 } 4123 4124 for (n = 0; !err && n < nsibling; n++) { 4125 ce = intel_context_create(siblings[n]); 4126 if (IS_ERR(ce)) { 4127 err = PTR_ERR(ce); 4128 goto out; 4129 } 4130 4131 rq = intel_context_create_request(ce); 4132 intel_context_put(ce); 4133 if (IS_ERR(rq)) { 4134 err = PTR_ERR(rq); 4135 goto out; 4136 } 4137 4138 i915_request_get(rq); 4139 i915_request_add(rq); 4140 if (i915_request_wait(rq, 0, timeout) < 0) { 4141 GEM_TRACE_ERR("%s(%s) failed to slice out virtual request\n", 4142 __func__, siblings[n]->name); 4143 GEM_TRACE_DUMP(); 4144 intel_gt_set_wedged(gt); 4145 err = -EIO; 4146 } 4147 i915_request_put(rq); 4148 } 4149 4150 out: 4151 igt_spinner_end(&spin); 4152 if (igt_flush_test(gt->i915)) 4153 err = -EIO; 4154 igt_spinner_fini(&spin); 4155 return err; 4156 } 4157 4158 static int live_virtual_slice(void *arg) 4159 { 4160 struct intel_gt *gt = arg; 4161 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 4162 unsigned int class; 4163 int err; 4164 4165 if (intel_uc_uses_guc_submission(>->uc)) 4166 return 0; 4167 4168 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 4169 unsigned int nsibling; 4170 4171 nsibling = __select_siblings(gt, class, siblings, 4172 intel_engine_has_timeslices); 4173 if (nsibling < 2) 4174 continue; 4175 4176 err = slicein_virtual_engine(gt, siblings, nsibling); 4177 if (err) 4178 return err; 4179 4180 err = sliceout_virtual_engine(gt, siblings, nsibling); 4181 if (err) 4182 return err; 4183 } 4184 4185 return 0; 4186 } 4187 4188 static int preserved_virtual_engine(struct intel_gt *gt, 4189 struct intel_engine_cs **siblings, 4190 unsigned int nsibling) 4191 { 4192 struct i915_request *last = NULL; 4193 struct intel_context *ve; 4194 struct i915_vma *scratch; 4195 struct igt_live_test t; 4196 unsigned int n; 4197 int err = 0; 4198 u32 *cs; 4199 4200 scratch = 4201 __vm_create_scratch_for_read_pinned(&siblings[0]->gt->ggtt->vm, 4202 PAGE_SIZE); 4203 if (IS_ERR(scratch)) 4204 return PTR_ERR(scratch); 4205 4206 err = i915_vma_sync(scratch); 4207 if (err) 4208 goto out_scratch; 4209 4210 ve = intel_engine_create_virtual(siblings, nsibling, 0); 4211 if (IS_ERR(ve)) { 4212 err = PTR_ERR(ve); 4213 goto out_scratch; 4214 } 4215 4216 err = intel_context_pin(ve); 4217 if (err) 4218 goto out_put; 4219 4220 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name); 4221 if (err) 4222 goto out_unpin; 4223 4224 for (n = 0; n < NUM_GPR_DW; n++) { 4225 struct intel_engine_cs *engine = siblings[n % nsibling]; 4226 struct i915_request *rq; 4227 4228 rq = i915_request_create(ve); 4229 if (IS_ERR(rq)) { 4230 err = PTR_ERR(rq); 4231 goto out_end; 4232 } 4233 4234 i915_request_put(last); 4235 last = i915_request_get(rq); 4236 4237 cs = intel_ring_begin(rq, 8); 4238 if (IS_ERR(cs)) { 4239 i915_request_add(rq); 4240 err = PTR_ERR(cs); 4241 goto out_end; 4242 } 4243 4244 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 4245 *cs++ = CS_GPR(engine, n); 4246 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32); 4247 *cs++ = 0; 4248 4249 *cs++ = MI_LOAD_REGISTER_IMM(1); 4250 *cs++ = CS_GPR(engine, (n + 1) % NUM_GPR_DW); 4251 *cs++ = n + 1; 4252 4253 *cs++ = MI_NOOP; 4254 intel_ring_advance(rq, cs); 4255 4256 /* Restrict this request to run on a particular engine */ 4257 rq->execution_mask = engine->mask; 4258 i915_request_add(rq); 4259 } 4260 4261 if (i915_request_wait(last, 0, HZ / 5) < 0) { 4262 err = -ETIME; 4263 goto out_end; 4264 } 4265 4266 cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB); 4267 if (IS_ERR(cs)) { 4268 err = PTR_ERR(cs); 4269 goto out_end; 4270 } 4271 4272 for (n = 0; n < NUM_GPR_DW; n++) { 4273 if (cs[n] != n) { 4274 pr_err("Incorrect value[%d] found for GPR[%d]\n", 4275 cs[n], n); 4276 err = -EINVAL; 4277 break; 4278 } 4279 } 4280 4281 i915_gem_object_unpin_map(scratch->obj); 4282 4283 out_end: 4284 if (igt_live_test_end(&t)) 4285 err = -EIO; 4286 i915_request_put(last); 4287 out_unpin: 4288 intel_context_unpin(ve); 4289 out_put: 4290 intel_context_put(ve); 4291 out_scratch: 4292 i915_vma_unpin_and_release(&scratch, 0); 4293 return err; 4294 } 4295 4296 static int live_virtual_preserved(void *arg) 4297 { 4298 struct intel_gt *gt = arg; 4299 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 4300 unsigned int class; 4301 4302 /* 4303 * Check that the context image retains non-privileged (user) registers 4304 * from one engine to the next. For this we check that the CS_GPR 4305 * are preserved. 4306 */ 4307 4308 if (intel_uc_uses_guc_submission(>->uc)) 4309 return 0; 4310 4311 /* As we use CS_GPR we cannot run before they existed on all engines. */ 4312 if (GRAPHICS_VER(gt->i915) < 9) 4313 return 0; 4314 4315 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 4316 int nsibling, err; 4317 4318 nsibling = select_siblings(gt, class, siblings); 4319 if (nsibling < 2) 4320 continue; 4321 4322 err = preserved_virtual_engine(gt, siblings, nsibling); 4323 if (err) 4324 return err; 4325 } 4326 4327 return 0; 4328 } 4329 4330 static int reset_virtual_engine(struct intel_gt *gt, 4331 struct intel_engine_cs **siblings, 4332 unsigned int nsibling) 4333 { 4334 struct intel_engine_cs *engine; 4335 struct intel_context *ve; 4336 struct igt_spinner spin; 4337 struct i915_request *rq; 4338 unsigned int n; 4339 int err = 0; 4340 4341 /* 4342 * In order to support offline error capture for fast preempt reset, 4343 * we need to decouple the guilty request and ensure that it and its 4344 * descendents are not executed while the capture is in progress. 4345 */ 4346 4347 if (igt_spinner_init(&spin, gt)) 4348 return -ENOMEM; 4349 4350 ve = intel_engine_create_virtual(siblings, nsibling, 0); 4351 if (IS_ERR(ve)) { 4352 err = PTR_ERR(ve); 4353 goto out_spin; 4354 } 4355 4356 for (n = 0; n < nsibling; n++) 4357 st_engine_heartbeat_disable(siblings[n]); 4358 4359 rq = igt_spinner_create_request(&spin, ve, MI_ARB_CHECK); 4360 if (IS_ERR(rq)) { 4361 err = PTR_ERR(rq); 4362 goto out_heartbeat; 4363 } 4364 i915_request_add(rq); 4365 4366 if (!igt_wait_for_spinner(&spin, rq)) { 4367 intel_gt_set_wedged(gt); 4368 err = -ETIME; 4369 goto out_heartbeat; 4370 } 4371 4372 engine = rq->engine; 4373 GEM_BUG_ON(engine == ve->engine); 4374 4375 /* Take ownership of the reset and tasklet */ 4376 err = engine_lock_reset_tasklet(engine); 4377 if (err) 4378 goto out_heartbeat; 4379 4380 engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet); 4381 GEM_BUG_ON(execlists_active(&engine->execlists) != rq); 4382 4383 /* Fake a preemption event; failed of course */ 4384 spin_lock_irq(&engine->sched_engine->lock); 4385 __unwind_incomplete_requests(engine); 4386 spin_unlock_irq(&engine->sched_engine->lock); 4387 GEM_BUG_ON(rq->engine != engine); 4388 4389 /* Reset the engine while keeping our active request on hold */ 4390 execlists_hold(engine, rq); 4391 GEM_BUG_ON(!i915_request_on_hold(rq)); 4392 4393 __intel_engine_reset_bh(engine, NULL); 4394 GEM_BUG_ON(rq->fence.error != -EIO); 4395 4396 /* Release our grasp on the engine, letting CS flow again */ 4397 engine_unlock_reset_tasklet(engine); 4398 4399 /* Check that we do not resubmit the held request */ 4400 i915_request_get(rq); 4401 if (!i915_request_wait(rq, 0, HZ / 5)) { 4402 pr_err("%s: on hold request completed!\n", 4403 engine->name); 4404 intel_gt_set_wedged(gt); 4405 err = -EIO; 4406 goto out_rq; 4407 } 4408 GEM_BUG_ON(!i915_request_on_hold(rq)); 4409 4410 /* But is resubmitted on release */ 4411 execlists_unhold(engine, rq); 4412 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 4413 pr_err("%s: held request did not complete!\n", 4414 engine->name); 4415 intel_gt_set_wedged(gt); 4416 err = -ETIME; 4417 } 4418 4419 out_rq: 4420 i915_request_put(rq); 4421 out_heartbeat: 4422 for (n = 0; n < nsibling; n++) 4423 st_engine_heartbeat_enable(siblings[n]); 4424 4425 intel_context_put(ve); 4426 out_spin: 4427 igt_spinner_fini(&spin); 4428 return err; 4429 } 4430 4431 static int live_virtual_reset(void *arg) 4432 { 4433 struct intel_gt *gt = arg; 4434 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 4435 unsigned int class; 4436 4437 /* 4438 * Check that we handle a reset event within a virtual engine. 4439 * Only the physical engine is reset, but we have to check the flow 4440 * of the virtual requests around the reset, and make sure it is not 4441 * forgotten. 4442 */ 4443 4444 if (intel_uc_uses_guc_submission(>->uc)) 4445 return 0; 4446 4447 if (!intel_has_reset_engine(gt)) 4448 return 0; 4449 4450 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 4451 int nsibling, err; 4452 4453 nsibling = select_siblings(gt, class, siblings); 4454 if (nsibling < 2) 4455 continue; 4456 4457 err = reset_virtual_engine(gt, siblings, nsibling); 4458 if (err) 4459 return err; 4460 } 4461 4462 return 0; 4463 } 4464 4465 int intel_execlists_live_selftests(struct drm_i915_private *i915) 4466 { 4467 static const struct i915_subtest tests[] = { 4468 SUBTEST(live_sanitycheck), 4469 SUBTEST(live_unlite_switch), 4470 SUBTEST(live_unlite_preempt), 4471 SUBTEST(live_unlite_ring), 4472 SUBTEST(live_pin_rewind), 4473 SUBTEST(live_hold_reset), 4474 SUBTEST(live_error_interrupt), 4475 SUBTEST(live_timeslice_preempt), 4476 SUBTEST(live_timeslice_rewind), 4477 SUBTEST(live_timeslice_queue), 4478 SUBTEST(live_timeslice_nopreempt), 4479 SUBTEST(live_busywait_preempt), 4480 SUBTEST(live_preempt), 4481 SUBTEST(live_late_preempt), 4482 SUBTEST(live_nopreempt), 4483 SUBTEST(live_preempt_cancel), 4484 SUBTEST(live_suppress_self_preempt), 4485 SUBTEST(live_chain_preempt), 4486 SUBTEST(live_preempt_ring), 4487 SUBTEST(live_preempt_gang), 4488 SUBTEST(live_preempt_timeout), 4489 SUBTEST(live_preempt_user), 4490 SUBTEST(live_preempt_smoke), 4491 SUBTEST(live_virtual_engine), 4492 SUBTEST(live_virtual_mask), 4493 SUBTEST(live_virtual_preserved), 4494 SUBTEST(live_virtual_slice), 4495 SUBTEST(live_virtual_reset), 4496 }; 4497 4498 if (to_gt(i915)->submission_method != INTEL_SUBMISSION_ELSP) 4499 return 0; 4500 4501 if (intel_gt_is_wedged(to_gt(i915))) 4502 return 0; 4503 4504 return intel_gt_live_subtests(tests, to_gt(i915)); 4505 } 4506