1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2018 Intel Corporation 5 */ 6 7 #include <linux/prime_numbers.h> 8 9 #include "gem/i915_gem_pm.h" 10 #include "gt/intel_reset.h" 11 12 #include "i915_selftest.h" 13 #include "selftests/i915_random.h" 14 #include "selftests/igt_flush_test.h" 15 #include "selftests/igt_live_test.h" 16 #include "selftests/igt_spinner.h" 17 #include "selftests/lib_sw_fence.h" 18 19 #include "gem/selftests/igt_gem_utils.h" 20 #include "gem/selftests/mock_context.h" 21 22 static int live_sanitycheck(void *arg) 23 { 24 struct drm_i915_private *i915 = arg; 25 struct i915_gem_engines_iter it; 26 struct i915_gem_context *ctx; 27 struct intel_context *ce; 28 struct igt_spinner spin; 29 int err = -ENOMEM; 30 31 if (!HAS_LOGICAL_RING_CONTEXTS(i915)) 32 return 0; 33 34 if (igt_spinner_init(&spin, &i915->gt)) 35 return -ENOMEM; 36 37 ctx = kernel_context(i915); 38 if (!ctx) 39 goto err_spin; 40 41 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 42 struct i915_request *rq; 43 44 rq = igt_spinner_create_request(&spin, ce, MI_NOOP); 45 if (IS_ERR(rq)) { 46 err = PTR_ERR(rq); 47 goto err_ctx; 48 } 49 50 i915_request_add(rq); 51 if (!igt_wait_for_spinner(&spin, rq)) { 52 GEM_TRACE("spinner failed to start\n"); 53 GEM_TRACE_DUMP(); 54 intel_gt_set_wedged(&i915->gt); 55 err = -EIO; 56 goto err_ctx; 57 } 58 59 igt_spinner_end(&spin); 60 if (igt_flush_test(i915)) { 61 err = -EIO; 62 goto err_ctx; 63 } 64 } 65 66 err = 0; 67 err_ctx: 68 i915_gem_context_unlock_engines(ctx); 69 kernel_context_close(ctx); 70 err_spin: 71 igt_spinner_fini(&spin); 72 return err; 73 } 74 75 static int live_unlite_restore(struct drm_i915_private *i915, int prio) 76 { 77 struct intel_engine_cs *engine; 78 struct i915_gem_context *ctx; 79 enum intel_engine_id id; 80 intel_wakeref_t wakeref; 81 struct igt_spinner spin; 82 int err = -ENOMEM; 83 84 /* 85 * Check that we can correctly context switch between 2 instances 86 * on the same engine from the same parent context. 87 */ 88 89 mutex_lock(&i915->drm.struct_mutex); 90 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 91 92 if (igt_spinner_init(&spin, &i915->gt)) 93 goto err_unlock; 94 95 ctx = kernel_context(i915); 96 if (!ctx) 97 goto err_spin; 98 99 err = 0; 100 for_each_engine(engine, i915, id) { 101 struct intel_context *ce[2] = {}; 102 struct i915_request *rq[2]; 103 struct igt_live_test t; 104 int n; 105 106 if (prio && !intel_engine_has_preemption(engine)) 107 continue; 108 109 if (!intel_engine_can_store_dword(engine)) 110 continue; 111 112 if (igt_live_test_begin(&t, i915, __func__, engine->name)) { 113 err = -EIO; 114 break; 115 } 116 117 for (n = 0; n < ARRAY_SIZE(ce); n++) { 118 struct intel_context *tmp; 119 120 tmp = intel_context_create(ctx, engine); 121 if (IS_ERR(tmp)) { 122 err = PTR_ERR(tmp); 123 goto err_ce; 124 } 125 126 err = intel_context_pin(tmp); 127 if (err) { 128 intel_context_put(tmp); 129 goto err_ce; 130 } 131 132 /* 133 * Setup the pair of contexts such that if we 134 * lite-restore using the RING_TAIL from ce[1] it 135 * will execute garbage from ce[0]->ring. 136 */ 137 memset(tmp->ring->vaddr, 138 POISON_INUSE, /* IPEHR: 0x5a5a5a5a [hung!] */ 139 tmp->ring->vma->size); 140 141 ce[n] = tmp; 142 } 143 GEM_BUG_ON(!ce[1]->ring->size); 144 intel_ring_reset(ce[1]->ring, ce[1]->ring->size / 2); 145 146 local_bh_disable(); /* appease lockdep */ 147 __context_pin_acquire(ce[1]); 148 __execlists_update_reg_state(ce[1], engine); 149 __context_pin_release(ce[1]); 150 local_bh_enable(); 151 152 rq[0] = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK); 153 if (IS_ERR(rq[0])) { 154 err = PTR_ERR(rq[0]); 155 goto err_ce; 156 } 157 158 i915_request_get(rq[0]); 159 i915_request_add(rq[0]); 160 GEM_BUG_ON(rq[0]->postfix > ce[1]->ring->emit); 161 162 if (!igt_wait_for_spinner(&spin, rq[0])) { 163 i915_request_put(rq[0]); 164 goto err_ce; 165 } 166 167 rq[1] = i915_request_create(ce[1]); 168 if (IS_ERR(rq[1])) { 169 err = PTR_ERR(rq[1]); 170 i915_request_put(rq[0]); 171 goto err_ce; 172 } 173 174 if (!prio) { 175 /* 176 * Ensure we do the switch to ce[1] on completion. 177 * 178 * rq[0] is already submitted, so this should reduce 179 * to a no-op (a wait on a request on the same engine 180 * uses the submit fence, not the completion fence), 181 * but it will install a dependency on rq[1] for rq[0] 182 * that will prevent the pair being reordered by 183 * timeslicing. 184 */ 185 i915_request_await_dma_fence(rq[1], &rq[0]->fence); 186 } 187 188 i915_request_get(rq[1]); 189 i915_request_add(rq[1]); 190 GEM_BUG_ON(rq[1]->postfix <= rq[0]->postfix); 191 i915_request_put(rq[0]); 192 193 if (prio) { 194 struct i915_sched_attr attr = { 195 .priority = prio, 196 }; 197 198 /* Alternatively preempt the spinner with ce[1] */ 199 engine->schedule(rq[1], &attr); 200 } 201 202 /* And switch back to ce[0] for good measure */ 203 rq[0] = i915_request_create(ce[0]); 204 if (IS_ERR(rq[0])) { 205 err = PTR_ERR(rq[0]); 206 i915_request_put(rq[1]); 207 goto err_ce; 208 } 209 210 i915_request_await_dma_fence(rq[0], &rq[1]->fence); 211 i915_request_get(rq[0]); 212 i915_request_add(rq[0]); 213 GEM_BUG_ON(rq[0]->postfix > rq[1]->postfix); 214 i915_request_put(rq[1]); 215 i915_request_put(rq[0]); 216 217 err_ce: 218 tasklet_kill(&engine->execlists.tasklet); /* flush submission */ 219 igt_spinner_end(&spin); 220 for (n = 0; n < ARRAY_SIZE(ce); n++) { 221 if (IS_ERR_OR_NULL(ce[n])) 222 break; 223 224 intel_context_unpin(ce[n]); 225 intel_context_put(ce[n]); 226 } 227 228 if (igt_live_test_end(&t)) 229 err = -EIO; 230 if (err) 231 break; 232 } 233 234 kernel_context_close(ctx); 235 err_spin: 236 igt_spinner_fini(&spin); 237 err_unlock: 238 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 239 mutex_unlock(&i915->drm.struct_mutex); 240 return err; 241 } 242 243 static int live_unlite_switch(void *arg) 244 { 245 return live_unlite_restore(arg, 0); 246 } 247 248 static int live_unlite_preempt(void *arg) 249 { 250 return live_unlite_restore(arg, I915_USER_PRIORITY(I915_PRIORITY_MAX)); 251 } 252 253 static int 254 emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx) 255 { 256 u32 *cs; 257 258 cs = intel_ring_begin(rq, 10); 259 if (IS_ERR(cs)) 260 return PTR_ERR(cs); 261 262 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 263 264 *cs++ = MI_SEMAPHORE_WAIT | 265 MI_SEMAPHORE_GLOBAL_GTT | 266 MI_SEMAPHORE_POLL | 267 MI_SEMAPHORE_SAD_NEQ_SDD; 268 *cs++ = 0; 269 *cs++ = i915_ggtt_offset(vma) + 4 * idx; 270 *cs++ = 0; 271 272 if (idx > 0) { 273 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 274 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1); 275 *cs++ = 0; 276 *cs++ = 1; 277 } else { 278 *cs++ = MI_NOOP; 279 *cs++ = MI_NOOP; 280 *cs++ = MI_NOOP; 281 *cs++ = MI_NOOP; 282 } 283 284 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 285 286 intel_ring_advance(rq, cs); 287 return 0; 288 } 289 290 static struct i915_request * 291 semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx) 292 { 293 struct i915_gem_context *ctx; 294 struct i915_request *rq; 295 int err; 296 297 ctx = kernel_context(engine->i915); 298 if (!ctx) 299 return ERR_PTR(-ENOMEM); 300 301 rq = igt_request_alloc(ctx, engine); 302 if (IS_ERR(rq)) 303 goto out_ctx; 304 305 err = emit_semaphore_chain(rq, vma, idx); 306 i915_request_add(rq); 307 if (err) 308 rq = ERR_PTR(err); 309 310 out_ctx: 311 kernel_context_close(ctx); 312 return rq; 313 } 314 315 static int 316 release_queue(struct intel_engine_cs *engine, 317 struct i915_vma *vma, 318 int idx) 319 { 320 struct i915_sched_attr attr = { 321 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX), 322 }; 323 struct i915_request *rq; 324 u32 *cs; 325 326 rq = i915_request_create(engine->kernel_context); 327 if (IS_ERR(rq)) 328 return PTR_ERR(rq); 329 330 cs = intel_ring_begin(rq, 4); 331 if (IS_ERR(cs)) { 332 i915_request_add(rq); 333 return PTR_ERR(cs); 334 } 335 336 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 337 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1); 338 *cs++ = 0; 339 *cs++ = 1; 340 341 intel_ring_advance(rq, cs); 342 i915_request_add(rq); 343 344 engine->schedule(rq, &attr); 345 346 return 0; 347 } 348 349 static int 350 slice_semaphore_queue(struct intel_engine_cs *outer, 351 struct i915_vma *vma, 352 int count) 353 { 354 struct intel_engine_cs *engine; 355 struct i915_request *head; 356 enum intel_engine_id id; 357 int err, i, n = 0; 358 359 head = semaphore_queue(outer, vma, n++); 360 if (IS_ERR(head)) 361 return PTR_ERR(head); 362 363 i915_request_get(head); 364 for_each_engine(engine, outer->i915, id) { 365 for (i = 0; i < count; i++) { 366 struct i915_request *rq; 367 368 rq = semaphore_queue(engine, vma, n++); 369 if (IS_ERR(rq)) { 370 err = PTR_ERR(rq); 371 goto out; 372 } 373 } 374 } 375 376 err = release_queue(outer, vma, n); 377 if (err) 378 goto out; 379 380 if (i915_request_wait(head, 0, 381 2 * RUNTIME_INFO(outer->i915)->num_engines * (count + 2) * (count + 3)) < 0) { 382 pr_err("Failed to slice along semaphore chain of length (%d, %d)!\n", 383 count, n); 384 GEM_TRACE_DUMP(); 385 intel_gt_set_wedged(outer->gt); 386 err = -EIO; 387 } 388 389 out: 390 i915_request_put(head); 391 return err; 392 } 393 394 static int live_timeslice_preempt(void *arg) 395 { 396 struct drm_i915_private *i915 = arg; 397 struct drm_i915_gem_object *obj; 398 struct i915_vma *vma; 399 void *vaddr; 400 int err = 0; 401 int count; 402 403 /* 404 * If a request takes too long, we would like to give other users 405 * a fair go on the GPU. In particular, users may create batches 406 * that wait upon external input, where that input may even be 407 * supplied by another GPU job. To avoid blocking forever, we 408 * need to preempt the current task and replace it with another 409 * ready task. 410 */ 411 412 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 413 if (IS_ERR(obj)) 414 return PTR_ERR(obj); 415 416 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); 417 if (IS_ERR(vma)) { 418 err = PTR_ERR(vma); 419 goto err_obj; 420 } 421 422 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC); 423 if (IS_ERR(vaddr)) { 424 err = PTR_ERR(vaddr); 425 goto err_obj; 426 } 427 428 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 429 if (err) 430 goto err_map; 431 432 for_each_prime_number_from(count, 1, 16) { 433 struct intel_engine_cs *engine; 434 enum intel_engine_id id; 435 436 for_each_engine(engine, i915, id) { 437 if (!intel_engine_has_preemption(engine)) 438 continue; 439 440 memset(vaddr, 0, PAGE_SIZE); 441 442 err = slice_semaphore_queue(engine, vma, count); 443 if (err) 444 goto err_pin; 445 446 if (igt_flush_test(i915)) { 447 err = -EIO; 448 goto err_pin; 449 } 450 } 451 } 452 453 err_pin: 454 i915_vma_unpin(vma); 455 err_map: 456 i915_gem_object_unpin_map(obj); 457 err_obj: 458 i915_gem_object_put(obj); 459 return err; 460 } 461 462 static int live_busywait_preempt(void *arg) 463 { 464 struct drm_i915_private *i915 = arg; 465 struct i915_gem_context *ctx_hi, *ctx_lo; 466 struct intel_engine_cs *engine; 467 struct drm_i915_gem_object *obj; 468 struct i915_vma *vma; 469 enum intel_engine_id id; 470 int err = -ENOMEM; 471 u32 *map; 472 473 /* 474 * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can 475 * preempt the busywaits used to synchronise between rings. 476 */ 477 478 ctx_hi = kernel_context(i915); 479 if (!ctx_hi) 480 return -ENOMEM; 481 ctx_hi->sched.priority = 482 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY); 483 484 ctx_lo = kernel_context(i915); 485 if (!ctx_lo) 486 goto err_ctx_hi; 487 ctx_lo->sched.priority = 488 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY); 489 490 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 491 if (IS_ERR(obj)) { 492 err = PTR_ERR(obj); 493 goto err_ctx_lo; 494 } 495 496 map = i915_gem_object_pin_map(obj, I915_MAP_WC); 497 if (IS_ERR(map)) { 498 err = PTR_ERR(map); 499 goto err_obj; 500 } 501 502 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); 503 if (IS_ERR(vma)) { 504 err = PTR_ERR(vma); 505 goto err_map; 506 } 507 508 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 509 if (err) 510 goto err_map; 511 512 for_each_engine(engine, i915, id) { 513 struct i915_request *lo, *hi; 514 struct igt_live_test t; 515 u32 *cs; 516 517 if (!intel_engine_has_preemption(engine)) 518 continue; 519 520 if (!intel_engine_can_store_dword(engine)) 521 continue; 522 523 if (igt_live_test_begin(&t, i915, __func__, engine->name)) { 524 err = -EIO; 525 goto err_vma; 526 } 527 528 /* 529 * We create two requests. The low priority request 530 * busywaits on a semaphore (inside the ringbuffer where 531 * is should be preemptible) and the high priority requests 532 * uses a MI_STORE_DWORD_IMM to update the semaphore value 533 * allowing the first request to complete. If preemption 534 * fails, we hang instead. 535 */ 536 537 lo = igt_request_alloc(ctx_lo, engine); 538 if (IS_ERR(lo)) { 539 err = PTR_ERR(lo); 540 goto err_vma; 541 } 542 543 cs = intel_ring_begin(lo, 8); 544 if (IS_ERR(cs)) { 545 err = PTR_ERR(cs); 546 i915_request_add(lo); 547 goto err_vma; 548 } 549 550 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 551 *cs++ = i915_ggtt_offset(vma); 552 *cs++ = 0; 553 *cs++ = 1; 554 555 /* XXX Do we need a flush + invalidate here? */ 556 557 *cs++ = MI_SEMAPHORE_WAIT | 558 MI_SEMAPHORE_GLOBAL_GTT | 559 MI_SEMAPHORE_POLL | 560 MI_SEMAPHORE_SAD_EQ_SDD; 561 *cs++ = 0; 562 *cs++ = i915_ggtt_offset(vma); 563 *cs++ = 0; 564 565 intel_ring_advance(lo, cs); 566 i915_request_add(lo); 567 568 if (wait_for(READ_ONCE(*map), 10)) { 569 err = -ETIMEDOUT; 570 goto err_vma; 571 } 572 573 /* Low priority request should be busywaiting now */ 574 if (i915_request_wait(lo, 0, 1) != -ETIME) { 575 pr_err("%s: Busywaiting request did not!\n", 576 engine->name); 577 err = -EIO; 578 goto err_vma; 579 } 580 581 hi = igt_request_alloc(ctx_hi, engine); 582 if (IS_ERR(hi)) { 583 err = PTR_ERR(hi); 584 goto err_vma; 585 } 586 587 cs = intel_ring_begin(hi, 4); 588 if (IS_ERR(cs)) { 589 err = PTR_ERR(cs); 590 i915_request_add(hi); 591 goto err_vma; 592 } 593 594 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 595 *cs++ = i915_ggtt_offset(vma); 596 *cs++ = 0; 597 *cs++ = 0; 598 599 intel_ring_advance(hi, cs); 600 i915_request_add(hi); 601 602 if (i915_request_wait(lo, 0, HZ / 5) < 0) { 603 struct drm_printer p = drm_info_printer(i915->drm.dev); 604 605 pr_err("%s: Failed to preempt semaphore busywait!\n", 606 engine->name); 607 608 intel_engine_dump(engine, &p, "%s\n", engine->name); 609 GEM_TRACE_DUMP(); 610 611 intel_gt_set_wedged(&i915->gt); 612 err = -EIO; 613 goto err_vma; 614 } 615 GEM_BUG_ON(READ_ONCE(*map)); 616 617 if (igt_live_test_end(&t)) { 618 err = -EIO; 619 goto err_vma; 620 } 621 } 622 623 err = 0; 624 err_vma: 625 i915_vma_unpin(vma); 626 err_map: 627 i915_gem_object_unpin_map(obj); 628 err_obj: 629 i915_gem_object_put(obj); 630 err_ctx_lo: 631 kernel_context_close(ctx_lo); 632 err_ctx_hi: 633 kernel_context_close(ctx_hi); 634 return err; 635 } 636 637 static struct i915_request * 638 spinner_create_request(struct igt_spinner *spin, 639 struct i915_gem_context *ctx, 640 struct intel_engine_cs *engine, 641 u32 arb) 642 { 643 struct intel_context *ce; 644 struct i915_request *rq; 645 646 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx); 647 if (IS_ERR(ce)) 648 return ERR_CAST(ce); 649 650 rq = igt_spinner_create_request(spin, ce, arb); 651 intel_context_put(ce); 652 return rq; 653 } 654 655 static int live_preempt(void *arg) 656 { 657 struct drm_i915_private *i915 = arg; 658 struct i915_gem_context *ctx_hi, *ctx_lo; 659 struct igt_spinner spin_hi, spin_lo; 660 struct intel_engine_cs *engine; 661 enum intel_engine_id id; 662 int err = -ENOMEM; 663 664 if (!HAS_LOGICAL_RING_PREEMPTION(i915)) 665 return 0; 666 667 if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION)) 668 pr_err("Logical preemption supported, but not exposed\n"); 669 670 if (igt_spinner_init(&spin_hi, &i915->gt)) 671 return -ENOMEM; 672 673 if (igt_spinner_init(&spin_lo, &i915->gt)) 674 goto err_spin_hi; 675 676 ctx_hi = kernel_context(i915); 677 if (!ctx_hi) 678 goto err_spin_lo; 679 ctx_hi->sched.priority = 680 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY); 681 682 ctx_lo = kernel_context(i915); 683 if (!ctx_lo) 684 goto err_ctx_hi; 685 ctx_lo->sched.priority = 686 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY); 687 688 for_each_engine(engine, i915, id) { 689 struct igt_live_test t; 690 struct i915_request *rq; 691 692 if (!intel_engine_has_preemption(engine)) 693 continue; 694 695 if (igt_live_test_begin(&t, i915, __func__, engine->name)) { 696 err = -EIO; 697 goto err_ctx_lo; 698 } 699 700 rq = spinner_create_request(&spin_lo, ctx_lo, engine, 701 MI_ARB_CHECK); 702 if (IS_ERR(rq)) { 703 err = PTR_ERR(rq); 704 goto err_ctx_lo; 705 } 706 707 i915_request_add(rq); 708 if (!igt_wait_for_spinner(&spin_lo, rq)) { 709 GEM_TRACE("lo spinner failed to start\n"); 710 GEM_TRACE_DUMP(); 711 intel_gt_set_wedged(&i915->gt); 712 err = -EIO; 713 goto err_ctx_lo; 714 } 715 716 rq = spinner_create_request(&spin_hi, ctx_hi, engine, 717 MI_ARB_CHECK); 718 if (IS_ERR(rq)) { 719 igt_spinner_end(&spin_lo); 720 err = PTR_ERR(rq); 721 goto err_ctx_lo; 722 } 723 724 i915_request_add(rq); 725 if (!igt_wait_for_spinner(&spin_hi, rq)) { 726 GEM_TRACE("hi spinner failed to start\n"); 727 GEM_TRACE_DUMP(); 728 intel_gt_set_wedged(&i915->gt); 729 err = -EIO; 730 goto err_ctx_lo; 731 } 732 733 igt_spinner_end(&spin_hi); 734 igt_spinner_end(&spin_lo); 735 736 if (igt_live_test_end(&t)) { 737 err = -EIO; 738 goto err_ctx_lo; 739 } 740 } 741 742 err = 0; 743 err_ctx_lo: 744 kernel_context_close(ctx_lo); 745 err_ctx_hi: 746 kernel_context_close(ctx_hi); 747 err_spin_lo: 748 igt_spinner_fini(&spin_lo); 749 err_spin_hi: 750 igt_spinner_fini(&spin_hi); 751 return err; 752 } 753 754 static int live_late_preempt(void *arg) 755 { 756 struct drm_i915_private *i915 = arg; 757 struct i915_gem_context *ctx_hi, *ctx_lo; 758 struct igt_spinner spin_hi, spin_lo; 759 struct intel_engine_cs *engine; 760 struct i915_sched_attr attr = {}; 761 enum intel_engine_id id; 762 int err = -ENOMEM; 763 764 if (!HAS_LOGICAL_RING_PREEMPTION(i915)) 765 return 0; 766 767 if (igt_spinner_init(&spin_hi, &i915->gt)) 768 return -ENOMEM; 769 770 if (igt_spinner_init(&spin_lo, &i915->gt)) 771 goto err_spin_hi; 772 773 ctx_hi = kernel_context(i915); 774 if (!ctx_hi) 775 goto err_spin_lo; 776 777 ctx_lo = kernel_context(i915); 778 if (!ctx_lo) 779 goto err_ctx_hi; 780 781 /* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */ 782 ctx_lo->sched.priority = I915_USER_PRIORITY(1); 783 784 for_each_engine(engine, i915, id) { 785 struct igt_live_test t; 786 struct i915_request *rq; 787 788 if (!intel_engine_has_preemption(engine)) 789 continue; 790 791 if (igt_live_test_begin(&t, i915, __func__, engine->name)) { 792 err = -EIO; 793 goto err_ctx_lo; 794 } 795 796 rq = spinner_create_request(&spin_lo, ctx_lo, engine, 797 MI_ARB_CHECK); 798 if (IS_ERR(rq)) { 799 err = PTR_ERR(rq); 800 goto err_ctx_lo; 801 } 802 803 i915_request_add(rq); 804 if (!igt_wait_for_spinner(&spin_lo, rq)) { 805 pr_err("First context failed to start\n"); 806 goto err_wedged; 807 } 808 809 rq = spinner_create_request(&spin_hi, ctx_hi, engine, 810 MI_NOOP); 811 if (IS_ERR(rq)) { 812 igt_spinner_end(&spin_lo); 813 err = PTR_ERR(rq); 814 goto err_ctx_lo; 815 } 816 817 i915_request_add(rq); 818 if (igt_wait_for_spinner(&spin_hi, rq)) { 819 pr_err("Second context overtook first?\n"); 820 goto err_wedged; 821 } 822 823 attr.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX); 824 engine->schedule(rq, &attr); 825 826 if (!igt_wait_for_spinner(&spin_hi, rq)) { 827 pr_err("High priority context failed to preempt the low priority context\n"); 828 GEM_TRACE_DUMP(); 829 goto err_wedged; 830 } 831 832 igt_spinner_end(&spin_hi); 833 igt_spinner_end(&spin_lo); 834 835 if (igt_live_test_end(&t)) { 836 err = -EIO; 837 goto err_ctx_lo; 838 } 839 } 840 841 err = 0; 842 err_ctx_lo: 843 kernel_context_close(ctx_lo); 844 err_ctx_hi: 845 kernel_context_close(ctx_hi); 846 err_spin_lo: 847 igt_spinner_fini(&spin_lo); 848 err_spin_hi: 849 igt_spinner_fini(&spin_hi); 850 return err; 851 852 err_wedged: 853 igt_spinner_end(&spin_hi); 854 igt_spinner_end(&spin_lo); 855 intel_gt_set_wedged(&i915->gt); 856 err = -EIO; 857 goto err_ctx_lo; 858 } 859 860 struct preempt_client { 861 struct igt_spinner spin; 862 struct i915_gem_context *ctx; 863 }; 864 865 static int preempt_client_init(struct drm_i915_private *i915, 866 struct preempt_client *c) 867 { 868 c->ctx = kernel_context(i915); 869 if (!c->ctx) 870 return -ENOMEM; 871 872 if (igt_spinner_init(&c->spin, &i915->gt)) 873 goto err_ctx; 874 875 return 0; 876 877 err_ctx: 878 kernel_context_close(c->ctx); 879 return -ENOMEM; 880 } 881 882 static void preempt_client_fini(struct preempt_client *c) 883 { 884 igt_spinner_fini(&c->spin); 885 kernel_context_close(c->ctx); 886 } 887 888 static int live_nopreempt(void *arg) 889 { 890 struct drm_i915_private *i915 = arg; 891 struct intel_engine_cs *engine; 892 struct preempt_client a, b; 893 enum intel_engine_id id; 894 int err = -ENOMEM; 895 896 /* 897 * Verify that we can disable preemption for an individual request 898 * that may be being observed and not want to be interrupted. 899 */ 900 901 if (!HAS_LOGICAL_RING_PREEMPTION(i915)) 902 return 0; 903 904 if (preempt_client_init(i915, &a)) 905 return -ENOMEM; 906 if (preempt_client_init(i915, &b)) 907 goto err_client_a; 908 b.ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX); 909 910 for_each_engine(engine, i915, id) { 911 struct i915_request *rq_a, *rq_b; 912 913 if (!intel_engine_has_preemption(engine)) 914 continue; 915 916 engine->execlists.preempt_hang.count = 0; 917 918 rq_a = spinner_create_request(&a.spin, 919 a.ctx, engine, 920 MI_ARB_CHECK); 921 if (IS_ERR(rq_a)) { 922 err = PTR_ERR(rq_a); 923 goto err_client_b; 924 } 925 926 /* Low priority client, but unpreemptable! */ 927 rq_a->flags |= I915_REQUEST_NOPREEMPT; 928 929 i915_request_add(rq_a); 930 if (!igt_wait_for_spinner(&a.spin, rq_a)) { 931 pr_err("First client failed to start\n"); 932 goto err_wedged; 933 } 934 935 rq_b = spinner_create_request(&b.spin, 936 b.ctx, engine, 937 MI_ARB_CHECK); 938 if (IS_ERR(rq_b)) { 939 err = PTR_ERR(rq_b); 940 goto err_client_b; 941 } 942 943 i915_request_add(rq_b); 944 945 /* B is much more important than A! (But A is unpreemptable.) */ 946 GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a)); 947 948 /* Wait long enough for preemption and timeslicing */ 949 if (igt_wait_for_spinner(&b.spin, rq_b)) { 950 pr_err("Second client started too early!\n"); 951 goto err_wedged; 952 } 953 954 igt_spinner_end(&a.spin); 955 956 if (!igt_wait_for_spinner(&b.spin, rq_b)) { 957 pr_err("Second client failed to start\n"); 958 goto err_wedged; 959 } 960 961 igt_spinner_end(&b.spin); 962 963 if (engine->execlists.preempt_hang.count) { 964 pr_err("Preemption recorded x%d; should have been suppressed!\n", 965 engine->execlists.preempt_hang.count); 966 err = -EINVAL; 967 goto err_wedged; 968 } 969 970 if (igt_flush_test(i915)) 971 goto err_wedged; 972 } 973 974 err = 0; 975 err_client_b: 976 preempt_client_fini(&b); 977 err_client_a: 978 preempt_client_fini(&a); 979 return err; 980 981 err_wedged: 982 igt_spinner_end(&b.spin); 983 igt_spinner_end(&a.spin); 984 intel_gt_set_wedged(&i915->gt); 985 err = -EIO; 986 goto err_client_b; 987 } 988 989 static int live_suppress_self_preempt(void *arg) 990 { 991 struct drm_i915_private *i915 = arg; 992 struct intel_engine_cs *engine; 993 struct i915_sched_attr attr = { 994 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX) 995 }; 996 struct preempt_client a, b; 997 enum intel_engine_id id; 998 int err = -ENOMEM; 999 1000 /* 1001 * Verify that if a preemption request does not cause a change in 1002 * the current execution order, the preempt-to-idle injection is 1003 * skipped and that we do not accidentally apply it after the CS 1004 * completion event. 1005 */ 1006 1007 if (!HAS_LOGICAL_RING_PREEMPTION(i915)) 1008 return 0; 1009 1010 if (USES_GUC_SUBMISSION(i915)) 1011 return 0; /* presume black blox */ 1012 1013 if (intel_vgpu_active(i915)) 1014 return 0; /* GVT forces single port & request submission */ 1015 1016 if (preempt_client_init(i915, &a)) 1017 return -ENOMEM; 1018 if (preempt_client_init(i915, &b)) 1019 goto err_client_a; 1020 1021 for_each_engine(engine, i915, id) { 1022 struct i915_request *rq_a, *rq_b; 1023 int depth; 1024 1025 if (!intel_engine_has_preemption(engine)) 1026 continue; 1027 1028 if (igt_flush_test(i915)) 1029 goto err_wedged; 1030 1031 intel_engine_pm_get(engine); 1032 engine->execlists.preempt_hang.count = 0; 1033 1034 rq_a = spinner_create_request(&a.spin, 1035 a.ctx, engine, 1036 MI_NOOP); 1037 if (IS_ERR(rq_a)) { 1038 err = PTR_ERR(rq_a); 1039 intel_engine_pm_put(engine); 1040 goto err_client_b; 1041 } 1042 1043 i915_request_add(rq_a); 1044 if (!igt_wait_for_spinner(&a.spin, rq_a)) { 1045 pr_err("First client failed to start\n"); 1046 intel_engine_pm_put(engine); 1047 goto err_wedged; 1048 } 1049 1050 /* Keep postponing the timer to avoid premature slicing */ 1051 mod_timer(&engine->execlists.timer, jiffies + HZ); 1052 for (depth = 0; depth < 8; depth++) { 1053 rq_b = spinner_create_request(&b.spin, 1054 b.ctx, engine, 1055 MI_NOOP); 1056 if (IS_ERR(rq_b)) { 1057 err = PTR_ERR(rq_b); 1058 intel_engine_pm_put(engine); 1059 goto err_client_b; 1060 } 1061 i915_request_add(rq_b); 1062 1063 GEM_BUG_ON(i915_request_completed(rq_a)); 1064 engine->schedule(rq_a, &attr); 1065 igt_spinner_end(&a.spin); 1066 1067 if (!igt_wait_for_spinner(&b.spin, rq_b)) { 1068 pr_err("Second client failed to start\n"); 1069 intel_engine_pm_put(engine); 1070 goto err_wedged; 1071 } 1072 1073 swap(a, b); 1074 rq_a = rq_b; 1075 } 1076 igt_spinner_end(&a.spin); 1077 1078 if (engine->execlists.preempt_hang.count) { 1079 pr_err("Preemption on %s recorded x%d, depth %d; should have been suppressed!\n", 1080 engine->name, 1081 engine->execlists.preempt_hang.count, 1082 depth); 1083 intel_engine_pm_put(engine); 1084 err = -EINVAL; 1085 goto err_client_b; 1086 } 1087 1088 intel_engine_pm_put(engine); 1089 if (igt_flush_test(i915)) 1090 goto err_wedged; 1091 } 1092 1093 err = 0; 1094 err_client_b: 1095 preempt_client_fini(&b); 1096 err_client_a: 1097 preempt_client_fini(&a); 1098 return err; 1099 1100 err_wedged: 1101 igt_spinner_end(&b.spin); 1102 igt_spinner_end(&a.spin); 1103 intel_gt_set_wedged(&i915->gt); 1104 err = -EIO; 1105 goto err_client_b; 1106 } 1107 1108 static int __i915_sw_fence_call 1109 dummy_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) 1110 { 1111 return NOTIFY_DONE; 1112 } 1113 1114 static struct i915_request *dummy_request(struct intel_engine_cs *engine) 1115 { 1116 struct i915_request *rq; 1117 1118 rq = kzalloc(sizeof(*rq), GFP_KERNEL); 1119 if (!rq) 1120 return NULL; 1121 1122 rq->engine = engine; 1123 1124 spin_lock_init(&rq->lock); 1125 INIT_LIST_HEAD(&rq->fence.cb_list); 1126 rq->fence.lock = &rq->lock; 1127 rq->fence.ops = &i915_fence_ops; 1128 1129 i915_sched_node_init(&rq->sched); 1130 1131 /* mark this request as permanently incomplete */ 1132 rq->fence.seqno = 1; 1133 BUILD_BUG_ON(sizeof(rq->fence.seqno) != 8); /* upper 32b == 0 */ 1134 rq->hwsp_seqno = (u32 *)&rq->fence.seqno + 1; 1135 GEM_BUG_ON(i915_request_completed(rq)); 1136 1137 i915_sw_fence_init(&rq->submit, dummy_notify); 1138 set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); 1139 1140 spin_lock_init(&rq->lock); 1141 rq->fence.lock = &rq->lock; 1142 INIT_LIST_HEAD(&rq->fence.cb_list); 1143 1144 return rq; 1145 } 1146 1147 static void dummy_request_free(struct i915_request *dummy) 1148 { 1149 /* We have to fake the CS interrupt to kick the next request */ 1150 i915_sw_fence_commit(&dummy->submit); 1151 1152 i915_request_mark_complete(dummy); 1153 dma_fence_signal(&dummy->fence); 1154 1155 i915_sched_node_fini(&dummy->sched); 1156 i915_sw_fence_fini(&dummy->submit); 1157 1158 dma_fence_free(&dummy->fence); 1159 } 1160 1161 static int live_suppress_wait_preempt(void *arg) 1162 { 1163 struct drm_i915_private *i915 = arg; 1164 struct preempt_client client[4]; 1165 struct intel_engine_cs *engine; 1166 enum intel_engine_id id; 1167 int err = -ENOMEM; 1168 int i; 1169 1170 /* 1171 * Waiters are given a little priority nudge, but not enough 1172 * to actually cause any preemption. Double check that we do 1173 * not needlessly generate preempt-to-idle cycles. 1174 */ 1175 1176 if (!HAS_LOGICAL_RING_PREEMPTION(i915)) 1177 return 0; 1178 1179 if (preempt_client_init(i915, &client[0])) /* ELSP[0] */ 1180 return -ENOMEM; 1181 if (preempt_client_init(i915, &client[1])) /* ELSP[1] */ 1182 goto err_client_0; 1183 if (preempt_client_init(i915, &client[2])) /* head of queue */ 1184 goto err_client_1; 1185 if (preempt_client_init(i915, &client[3])) /* bystander */ 1186 goto err_client_2; 1187 1188 for_each_engine(engine, i915, id) { 1189 int depth; 1190 1191 if (!intel_engine_has_preemption(engine)) 1192 continue; 1193 1194 if (!engine->emit_init_breadcrumb) 1195 continue; 1196 1197 for (depth = 0; depth < ARRAY_SIZE(client); depth++) { 1198 struct i915_request *rq[ARRAY_SIZE(client)]; 1199 struct i915_request *dummy; 1200 1201 engine->execlists.preempt_hang.count = 0; 1202 1203 dummy = dummy_request(engine); 1204 if (!dummy) 1205 goto err_client_3; 1206 1207 for (i = 0; i < ARRAY_SIZE(client); i++) { 1208 rq[i] = spinner_create_request(&client[i].spin, 1209 client[i].ctx, engine, 1210 MI_NOOP); 1211 if (IS_ERR(rq[i])) { 1212 err = PTR_ERR(rq[i]); 1213 goto err_wedged; 1214 } 1215 1216 /* Disable NEWCLIENT promotion */ 1217 __i915_active_fence_set(&i915_request_timeline(rq[i])->last_request, 1218 &dummy->fence); 1219 i915_request_add(rq[i]); 1220 } 1221 1222 dummy_request_free(dummy); 1223 1224 GEM_BUG_ON(i915_request_completed(rq[0])); 1225 if (!igt_wait_for_spinner(&client[0].spin, rq[0])) { 1226 pr_err("%s: First client failed to start\n", 1227 engine->name); 1228 goto err_wedged; 1229 } 1230 GEM_BUG_ON(!i915_request_started(rq[0])); 1231 1232 if (i915_request_wait(rq[depth], 1233 I915_WAIT_PRIORITY, 1234 1) != -ETIME) { 1235 pr_err("%s: Waiter depth:%d completed!\n", 1236 engine->name, depth); 1237 goto err_wedged; 1238 } 1239 1240 for (i = 0; i < ARRAY_SIZE(client); i++) 1241 igt_spinner_end(&client[i].spin); 1242 1243 if (igt_flush_test(i915)) 1244 goto err_wedged; 1245 1246 if (engine->execlists.preempt_hang.count) { 1247 pr_err("%s: Preemption recorded x%d, depth %d; should have been suppressed!\n", 1248 engine->name, 1249 engine->execlists.preempt_hang.count, 1250 depth); 1251 err = -EINVAL; 1252 goto err_client_3; 1253 } 1254 } 1255 } 1256 1257 err = 0; 1258 err_client_3: 1259 preempt_client_fini(&client[3]); 1260 err_client_2: 1261 preempt_client_fini(&client[2]); 1262 err_client_1: 1263 preempt_client_fini(&client[1]); 1264 err_client_0: 1265 preempt_client_fini(&client[0]); 1266 return err; 1267 1268 err_wedged: 1269 for (i = 0; i < ARRAY_SIZE(client); i++) 1270 igt_spinner_end(&client[i].spin); 1271 intel_gt_set_wedged(&i915->gt); 1272 err = -EIO; 1273 goto err_client_3; 1274 } 1275 1276 static int live_chain_preempt(void *arg) 1277 { 1278 struct drm_i915_private *i915 = arg; 1279 struct intel_engine_cs *engine; 1280 struct preempt_client hi, lo; 1281 enum intel_engine_id id; 1282 int err = -ENOMEM; 1283 1284 /* 1285 * Build a chain AB...BA between two contexts (A, B) and request 1286 * preemption of the last request. It should then complete before 1287 * the previously submitted spinner in B. 1288 */ 1289 1290 if (!HAS_LOGICAL_RING_PREEMPTION(i915)) 1291 return 0; 1292 1293 if (preempt_client_init(i915, &hi)) 1294 return -ENOMEM; 1295 1296 if (preempt_client_init(i915, &lo)) 1297 goto err_client_hi; 1298 1299 for_each_engine(engine, i915, id) { 1300 struct i915_sched_attr attr = { 1301 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX), 1302 }; 1303 struct igt_live_test t; 1304 struct i915_request *rq; 1305 int ring_size, count, i; 1306 1307 if (!intel_engine_has_preemption(engine)) 1308 continue; 1309 1310 rq = spinner_create_request(&lo.spin, 1311 lo.ctx, engine, 1312 MI_ARB_CHECK); 1313 if (IS_ERR(rq)) 1314 goto err_wedged; 1315 i915_request_add(rq); 1316 1317 ring_size = rq->wa_tail - rq->head; 1318 if (ring_size < 0) 1319 ring_size += rq->ring->size; 1320 ring_size = rq->ring->size / ring_size; 1321 pr_debug("%s(%s): Using maximum of %d requests\n", 1322 __func__, engine->name, ring_size); 1323 1324 igt_spinner_end(&lo.spin); 1325 if (i915_request_wait(rq, 0, HZ / 2) < 0) { 1326 pr_err("Timed out waiting to flush %s\n", engine->name); 1327 goto err_wedged; 1328 } 1329 1330 if (igt_live_test_begin(&t, i915, __func__, engine->name)) { 1331 err = -EIO; 1332 goto err_wedged; 1333 } 1334 1335 for_each_prime_number_from(count, 1, ring_size) { 1336 rq = spinner_create_request(&hi.spin, 1337 hi.ctx, engine, 1338 MI_ARB_CHECK); 1339 if (IS_ERR(rq)) 1340 goto err_wedged; 1341 i915_request_add(rq); 1342 if (!igt_wait_for_spinner(&hi.spin, rq)) 1343 goto err_wedged; 1344 1345 rq = spinner_create_request(&lo.spin, 1346 lo.ctx, engine, 1347 MI_ARB_CHECK); 1348 if (IS_ERR(rq)) 1349 goto err_wedged; 1350 i915_request_add(rq); 1351 1352 for (i = 0; i < count; i++) { 1353 rq = igt_request_alloc(lo.ctx, engine); 1354 if (IS_ERR(rq)) 1355 goto err_wedged; 1356 i915_request_add(rq); 1357 } 1358 1359 rq = igt_request_alloc(hi.ctx, engine); 1360 if (IS_ERR(rq)) 1361 goto err_wedged; 1362 i915_request_add(rq); 1363 engine->schedule(rq, &attr); 1364 1365 igt_spinner_end(&hi.spin); 1366 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 1367 struct drm_printer p = 1368 drm_info_printer(i915->drm.dev); 1369 1370 pr_err("Failed to preempt over chain of %d\n", 1371 count); 1372 intel_engine_dump(engine, &p, 1373 "%s\n", engine->name); 1374 goto err_wedged; 1375 } 1376 igt_spinner_end(&lo.spin); 1377 1378 rq = igt_request_alloc(lo.ctx, engine); 1379 if (IS_ERR(rq)) 1380 goto err_wedged; 1381 i915_request_add(rq); 1382 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 1383 struct drm_printer p = 1384 drm_info_printer(i915->drm.dev); 1385 1386 pr_err("Failed to flush low priority chain of %d requests\n", 1387 count); 1388 intel_engine_dump(engine, &p, 1389 "%s\n", engine->name); 1390 goto err_wedged; 1391 } 1392 } 1393 1394 if (igt_live_test_end(&t)) { 1395 err = -EIO; 1396 goto err_wedged; 1397 } 1398 } 1399 1400 err = 0; 1401 err_client_lo: 1402 preempt_client_fini(&lo); 1403 err_client_hi: 1404 preempt_client_fini(&hi); 1405 return err; 1406 1407 err_wedged: 1408 igt_spinner_end(&hi.spin); 1409 igt_spinner_end(&lo.spin); 1410 intel_gt_set_wedged(&i915->gt); 1411 err = -EIO; 1412 goto err_client_lo; 1413 } 1414 1415 static int live_preempt_hang(void *arg) 1416 { 1417 struct drm_i915_private *i915 = arg; 1418 struct i915_gem_context *ctx_hi, *ctx_lo; 1419 struct igt_spinner spin_hi, spin_lo; 1420 struct intel_engine_cs *engine; 1421 enum intel_engine_id id; 1422 int err = -ENOMEM; 1423 1424 if (!HAS_LOGICAL_RING_PREEMPTION(i915)) 1425 return 0; 1426 1427 if (!intel_has_reset_engine(&i915->gt)) 1428 return 0; 1429 1430 if (igt_spinner_init(&spin_hi, &i915->gt)) 1431 return -ENOMEM; 1432 1433 if (igt_spinner_init(&spin_lo, &i915->gt)) 1434 goto err_spin_hi; 1435 1436 ctx_hi = kernel_context(i915); 1437 if (!ctx_hi) 1438 goto err_spin_lo; 1439 ctx_hi->sched.priority = 1440 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY); 1441 1442 ctx_lo = kernel_context(i915); 1443 if (!ctx_lo) 1444 goto err_ctx_hi; 1445 ctx_lo->sched.priority = 1446 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY); 1447 1448 for_each_engine(engine, i915, id) { 1449 struct i915_request *rq; 1450 1451 if (!intel_engine_has_preemption(engine)) 1452 continue; 1453 1454 rq = spinner_create_request(&spin_lo, ctx_lo, engine, 1455 MI_ARB_CHECK); 1456 if (IS_ERR(rq)) { 1457 err = PTR_ERR(rq); 1458 goto err_ctx_lo; 1459 } 1460 1461 i915_request_add(rq); 1462 if (!igt_wait_for_spinner(&spin_lo, rq)) { 1463 GEM_TRACE("lo spinner failed to start\n"); 1464 GEM_TRACE_DUMP(); 1465 intel_gt_set_wedged(&i915->gt); 1466 err = -EIO; 1467 goto err_ctx_lo; 1468 } 1469 1470 rq = spinner_create_request(&spin_hi, ctx_hi, engine, 1471 MI_ARB_CHECK); 1472 if (IS_ERR(rq)) { 1473 igt_spinner_end(&spin_lo); 1474 err = PTR_ERR(rq); 1475 goto err_ctx_lo; 1476 } 1477 1478 init_completion(&engine->execlists.preempt_hang.completion); 1479 engine->execlists.preempt_hang.inject_hang = true; 1480 1481 i915_request_add(rq); 1482 1483 if (!wait_for_completion_timeout(&engine->execlists.preempt_hang.completion, 1484 HZ / 10)) { 1485 pr_err("Preemption did not occur within timeout!"); 1486 GEM_TRACE_DUMP(); 1487 intel_gt_set_wedged(&i915->gt); 1488 err = -EIO; 1489 goto err_ctx_lo; 1490 } 1491 1492 set_bit(I915_RESET_ENGINE + id, &i915->gt.reset.flags); 1493 intel_engine_reset(engine, NULL); 1494 clear_bit(I915_RESET_ENGINE + id, &i915->gt.reset.flags); 1495 1496 engine->execlists.preempt_hang.inject_hang = false; 1497 1498 if (!igt_wait_for_spinner(&spin_hi, rq)) { 1499 GEM_TRACE("hi spinner failed to start\n"); 1500 GEM_TRACE_DUMP(); 1501 intel_gt_set_wedged(&i915->gt); 1502 err = -EIO; 1503 goto err_ctx_lo; 1504 } 1505 1506 igt_spinner_end(&spin_hi); 1507 igt_spinner_end(&spin_lo); 1508 if (igt_flush_test(i915)) { 1509 err = -EIO; 1510 goto err_ctx_lo; 1511 } 1512 } 1513 1514 err = 0; 1515 err_ctx_lo: 1516 kernel_context_close(ctx_lo); 1517 err_ctx_hi: 1518 kernel_context_close(ctx_hi); 1519 err_spin_lo: 1520 igt_spinner_fini(&spin_lo); 1521 err_spin_hi: 1522 igt_spinner_fini(&spin_hi); 1523 return err; 1524 } 1525 1526 static int random_range(struct rnd_state *rnd, int min, int max) 1527 { 1528 return i915_prandom_u32_max_state(max - min, rnd) + min; 1529 } 1530 1531 static int random_priority(struct rnd_state *rnd) 1532 { 1533 return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX); 1534 } 1535 1536 struct preempt_smoke { 1537 struct drm_i915_private *i915; 1538 struct i915_gem_context **contexts; 1539 struct intel_engine_cs *engine; 1540 struct drm_i915_gem_object *batch; 1541 unsigned int ncontext; 1542 struct rnd_state prng; 1543 unsigned long count; 1544 }; 1545 1546 static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke) 1547 { 1548 return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext, 1549 &smoke->prng)]; 1550 } 1551 1552 static int smoke_submit(struct preempt_smoke *smoke, 1553 struct i915_gem_context *ctx, int prio, 1554 struct drm_i915_gem_object *batch) 1555 { 1556 struct i915_request *rq; 1557 struct i915_vma *vma = NULL; 1558 int err = 0; 1559 1560 if (batch) { 1561 struct i915_address_space *vm; 1562 1563 vm = i915_gem_context_get_vm_rcu(ctx); 1564 vma = i915_vma_instance(batch, vm, NULL); 1565 i915_vm_put(vm); 1566 if (IS_ERR(vma)) 1567 return PTR_ERR(vma); 1568 1569 err = i915_vma_pin(vma, 0, 0, PIN_USER); 1570 if (err) 1571 return err; 1572 } 1573 1574 ctx->sched.priority = prio; 1575 1576 rq = igt_request_alloc(ctx, smoke->engine); 1577 if (IS_ERR(rq)) { 1578 err = PTR_ERR(rq); 1579 goto unpin; 1580 } 1581 1582 if (vma) { 1583 i915_vma_lock(vma); 1584 err = i915_request_await_object(rq, vma->obj, false); 1585 if (!err) 1586 err = i915_vma_move_to_active(vma, rq, 0); 1587 if (!err) 1588 err = rq->engine->emit_bb_start(rq, 1589 vma->node.start, 1590 PAGE_SIZE, 0); 1591 i915_vma_unlock(vma); 1592 } 1593 1594 i915_request_add(rq); 1595 1596 unpin: 1597 if (vma) 1598 i915_vma_unpin(vma); 1599 1600 return err; 1601 } 1602 1603 static int smoke_crescendo_thread(void *arg) 1604 { 1605 struct preempt_smoke *smoke = arg; 1606 IGT_TIMEOUT(end_time); 1607 unsigned long count; 1608 1609 count = 0; 1610 do { 1611 struct i915_gem_context *ctx = smoke_context(smoke); 1612 int err; 1613 1614 err = smoke_submit(smoke, 1615 ctx, count % I915_PRIORITY_MAX, 1616 smoke->batch); 1617 if (err) 1618 return err; 1619 1620 count++; 1621 } while (!__igt_timeout(end_time, NULL)); 1622 1623 smoke->count = count; 1624 return 0; 1625 } 1626 1627 static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags) 1628 #define BATCH BIT(0) 1629 { 1630 struct task_struct *tsk[I915_NUM_ENGINES] = {}; 1631 struct preempt_smoke arg[I915_NUM_ENGINES]; 1632 struct intel_engine_cs *engine; 1633 enum intel_engine_id id; 1634 unsigned long count; 1635 int err = 0; 1636 1637 for_each_engine(engine, smoke->i915, id) { 1638 arg[id] = *smoke; 1639 arg[id].engine = engine; 1640 if (!(flags & BATCH)) 1641 arg[id].batch = NULL; 1642 arg[id].count = 0; 1643 1644 tsk[id] = kthread_run(smoke_crescendo_thread, &arg, 1645 "igt/smoke:%d", id); 1646 if (IS_ERR(tsk[id])) { 1647 err = PTR_ERR(tsk[id]); 1648 break; 1649 } 1650 get_task_struct(tsk[id]); 1651 } 1652 1653 count = 0; 1654 for_each_engine(engine, smoke->i915, id) { 1655 int status; 1656 1657 if (IS_ERR_OR_NULL(tsk[id])) 1658 continue; 1659 1660 status = kthread_stop(tsk[id]); 1661 if (status && !err) 1662 err = status; 1663 1664 count += arg[id].count; 1665 1666 put_task_struct(tsk[id]); 1667 } 1668 1669 pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n", 1670 count, flags, 1671 RUNTIME_INFO(smoke->i915)->num_engines, smoke->ncontext); 1672 return 0; 1673 } 1674 1675 static int smoke_random(struct preempt_smoke *smoke, unsigned int flags) 1676 { 1677 enum intel_engine_id id; 1678 IGT_TIMEOUT(end_time); 1679 unsigned long count; 1680 1681 count = 0; 1682 do { 1683 for_each_engine(smoke->engine, smoke->i915, id) { 1684 struct i915_gem_context *ctx = smoke_context(smoke); 1685 int err; 1686 1687 err = smoke_submit(smoke, 1688 ctx, random_priority(&smoke->prng), 1689 flags & BATCH ? smoke->batch : NULL); 1690 if (err) 1691 return err; 1692 1693 count++; 1694 } 1695 } while (!__igt_timeout(end_time, NULL)); 1696 1697 pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n", 1698 count, flags, 1699 RUNTIME_INFO(smoke->i915)->num_engines, smoke->ncontext); 1700 return 0; 1701 } 1702 1703 static int live_preempt_smoke(void *arg) 1704 { 1705 struct preempt_smoke smoke = { 1706 .i915 = arg, 1707 .prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed), 1708 .ncontext = 1024, 1709 }; 1710 const unsigned int phase[] = { 0, BATCH }; 1711 struct igt_live_test t; 1712 int err = -ENOMEM; 1713 u32 *cs; 1714 int n; 1715 1716 if (!HAS_LOGICAL_RING_PREEMPTION(smoke.i915)) 1717 return 0; 1718 1719 smoke.contexts = kmalloc_array(smoke.ncontext, 1720 sizeof(*smoke.contexts), 1721 GFP_KERNEL); 1722 if (!smoke.contexts) 1723 return -ENOMEM; 1724 1725 smoke.batch = i915_gem_object_create_internal(smoke.i915, PAGE_SIZE); 1726 if (IS_ERR(smoke.batch)) { 1727 err = PTR_ERR(smoke.batch); 1728 goto err_free; 1729 } 1730 1731 cs = i915_gem_object_pin_map(smoke.batch, I915_MAP_WB); 1732 if (IS_ERR(cs)) { 1733 err = PTR_ERR(cs); 1734 goto err_batch; 1735 } 1736 for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++) 1737 cs[n] = MI_ARB_CHECK; 1738 cs[n] = MI_BATCH_BUFFER_END; 1739 i915_gem_object_flush_map(smoke.batch); 1740 i915_gem_object_unpin_map(smoke.batch); 1741 1742 if (igt_live_test_begin(&t, smoke.i915, __func__, "all")) { 1743 err = -EIO; 1744 goto err_batch; 1745 } 1746 1747 for (n = 0; n < smoke.ncontext; n++) { 1748 smoke.contexts[n] = kernel_context(smoke.i915); 1749 if (!smoke.contexts[n]) 1750 goto err_ctx; 1751 } 1752 1753 for (n = 0; n < ARRAY_SIZE(phase); n++) { 1754 err = smoke_crescendo(&smoke, phase[n]); 1755 if (err) 1756 goto err_ctx; 1757 1758 err = smoke_random(&smoke, phase[n]); 1759 if (err) 1760 goto err_ctx; 1761 } 1762 1763 err_ctx: 1764 if (igt_live_test_end(&t)) 1765 err = -EIO; 1766 1767 for (n = 0; n < smoke.ncontext; n++) { 1768 if (!smoke.contexts[n]) 1769 break; 1770 kernel_context_close(smoke.contexts[n]); 1771 } 1772 1773 err_batch: 1774 i915_gem_object_put(smoke.batch); 1775 err_free: 1776 kfree(smoke.contexts); 1777 1778 return err; 1779 } 1780 1781 static int nop_virtual_engine(struct drm_i915_private *i915, 1782 struct intel_engine_cs **siblings, 1783 unsigned int nsibling, 1784 unsigned int nctx, 1785 unsigned int flags) 1786 #define CHAIN BIT(0) 1787 { 1788 IGT_TIMEOUT(end_time); 1789 struct i915_request *request[16]; 1790 struct i915_gem_context *ctx[16]; 1791 struct intel_context *ve[16]; 1792 unsigned long n, prime, nc; 1793 struct igt_live_test t; 1794 ktime_t times[2] = {}; 1795 int err; 1796 1797 GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ctx)); 1798 1799 for (n = 0; n < nctx; n++) { 1800 ctx[n] = kernel_context(i915); 1801 if (!ctx[n]) { 1802 err = -ENOMEM; 1803 nctx = n; 1804 goto out; 1805 } 1806 1807 ve[n] = intel_execlists_create_virtual(ctx[n], 1808 siblings, nsibling); 1809 if (IS_ERR(ve[n])) { 1810 kernel_context_close(ctx[n]); 1811 err = PTR_ERR(ve[n]); 1812 nctx = n; 1813 goto out; 1814 } 1815 1816 err = intel_context_pin(ve[n]); 1817 if (err) { 1818 intel_context_put(ve[n]); 1819 kernel_context_close(ctx[n]); 1820 nctx = n; 1821 goto out; 1822 } 1823 } 1824 1825 err = igt_live_test_begin(&t, i915, __func__, ve[0]->engine->name); 1826 if (err) 1827 goto out; 1828 1829 for_each_prime_number_from(prime, 1, 8192) { 1830 times[1] = ktime_get_raw(); 1831 1832 if (flags & CHAIN) { 1833 for (nc = 0; nc < nctx; nc++) { 1834 for (n = 0; n < prime; n++) { 1835 request[nc] = 1836 i915_request_create(ve[nc]); 1837 if (IS_ERR(request[nc])) { 1838 err = PTR_ERR(request[nc]); 1839 goto out; 1840 } 1841 1842 i915_request_add(request[nc]); 1843 } 1844 } 1845 } else { 1846 for (n = 0; n < prime; n++) { 1847 for (nc = 0; nc < nctx; nc++) { 1848 request[nc] = 1849 i915_request_create(ve[nc]); 1850 if (IS_ERR(request[nc])) { 1851 err = PTR_ERR(request[nc]); 1852 goto out; 1853 } 1854 1855 i915_request_add(request[nc]); 1856 } 1857 } 1858 } 1859 1860 for (nc = 0; nc < nctx; nc++) { 1861 if (i915_request_wait(request[nc], 0, HZ / 10) < 0) { 1862 pr_err("%s(%s): wait for %llx:%lld timed out\n", 1863 __func__, ve[0]->engine->name, 1864 request[nc]->fence.context, 1865 request[nc]->fence.seqno); 1866 1867 GEM_TRACE("%s(%s) failed at request %llx:%lld\n", 1868 __func__, ve[0]->engine->name, 1869 request[nc]->fence.context, 1870 request[nc]->fence.seqno); 1871 GEM_TRACE_DUMP(); 1872 intel_gt_set_wedged(&i915->gt); 1873 break; 1874 } 1875 } 1876 1877 times[1] = ktime_sub(ktime_get_raw(), times[1]); 1878 if (prime == 1) 1879 times[0] = times[1]; 1880 1881 if (__igt_timeout(end_time, NULL)) 1882 break; 1883 } 1884 1885 err = igt_live_test_end(&t); 1886 if (err) 1887 goto out; 1888 1889 pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n", 1890 nctx, ve[0]->engine->name, ktime_to_ns(times[0]), 1891 prime, div64_u64(ktime_to_ns(times[1]), prime)); 1892 1893 out: 1894 if (igt_flush_test(i915)) 1895 err = -EIO; 1896 1897 for (nc = 0; nc < nctx; nc++) { 1898 intel_context_unpin(ve[nc]); 1899 intel_context_put(ve[nc]); 1900 kernel_context_close(ctx[nc]); 1901 } 1902 return err; 1903 } 1904 1905 static int live_virtual_engine(void *arg) 1906 { 1907 struct drm_i915_private *i915 = arg; 1908 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 1909 struct intel_engine_cs *engine; 1910 struct intel_gt *gt = &i915->gt; 1911 enum intel_engine_id id; 1912 unsigned int class, inst; 1913 int err; 1914 1915 if (USES_GUC_SUBMISSION(i915)) 1916 return 0; 1917 1918 for_each_engine(engine, i915, id) { 1919 err = nop_virtual_engine(i915, &engine, 1, 1, 0); 1920 if (err) { 1921 pr_err("Failed to wrap engine %s: err=%d\n", 1922 engine->name, err); 1923 return err; 1924 } 1925 } 1926 1927 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 1928 int nsibling, n; 1929 1930 nsibling = 0; 1931 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) { 1932 if (!gt->engine_class[class][inst]) 1933 continue; 1934 1935 siblings[nsibling++] = gt->engine_class[class][inst]; 1936 } 1937 if (nsibling < 2) 1938 continue; 1939 1940 for (n = 1; n <= nsibling + 1; n++) { 1941 err = nop_virtual_engine(i915, siblings, nsibling, 1942 n, 0); 1943 if (err) 1944 return err; 1945 } 1946 1947 err = nop_virtual_engine(i915, siblings, nsibling, n, CHAIN); 1948 if (err) 1949 return err; 1950 } 1951 1952 return 0; 1953 } 1954 1955 static int mask_virtual_engine(struct drm_i915_private *i915, 1956 struct intel_engine_cs **siblings, 1957 unsigned int nsibling) 1958 { 1959 struct i915_request *request[MAX_ENGINE_INSTANCE + 1]; 1960 struct i915_gem_context *ctx; 1961 struct intel_context *ve; 1962 struct igt_live_test t; 1963 unsigned int n; 1964 int err; 1965 1966 /* 1967 * Check that by setting the execution mask on a request, we can 1968 * restrict it to our desired engine within the virtual engine. 1969 */ 1970 1971 ctx = kernel_context(i915); 1972 if (!ctx) 1973 return -ENOMEM; 1974 1975 ve = intel_execlists_create_virtual(ctx, siblings, nsibling); 1976 if (IS_ERR(ve)) { 1977 err = PTR_ERR(ve); 1978 goto out_close; 1979 } 1980 1981 err = intel_context_pin(ve); 1982 if (err) 1983 goto out_put; 1984 1985 err = igt_live_test_begin(&t, i915, __func__, ve->engine->name); 1986 if (err) 1987 goto out_unpin; 1988 1989 for (n = 0; n < nsibling; n++) { 1990 request[n] = i915_request_create(ve); 1991 if (IS_ERR(request[n])) { 1992 err = PTR_ERR(request[n]); 1993 nsibling = n; 1994 goto out; 1995 } 1996 1997 /* Reverse order as it's more likely to be unnatural */ 1998 request[n]->execution_mask = siblings[nsibling - n - 1]->mask; 1999 2000 i915_request_get(request[n]); 2001 i915_request_add(request[n]); 2002 } 2003 2004 for (n = 0; n < nsibling; n++) { 2005 if (i915_request_wait(request[n], 0, HZ / 10) < 0) { 2006 pr_err("%s(%s): wait for %llx:%lld timed out\n", 2007 __func__, ve->engine->name, 2008 request[n]->fence.context, 2009 request[n]->fence.seqno); 2010 2011 GEM_TRACE("%s(%s) failed at request %llx:%lld\n", 2012 __func__, ve->engine->name, 2013 request[n]->fence.context, 2014 request[n]->fence.seqno); 2015 GEM_TRACE_DUMP(); 2016 intel_gt_set_wedged(&i915->gt); 2017 err = -EIO; 2018 goto out; 2019 } 2020 2021 if (request[n]->engine != siblings[nsibling - n - 1]) { 2022 pr_err("Executed on wrong sibling '%s', expected '%s'\n", 2023 request[n]->engine->name, 2024 siblings[nsibling - n - 1]->name); 2025 err = -EINVAL; 2026 goto out; 2027 } 2028 } 2029 2030 err = igt_live_test_end(&t); 2031 out: 2032 if (igt_flush_test(i915)) 2033 err = -EIO; 2034 2035 for (n = 0; n < nsibling; n++) 2036 i915_request_put(request[n]); 2037 2038 out_unpin: 2039 intel_context_unpin(ve); 2040 out_put: 2041 intel_context_put(ve); 2042 out_close: 2043 kernel_context_close(ctx); 2044 return err; 2045 } 2046 2047 static int live_virtual_mask(void *arg) 2048 { 2049 struct drm_i915_private *i915 = arg; 2050 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 2051 struct intel_gt *gt = &i915->gt; 2052 unsigned int class, inst; 2053 int err; 2054 2055 if (USES_GUC_SUBMISSION(i915)) 2056 return 0; 2057 2058 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 2059 unsigned int nsibling; 2060 2061 nsibling = 0; 2062 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) { 2063 if (!gt->engine_class[class][inst]) 2064 break; 2065 2066 siblings[nsibling++] = gt->engine_class[class][inst]; 2067 } 2068 if (nsibling < 2) 2069 continue; 2070 2071 err = mask_virtual_engine(i915, siblings, nsibling); 2072 if (err) 2073 return err; 2074 } 2075 2076 return 0; 2077 } 2078 2079 static int bond_virtual_engine(struct drm_i915_private *i915, 2080 unsigned int class, 2081 struct intel_engine_cs **siblings, 2082 unsigned int nsibling, 2083 unsigned int flags) 2084 #define BOND_SCHEDULE BIT(0) 2085 { 2086 struct intel_engine_cs *master; 2087 struct i915_gem_context *ctx; 2088 struct i915_request *rq[16]; 2089 enum intel_engine_id id; 2090 unsigned long n; 2091 int err; 2092 2093 GEM_BUG_ON(nsibling >= ARRAY_SIZE(rq) - 1); 2094 2095 ctx = kernel_context(i915); 2096 if (!ctx) 2097 return -ENOMEM; 2098 2099 err = 0; 2100 rq[0] = ERR_PTR(-ENOMEM); 2101 for_each_engine(master, i915, id) { 2102 struct i915_sw_fence fence = {}; 2103 2104 if (master->class == class) 2105 continue; 2106 2107 memset_p((void *)rq, ERR_PTR(-EINVAL), ARRAY_SIZE(rq)); 2108 2109 rq[0] = igt_request_alloc(ctx, master); 2110 if (IS_ERR(rq[0])) { 2111 err = PTR_ERR(rq[0]); 2112 goto out; 2113 } 2114 i915_request_get(rq[0]); 2115 2116 if (flags & BOND_SCHEDULE) { 2117 onstack_fence_init(&fence); 2118 err = i915_sw_fence_await_sw_fence_gfp(&rq[0]->submit, 2119 &fence, 2120 GFP_KERNEL); 2121 } 2122 i915_request_add(rq[0]); 2123 if (err < 0) 2124 goto out; 2125 2126 for (n = 0; n < nsibling; n++) { 2127 struct intel_context *ve; 2128 2129 ve = intel_execlists_create_virtual(ctx, 2130 siblings, 2131 nsibling); 2132 if (IS_ERR(ve)) { 2133 err = PTR_ERR(ve); 2134 onstack_fence_fini(&fence); 2135 goto out; 2136 } 2137 2138 err = intel_virtual_engine_attach_bond(ve->engine, 2139 master, 2140 siblings[n]); 2141 if (err) { 2142 intel_context_put(ve); 2143 onstack_fence_fini(&fence); 2144 goto out; 2145 } 2146 2147 err = intel_context_pin(ve); 2148 intel_context_put(ve); 2149 if (err) { 2150 onstack_fence_fini(&fence); 2151 goto out; 2152 } 2153 2154 rq[n + 1] = i915_request_create(ve); 2155 intel_context_unpin(ve); 2156 if (IS_ERR(rq[n + 1])) { 2157 err = PTR_ERR(rq[n + 1]); 2158 onstack_fence_fini(&fence); 2159 goto out; 2160 } 2161 i915_request_get(rq[n + 1]); 2162 2163 err = i915_request_await_execution(rq[n + 1], 2164 &rq[0]->fence, 2165 ve->engine->bond_execute); 2166 i915_request_add(rq[n + 1]); 2167 if (err < 0) { 2168 onstack_fence_fini(&fence); 2169 goto out; 2170 } 2171 } 2172 onstack_fence_fini(&fence); 2173 2174 if (i915_request_wait(rq[0], 0, HZ / 10) < 0) { 2175 pr_err("Master request did not execute (on %s)!\n", 2176 rq[0]->engine->name); 2177 err = -EIO; 2178 goto out; 2179 } 2180 2181 for (n = 0; n < nsibling; n++) { 2182 if (i915_request_wait(rq[n + 1], 0, 2183 MAX_SCHEDULE_TIMEOUT) < 0) { 2184 err = -EIO; 2185 goto out; 2186 } 2187 2188 if (rq[n + 1]->engine != siblings[n]) { 2189 pr_err("Bonded request did not execute on target engine: expected %s, used %s; master was %s\n", 2190 siblings[n]->name, 2191 rq[n + 1]->engine->name, 2192 rq[0]->engine->name); 2193 err = -EINVAL; 2194 goto out; 2195 } 2196 } 2197 2198 for (n = 0; !IS_ERR(rq[n]); n++) 2199 i915_request_put(rq[n]); 2200 rq[0] = ERR_PTR(-ENOMEM); 2201 } 2202 2203 out: 2204 for (n = 0; !IS_ERR(rq[n]); n++) 2205 i915_request_put(rq[n]); 2206 if (igt_flush_test(i915)) 2207 err = -EIO; 2208 2209 kernel_context_close(ctx); 2210 return err; 2211 } 2212 2213 static int live_virtual_bond(void *arg) 2214 { 2215 static const struct phase { 2216 const char *name; 2217 unsigned int flags; 2218 } phases[] = { 2219 { "", 0 }, 2220 { "schedule", BOND_SCHEDULE }, 2221 { }, 2222 }; 2223 struct drm_i915_private *i915 = arg; 2224 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; 2225 struct intel_gt *gt = &i915->gt; 2226 unsigned int class, inst; 2227 int err; 2228 2229 if (USES_GUC_SUBMISSION(i915)) 2230 return 0; 2231 2232 for (class = 0; class <= MAX_ENGINE_CLASS; class++) { 2233 const struct phase *p; 2234 int nsibling; 2235 2236 nsibling = 0; 2237 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) { 2238 if (!gt->engine_class[class][inst]) 2239 break; 2240 2241 GEM_BUG_ON(nsibling == ARRAY_SIZE(siblings)); 2242 siblings[nsibling++] = gt->engine_class[class][inst]; 2243 } 2244 if (nsibling < 2) 2245 continue; 2246 2247 for (p = phases; p->name; p++) { 2248 err = bond_virtual_engine(i915, 2249 class, siblings, nsibling, 2250 p->flags); 2251 if (err) { 2252 pr_err("%s(%s): failed class=%d, nsibling=%d, err=%d\n", 2253 __func__, p->name, class, nsibling, err); 2254 return err; 2255 } 2256 } 2257 } 2258 2259 return 0; 2260 } 2261 2262 int intel_execlists_live_selftests(struct drm_i915_private *i915) 2263 { 2264 static const struct i915_subtest tests[] = { 2265 SUBTEST(live_sanitycheck), 2266 SUBTEST(live_unlite_switch), 2267 SUBTEST(live_unlite_preempt), 2268 SUBTEST(live_timeslice_preempt), 2269 SUBTEST(live_busywait_preempt), 2270 SUBTEST(live_preempt), 2271 SUBTEST(live_late_preempt), 2272 SUBTEST(live_nopreempt), 2273 SUBTEST(live_suppress_self_preempt), 2274 SUBTEST(live_suppress_wait_preempt), 2275 SUBTEST(live_chain_preempt), 2276 SUBTEST(live_preempt_hang), 2277 SUBTEST(live_preempt_smoke), 2278 SUBTEST(live_virtual_engine), 2279 SUBTEST(live_virtual_mask), 2280 SUBTEST(live_virtual_bond), 2281 }; 2282 2283 if (!HAS_EXECLISTS(i915)) 2284 return 0; 2285 2286 if (intel_gt_is_wedged(&i915->gt)) 2287 return 0; 2288 2289 return i915_live_subtests(tests, i915); 2290 } 2291 2292 static void hexdump(const void *buf, size_t len) 2293 { 2294 const size_t rowsize = 8 * sizeof(u32); 2295 const void *prev = NULL; 2296 bool skip = false; 2297 size_t pos; 2298 2299 for (pos = 0; pos < len; pos += rowsize) { 2300 char line[128]; 2301 2302 if (prev && !memcmp(prev, buf + pos, rowsize)) { 2303 if (!skip) { 2304 pr_info("*\n"); 2305 skip = true; 2306 } 2307 continue; 2308 } 2309 2310 WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos, 2311 rowsize, sizeof(u32), 2312 line, sizeof(line), 2313 false) >= sizeof(line)); 2314 pr_info("[%04zx] %s\n", pos, line); 2315 2316 prev = buf + pos; 2317 skip = false; 2318 } 2319 } 2320 2321 static int live_lrc_layout(void *arg) 2322 { 2323 struct intel_gt *gt = arg; 2324 struct intel_engine_cs *engine; 2325 enum intel_engine_id id; 2326 u32 *mem; 2327 int err; 2328 2329 /* 2330 * Check the registers offsets we use to create the initial reg state 2331 * match the layout saved by HW. 2332 */ 2333 2334 mem = kmalloc(PAGE_SIZE, GFP_KERNEL); 2335 if (!mem) 2336 return -ENOMEM; 2337 2338 err = 0; 2339 for_each_engine(engine, gt->i915, id) { 2340 u32 *hw, *lrc; 2341 int dw; 2342 2343 if (!engine->default_state) 2344 continue; 2345 2346 hw = i915_gem_object_pin_map(engine->default_state, 2347 I915_MAP_WB); 2348 if (IS_ERR(hw)) { 2349 err = PTR_ERR(hw); 2350 break; 2351 } 2352 hw += LRC_STATE_PN * PAGE_SIZE / sizeof(*hw); 2353 2354 lrc = memset(mem, 0, PAGE_SIZE); 2355 execlists_init_reg_state(lrc, 2356 engine->kernel_context, 2357 engine, 2358 engine->kernel_context->ring, 2359 true); 2360 2361 dw = 0; 2362 do { 2363 u32 lri = hw[dw]; 2364 2365 if (lri == 0) { 2366 dw++; 2367 continue; 2368 } 2369 2370 if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 2371 pr_err("%s: Expected LRI command at dword %d, found %08x\n", 2372 engine->name, dw, lri); 2373 err = -EINVAL; 2374 break; 2375 } 2376 2377 if (lrc[dw] != lri) { 2378 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n", 2379 engine->name, dw, lri, lrc[dw]); 2380 err = -EINVAL; 2381 break; 2382 } 2383 2384 lri &= 0x7f; 2385 lri++; 2386 dw++; 2387 2388 while (lri) { 2389 if (hw[dw] != lrc[dw]) { 2390 pr_err("%s: Different registers found at dword %d, expected %x, found %x\n", 2391 engine->name, dw, hw[dw], lrc[dw]); 2392 err = -EINVAL; 2393 break; 2394 } 2395 2396 /* 2397 * Skip over the actual register value as we 2398 * expect that to differ. 2399 */ 2400 dw += 2; 2401 lri -= 2; 2402 } 2403 } while ((lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 2404 2405 if (err) { 2406 pr_info("%s: HW register image:\n", engine->name); 2407 hexdump(hw, PAGE_SIZE); 2408 2409 pr_info("%s: SW register image:\n", engine->name); 2410 hexdump(lrc, PAGE_SIZE); 2411 } 2412 2413 i915_gem_object_unpin_map(engine->default_state); 2414 if (err) 2415 break; 2416 } 2417 2418 kfree(mem); 2419 return err; 2420 } 2421 2422 int intel_lrc_live_selftests(struct drm_i915_private *i915) 2423 { 2424 static const struct i915_subtest tests[] = { 2425 SUBTEST(live_lrc_layout), 2426 }; 2427 2428 if (!HAS_LOGICAL_RING_CONTEXTS(i915)) 2429 return 0; 2430 2431 return intel_gt_live_subtests(tests, &i915->gt); 2432 } 2433