1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2016 Intel Corporation 4 */ 5 6 #include <linux/kthread.h> 7 8 #include "gem/i915_gem_context.h" 9 10 #include "intel_gt.h" 11 #include "intel_engine_heartbeat.h" 12 #include "intel_engine_pm.h" 13 #include "selftest_engine_heartbeat.h" 14 15 #include "i915_selftest.h" 16 #include "selftests/i915_random.h" 17 #include "selftests/igt_flush_test.h" 18 #include "selftests/igt_reset.h" 19 #include "selftests/igt_atomic.h" 20 #include "selftests/igt_spinner.h" 21 #include "selftests/intel_scheduler_helpers.h" 22 23 #include "selftests/mock_drm.h" 24 25 #include "gem/selftests/mock_context.h" 26 #include "gem/selftests/igt_gem_utils.h" 27 28 #define IGT_IDLE_TIMEOUT 50 /* ms; time to wait after flushing between tests */ 29 30 struct hang { 31 struct intel_gt *gt; 32 struct drm_i915_gem_object *hws; 33 struct drm_i915_gem_object *obj; 34 struct i915_gem_context *ctx; 35 u32 *seqno; 36 u32 *batch; 37 }; 38 39 static int hang_init(struct hang *h, struct intel_gt *gt) 40 { 41 void *vaddr; 42 int err; 43 44 memset(h, 0, sizeof(*h)); 45 h->gt = gt; 46 47 h->ctx = kernel_context(gt->i915, NULL); 48 if (IS_ERR(h->ctx)) 49 return PTR_ERR(h->ctx); 50 51 GEM_BUG_ON(i915_gem_context_is_bannable(h->ctx)); 52 53 h->hws = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); 54 if (IS_ERR(h->hws)) { 55 err = PTR_ERR(h->hws); 56 goto err_ctx; 57 } 58 59 h->obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); 60 if (IS_ERR(h->obj)) { 61 err = PTR_ERR(h->obj); 62 goto err_hws; 63 } 64 65 i915_gem_object_set_cache_coherency(h->hws, I915_CACHE_LLC); 66 vaddr = i915_gem_object_pin_map_unlocked(h->hws, I915_MAP_WB); 67 if (IS_ERR(vaddr)) { 68 err = PTR_ERR(vaddr); 69 goto err_obj; 70 } 71 h->seqno = memset(vaddr, 0xff, PAGE_SIZE); 72 73 vaddr = i915_gem_object_pin_map_unlocked(h->obj, 74 i915_coherent_map_type(gt->i915, h->obj, false)); 75 if (IS_ERR(vaddr)) { 76 err = PTR_ERR(vaddr); 77 goto err_unpin_hws; 78 } 79 h->batch = vaddr; 80 81 return 0; 82 83 err_unpin_hws: 84 i915_gem_object_unpin_map(h->hws); 85 err_obj: 86 i915_gem_object_put(h->obj); 87 err_hws: 88 i915_gem_object_put(h->hws); 89 err_ctx: 90 kernel_context_close(h->ctx); 91 return err; 92 } 93 94 static u64 hws_address(const struct i915_vma *hws, 95 const struct i915_request *rq) 96 { 97 return hws->node.start + offset_in_page(sizeof(u32)*rq->fence.context); 98 } 99 100 static int move_to_active(struct i915_vma *vma, 101 struct i915_request *rq, 102 unsigned int flags) 103 { 104 int err; 105 106 i915_vma_lock(vma); 107 err = i915_request_await_object(rq, vma->obj, 108 flags & EXEC_OBJECT_WRITE); 109 if (err == 0) 110 err = i915_vma_move_to_active(vma, rq, flags); 111 i915_vma_unlock(vma); 112 113 return err; 114 } 115 116 static struct i915_request * 117 hang_create_request(struct hang *h, struct intel_engine_cs *engine) 118 { 119 struct intel_gt *gt = h->gt; 120 struct i915_address_space *vm = i915_gem_context_get_eb_vm(h->ctx); 121 struct drm_i915_gem_object *obj; 122 struct i915_request *rq = NULL; 123 struct i915_vma *hws, *vma; 124 unsigned int flags; 125 void *vaddr; 126 u32 *batch; 127 int err; 128 129 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); 130 if (IS_ERR(obj)) { 131 i915_vm_put(vm); 132 return ERR_CAST(obj); 133 } 134 135 vaddr = i915_gem_object_pin_map_unlocked(obj, i915_coherent_map_type(gt->i915, obj, false)); 136 if (IS_ERR(vaddr)) { 137 i915_gem_object_put(obj); 138 i915_vm_put(vm); 139 return ERR_CAST(vaddr); 140 } 141 142 i915_gem_object_unpin_map(h->obj); 143 i915_gem_object_put(h->obj); 144 145 h->obj = obj; 146 h->batch = vaddr; 147 148 vma = i915_vma_instance(h->obj, vm, NULL); 149 if (IS_ERR(vma)) { 150 i915_vm_put(vm); 151 return ERR_CAST(vma); 152 } 153 154 hws = i915_vma_instance(h->hws, vm, NULL); 155 if (IS_ERR(hws)) { 156 i915_vm_put(vm); 157 return ERR_CAST(hws); 158 } 159 160 err = i915_vma_pin(vma, 0, 0, PIN_USER); 161 if (err) { 162 i915_vm_put(vm); 163 return ERR_PTR(err); 164 } 165 166 err = i915_vma_pin(hws, 0, 0, PIN_USER); 167 if (err) 168 goto unpin_vma; 169 170 rq = igt_request_alloc(h->ctx, engine); 171 if (IS_ERR(rq)) { 172 err = PTR_ERR(rq); 173 goto unpin_hws; 174 } 175 176 err = move_to_active(vma, rq, 0); 177 if (err) 178 goto cancel_rq; 179 180 err = move_to_active(hws, rq, 0); 181 if (err) 182 goto cancel_rq; 183 184 batch = h->batch; 185 if (GRAPHICS_VER(gt->i915) >= 8) { 186 *batch++ = MI_STORE_DWORD_IMM_GEN4; 187 *batch++ = lower_32_bits(hws_address(hws, rq)); 188 *batch++ = upper_32_bits(hws_address(hws, rq)); 189 *batch++ = rq->fence.seqno; 190 *batch++ = MI_NOOP; 191 192 memset(batch, 0, 1024); 193 batch += 1024 / sizeof(*batch); 194 195 *batch++ = MI_NOOP; 196 *batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1; 197 *batch++ = lower_32_bits(vma->node.start); 198 *batch++ = upper_32_bits(vma->node.start); 199 } else if (GRAPHICS_VER(gt->i915) >= 6) { 200 *batch++ = MI_STORE_DWORD_IMM_GEN4; 201 *batch++ = 0; 202 *batch++ = lower_32_bits(hws_address(hws, rq)); 203 *batch++ = rq->fence.seqno; 204 *batch++ = MI_NOOP; 205 206 memset(batch, 0, 1024); 207 batch += 1024 / sizeof(*batch); 208 209 *batch++ = MI_NOOP; 210 *batch++ = MI_BATCH_BUFFER_START | 1 << 8; 211 *batch++ = lower_32_bits(vma->node.start); 212 } else if (GRAPHICS_VER(gt->i915) >= 4) { 213 *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 214 *batch++ = 0; 215 *batch++ = lower_32_bits(hws_address(hws, rq)); 216 *batch++ = rq->fence.seqno; 217 *batch++ = MI_NOOP; 218 219 memset(batch, 0, 1024); 220 batch += 1024 / sizeof(*batch); 221 222 *batch++ = MI_NOOP; 223 *batch++ = MI_BATCH_BUFFER_START | 2 << 6; 224 *batch++ = lower_32_bits(vma->node.start); 225 } else { 226 *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; 227 *batch++ = lower_32_bits(hws_address(hws, rq)); 228 *batch++ = rq->fence.seqno; 229 *batch++ = MI_NOOP; 230 231 memset(batch, 0, 1024); 232 batch += 1024 / sizeof(*batch); 233 234 *batch++ = MI_NOOP; 235 *batch++ = MI_BATCH_BUFFER_START | 2 << 6; 236 *batch++ = lower_32_bits(vma->node.start); 237 } 238 *batch++ = MI_BATCH_BUFFER_END; /* not reached */ 239 intel_gt_chipset_flush(engine->gt); 240 241 if (rq->engine->emit_init_breadcrumb) { 242 err = rq->engine->emit_init_breadcrumb(rq); 243 if (err) 244 goto cancel_rq; 245 } 246 247 flags = 0; 248 if (GRAPHICS_VER(gt->i915) <= 5) 249 flags |= I915_DISPATCH_SECURE; 250 251 err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags); 252 253 cancel_rq: 254 if (err) { 255 i915_request_set_error_once(rq, err); 256 i915_request_add(rq); 257 } 258 unpin_hws: 259 i915_vma_unpin(hws); 260 unpin_vma: 261 i915_vma_unpin(vma); 262 i915_vm_put(vm); 263 return err ? ERR_PTR(err) : rq; 264 } 265 266 static u32 hws_seqno(const struct hang *h, const struct i915_request *rq) 267 { 268 return READ_ONCE(h->seqno[rq->fence.context % (PAGE_SIZE/sizeof(u32))]); 269 } 270 271 static void hang_fini(struct hang *h) 272 { 273 *h->batch = MI_BATCH_BUFFER_END; 274 intel_gt_chipset_flush(h->gt); 275 276 i915_gem_object_unpin_map(h->obj); 277 i915_gem_object_put(h->obj); 278 279 i915_gem_object_unpin_map(h->hws); 280 i915_gem_object_put(h->hws); 281 282 kernel_context_close(h->ctx); 283 284 igt_flush_test(h->gt->i915); 285 } 286 287 static bool wait_until_running(struct hang *h, struct i915_request *rq) 288 { 289 return !(wait_for_us(i915_seqno_passed(hws_seqno(h, rq), 290 rq->fence.seqno), 291 10) && 292 wait_for(i915_seqno_passed(hws_seqno(h, rq), 293 rq->fence.seqno), 294 1000)); 295 } 296 297 static int igt_hang_sanitycheck(void *arg) 298 { 299 struct intel_gt *gt = arg; 300 struct i915_request *rq; 301 struct intel_engine_cs *engine; 302 enum intel_engine_id id; 303 struct hang h; 304 int err; 305 306 /* Basic check that we can execute our hanging batch */ 307 308 err = hang_init(&h, gt); 309 if (err) 310 return err; 311 312 for_each_engine(engine, gt, id) { 313 struct intel_wedge_me w; 314 long timeout; 315 316 if (!intel_engine_can_store_dword(engine)) 317 continue; 318 319 rq = hang_create_request(&h, engine); 320 if (IS_ERR(rq)) { 321 err = PTR_ERR(rq); 322 pr_err("Failed to create request for %s, err=%d\n", 323 engine->name, err); 324 goto fini; 325 } 326 327 i915_request_get(rq); 328 329 *h.batch = MI_BATCH_BUFFER_END; 330 intel_gt_chipset_flush(engine->gt); 331 332 i915_request_add(rq); 333 334 timeout = 0; 335 intel_wedge_on_timeout(&w, gt, HZ / 10 /* 100ms */) 336 timeout = i915_request_wait(rq, 0, 337 MAX_SCHEDULE_TIMEOUT); 338 if (intel_gt_is_wedged(gt)) 339 timeout = -EIO; 340 341 i915_request_put(rq); 342 343 if (timeout < 0) { 344 err = timeout; 345 pr_err("Wait for request failed on %s, err=%d\n", 346 engine->name, err); 347 goto fini; 348 } 349 } 350 351 fini: 352 hang_fini(&h); 353 return err; 354 } 355 356 static bool wait_for_idle(struct intel_engine_cs *engine) 357 { 358 return wait_for(intel_engine_is_idle(engine), IGT_IDLE_TIMEOUT) == 0; 359 } 360 361 static int igt_reset_nop(void *arg) 362 { 363 struct intel_gt *gt = arg; 364 struct i915_gpu_error *global = >->i915->gpu_error; 365 struct intel_engine_cs *engine; 366 unsigned int reset_count, count; 367 enum intel_engine_id id; 368 IGT_TIMEOUT(end_time); 369 int err = 0; 370 371 /* Check that we can reset during non-user portions of requests */ 372 373 reset_count = i915_reset_count(global); 374 count = 0; 375 do { 376 for_each_engine(engine, gt, id) { 377 struct intel_context *ce; 378 int i; 379 380 ce = intel_context_create(engine); 381 if (IS_ERR(ce)) { 382 err = PTR_ERR(ce); 383 pr_err("[%s] Create context failed: %d!\n", engine->name, err); 384 break; 385 } 386 387 for (i = 0; i < 16; i++) { 388 struct i915_request *rq; 389 390 rq = intel_context_create_request(ce); 391 if (IS_ERR(rq)) { 392 err = PTR_ERR(rq); 393 pr_err("[%s] Create request failed: %d!\n", 394 engine->name, err); 395 break; 396 } 397 398 i915_request_add(rq); 399 } 400 401 intel_context_put(ce); 402 } 403 404 igt_global_reset_lock(gt); 405 intel_gt_reset(gt, ALL_ENGINES, NULL); 406 igt_global_reset_unlock(gt); 407 408 if (intel_gt_is_wedged(gt)) { 409 pr_err("[%s] GT is wedged!\n", engine->name); 410 err = -EIO; 411 break; 412 } 413 414 if (i915_reset_count(global) != reset_count + ++count) { 415 pr_err("[%s] Reset not recorded: %d vs %d + %d!\n", 416 engine->name, i915_reset_count(global), reset_count, count); 417 err = -EINVAL; 418 break; 419 } 420 421 err = igt_flush_test(gt->i915); 422 if (err) { 423 pr_err("[%s] Flush failed: %d!\n", engine->name, err); 424 break; 425 } 426 } while (time_before(jiffies, end_time)); 427 pr_info("%s: %d resets\n", __func__, count); 428 429 if (igt_flush_test(gt->i915)) { 430 pr_err("Post flush failed: %d!\n", err); 431 err = -EIO; 432 } 433 434 return err; 435 } 436 437 static int igt_reset_nop_engine(void *arg) 438 { 439 struct intel_gt *gt = arg; 440 struct i915_gpu_error *global = >->i915->gpu_error; 441 struct intel_engine_cs *engine; 442 enum intel_engine_id id; 443 444 /* Check that we can engine-reset during non-user portions */ 445 446 if (!intel_has_reset_engine(gt)) 447 return 0; 448 449 for_each_engine(engine, gt, id) { 450 unsigned int reset_count, reset_engine_count, count; 451 struct intel_context *ce; 452 IGT_TIMEOUT(end_time); 453 int err; 454 455 if (intel_engine_uses_guc(engine)) { 456 /* Engine level resets are triggered by GuC when a hang 457 * is detected. They can't be triggered by the KMD any 458 * more. Thus a nop batch cannot be used as a reset test 459 */ 460 continue; 461 } 462 463 ce = intel_context_create(engine); 464 if (IS_ERR(ce)) { 465 pr_err("[%s] Create context failed: %pe!\n", engine->name, ce); 466 return PTR_ERR(ce); 467 } 468 469 reset_count = i915_reset_count(global); 470 reset_engine_count = i915_reset_engine_count(global, engine); 471 count = 0; 472 473 st_engine_heartbeat_disable(engine); 474 set_bit(I915_RESET_ENGINE + id, >->reset.flags); 475 do { 476 int i; 477 478 if (!wait_for_idle(engine)) { 479 pr_err("%s failed to idle before reset\n", 480 engine->name); 481 err = -EIO; 482 break; 483 } 484 485 for (i = 0; i < 16; i++) { 486 struct i915_request *rq; 487 488 rq = intel_context_create_request(ce); 489 if (IS_ERR(rq)) { 490 struct drm_printer p = 491 drm_info_printer(gt->i915->drm.dev); 492 intel_engine_dump(engine, &p, 493 "%s(%s): failed to submit request\n", 494 __func__, 495 engine->name); 496 497 GEM_TRACE("%s(%s): failed to submit request\n", 498 __func__, 499 engine->name); 500 GEM_TRACE_DUMP(); 501 502 intel_gt_set_wedged(gt); 503 504 err = PTR_ERR(rq); 505 break; 506 } 507 508 i915_request_add(rq); 509 } 510 err = intel_engine_reset(engine, NULL); 511 if (err) { 512 pr_err("intel_engine_reset(%s) failed, err:%d\n", 513 engine->name, err); 514 break; 515 } 516 517 if (i915_reset_count(global) != reset_count) { 518 pr_err("Full GPU reset recorded! (engine reset expected)\n"); 519 err = -EINVAL; 520 break; 521 } 522 523 if (i915_reset_engine_count(global, engine) != 524 reset_engine_count + ++count) { 525 pr_err("%s engine reset not recorded!\n", 526 engine->name); 527 err = -EINVAL; 528 break; 529 } 530 } while (time_before(jiffies, end_time)); 531 clear_bit(I915_RESET_ENGINE + id, >->reset.flags); 532 st_engine_heartbeat_enable(engine); 533 534 pr_info("%s(%s): %d resets\n", __func__, engine->name, count); 535 536 intel_context_put(ce); 537 if (igt_flush_test(gt->i915)) 538 err = -EIO; 539 if (err) 540 return err; 541 } 542 543 return 0; 544 } 545 546 static void force_reset_timeout(struct intel_engine_cs *engine) 547 { 548 engine->reset_timeout.probability = 999; 549 atomic_set(&engine->reset_timeout.times, -1); 550 } 551 552 static void cancel_reset_timeout(struct intel_engine_cs *engine) 553 { 554 memset(&engine->reset_timeout, 0, sizeof(engine->reset_timeout)); 555 } 556 557 static int igt_reset_fail_engine(void *arg) 558 { 559 struct intel_gt *gt = arg; 560 struct intel_engine_cs *engine; 561 enum intel_engine_id id; 562 563 /* Check that we can recover from engine-reset failues */ 564 565 if (!intel_has_reset_engine(gt)) 566 return 0; 567 568 for_each_engine(engine, gt, id) { 569 unsigned int count; 570 struct intel_context *ce; 571 IGT_TIMEOUT(end_time); 572 int err; 573 574 /* Can't manually break the reset if i915 doesn't perform it */ 575 if (intel_engine_uses_guc(engine)) 576 continue; 577 578 ce = intel_context_create(engine); 579 if (IS_ERR(ce)) { 580 pr_err("[%s] Create context failed: %pe!\n", engine->name, ce); 581 return PTR_ERR(ce); 582 } 583 584 st_engine_heartbeat_disable(engine); 585 set_bit(I915_RESET_ENGINE + id, >->reset.flags); 586 587 force_reset_timeout(engine); 588 err = intel_engine_reset(engine, NULL); 589 cancel_reset_timeout(engine); 590 if (err == 0) /* timeouts only generated on gen8+ */ 591 goto skip; 592 593 count = 0; 594 do { 595 struct i915_request *last = NULL; 596 int i; 597 598 if (!wait_for_idle(engine)) { 599 pr_err("%s failed to idle before reset\n", 600 engine->name); 601 err = -EIO; 602 break; 603 } 604 605 for (i = 0; i < count % 15; i++) { 606 struct i915_request *rq; 607 608 rq = intel_context_create_request(ce); 609 if (IS_ERR(rq)) { 610 struct drm_printer p = 611 drm_info_printer(gt->i915->drm.dev); 612 intel_engine_dump(engine, &p, 613 "%s(%s): failed to submit request\n", 614 __func__, 615 engine->name); 616 617 GEM_TRACE("%s(%s): failed to submit request\n", 618 __func__, 619 engine->name); 620 GEM_TRACE_DUMP(); 621 622 intel_gt_set_wedged(gt); 623 if (last) 624 i915_request_put(last); 625 626 err = PTR_ERR(rq); 627 goto out; 628 } 629 630 if (last) 631 i915_request_put(last); 632 last = i915_request_get(rq); 633 i915_request_add(rq); 634 } 635 636 if (count & 1) { 637 err = intel_engine_reset(engine, NULL); 638 if (err) { 639 GEM_TRACE_ERR("intel_engine_reset(%s) failed, err:%d\n", 640 engine->name, err); 641 GEM_TRACE_DUMP(); 642 i915_request_put(last); 643 break; 644 } 645 } else { 646 force_reset_timeout(engine); 647 err = intel_engine_reset(engine, NULL); 648 cancel_reset_timeout(engine); 649 if (err != -ETIMEDOUT) { 650 pr_err("intel_engine_reset(%s) did not fail, err:%d\n", 651 engine->name, err); 652 i915_request_put(last); 653 break; 654 } 655 } 656 657 err = 0; 658 if (last) { 659 if (i915_request_wait(last, 0, HZ / 2) < 0) { 660 struct drm_printer p = 661 drm_info_printer(gt->i915->drm.dev); 662 663 intel_engine_dump(engine, &p, 664 "%s(%s): failed to complete request\n", 665 __func__, 666 engine->name); 667 668 GEM_TRACE("%s(%s): failed to complete request\n", 669 __func__, 670 engine->name); 671 GEM_TRACE_DUMP(); 672 673 err = -EIO; 674 } 675 i915_request_put(last); 676 } 677 count++; 678 } while (err == 0 && time_before(jiffies, end_time)); 679 out: 680 pr_info("%s(%s): %d resets\n", __func__, engine->name, count); 681 skip: 682 clear_bit(I915_RESET_ENGINE + id, >->reset.flags); 683 st_engine_heartbeat_enable(engine); 684 intel_context_put(ce); 685 686 if (igt_flush_test(gt->i915)) 687 err = -EIO; 688 if (err) 689 return err; 690 } 691 692 return 0; 693 } 694 695 static int __igt_reset_engine(struct intel_gt *gt, bool active) 696 { 697 struct i915_gpu_error *global = >->i915->gpu_error; 698 struct intel_engine_cs *engine; 699 enum intel_engine_id id; 700 struct hang h; 701 int err = 0; 702 703 /* Check that we can issue an engine reset on an idle engine (no-op) */ 704 705 if (!intel_has_reset_engine(gt)) 706 return 0; 707 708 if (active) { 709 err = hang_init(&h, gt); 710 if (err) 711 return err; 712 } 713 714 for_each_engine(engine, gt, id) { 715 unsigned int reset_count, reset_engine_count; 716 unsigned long count; 717 bool using_guc = intel_engine_uses_guc(engine); 718 IGT_TIMEOUT(end_time); 719 720 if (using_guc && !active) 721 continue; 722 723 if (active && !intel_engine_can_store_dword(engine)) 724 continue; 725 726 if (!wait_for_idle(engine)) { 727 pr_err("%s failed to idle before reset\n", 728 engine->name); 729 err = -EIO; 730 break; 731 } 732 733 reset_count = i915_reset_count(global); 734 reset_engine_count = i915_reset_engine_count(global, engine); 735 736 st_engine_heartbeat_disable(engine); 737 set_bit(I915_RESET_ENGINE + id, >->reset.flags); 738 count = 0; 739 do { 740 struct i915_request *rq = NULL; 741 struct intel_selftest_saved_policy saved; 742 int err2; 743 744 err = intel_selftest_modify_policy(engine, &saved, 745 SELFTEST_SCHEDULER_MODIFY_FAST_RESET); 746 if (err) { 747 pr_err("[%s] Modify policy failed: %d!\n", engine->name, err); 748 break; 749 } 750 751 if (active) { 752 rq = hang_create_request(&h, engine); 753 if (IS_ERR(rq)) { 754 err = PTR_ERR(rq); 755 pr_err("[%s] Create hang request failed: %d!\n", 756 engine->name, err); 757 goto restore; 758 } 759 760 i915_request_get(rq); 761 i915_request_add(rq); 762 763 if (!wait_until_running(&h, rq)) { 764 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 765 766 pr_err("%s: Failed to start request %llx, at %x\n", 767 __func__, rq->fence.seqno, hws_seqno(&h, rq)); 768 intel_engine_dump(engine, &p, 769 "%s\n", engine->name); 770 771 i915_request_put(rq); 772 err = -EIO; 773 goto restore; 774 } 775 } 776 777 if (!using_guc) { 778 err = intel_engine_reset(engine, NULL); 779 if (err) { 780 pr_err("intel_engine_reset(%s) failed, err:%d\n", 781 engine->name, err); 782 goto skip; 783 } 784 } 785 786 if (rq) { 787 /* Ensure the reset happens and kills the engine */ 788 err = intel_selftest_wait_for_rq(rq); 789 if (err) 790 pr_err("[%s] Wait for request %lld:%lld [0x%04X] failed: %d!\n", 791 engine->name, rq->fence.context, 792 rq->fence.seqno, rq->context->guc_id.id, err); 793 } 794 795 skip: 796 if (rq) 797 i915_request_put(rq); 798 799 if (i915_reset_count(global) != reset_count) { 800 pr_err("Full GPU reset recorded! (engine reset expected)\n"); 801 err = -EINVAL; 802 goto restore; 803 } 804 805 /* GuC based resets are not logged per engine */ 806 if (!using_guc) { 807 if (i915_reset_engine_count(global, engine) != 808 ++reset_engine_count) { 809 pr_err("%s engine reset not recorded!\n", 810 engine->name); 811 err = -EINVAL; 812 goto restore; 813 } 814 } 815 816 count++; 817 818 restore: 819 err2 = intel_selftest_restore_policy(engine, &saved); 820 if (err2) 821 pr_err("[%s] Restore policy failed: %d!\n", engine->name, err); 822 if (err == 0) 823 err = err2; 824 if (err) 825 break; 826 } while (time_before(jiffies, end_time)); 827 clear_bit(I915_RESET_ENGINE + id, >->reset.flags); 828 st_engine_heartbeat_enable(engine); 829 pr_info("%s: Completed %lu %s resets\n", 830 engine->name, count, active ? "active" : "idle"); 831 832 if (err) 833 break; 834 835 err = igt_flush_test(gt->i915); 836 if (err) { 837 pr_err("[%s] Flush failed: %d!\n", engine->name, err); 838 break; 839 } 840 } 841 842 if (intel_gt_is_wedged(gt)) { 843 pr_err("GT is wedged!\n"); 844 err = -EIO; 845 } 846 847 if (active) 848 hang_fini(&h); 849 850 return err; 851 } 852 853 static int igt_reset_idle_engine(void *arg) 854 { 855 return __igt_reset_engine(arg, false); 856 } 857 858 static int igt_reset_active_engine(void *arg) 859 { 860 return __igt_reset_engine(arg, true); 861 } 862 863 struct active_engine { 864 struct task_struct *task; 865 struct intel_engine_cs *engine; 866 unsigned long resets; 867 unsigned int flags; 868 }; 869 870 #define TEST_ACTIVE BIT(0) 871 #define TEST_OTHERS BIT(1) 872 #define TEST_SELF BIT(2) 873 #define TEST_PRIORITY BIT(3) 874 875 static int active_request_put(struct i915_request *rq) 876 { 877 int err = 0; 878 879 if (!rq) 880 return 0; 881 882 if (i915_request_wait(rq, 0, 10 * HZ) < 0) { 883 GEM_TRACE("%s timed out waiting for completion of fence %llx:%lld\n", 884 rq->engine->name, 885 rq->fence.context, 886 rq->fence.seqno); 887 GEM_TRACE_DUMP(); 888 889 intel_gt_set_wedged(rq->engine->gt); 890 err = -EIO; 891 } 892 893 i915_request_put(rq); 894 895 return err; 896 } 897 898 static int active_engine(void *data) 899 { 900 I915_RND_STATE(prng); 901 struct active_engine *arg = data; 902 struct intel_engine_cs *engine = arg->engine; 903 struct i915_request *rq[8] = {}; 904 struct intel_context *ce[ARRAY_SIZE(rq)]; 905 unsigned long count; 906 int err = 0; 907 908 for (count = 0; count < ARRAY_SIZE(ce); count++) { 909 ce[count] = intel_context_create(engine); 910 if (IS_ERR(ce[count])) { 911 err = PTR_ERR(ce[count]); 912 pr_err("[%s] Create context #%ld failed: %d!\n", engine->name, count, err); 913 while (--count) 914 intel_context_put(ce[count]); 915 return err; 916 } 917 } 918 919 count = 0; 920 while (!kthread_should_stop()) { 921 unsigned int idx = count++ & (ARRAY_SIZE(rq) - 1); 922 struct i915_request *old = rq[idx]; 923 struct i915_request *new; 924 925 new = intel_context_create_request(ce[idx]); 926 if (IS_ERR(new)) { 927 err = PTR_ERR(new); 928 pr_err("[%s] Create request #%d failed: %d!\n", engine->name, idx, err); 929 break; 930 } 931 932 rq[idx] = i915_request_get(new); 933 i915_request_add(new); 934 935 if (engine->sched_engine->schedule && arg->flags & TEST_PRIORITY) { 936 struct i915_sched_attr attr = { 937 .priority = 938 i915_prandom_u32_max_state(512, &prng), 939 }; 940 engine->sched_engine->schedule(rq[idx], &attr); 941 } 942 943 err = active_request_put(old); 944 if (err) { 945 pr_err("[%s] Request put failed: %d!\n", engine->name, err); 946 break; 947 } 948 949 cond_resched(); 950 } 951 952 for (count = 0; count < ARRAY_SIZE(rq); count++) { 953 int err__ = active_request_put(rq[count]); 954 955 if (err) 956 pr_err("[%s] Request put #%ld failed: %d!\n", engine->name, count, err); 957 958 /* Keep the first error */ 959 if (!err) 960 err = err__; 961 962 intel_context_put(ce[count]); 963 } 964 965 return err; 966 } 967 968 static int __igt_reset_engines(struct intel_gt *gt, 969 const char *test_name, 970 unsigned int flags) 971 { 972 struct i915_gpu_error *global = >->i915->gpu_error; 973 struct intel_engine_cs *engine, *other; 974 enum intel_engine_id id, tmp; 975 struct hang h; 976 int err = 0; 977 978 /* Check that issuing a reset on one engine does not interfere 979 * with any other engine. 980 */ 981 982 if (!intel_has_reset_engine(gt)) 983 return 0; 984 985 if (flags & TEST_ACTIVE) { 986 err = hang_init(&h, gt); 987 if (err) 988 return err; 989 990 if (flags & TEST_PRIORITY) 991 h.ctx->sched.priority = 1024; 992 } 993 994 for_each_engine(engine, gt, id) { 995 struct active_engine threads[I915_NUM_ENGINES] = {}; 996 unsigned long device = i915_reset_count(global); 997 unsigned long count = 0, reported; 998 bool using_guc = intel_engine_uses_guc(engine); 999 IGT_TIMEOUT(end_time); 1000 1001 if (flags & TEST_ACTIVE) { 1002 if (!intel_engine_can_store_dword(engine)) 1003 continue; 1004 } else if (using_guc) 1005 continue; 1006 1007 if (!wait_for_idle(engine)) { 1008 pr_err("i915_reset_engine(%s:%s): failed to idle before reset\n", 1009 engine->name, test_name); 1010 err = -EIO; 1011 break; 1012 } 1013 1014 memset(threads, 0, sizeof(threads)); 1015 for_each_engine(other, gt, tmp) { 1016 struct task_struct *tsk; 1017 1018 threads[tmp].resets = 1019 i915_reset_engine_count(global, other); 1020 1021 if (other == engine && !(flags & TEST_SELF)) 1022 continue; 1023 1024 if (other != engine && !(flags & TEST_OTHERS)) 1025 continue; 1026 1027 threads[tmp].engine = other; 1028 threads[tmp].flags = flags; 1029 1030 tsk = kthread_run(active_engine, &threads[tmp], 1031 "igt/%s", other->name); 1032 if (IS_ERR(tsk)) { 1033 err = PTR_ERR(tsk); 1034 pr_err("[%s] Thread spawn failed: %d!\n", engine->name, err); 1035 goto unwind; 1036 } 1037 1038 threads[tmp].task = tsk; 1039 get_task_struct(tsk); 1040 } 1041 1042 yield(); /* start all threads before we begin */ 1043 1044 st_engine_heartbeat_disable_no_pm(engine); 1045 set_bit(I915_RESET_ENGINE + id, >->reset.flags); 1046 do { 1047 struct i915_request *rq = NULL; 1048 struct intel_selftest_saved_policy saved; 1049 int err2; 1050 1051 err = intel_selftest_modify_policy(engine, &saved, 1052 SELFTEST_SCHEDULER_MODIFY_FAST_RESET); 1053 if (err) { 1054 pr_err("[%s] Modify policy failed: %d!\n", engine->name, err); 1055 break; 1056 } 1057 1058 if (flags & TEST_ACTIVE) { 1059 rq = hang_create_request(&h, engine); 1060 if (IS_ERR(rq)) { 1061 err = PTR_ERR(rq); 1062 pr_err("[%s] Create hang request failed: %d!\n", 1063 engine->name, err); 1064 goto restore; 1065 } 1066 1067 i915_request_get(rq); 1068 i915_request_add(rq); 1069 1070 if (!wait_until_running(&h, rq)) { 1071 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 1072 1073 pr_err("%s: Failed to start request %llx, at %x\n", 1074 __func__, rq->fence.seqno, hws_seqno(&h, rq)); 1075 intel_engine_dump(engine, &p, 1076 "%s\n", engine->name); 1077 1078 i915_request_put(rq); 1079 err = -EIO; 1080 goto restore; 1081 } 1082 } else { 1083 intel_engine_pm_get(engine); 1084 } 1085 1086 if (!using_guc) { 1087 err = intel_engine_reset(engine, NULL); 1088 if (err) { 1089 pr_err("i915_reset_engine(%s:%s): failed, err=%d\n", 1090 engine->name, test_name, err); 1091 goto restore; 1092 } 1093 } 1094 1095 if (rq) { 1096 /* Ensure the reset happens and kills the engine */ 1097 err = intel_selftest_wait_for_rq(rq); 1098 if (err) 1099 pr_err("[%s] Wait for request %lld:%lld [0x%04X] failed: %d!\n", 1100 engine->name, rq->fence.context, 1101 rq->fence.seqno, rq->context->guc_id.id, err); 1102 } 1103 1104 count++; 1105 1106 if (rq) { 1107 if (rq->fence.error != -EIO) { 1108 pr_err("i915_reset_engine(%s:%s): failed to reset request %lld:%lld [0x%04X]\n", 1109 engine->name, test_name, 1110 rq->fence.context, 1111 rq->fence.seqno, rq->context->guc_id.id); 1112 i915_request_put(rq); 1113 1114 GEM_TRACE_DUMP(); 1115 intel_gt_set_wedged(gt); 1116 err = -EIO; 1117 goto restore; 1118 } 1119 1120 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 1121 struct drm_printer p = 1122 drm_info_printer(gt->i915->drm.dev); 1123 1124 pr_err("i915_reset_engine(%s:%s):" 1125 " failed to complete request %llx:%lld after reset\n", 1126 engine->name, test_name, 1127 rq->fence.context, 1128 rq->fence.seqno); 1129 intel_engine_dump(engine, &p, 1130 "%s\n", engine->name); 1131 i915_request_put(rq); 1132 1133 GEM_TRACE_DUMP(); 1134 intel_gt_set_wedged(gt); 1135 err = -EIO; 1136 goto restore; 1137 } 1138 1139 i915_request_put(rq); 1140 } 1141 1142 if (!(flags & TEST_ACTIVE)) 1143 intel_engine_pm_put(engine); 1144 1145 if (!(flags & TEST_SELF) && !wait_for_idle(engine)) { 1146 struct drm_printer p = 1147 drm_info_printer(gt->i915->drm.dev); 1148 1149 pr_err("i915_reset_engine(%s:%s):" 1150 " failed to idle after reset\n", 1151 engine->name, test_name); 1152 intel_engine_dump(engine, &p, 1153 "%s\n", engine->name); 1154 1155 err = -EIO; 1156 goto restore; 1157 } 1158 1159 restore: 1160 err2 = intel_selftest_restore_policy(engine, &saved); 1161 if (err2) 1162 pr_err("[%s] Restore policy failed: %d!\n", engine->name, err2); 1163 if (err == 0) 1164 err = err2; 1165 if (err) 1166 break; 1167 } while (time_before(jiffies, end_time)); 1168 clear_bit(I915_RESET_ENGINE + id, >->reset.flags); 1169 st_engine_heartbeat_enable_no_pm(engine); 1170 1171 pr_info("i915_reset_engine(%s:%s): %lu resets\n", 1172 engine->name, test_name, count); 1173 1174 /* GuC based resets are not logged per engine */ 1175 if (!using_guc) { 1176 reported = i915_reset_engine_count(global, engine); 1177 reported -= threads[engine->id].resets; 1178 if (reported != count) { 1179 pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu\n", 1180 engine->name, test_name, count, reported); 1181 if (!err) 1182 err = -EINVAL; 1183 } 1184 } 1185 1186 unwind: 1187 for_each_engine(other, gt, tmp) { 1188 int ret; 1189 1190 if (!threads[tmp].task) 1191 continue; 1192 1193 ret = kthread_stop(threads[tmp].task); 1194 if (ret) { 1195 pr_err("kthread for other engine %s failed, err=%d\n", 1196 other->name, ret); 1197 if (!err) 1198 err = ret; 1199 } 1200 put_task_struct(threads[tmp].task); 1201 1202 /* GuC based resets are not logged per engine */ 1203 if (!using_guc) { 1204 if (other->uabi_class != engine->uabi_class && 1205 threads[tmp].resets != 1206 i915_reset_engine_count(global, other)) { 1207 pr_err("Innocent engine %s was reset (count=%ld)\n", 1208 other->name, 1209 i915_reset_engine_count(global, other) - 1210 threads[tmp].resets); 1211 if (!err) 1212 err = -EINVAL; 1213 } 1214 } 1215 } 1216 1217 if (device != i915_reset_count(global)) { 1218 pr_err("Global reset (count=%ld)!\n", 1219 i915_reset_count(global) - device); 1220 if (!err) 1221 err = -EINVAL; 1222 } 1223 1224 if (err) 1225 break; 1226 1227 err = igt_flush_test(gt->i915); 1228 if (err) { 1229 pr_err("[%s] Flush failed: %d!\n", engine->name, err); 1230 break; 1231 } 1232 } 1233 1234 if (intel_gt_is_wedged(gt)) 1235 err = -EIO; 1236 1237 if (flags & TEST_ACTIVE) 1238 hang_fini(&h); 1239 1240 return err; 1241 } 1242 1243 static int igt_reset_engines(void *arg) 1244 { 1245 static const struct { 1246 const char *name; 1247 unsigned int flags; 1248 } phases[] = { 1249 { "idle", 0 }, 1250 { "active", TEST_ACTIVE }, 1251 { "others-idle", TEST_OTHERS }, 1252 { "others-active", TEST_OTHERS | TEST_ACTIVE }, 1253 { 1254 "others-priority", 1255 TEST_OTHERS | TEST_ACTIVE | TEST_PRIORITY 1256 }, 1257 { 1258 "self-priority", 1259 TEST_ACTIVE | TEST_PRIORITY | TEST_SELF, 1260 }, 1261 { } 1262 }; 1263 struct intel_gt *gt = arg; 1264 typeof(*phases) *p; 1265 int err; 1266 1267 for (p = phases; p->name; p++) { 1268 if (p->flags & TEST_PRIORITY) { 1269 if (!(gt->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY)) 1270 continue; 1271 } 1272 1273 err = __igt_reset_engines(arg, p->name, p->flags); 1274 if (err) 1275 return err; 1276 } 1277 1278 return 0; 1279 } 1280 1281 static u32 fake_hangcheck(struct intel_gt *gt, intel_engine_mask_t mask) 1282 { 1283 u32 count = i915_reset_count(>->i915->gpu_error); 1284 1285 intel_gt_reset(gt, mask, NULL); 1286 1287 return count; 1288 } 1289 1290 static int igt_reset_wait(void *arg) 1291 { 1292 struct intel_gt *gt = arg; 1293 struct i915_gpu_error *global = >->i915->gpu_error; 1294 struct intel_engine_cs *engine = gt->engine[RCS0]; 1295 struct i915_request *rq; 1296 unsigned int reset_count; 1297 struct hang h; 1298 long timeout; 1299 int err; 1300 1301 if (!engine || !intel_engine_can_store_dword(engine)) 1302 return 0; 1303 1304 /* Check that we detect a stuck waiter and issue a reset */ 1305 1306 igt_global_reset_lock(gt); 1307 1308 err = hang_init(&h, gt); 1309 if (err) { 1310 pr_err("[%s] Hang init failed: %d!\n", engine->name, err); 1311 goto unlock; 1312 } 1313 1314 rq = hang_create_request(&h, engine); 1315 if (IS_ERR(rq)) { 1316 err = PTR_ERR(rq); 1317 pr_err("[%s] Create hang request failed: %d!\n", engine->name, err); 1318 goto fini; 1319 } 1320 1321 i915_request_get(rq); 1322 i915_request_add(rq); 1323 1324 if (!wait_until_running(&h, rq)) { 1325 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 1326 1327 pr_err("%s: Failed to start request %llx, at %x\n", 1328 __func__, rq->fence.seqno, hws_seqno(&h, rq)); 1329 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name); 1330 1331 intel_gt_set_wedged(gt); 1332 1333 err = -EIO; 1334 goto out_rq; 1335 } 1336 1337 reset_count = fake_hangcheck(gt, ALL_ENGINES); 1338 1339 timeout = i915_request_wait(rq, 0, 10); 1340 if (timeout < 0) { 1341 pr_err("i915_request_wait failed on a stuck request: err=%ld\n", 1342 timeout); 1343 err = timeout; 1344 goto out_rq; 1345 } 1346 1347 if (i915_reset_count(global) == reset_count) { 1348 pr_err("No GPU reset recorded!\n"); 1349 err = -EINVAL; 1350 goto out_rq; 1351 } 1352 1353 out_rq: 1354 i915_request_put(rq); 1355 fini: 1356 hang_fini(&h); 1357 unlock: 1358 igt_global_reset_unlock(gt); 1359 1360 if (intel_gt_is_wedged(gt)) 1361 return -EIO; 1362 1363 return err; 1364 } 1365 1366 struct evict_vma { 1367 struct completion completion; 1368 struct i915_vma *vma; 1369 }; 1370 1371 static int evict_vma(void *data) 1372 { 1373 struct evict_vma *arg = data; 1374 struct i915_address_space *vm = arg->vma->vm; 1375 struct drm_mm_node evict = arg->vma->node; 1376 int err; 1377 1378 complete(&arg->completion); 1379 1380 mutex_lock(&vm->mutex); 1381 err = i915_gem_evict_for_node(vm, &evict, 0); 1382 mutex_unlock(&vm->mutex); 1383 1384 return err; 1385 } 1386 1387 static int evict_fence(void *data) 1388 { 1389 struct evict_vma *arg = data; 1390 int err; 1391 1392 complete(&arg->completion); 1393 1394 /* Mark the fence register as dirty to force the mmio update. */ 1395 err = i915_gem_object_set_tiling(arg->vma->obj, I915_TILING_Y, 512); 1396 if (err) { 1397 pr_err("Invalid Y-tiling settings; err:%d\n", err); 1398 return err; 1399 } 1400 1401 err = i915_vma_pin(arg->vma, 0, 0, PIN_GLOBAL | PIN_MAPPABLE); 1402 if (err) { 1403 pr_err("Unable to pin vma for Y-tiled fence; err:%d\n", err); 1404 return err; 1405 } 1406 1407 err = i915_vma_pin_fence(arg->vma); 1408 i915_vma_unpin(arg->vma); 1409 if (err) { 1410 pr_err("Unable to pin Y-tiled fence; err:%d\n", err); 1411 return err; 1412 } 1413 1414 i915_vma_unpin_fence(arg->vma); 1415 1416 return 0; 1417 } 1418 1419 static int __igt_reset_evict_vma(struct intel_gt *gt, 1420 struct i915_address_space *vm, 1421 int (*fn)(void *), 1422 unsigned int flags) 1423 { 1424 struct intel_engine_cs *engine = gt->engine[RCS0]; 1425 struct drm_i915_gem_object *obj; 1426 struct task_struct *tsk = NULL; 1427 struct i915_request *rq; 1428 struct evict_vma arg; 1429 struct hang h; 1430 unsigned int pin_flags; 1431 int err; 1432 1433 if (!gt->ggtt->num_fences && flags & EXEC_OBJECT_NEEDS_FENCE) 1434 return 0; 1435 1436 if (!engine || !intel_engine_can_store_dword(engine)) 1437 return 0; 1438 1439 /* Check that we can recover an unbind stuck on a hanging request */ 1440 1441 err = hang_init(&h, gt); 1442 if (err) { 1443 pr_err("[%s] Hang init failed: %d!\n", engine->name, err); 1444 return err; 1445 } 1446 1447 obj = i915_gem_object_create_internal(gt->i915, SZ_1M); 1448 if (IS_ERR(obj)) { 1449 err = PTR_ERR(obj); 1450 pr_err("[%s] Create object failed: %d!\n", engine->name, err); 1451 goto fini; 1452 } 1453 1454 if (flags & EXEC_OBJECT_NEEDS_FENCE) { 1455 err = i915_gem_object_set_tiling(obj, I915_TILING_X, 512); 1456 if (err) { 1457 pr_err("Invalid X-tiling settings; err:%d\n", err); 1458 goto out_obj; 1459 } 1460 } 1461 1462 arg.vma = i915_vma_instance(obj, vm, NULL); 1463 if (IS_ERR(arg.vma)) { 1464 err = PTR_ERR(arg.vma); 1465 pr_err("[%s] VMA instance failed: %d!\n", engine->name, err); 1466 goto out_obj; 1467 } 1468 1469 rq = hang_create_request(&h, engine); 1470 if (IS_ERR(rq)) { 1471 err = PTR_ERR(rq); 1472 pr_err("[%s] Create hang request failed: %d!\n", engine->name, err); 1473 goto out_obj; 1474 } 1475 1476 pin_flags = i915_vma_is_ggtt(arg.vma) ? PIN_GLOBAL : PIN_USER; 1477 1478 if (flags & EXEC_OBJECT_NEEDS_FENCE) 1479 pin_flags |= PIN_MAPPABLE; 1480 1481 err = i915_vma_pin(arg.vma, 0, 0, pin_flags); 1482 if (err) { 1483 i915_request_add(rq); 1484 pr_err("[%s] VMA pin failed: %d!\n", engine->name, err); 1485 goto out_obj; 1486 } 1487 1488 if (flags & EXEC_OBJECT_NEEDS_FENCE) { 1489 err = i915_vma_pin_fence(arg.vma); 1490 if (err) { 1491 pr_err("Unable to pin X-tiled fence; err:%d\n", err); 1492 i915_vma_unpin(arg.vma); 1493 i915_request_add(rq); 1494 goto out_obj; 1495 } 1496 } 1497 1498 i915_vma_lock(arg.vma); 1499 err = i915_request_await_object(rq, arg.vma->obj, 1500 flags & EXEC_OBJECT_WRITE); 1501 if (err == 0) { 1502 err = i915_vma_move_to_active(arg.vma, rq, flags); 1503 if (err) 1504 pr_err("[%s] Move to active failed: %d!\n", engine->name, err); 1505 } else { 1506 pr_err("[%s] Request await failed: %d!\n", engine->name, err); 1507 } 1508 1509 i915_vma_unlock(arg.vma); 1510 1511 if (flags & EXEC_OBJECT_NEEDS_FENCE) 1512 i915_vma_unpin_fence(arg.vma); 1513 i915_vma_unpin(arg.vma); 1514 1515 i915_request_get(rq); 1516 i915_request_add(rq); 1517 if (err) 1518 goto out_rq; 1519 1520 if (!wait_until_running(&h, rq)) { 1521 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 1522 1523 pr_err("%s: Failed to start request %llx, at %x\n", 1524 __func__, rq->fence.seqno, hws_seqno(&h, rq)); 1525 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name); 1526 1527 intel_gt_set_wedged(gt); 1528 goto out_reset; 1529 } 1530 1531 init_completion(&arg.completion); 1532 1533 tsk = kthread_run(fn, &arg, "igt/evict_vma"); 1534 if (IS_ERR(tsk)) { 1535 err = PTR_ERR(tsk); 1536 pr_err("[%s] Thread spawn failed: %d!\n", engine->name, err); 1537 tsk = NULL; 1538 goto out_reset; 1539 } 1540 get_task_struct(tsk); 1541 1542 wait_for_completion(&arg.completion); 1543 1544 if (wait_for(!list_empty(&rq->fence.cb_list), 10)) { 1545 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 1546 1547 pr_err("igt/evict_vma kthread did not wait\n"); 1548 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name); 1549 1550 intel_gt_set_wedged(gt); 1551 goto out_reset; 1552 } 1553 1554 out_reset: 1555 igt_global_reset_lock(gt); 1556 fake_hangcheck(gt, rq->engine->mask); 1557 igt_global_reset_unlock(gt); 1558 1559 if (tsk) { 1560 struct intel_wedge_me w; 1561 1562 /* The reset, even indirectly, should take less than 10ms. */ 1563 intel_wedge_on_timeout(&w, gt, HZ / 10 /* 100ms */) 1564 err = kthread_stop(tsk); 1565 1566 put_task_struct(tsk); 1567 } 1568 1569 out_rq: 1570 i915_request_put(rq); 1571 out_obj: 1572 i915_gem_object_put(obj); 1573 fini: 1574 hang_fini(&h); 1575 if (intel_gt_is_wedged(gt)) 1576 return -EIO; 1577 1578 return err; 1579 } 1580 1581 static int igt_reset_evict_ggtt(void *arg) 1582 { 1583 struct intel_gt *gt = arg; 1584 1585 return __igt_reset_evict_vma(gt, >->ggtt->vm, 1586 evict_vma, EXEC_OBJECT_WRITE); 1587 } 1588 1589 static int igt_reset_evict_ppgtt(void *arg) 1590 { 1591 struct intel_gt *gt = arg; 1592 struct i915_ppgtt *ppgtt; 1593 int err; 1594 1595 /* aliasing == global gtt locking, covered above */ 1596 if (INTEL_PPGTT(gt->i915) < INTEL_PPGTT_FULL) 1597 return 0; 1598 1599 ppgtt = i915_ppgtt_create(gt); 1600 if (IS_ERR(ppgtt)) 1601 return PTR_ERR(ppgtt); 1602 1603 err = __igt_reset_evict_vma(gt, &ppgtt->vm, 1604 evict_vma, EXEC_OBJECT_WRITE); 1605 i915_vm_put(&ppgtt->vm); 1606 1607 return err; 1608 } 1609 1610 static int igt_reset_evict_fence(void *arg) 1611 { 1612 struct intel_gt *gt = arg; 1613 1614 return __igt_reset_evict_vma(gt, >->ggtt->vm, 1615 evict_fence, EXEC_OBJECT_NEEDS_FENCE); 1616 } 1617 1618 static int wait_for_others(struct intel_gt *gt, 1619 struct intel_engine_cs *exclude) 1620 { 1621 struct intel_engine_cs *engine; 1622 enum intel_engine_id id; 1623 1624 for_each_engine(engine, gt, id) { 1625 if (engine == exclude) 1626 continue; 1627 1628 if (!wait_for_idle(engine)) 1629 return -EIO; 1630 } 1631 1632 return 0; 1633 } 1634 1635 static int igt_reset_queue(void *arg) 1636 { 1637 struct intel_gt *gt = arg; 1638 struct i915_gpu_error *global = >->i915->gpu_error; 1639 struct intel_engine_cs *engine; 1640 enum intel_engine_id id; 1641 struct hang h; 1642 int err; 1643 1644 /* Check that we replay pending requests following a hang */ 1645 1646 igt_global_reset_lock(gt); 1647 1648 err = hang_init(&h, gt); 1649 if (err) 1650 goto unlock; 1651 1652 for_each_engine(engine, gt, id) { 1653 struct intel_selftest_saved_policy saved; 1654 struct i915_request *prev; 1655 IGT_TIMEOUT(end_time); 1656 unsigned int count; 1657 bool using_guc = intel_engine_uses_guc(engine); 1658 1659 if (!intel_engine_can_store_dword(engine)) 1660 continue; 1661 1662 if (using_guc) { 1663 err = intel_selftest_modify_policy(engine, &saved, 1664 SELFTEST_SCHEDULER_MODIFY_NO_HANGCHECK); 1665 if (err) { 1666 pr_err("[%s] Modify policy failed: %d!\n", engine->name, err); 1667 goto fini; 1668 } 1669 } 1670 1671 prev = hang_create_request(&h, engine); 1672 if (IS_ERR(prev)) { 1673 err = PTR_ERR(prev); 1674 pr_err("[%s] Create 'prev' hang request failed: %d!\n", engine->name, err); 1675 goto restore; 1676 } 1677 1678 i915_request_get(prev); 1679 i915_request_add(prev); 1680 1681 count = 0; 1682 do { 1683 struct i915_request *rq; 1684 unsigned int reset_count; 1685 1686 rq = hang_create_request(&h, engine); 1687 if (IS_ERR(rq)) { 1688 err = PTR_ERR(rq); 1689 pr_err("[%s] Create hang request failed: %d!\n", engine->name, err); 1690 goto restore; 1691 } 1692 1693 i915_request_get(rq); 1694 i915_request_add(rq); 1695 1696 /* 1697 * XXX We don't handle resetting the kernel context 1698 * very well. If we trigger a device reset twice in 1699 * quick succession while the kernel context is 1700 * executing, we may end up skipping the breadcrumb. 1701 * This is really only a problem for the selftest as 1702 * normally there is a large interlude between resets 1703 * (hangcheck), or we focus on resetting just one 1704 * engine and so avoid repeatedly resetting innocents. 1705 */ 1706 err = wait_for_others(gt, engine); 1707 if (err) { 1708 pr_err("%s(%s): Failed to idle other inactive engines after device reset\n", 1709 __func__, engine->name); 1710 i915_request_put(rq); 1711 i915_request_put(prev); 1712 1713 GEM_TRACE_DUMP(); 1714 intel_gt_set_wedged(gt); 1715 goto restore; 1716 } 1717 1718 if (!wait_until_running(&h, prev)) { 1719 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 1720 1721 pr_err("%s(%s): Failed to start request %llx, at %x\n", 1722 __func__, engine->name, 1723 prev->fence.seqno, hws_seqno(&h, prev)); 1724 intel_engine_dump(engine, &p, 1725 "%s\n", engine->name); 1726 1727 i915_request_put(rq); 1728 i915_request_put(prev); 1729 1730 intel_gt_set_wedged(gt); 1731 1732 err = -EIO; 1733 goto restore; 1734 } 1735 1736 reset_count = fake_hangcheck(gt, BIT(id)); 1737 1738 if (prev->fence.error != -EIO) { 1739 pr_err("GPU reset not recorded on hanging request [fence.error=%d]!\n", 1740 prev->fence.error); 1741 i915_request_put(rq); 1742 i915_request_put(prev); 1743 err = -EINVAL; 1744 goto restore; 1745 } 1746 1747 if (rq->fence.error) { 1748 pr_err("Fence error status not zero [%d] after unrelated reset\n", 1749 rq->fence.error); 1750 i915_request_put(rq); 1751 i915_request_put(prev); 1752 err = -EINVAL; 1753 goto restore; 1754 } 1755 1756 if (i915_reset_count(global) == reset_count) { 1757 pr_err("No GPU reset recorded!\n"); 1758 i915_request_put(rq); 1759 i915_request_put(prev); 1760 err = -EINVAL; 1761 goto restore; 1762 } 1763 1764 i915_request_put(prev); 1765 prev = rq; 1766 count++; 1767 } while (time_before(jiffies, end_time)); 1768 pr_info("%s: Completed %d queued resets\n", 1769 engine->name, count); 1770 1771 *h.batch = MI_BATCH_BUFFER_END; 1772 intel_gt_chipset_flush(engine->gt); 1773 1774 i915_request_put(prev); 1775 1776 restore: 1777 if (using_guc) { 1778 int err2 = intel_selftest_restore_policy(engine, &saved); 1779 1780 if (err2) 1781 pr_err("%s:%d> [%s] Restore policy failed: %d!\n", 1782 __func__, __LINE__, engine->name, err2); 1783 if (err == 0) 1784 err = err2; 1785 } 1786 if (err) 1787 goto fini; 1788 1789 err = igt_flush_test(gt->i915); 1790 if (err) { 1791 pr_err("[%s] Flush failed: %d!\n", engine->name, err); 1792 break; 1793 } 1794 } 1795 1796 fini: 1797 hang_fini(&h); 1798 unlock: 1799 igt_global_reset_unlock(gt); 1800 1801 if (intel_gt_is_wedged(gt)) 1802 return -EIO; 1803 1804 return err; 1805 } 1806 1807 static int igt_handle_error(void *arg) 1808 { 1809 struct intel_gt *gt = arg; 1810 struct i915_gpu_error *global = >->i915->gpu_error; 1811 struct intel_engine_cs *engine = gt->engine[RCS0]; 1812 struct hang h; 1813 struct i915_request *rq; 1814 struct i915_gpu_coredump *error; 1815 int err; 1816 1817 /* Check that we can issue a global GPU and engine reset */ 1818 1819 if (!intel_has_reset_engine(gt)) 1820 return 0; 1821 1822 if (!engine || !intel_engine_can_store_dword(engine)) 1823 return 0; 1824 1825 err = hang_init(&h, gt); 1826 if (err) { 1827 pr_err("[%s] Hang init failed: %d!\n", engine->name, err); 1828 return err; 1829 } 1830 1831 rq = hang_create_request(&h, engine); 1832 if (IS_ERR(rq)) { 1833 err = PTR_ERR(rq); 1834 pr_err("[%s] Create hang request failed: %d!\n", engine->name, err); 1835 goto err_fini; 1836 } 1837 1838 i915_request_get(rq); 1839 i915_request_add(rq); 1840 1841 if (!wait_until_running(&h, rq)) { 1842 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 1843 1844 pr_err("%s: Failed to start request %llx, at %x\n", 1845 __func__, rq->fence.seqno, hws_seqno(&h, rq)); 1846 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name); 1847 1848 intel_gt_set_wedged(gt); 1849 1850 err = -EIO; 1851 goto err_request; 1852 } 1853 1854 /* Temporarily disable error capture */ 1855 error = xchg(&global->first_error, (void *)-1); 1856 1857 intel_gt_handle_error(gt, engine->mask, 0, NULL); 1858 1859 xchg(&global->first_error, error); 1860 1861 if (rq->fence.error != -EIO) { 1862 pr_err("Guilty request not identified!\n"); 1863 err = -EINVAL; 1864 goto err_request; 1865 } 1866 1867 err_request: 1868 i915_request_put(rq); 1869 err_fini: 1870 hang_fini(&h); 1871 return err; 1872 } 1873 1874 static int __igt_atomic_reset_engine(struct intel_engine_cs *engine, 1875 const struct igt_atomic_section *p, 1876 const char *mode) 1877 { 1878 struct tasklet_struct * const t = &engine->sched_engine->tasklet; 1879 int err; 1880 1881 GEM_TRACE("i915_reset_engine(%s:%s) under %s\n", 1882 engine->name, mode, p->name); 1883 1884 if (t->func) 1885 tasklet_disable(t); 1886 if (strcmp(p->name, "softirq")) 1887 local_bh_disable(); 1888 p->critical_section_begin(); 1889 1890 err = __intel_engine_reset_bh(engine, NULL); 1891 1892 p->critical_section_end(); 1893 if (strcmp(p->name, "softirq")) 1894 local_bh_enable(); 1895 if (t->func) { 1896 tasklet_enable(t); 1897 tasklet_hi_schedule(t); 1898 } 1899 1900 if (err) 1901 pr_err("i915_reset_engine(%s:%s) failed under %s\n", 1902 engine->name, mode, p->name); 1903 1904 return err; 1905 } 1906 1907 static int igt_atomic_reset_engine(struct intel_engine_cs *engine, 1908 const struct igt_atomic_section *p) 1909 { 1910 struct i915_request *rq; 1911 struct hang h; 1912 int err; 1913 1914 err = __igt_atomic_reset_engine(engine, p, "idle"); 1915 if (err) 1916 return err; 1917 1918 err = hang_init(&h, engine->gt); 1919 if (err) { 1920 pr_err("[%s] Hang init failed: %d!\n", engine->name, err); 1921 return err; 1922 } 1923 1924 rq = hang_create_request(&h, engine); 1925 if (IS_ERR(rq)) { 1926 err = PTR_ERR(rq); 1927 pr_err("[%s] Create hang request failed: %d!\n", engine->name, err); 1928 goto out; 1929 } 1930 1931 i915_request_get(rq); 1932 i915_request_add(rq); 1933 1934 if (wait_until_running(&h, rq)) { 1935 err = __igt_atomic_reset_engine(engine, p, "active"); 1936 } else { 1937 pr_err("%s(%s): Failed to start request %llx, at %x\n", 1938 __func__, engine->name, 1939 rq->fence.seqno, hws_seqno(&h, rq)); 1940 intel_gt_set_wedged(engine->gt); 1941 err = -EIO; 1942 } 1943 1944 if (err == 0) { 1945 struct intel_wedge_me w; 1946 1947 intel_wedge_on_timeout(&w, engine->gt, HZ / 20 /* 50ms */) 1948 i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT); 1949 if (intel_gt_is_wedged(engine->gt)) 1950 err = -EIO; 1951 } 1952 1953 i915_request_put(rq); 1954 out: 1955 hang_fini(&h); 1956 return err; 1957 } 1958 1959 static int igt_reset_engines_atomic(void *arg) 1960 { 1961 struct intel_gt *gt = arg; 1962 const typeof(*igt_atomic_phases) *p; 1963 int err = 0; 1964 1965 /* Check that the engines resets are usable from atomic context */ 1966 1967 if (!intel_has_reset_engine(gt)) 1968 return 0; 1969 1970 if (intel_uc_uses_guc_submission(>->uc)) 1971 return 0; 1972 1973 igt_global_reset_lock(gt); 1974 1975 /* Flush any requests before we get started and check basics */ 1976 if (!igt_force_reset(gt)) 1977 goto unlock; 1978 1979 for (p = igt_atomic_phases; p->name; p++) { 1980 struct intel_engine_cs *engine; 1981 enum intel_engine_id id; 1982 1983 for_each_engine(engine, gt, id) { 1984 err = igt_atomic_reset_engine(engine, p); 1985 if (err) 1986 goto out; 1987 } 1988 } 1989 1990 out: 1991 /* As we poke around the guts, do a full reset before continuing. */ 1992 igt_force_reset(gt); 1993 unlock: 1994 igt_global_reset_unlock(gt); 1995 1996 return err; 1997 } 1998 1999 int intel_hangcheck_live_selftests(struct drm_i915_private *i915) 2000 { 2001 static const struct i915_subtest tests[] = { 2002 SUBTEST(igt_hang_sanitycheck), 2003 SUBTEST(igt_reset_nop), 2004 SUBTEST(igt_reset_nop_engine), 2005 SUBTEST(igt_reset_idle_engine), 2006 SUBTEST(igt_reset_active_engine), 2007 SUBTEST(igt_reset_fail_engine), 2008 SUBTEST(igt_reset_engines), 2009 SUBTEST(igt_reset_engines_atomic), 2010 SUBTEST(igt_reset_queue), 2011 SUBTEST(igt_reset_wait), 2012 SUBTEST(igt_reset_evict_ggtt), 2013 SUBTEST(igt_reset_evict_ppgtt), 2014 SUBTEST(igt_reset_evict_fence), 2015 SUBTEST(igt_handle_error), 2016 }; 2017 struct intel_gt *gt = &i915->gt; 2018 intel_wakeref_t wakeref; 2019 int err; 2020 2021 if (!intel_has_gpu_reset(gt)) 2022 return 0; 2023 2024 if (intel_gt_is_wedged(gt)) 2025 return -EIO; /* we're long past hope of a successful reset */ 2026 2027 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 2028 2029 err = intel_gt_live_subtests(tests, gt); 2030 2031 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 2032 2033 return err; 2034 } 2035